No OneTemporary
Actions

Size

6 MB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
	index 9e2baf525f22..6db60ca13866 100644
	--- a/ObsoleteFiles.inc
	+++ b/ObsoleteFiles.inc
	@@ -1,11300 +1,11305 @@
	#
	# $FreeBSD$
	#
	# This file lists old files (OLD_FILES), libraries (OLD_LIBS) and
	# directories (OLD_DIRS) which should get removed at an update. Recently
	# removed entries first (with the date as a comment). Dynamic libraries are
	# special cased (OLD_LIBS). Static libraries or the generic links to
	# the dynamic libraries (lib*.so) should (if you don't know why to make an
	# exception, make this a "must") be viewed as normal files (OLD_FILES).
	#
	# In case of a complete directory hierarchy the sorting is in depth first
	# order.
	#
	# Before you commit changes to this file please check if any entries in
	# tools/build/mk/OptionalObsoleteFiles.inc can be removed. The following
	# command tells which files are listed more than once regardless of some
	# architecture specific conditionals, so you can not blindly trust the
	# output:
	# ( grep '+=' /usr/src/ObsoleteFiles.inc \| sort -u ; \
	# grep '+=' /usr/src/tools/build/mk/OptionalObsoleteFiles.inc \| sort -u) \| \
	# sort \| uniq -d
	#
	# To find regular duplicates not dependent on optional components, you can
	# also use something that will not give you false positives, e.g.:
	# for t in `make -V TARGETS universe`; do
	# __MAKE_CONF=/dev/null make -f Makefile.inc1 TARGET=$t \
	# -V OLD_FILES -V OLD_LIBS -V OLD_DIRS check-old \| \
	# xargs -n1 \| sort \| uniq -d;
	# done
	#
	# For optional components, you can use the following to see if some entries
	# in OptionalObsoleteFiles.inc have been obsoleted by ObsoleteFiles.inc
	# for o in tools/build/options/WITH*; do
	# __MAKE_CONF=/dev/null make -f Makefile.inc1 -D${o##*/} \
	# -V OLD_FILES -V OLD_LIBS -V OLD_DIRS check-old \| \
	# xargs -n1 \| sort \| uniq -d;
	# done

	+# 20200729: remove long expired serial drivers
	+OLD_FILES+=usr/share/man/man4/cy.4.gz
	+OLD_FILES+=usr/share/man/man4/rc.4.gz
	+OLD_FILES+=usr/share/man/man4/rp.4.gz
	+
	# 20200715: rework of devstat(9) man page
	OLD_FILES+=usr/share/man/man9/devstat_add_entry.9.gz

	# 20200714: update byacc to 20200330
	OLD_FILES+=usr/tests/usr.bin/yacc/btyacc_calc1.y
	OLD_FILES+=usr/tests/usr.bin/yacc/btyacc_demo.y
	OLD_FILES+=usr/tests/usr.bin/yacc/btyacc_destroy1.y
	OLD_FILES+=usr/tests/usr.bin/yacc/btyacc_destroy2.y
	OLD_FILES+=usr/tests/usr.bin/yacc/btyacc_destroy3.y
	OLD_FILES+=usr/tests/usr.bin/yacc/err_inherit1.y
	OLD_FILES+=usr/tests/usr.bin/yacc/err_inherit2.y
	OLD_FILES+=usr/tests/usr.bin/yacc/err_inherit3.y
	OLD_FILES+=usr/tests/usr.bin/yacc/err_inherit4.y
	OLD_FILES+=usr/tests/usr.bin/yacc/err_inherit5.y
	OLD_FILES+=usr/tests/usr.bin/yacc/inherit0.y
	OLD_FILES+=usr/tests/usr.bin/yacc/inherit1.y
	OLD_FILES+=usr/tests/usr.bin/yacc/inherit2.y

	# 20200706: update of sglist(9), r360574
	OLD_FILES+=usr/share/man/man9/sglist_append_ext_pgs.9.gz
	OLD_FILES+=usr/share/man/man9/sglist_append_mb_ext_pgs.9.gz
	OLD_FILES+=usr/share/man/man9/sglist_count_ext_pgs.9.gz
	OLD_FILES+=usr/share/man/man9/sglist_count_mb_ext_pgs.9.gz

	# 20200617: update opencsd to 0.14.2
	OLD_FILES+=usr/include/opencsd/etmv4/trc_pkt_elem_etmv4d.h

	# 20200606: retire binutils build infrastructure
	.if !defined(WITH_PORT_BASE_BINUTILS)
	OLD_FILES+=usr/bin/as
	OLD_FILES+=usr/bin/ld.bfd
	OLD_FILES+=usr/share/man/man1/as.1.gz
	OLD_FILES+=usr/share/man/man7/as.7.gz
	OLD_FILES+=usr/share/man/man7/ld.7.gz
	OLD_FILES+=usr/share/man/man7/ldint.7.gz
	OLD_FILES+=usr/share/man/man7/binutils.7.gz
	.endif
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/armelf_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/armelfb_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.x
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xc
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xd
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xn
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xr
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xs
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xu
	OLD_FILES+=usr/libdata/ldscripts/elf32_sparc.xw
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmip_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf32btsmipn32_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmip_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf32ltsmipn32_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf32ppc_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.x
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xc
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xd
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xn
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xr
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xs
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xu
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc.xw
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf64_sparc_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf64btsmip_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf64ltsmip_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf64ppc_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf_i386_fbsd.xw
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.x
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xbn
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xc
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xd
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xdc
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xdw
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xn
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xr
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xs
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xsc
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xsw
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xu
	OLD_FILES+=usr/libdata/ldscripts/elf_x86_64_fbsd.xw

	# 20200601: OpenSSL 32-bit compat engines moved to /usr/lib32/engines
	OLD_LIBS+=usr/lib32/capi.so
	OLD_LIBS+=usr/lib32/padlock.so

	# 20200528: libevent renamed libevent1
	OLD_FILES+=usr/include/private/event/event.h
	OLD_DIRS+=usr/include/private/event

	# 20200523: new clang import which bumps version from 10.0.0 to 10.0.1.s
	OLD_FILES+=usr/lib/clang/10.0.0/include/cuda_wrappers/algorithm
	OLD_FILES+=usr/lib/clang/10.0.0/include/cuda_wrappers/complex
	OLD_FILES+=usr/lib/clang/10.0.0/include/cuda_wrappers/new
	OLD_DIRS+=usr/lib/clang/10.0.0/include/cuda_wrappers
	OLD_FILES+=usr/lib/clang/10.0.0/include/fuzzer/FuzzedDataProvider.h
	OLD_DIRS+=usr/lib/clang/10.0.0/include/fuzzer
	OLD_FILES+=usr/lib/clang/10.0.0/include/openmp_wrappers/__clang_openmp_math.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/openmp_wrappers/__clang_openmp_math_declares.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/openmp_wrappers/cmath
	OLD_FILES+=usr/lib/clang/10.0.0/include/openmp_wrappers/math.h
	OLD_DIRS+=usr/lib/clang/10.0.0/include/openmp_wrappers
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/emmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/mm_malloc.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/mmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/pmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/smmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/tmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ppc_wrappers/xmmintrin.h
	OLD_DIRS+=usr/lib/clang/10.0.0/include/ppc_wrappers
	OLD_FILES+=usr/lib/clang/10.0.0/include/profile/InstrProfData.inc
	OLD_DIRS+=usr/lib/clang/10.0.0/include/profile
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/netbsd_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/tsan_interface_atomic.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sanitizer/ubsan_interface.h
	OLD_DIRS+=usr/lib/clang/10.0.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/10.0.0/include/xray/xray_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xray/xray_log_interface.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xray/xray_records.h
	OLD_DIRS+=usr/lib/clang/10.0.0/include/xray
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_device_functions.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_libdevice_declares.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/arm_cmse.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/arm_fp16.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/arm_mve.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/armintr.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512bf16intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlbf16intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vlvp2intersectintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vp2intersectintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/cldemoteintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/enqcmdintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/invpcidintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/10.0.0/include/movdirintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/msa.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/opencl-c-base.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/pconfigintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/ptwriteintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/sgxintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/waitpkgintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/wbnoinvdintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/10.0.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/10.0.0/include
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-aarch64.so
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-arm.so
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-armhf.so
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-preinit-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-preinit-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-preinit-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan_cxx-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan_cxx-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi_diag-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi_diag-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi_diag-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi_diag-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.cfi_diag-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.dd-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.dd-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.fuzzer-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.fuzzer-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.fuzzer_no_main-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.fuzzer_no_main-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.msan-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.msan-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.msan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-powerpc.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-powerpc64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.safestack-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats_client-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats_client-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats_client-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.tsan-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.tsan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_minimal-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_minimal-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_minimal-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-basic-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-basic-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-basic-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-basic-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-fdr-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-fdr-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-fdr-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-fdr-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-profiling-aarch64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-profiling-arm.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-profiling-armhf.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-profiling-x86_64.a
	OLD_FILES+=usr/lib/clang/10.0.0/lib/freebsd/libclang_rt.xray-x86_64.a
	OLD_DIRS+=usr/lib/clang/10.0.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/10.0.0/lib
	OLD_DIRS+=usr/lib/clang/10.0.0

	# 20200515: libalias cuseeme protocol support retired
	OLD_LIBS+=lib/libalias_cuseeme.so
	OLD_FILES+=usr/lib/libalias_cuseeme.a
	OLD_FILES+=usr/lib/libalias_cuseeme_p.a

	# 20200511: Remove ubsec(4)
	OLD_FILES+=usr/share/man/man4/ubsec.4.gz

	# 20200506: GNU objdump 2.17.50 retired
	OLD_FILES+=usr/bin/objdump
	OLD_FILES+=usr/share/man/man1/objdump.1.gz

	# 20200428: route_var.h moved to net/route
	OLD_FILES+=usr/include/net/route_var.h

	# 20200418: Make libauditd private
	OLD_FILES+=usr/lib/libauditd.a
	OLD_FILES+=usr/lib/libauditd.so
	OLD_LIBS+=usr/lib/libauditd.so.5
	OLD_FILES+=usr/lib/libauditd_p.a

	# 20200418: Remove bogus man links
	OLD_FILES+=usr/share/man/man3/getauusernam_R.3.gz
	OLD_FILES+=usr/share/man/man3/getauclassnam_3.3.gz

	# 20200414: NFS file handle affinity code for the NFS server re-organized
	OLD_FILES+=usr/include/nfs/nfs_fha.h

	# 20200401: Remove procfs-based process debugging
	OLD_FILES+=usr/include/sys/pioctl.h

	# 20200330: GDB_LIBEXEC option retired (always true)
	OLD_FILES+=usr/bin/gdb
	OLD_FILES+=usr/bin/gdbserver
	OLD_FILES+=usr/bin/kgdb
	OLD_FILES+=usr/share/man/man1/gdb.1.gz
	OLD_FILES+=usr/share/man/man1/gdbserver.1.gz
	OLD_FILES+=usr/share/man/man1/kgdb.1.gz

	# 20200327: OCF refactoring
	OLD_FILES+=usr/share/man/man9/crypto_find_driver.9
	OLD_FILES+=usr/share/man/man9/crypto_register.9
	OLD_FILES+=usr/share/man/man9/crypto_unregister.9

	# 20200323: INTERNALLIB don't install headers anymore
	OLD_FILES+=usr/include/libelftc.h
	OLD_FILES+=usr/include/libifconfig.h
	OLD_FILES+=usr/include/libpmcstat.h

	# 20200320: cx and ctau drivers retired
	OLD_FILES+=usr/share/man/man4/ctau.4.gz
	OLD_FILES+=usr/share/man/man4/cx.4.gz

	# 20200318: host.conf was deprecated a long time ago.
	OLD_FILES+=etc/host.conf
	OLD_FILES+=etc/rc.d/nsswitch

	# 20200310: new clang import which bumps version from 9.0.1 to 10.0.0.
	OLD_FILES+=usr/lib/clang/9.0.1/include/cuda_wrappers/algorithm
	OLD_FILES+=usr/lib/clang/9.0.1/include/cuda_wrappers/complex
	OLD_FILES+=usr/lib/clang/9.0.1/include/cuda_wrappers/new
	OLD_DIRS+=usr/lib/clang/9.0.1/include/cuda_wrappers
	OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/__clang_openmp_math.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/__clang_openmp_math_declares.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/cmath
	OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/math.h
	OLD_DIRS+=usr/lib/clang/9.0.1/include/openmp_wrappers
	OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/emmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/mm_malloc.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/mmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/xmmintrin.h
	OLD_DIRS+=usr/lib/clang/9.0.1/include/ppc_wrappers
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/netbsd_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/9.0.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_device_functions.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_libdevice_declares.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/arm_fp16.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/armintr.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512bf16intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlbf16intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlvp2intersectintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vp2intersectintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/cldemoteintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/enqcmdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/invpcidintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/9.0.1/include/movdirintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/msa.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/opencl-c-base.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/pconfigintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/ptwriteintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/sgxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/waitpkgintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/wbnoinvdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/9.0.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/9.0.1/include
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-aarch64.so
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-arm.so
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-armhf.so
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.dd-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.dd-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer_no_main-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer_no_main-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-powerpc.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-powerpc64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.safestack-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-arm.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-x86_64.a
	OLD_DIRS+=usr/lib/clang/9.0.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/9.0.1/lib
	OLD_DIRS+=usr/lib/clang/9.0.1

	# 20200309: amd(8) retired
	OLD_FILES+=etc/amd.map
	OLD_FILES+=etc/newsyslog.conf.d/amd.conf
	OLD_FILES+=etc/rc.d/amd
	OLD_FILES+=usr/bin/pawd
	OLD_FILES+=usr/sbin/amd
	OLD_FILES+=usr/sbin/amq
	OLD_FILES+=usr/sbin/fixmount
	OLD_FILES+=usr/sbin/fsinfo
	OLD_FILES+=usr/sbin/hlfsd
	OLD_FILES+=usr/sbin/mk-amd-map
	OLD_FILES+=usr/sbin/wire-test
	OLD_FILES+=usr/share/examples/etc/amd.map
	OLD_FILES+=usr/share/man/man1/pawd.1.gz
	OLD_FILES+=usr/share/man/man5/amd.conf.5.gz
	OLD_FILES+=usr/share/man/man8/amd.8.gz
	OLD_FILES+=usr/share/man/man8/amq.8.gz
	OLD_FILES+=usr/share/man/man8/fixmount.8.gz
	OLD_FILES+=usr/share/man/man8/fsinfo.8.gz
	OLD_FILES+=usr/share/man/man8/hlfsd.8.gz
	OLD_FILES+=usr/share/man/man8/mk-amd-map.8.gz
	OLD_FILES+=usr/share/man/man8/wire-test.8.gz

	# 20200301: bktr removed
	OLD_DIRS+=usr/include/dev/bktr
	OLD_FILES+=usr/include/dev/bktr/ioctl_bktr.h
	OLD_FILES+=usr/include/dev/bktr/ioctl_bt848.h
	OLD_FILES+=usr/include/dev/bktr/ioctl_meteor.h
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/machine/ioctl_bktr.h
	OLD_FILES+=usr/include/machine/ioctl_meteor.h
	.endif
	OLD_FILES+=usr/share/man/man4/bktr.4.gz

	# 20200229: GCC 4.2.1 removed
	.if !defined(WITH_PORT_BASE_GCC)
	OLD_FILES+=usr/bin/g++
	OLD_FILES+=usr/bin/gcc
	OLD_FILES+=usr/share/man/man1/g++.1.gz
	OLD_FILES+=usr/share/man/man1/gcc.1.gz
	.endif
	OLD_FILES+=usr/bin/gcpp
	OLD_FILES+=usr/bin/gperf
	OLD_FILES+=usr/include/c++/4.2/algorithm
	OLD_FILES+=usr/include/c++/4.2/backward/algo.h
	OLD_FILES+=usr/include/c++/4.2/backward/algobase.h
	OLD_FILES+=usr/include/c++/4.2/backward/alloc.h
	OLD_FILES+=usr/include/c++/4.2/backward/backward_warning.h
	OLD_FILES+=usr/include/c++/4.2/backward/bvector.h
	OLD_FILES+=usr/include/c++/4.2/backward/complex.h
	OLD_FILES+=usr/include/c++/4.2/backward/defalloc.h
	OLD_FILES+=usr/include/c++/4.2/backward/deque.h
	OLD_FILES+=usr/include/c++/4.2/backward/fstream.h
	OLD_FILES+=usr/include/c++/4.2/backward/function.h
	OLD_FILES+=usr/include/c++/4.2/backward/hash_map.h
	OLD_FILES+=usr/include/c++/4.2/backward/hash_set.h
	OLD_FILES+=usr/include/c++/4.2/backward/hashtable.h
	OLD_FILES+=usr/include/c++/4.2/backward/heap.h
	OLD_FILES+=usr/include/c++/4.2/backward/iomanip.h
	OLD_FILES+=usr/include/c++/4.2/backward/iostream.h
	OLD_FILES+=usr/include/c++/4.2/backward/istream.h
	OLD_FILES+=usr/include/c++/4.2/backward/iterator.h
	OLD_FILES+=usr/include/c++/4.2/backward/list.h
	OLD_FILES+=usr/include/c++/4.2/backward/map.h
	OLD_FILES+=usr/include/c++/4.2/backward/multimap.h
	OLD_FILES+=usr/include/c++/4.2/backward/multiset.h
	OLD_FILES+=usr/include/c++/4.2/backward/new.h
	OLD_FILES+=usr/include/c++/4.2/backward/ostream.h
	OLD_FILES+=usr/include/c++/4.2/backward/pair.h
	OLD_FILES+=usr/include/c++/4.2/backward/queue.h
	OLD_FILES+=usr/include/c++/4.2/backward/rope.h
	OLD_FILES+=usr/include/c++/4.2/backward/set.h
	OLD_FILES+=usr/include/c++/4.2/backward/slist.h
	OLD_FILES+=usr/include/c++/4.2/backward/stack.h
	OLD_FILES+=usr/include/c++/4.2/backward/stream.h
	OLD_FILES+=usr/include/c++/4.2/backward/streambuf.h
	OLD_FILES+=usr/include/c++/4.2/backward/strstream
	OLD_FILES+=usr/include/c++/4.2/backward/tempbuf.h
	OLD_FILES+=usr/include/c++/4.2/backward/tree.h
	OLD_FILES+=usr/include/c++/4.2/backward/vector.h
	OLD_FILES+=usr/include/c++/4.2/bits/allocator.h
	OLD_FILES+=usr/include/c++/4.2/bits/atomic_word.h
	OLD_FILES+=usr/include/c++/4.2/bits/basic_file.h
	OLD_FILES+=usr/include/c++/4.2/bits/basic_ios.h
	OLD_FILES+=usr/include/c++/4.2/bits/basic_ios.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/basic_string.h
	OLD_FILES+=usr/include/c++/4.2/bits/basic_string.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/boost_concept_check.h
	OLD_FILES+=usr/include/c++/4.2/bits/c++allocator.h
	OLD_FILES+=usr/include/c++/4.2/bits/c++config.h
	OLD_FILES+=usr/include/c++/4.2/bits/c++io.h
	OLD_FILES+=usr/include/c++/4.2/bits/c++locale.h
	OLD_FILES+=usr/include/c++/4.2/bits/c++locale_internal.h
	OLD_FILES+=usr/include/c++/4.2/bits/char_traits.h
	OLD_FILES+=usr/include/c++/4.2/bits/cmath.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/codecvt.h
	OLD_FILES+=usr/include/c++/4.2/bits/compatibility.h
	OLD_FILES+=usr/include/c++/4.2/bits/concept_check.h
	OLD_FILES+=usr/include/c++/4.2/bits/cpp_type_traits.h
	OLD_FILES+=usr/include/c++/4.2/bits/cpu_defines.h
	OLD_FILES+=usr/include/c++/4.2/bits/ctype_base.h
	OLD_FILES+=usr/include/c++/4.2/bits/ctype_inline.h
	OLD_FILES+=usr/include/c++/4.2/bits/ctype_noninline.h
	OLD_FILES+=usr/include/c++/4.2/bits/cxxabi_tweaks.h
	OLD_FILES+=usr/include/c++/4.2/bits/deque.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/fstream.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/functexcept.h
	OLD_FILES+=usr/include/c++/4.2/bits/gslice.h
	OLD_FILES+=usr/include/c++/4.2/bits/gslice_array.h
	OLD_FILES+=usr/include/c++/4.2/bits/gthr-default.h
	OLD_FILES+=usr/include/c++/4.2/bits/gthr-posix.h
	OLD_FILES+=usr/include/c++/4.2/bits/gthr-single.h
	OLD_FILES+=usr/include/c++/4.2/bits/gthr-tpf.h
	OLD_FILES+=usr/include/c++/4.2/bits/gthr.h
	OLD_FILES+=usr/include/c++/4.2/bits/indirect_array.h
	OLD_FILES+=usr/include/c++/4.2/bits/ios_base.h
	OLD_FILES+=usr/include/c++/4.2/bits/istream.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/list.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/locale_classes.h
	OLD_FILES+=usr/include/c++/4.2/bits/locale_facets.h
	OLD_FILES+=usr/include/c++/4.2/bits/locale_facets.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/localefwd.h
	OLD_FILES+=usr/include/c++/4.2/bits/mask_array.h
	OLD_FILES+=usr/include/c++/4.2/bits/messages_members.h
	OLD_FILES+=usr/include/c++/4.2/bits/os_defines.h
	OLD_FILES+=usr/include/c++/4.2/bits/ostream.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/ostream_insert.h
	OLD_FILES+=usr/include/c++/4.2/bits/postypes.h
	OLD_FILES+=usr/include/c++/4.2/bits/slice_array.h
	OLD_FILES+=usr/include/c++/4.2/bits/sstream.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/stl_algo.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_algobase.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_bvector.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_construct.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_deque.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_function.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_heap.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_iterator.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_iterator_base_funcs.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_iterator_base_types.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_list.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_map.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_multimap.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_multiset.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_numeric.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_pair.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_queue.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_raw_storage_iter.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_relops.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_set.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_stack.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_tempbuf.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_tree.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_uninitialized.h
	OLD_FILES+=usr/include/c++/4.2/bits/stl_vector.h
	OLD_FILES+=usr/include/c++/4.2/bits/stream_iterator.h
	OLD_FILES+=usr/include/c++/4.2/bits/streambuf.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/streambuf_iterator.h
	OLD_FILES+=usr/include/c++/4.2/bits/stringfwd.h
	OLD_FILES+=usr/include/c++/4.2/bits/time_members.h
	OLD_FILES+=usr/include/c++/4.2/bits/valarray_after.h
	OLD_FILES+=usr/include/c++/4.2/bits/valarray_array.h
	OLD_FILES+=usr/include/c++/4.2/bits/valarray_array.tcc
	OLD_FILES+=usr/include/c++/4.2/bits/valarray_before.h
	OLD_FILES+=usr/include/c++/4.2/bits/vector.tcc
	OLD_FILES+=usr/include/c++/4.2/bitset
	OLD_FILES+=usr/include/c++/4.2/cassert
	OLD_FILES+=usr/include/c++/4.2/cctype
	OLD_FILES+=usr/include/c++/4.2/cerrno
	OLD_FILES+=usr/include/c++/4.2/cfloat
	OLD_FILES+=usr/include/c++/4.2/ciso646
	OLD_FILES+=usr/include/c++/4.2/climits
	OLD_FILES+=usr/include/c++/4.2/clocale
	OLD_FILES+=usr/include/c++/4.2/cmath
	OLD_FILES+=usr/include/c++/4.2/complex
	OLD_FILES+=usr/include/c++/4.2/csetjmp
	OLD_FILES+=usr/include/c++/4.2/csignal
	OLD_FILES+=usr/include/c++/4.2/cstdarg
	OLD_FILES+=usr/include/c++/4.2/cstddef
	OLD_FILES+=usr/include/c++/4.2/cstdio
	OLD_FILES+=usr/include/c++/4.2/cstdlib
	OLD_FILES+=usr/include/c++/4.2/cstring
	OLD_FILES+=usr/include/c++/4.2/ctime
	OLD_FILES+=usr/include/c++/4.2/cwchar
	OLD_FILES+=usr/include/c++/4.2/cwctype
	OLD_FILES+=usr/include/c++/4.2/cxxabi.h
	OLD_FILES+=usr/include/c++/4.2/debug/bitset
	OLD_FILES+=usr/include/c++/4.2/debug/debug.h
	OLD_FILES+=usr/include/c++/4.2/debug/deque
	OLD_FILES+=usr/include/c++/4.2/debug/formatter.h
	OLD_FILES+=usr/include/c++/4.2/debug/functions.h
	OLD_FILES+=usr/include/c++/4.2/debug/hash_map
	OLD_FILES+=usr/include/c++/4.2/debug/hash_map.h
	OLD_FILES+=usr/include/c++/4.2/debug/hash_multimap.h
	OLD_FILES+=usr/include/c++/4.2/debug/hash_multiset.h
	OLD_FILES+=usr/include/c++/4.2/debug/hash_set
	OLD_FILES+=usr/include/c++/4.2/debug/hash_set.h
	OLD_FILES+=usr/include/c++/4.2/debug/list
	OLD_FILES+=usr/include/c++/4.2/debug/macros.h
	OLD_FILES+=usr/include/c++/4.2/debug/map
	OLD_FILES+=usr/include/c++/4.2/debug/map.h
	OLD_FILES+=usr/include/c++/4.2/debug/multimap.h
	OLD_FILES+=usr/include/c++/4.2/debug/multiset.h
	OLD_FILES+=usr/include/c++/4.2/debug/safe_base.h
	OLD_FILES+=usr/include/c++/4.2/debug/safe_iterator.h
	OLD_FILES+=usr/include/c++/4.2/debug/safe_iterator.tcc
	OLD_FILES+=usr/include/c++/4.2/debug/safe_sequence.h
	OLD_FILES+=usr/include/c++/4.2/debug/set
	OLD_FILES+=usr/include/c++/4.2/debug/set.h
	OLD_FILES+=usr/include/c++/4.2/debug/string
	OLD_FILES+=usr/include/c++/4.2/debug/vector
	OLD_FILES+=usr/include/c++/4.2/deque
	OLD_FILES+=usr/include/c++/4.2/exception
	OLD_FILES+=usr/include/c++/4.2/exception_defines.h
	OLD_FILES+=usr/include/c++/4.2/ext/algorithm
	OLD_FILES+=usr/include/c++/4.2/ext/array_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/atomicity.h
	OLD_FILES+=usr/include/c++/4.2/ext/bitmap_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/codecvt_specializations.h
	OLD_FILES+=usr/include/c++/4.2/ext/concurrence.h
	OLD_FILES+=usr/include/c++/4.2/ext/debug_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/functional
	OLD_FILES+=usr/include/c++/4.2/ext/hash_fun.h
	OLD_FILES+=usr/include/c++/4.2/ext/hash_map
	OLD_FILES+=usr/include/c++/4.2/ext/hash_set
	OLD_FILES+=usr/include/c++/4.2/ext/hashtable.h
	OLD_FILES+=usr/include/c++/4.2/ext/iterator
	OLD_FILES+=usr/include/c++/4.2/ext/malloc_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/memory
	OLD_FILES+=usr/include/c++/4.2/ext/mt_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/new_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/numeric
	OLD_FILES+=usr/include/c++/4.2/ext/numeric_traits.h
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/assoc_container.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_tree_policy/basic_tree_policy_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_tree_policy/null_node_metadata.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_tree_policy/traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_types.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/bin_search_tree_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/cond_dtor_entry_dealtor.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/cond_key_dtor_entry_dealtor.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/node_iterators.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/point_iterators.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/r_erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/rotate_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/binary_heap_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/const_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/const_point_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/entry_cmp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/entry_pred.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/resize_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_/binomial_heap_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/binomial_heap_base_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/cc_ht_map_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/cmp_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/cond_key_dtor_entry_dealtor.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/constructor_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/constructor_destructor_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/constructor_destructor_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/debug_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/debug_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/entry_list_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/erase_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/erase_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/find_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/insert_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/insert_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/resize_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/resize_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/resize_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/size_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/standard_policies.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cond_dealtor.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/container_base_dispatch.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/eq_fn/eq_by_less.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/eq_fn/hash_eq_fn.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/constructor_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/constructor_destructor_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/constructor_destructor_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/debug_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/debug_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/erase_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/erase_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/find_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/find_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/gp_ht_map_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/insert_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/insert_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/iterator_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/resize_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/resize_no_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/resize_store_hash_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/standard_policies.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/direct_mask_range_hashing_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/direct_mod_range_hashing_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/linear_probe_fn_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/mask_based_range_hashing.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/mod_based_range_hashing.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/probe_fn_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/quadratic_probe_fn_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/ranged_hash_fn.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/ranged_probe_fn.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_probe_fn.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_range_hashing.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_ranged_hash_fn.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_ranged_probe_fn.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/const_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/const_point_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/left_child_next_sibling_heap_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/node.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/null_metadata.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/constructor_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/entry_metadata_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/lu_map_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/counter_lu_metadata.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/counter_lu_policy_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/mtf_lu_policy_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/sample_update_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/map_debug_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/cond_dtor.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/node_iterators.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/ov_tree_map_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/pairing_heap_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/child_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/cond_dtor_entry_dealtor.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/const_child_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/head.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/insert_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/internal_node.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/iterators_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/leaf.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/node_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/node_iterators.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/node_metadata_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/pat_trie_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/point_iterators.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/policy_access_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/r_erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/rotate_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/split_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/split_join_branch_bag.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/synth_e_access_traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/update_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/priority_queue_base_dispatch.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/node.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/rb_tree_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/rc.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/rc_binomial_heap_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/cc_hash_max_collision_check_resize_trigger_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_exponential_size_policy_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_load_check_resize_trigger_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_load_check_resize_trigger_size_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_prime_size_policy_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_standard_resize_policy_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/sample_resize_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/sample_resize_trigger.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/sample_size_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/info_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/node.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/splay_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/splay_tree_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/standard_policies.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/constructors_destructor_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/debug_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/erase_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/find_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/insert_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/split_join_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/thin_heap_.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/trace_fn_imps.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/node_metadata_selector.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/null_node_update_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/order_statistics_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/sample_tree_node_update.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_trace_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/node_metadata_selector.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/null_node_update_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/order_statistics_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/prefix_search_node_update_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/sample_trie_e_access_traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/sample_trie_node_update.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/string_trie_e_access_traits_imp.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/trie_policy_base.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/type_utils.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/types_traits.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/const_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/const_point_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/point_iterator.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/exception.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/hash_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/list_update_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/priority_queue.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/tag_and_trait.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/tree_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/trie_policy.hpp
	OLD_FILES+=usr/include/c++/4.2/ext/pod_char_traits.h
	OLD_FILES+=usr/include/c++/4.2/ext/pool_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/rb_tree
	OLD_FILES+=usr/include/c++/4.2/ext/rc_string_base.h
	OLD_FILES+=usr/include/c++/4.2/ext/rope
	OLD_FILES+=usr/include/c++/4.2/ext/ropeimpl.h
	OLD_FILES+=usr/include/c++/4.2/ext/slist
	OLD_FILES+=usr/include/c++/4.2/ext/sso_string_base.h
	OLD_FILES+=usr/include/c++/4.2/ext/stdio_filebuf.h
	OLD_FILES+=usr/include/c++/4.2/ext/stdio_sync_filebuf.h
	OLD_FILES+=usr/include/c++/4.2/ext/throw_allocator.h
	OLD_FILES+=usr/include/c++/4.2/ext/type_traits.h
	OLD_FILES+=usr/include/c++/4.2/ext/typelist.h
	OLD_FILES+=usr/include/c++/4.2/ext/vstring.h
	OLD_FILES+=usr/include/c++/4.2/ext/vstring.tcc
	OLD_FILES+=usr/include/c++/4.2/ext/vstring_fwd.h
	OLD_FILES+=usr/include/c++/4.2/ext/vstring_util.h
	OLD_FILES+=usr/include/c++/4.2/fstream
	OLD_FILES+=usr/include/c++/4.2/functional
	OLD_FILES+=usr/include/c++/4.2/iomanip
	OLD_FILES+=usr/include/c++/4.2/ios
	OLD_FILES+=usr/include/c++/4.2/iosfwd
	OLD_FILES+=usr/include/c++/4.2/iostream
	OLD_FILES+=usr/include/c++/4.2/istream
	OLD_FILES+=usr/include/c++/4.2/iterator
	OLD_FILES+=usr/include/c++/4.2/limits
	OLD_FILES+=usr/include/c++/4.2/list
	OLD_FILES+=usr/include/c++/4.2/locale
	OLD_FILES+=usr/include/c++/4.2/map
	OLD_FILES+=usr/include/c++/4.2/memory
	OLD_FILES+=usr/include/c++/4.2/new
	OLD_FILES+=usr/include/c++/4.2/numeric
	OLD_FILES+=usr/include/c++/4.2/ostream
	OLD_FILES+=usr/include/c++/4.2/queue
	OLD_FILES+=usr/include/c++/4.2/set
	OLD_FILES+=usr/include/c++/4.2/sstream
	OLD_FILES+=usr/include/c++/4.2/stack
	OLD_FILES+=usr/include/c++/4.2/stdexcept
	OLD_FILES+=usr/include/c++/4.2/streambuf
	OLD_FILES+=usr/include/c++/4.2/string
	OLD_FILES+=usr/include/c++/4.2/tr1/array
	OLD_FILES+=usr/include/c++/4.2/tr1/bind_iterate.h
	OLD_FILES+=usr/include/c++/4.2/tr1/bind_repeat.h
	OLD_FILES+=usr/include/c++/4.2/tr1/boost_shared_ptr.h
	OLD_FILES+=usr/include/c++/4.2/tr1/cctype
	OLD_FILES+=usr/include/c++/4.2/tr1/cfenv
	OLD_FILES+=usr/include/c++/4.2/tr1/cfloat
	OLD_FILES+=usr/include/c++/4.2/tr1/cinttypes
	OLD_FILES+=usr/include/c++/4.2/tr1/climits
	OLD_FILES+=usr/include/c++/4.2/tr1/cmath
	OLD_FILES+=usr/include/c++/4.2/tr1/common.h
	OLD_FILES+=usr/include/c++/4.2/tr1/complex
	OLD_FILES+=usr/include/c++/4.2/tr1/cstdarg
	OLD_FILES+=usr/include/c++/4.2/tr1/cstdbool
	OLD_FILES+=usr/include/c++/4.2/tr1/cstdint
	OLD_FILES+=usr/include/c++/4.2/tr1/cstdio
	OLD_FILES+=usr/include/c++/4.2/tr1/cstdlib
	OLD_FILES+=usr/include/c++/4.2/tr1/ctgmath
	OLD_FILES+=usr/include/c++/4.2/tr1/ctime
	OLD_FILES+=usr/include/c++/4.2/tr1/ctype.h
	OLD_FILES+=usr/include/c++/4.2/tr1/cwchar
	OLD_FILES+=usr/include/c++/4.2/tr1/cwctype
	OLD_FILES+=usr/include/c++/4.2/tr1/fenv.h
	OLD_FILES+=usr/include/c++/4.2/tr1/float.h
	OLD_FILES+=usr/include/c++/4.2/tr1/functional
	OLD_FILES+=usr/include/c++/4.2/tr1/functional_hash.h
	OLD_FILES+=usr/include/c++/4.2/tr1/functional_iterate.h
	OLD_FILES+=usr/include/c++/4.2/tr1/hashtable
	OLD_FILES+=usr/include/c++/4.2/tr1/hashtable_policy.h
	OLD_FILES+=usr/include/c++/4.2/tr1/inttypes.h
	OLD_FILES+=usr/include/c++/4.2/tr1/limits.h
	OLD_FILES+=usr/include/c++/4.2/tr1/math.h
	OLD_FILES+=usr/include/c++/4.2/tr1/memory
	OLD_FILES+=usr/include/c++/4.2/tr1/mu_iterate.h
	OLD_FILES+=usr/include/c++/4.2/tr1/random
	OLD_FILES+=usr/include/c++/4.2/tr1/random.tcc
	OLD_FILES+=usr/include/c++/4.2/tr1/ref_fwd.h
	OLD_FILES+=usr/include/c++/4.2/tr1/ref_wrap_iterate.h
	OLD_FILES+=usr/include/c++/4.2/tr1/repeat.h
	OLD_FILES+=usr/include/c++/4.2/tr1/stdarg.h
	OLD_FILES+=usr/include/c++/4.2/tr1/stdbool.h
	OLD_FILES+=usr/include/c++/4.2/tr1/stdint.h
	OLD_FILES+=usr/include/c++/4.2/tr1/stdio.h
	OLD_FILES+=usr/include/c++/4.2/tr1/stdlib.h
	OLD_FILES+=usr/include/c++/4.2/tr1/tgmath.h
	OLD_FILES+=usr/include/c++/4.2/tr1/tuple
	OLD_FILES+=usr/include/c++/4.2/tr1/tuple_defs.h
	OLD_FILES+=usr/include/c++/4.2/tr1/tuple_iterate.h
	OLD_FILES+=usr/include/c++/4.2/tr1/type_traits
	OLD_FILES+=usr/include/c++/4.2/tr1/type_traits_fwd.h
	OLD_FILES+=usr/include/c++/4.2/tr1/unordered_map
	OLD_FILES+=usr/include/c++/4.2/tr1/unordered_set
	OLD_FILES+=usr/include/c++/4.2/tr1/utility
	OLD_FILES+=usr/include/c++/4.2/tr1/wchar.h
	OLD_FILES+=usr/include/c++/4.2/tr1/wctype.h
	OLD_FILES+=usr/include/c++/4.2/typeinfo
	OLD_FILES+=usr/include/c++/4.2/utility
	OLD_FILES+=usr/include/c++/4.2/valarray
	OLD_FILES+=usr/include/c++/4.2/vector
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/gcc/4.2/__wmmintrin_aes.h
	OLD_FILES+=usr/include/gcc/4.2/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/gcc/4.2/ammintrin.h
	OLD_FILES+=usr/include/gcc/4.2/emmintrin.h
	OLD_FILES+=usr/include/gcc/4.2/mm3dnow.h
	OLD_FILES+=usr/include/gcc/4.2/mm_malloc.h
	OLD_FILES+=usr/include/gcc/4.2/mmintrin.h
	OLD_FILES+=usr/include/gcc/4.2/pmmintrin.h
	OLD_FILES+=usr/include/gcc/4.2/tmmintrin.h
	OLD_FILES+=usr/include/gcc/4.2/wmmintrin.h
	OLD_FILES+=usr/include/gcc/4.2/xmmintrin.h
	.elif ${TARGET_ARCH} == "arm"
	OLD_FILES+=usr/include/gcc/4.2/mmintrin.h
	.elif ${TARGET_ARCH} == "powerpc" \|\| ${TARGET_ARCH} == "powerpc64"
	OLD_FILES+=usr/include/gcc/4.2/altivec.h
	OLD_FILES+=usr/include/gcc/4.2/ppc-asm.h
	OLD_FILES+=usr/include/gcc/4.2/spe.h
	.endif
	OLD_FILES+=usr/lib/libgcov.a
	OLD_FILES+=usr/lib/libgomp.a
	OLD_FILES+=usr/lib/libstdc++.a
	OLD_FILES+=usr/lib/libstdc++.so
	OLD_LIBS+=usr/lib/libstdc++.so.6
	OLD_FILES+=usr/lib/libstdc++_p.a
	OLD_FILES+=usr/lib/libsupc++.a
	OLD_FILES+=usr/lib/libsupc++.so
	OLD_LIBS+=usr/lib/libsupc++.so.1
	OLD_FILES+=usr/lib/libsupc++_p.a
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "powerpc64"
	OLD_FILES+=usr/lib32/libstdc++.a
	OLD_FILES+=usr/lib32/libstdc++.so
	OLD_LIBS+=usr/lib32/libstdc++.so.6
	OLD_FILES+=usr/lib32/libstdc++_p.a
	OLD_FILES+=usr/lib32/libsupc++.a
	OLD_FILES+=usr/lib32/libsupc++.so
	OLD_LIBS+=usr/lib32/libsupc++.so.1
	OLD_FILES+=usr/lib32/libsupc++_p.a
	.endif
	OLD_LIBS+=usr/lib/libgomp.so.1
	OLD_FILES+=usr/lib/libgomp_p.a
	OLD_FILES+=usr/lib32/libgcov.a
	OLD_FILES+=usr/lib32/libgomp.a
	OLD_LIBS+=usr/lib32/libgomp.so.1
	OLD_FILES+=usr/lib32/libgomp_p.a
	OLD_FILES+=usr/libexec/cc1
	OLD_FILES+=usr/libexec/cc1plus
	OLD_FILES+=usr/share/man/man1/gcpp.1.gz
	OLD_FILES+=usr/share/man/man1/gperf.1.gz
	OLD_FILES+=usr/share/man/man1/gperf.7.gz

	# 20200220: Upgrade of ncurses, shlib bumped to version 9
	OLD_LIBS+=lib/libncurses.so.8
	OLD_LIBS+=lib/libncursesw.so.8
	OLD_LIBS+=usr/lib32/libncurses.so.8
	OLD_LIBS+=usr/lib32/libncursesw.so.8

	# 20200206: Remove elf2aout
	OLD_FILES+=usr/bin/elf2aout
	OLD_FILES+=usr/share/man/man1/elf2aout.1.gz

	# 20200204: simple_httpd removed
	OLD_FILES+=usr/sbin/simple_httpd

	# 20200127: vpo removed
	OLD_FILES+=usr/share/man/man4/imm.4.gz
	OLD_FILES+=usr/share/man/man4/vpo.4.gz

	# 20200104: gcc libssp removed
	OLD_FILES+=usr/include/ssp/ssp.h
	OLD_FILES+=usr/include/ssp/stdio.h
	OLD_FILES+=usr/include/ssp/string.h
	OLD_FILES+=usr/include/ssp/unistd.h
	OLD_DIRS+=usr/include/ssp

	# 20191229: GEOM_SCHED class and gsched tool removed
	OLD_FILES+=sbin/gsched
	OLD_LIBS+=lib/geom/geom_sched.so

	# 20191222: new clang import which bumps version from 9.0.0 to 9.0.1.
	OLD_FILES+=usr/lib/clang/9.0.0/include/cuda_wrappers/algorithm
	OLD_FILES+=usr/lib/clang/9.0.0/include/cuda_wrappers/complex
	OLD_FILES+=usr/lib/clang/9.0.0/include/cuda_wrappers/new
	OLD_DIRS+=usr/lib/clang/9.0.0/include/cuda_wrappers
	OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/__clang_openmp_math.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/__clang_openmp_math_declares.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/cmath
	OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/math.h
	OLD_DIRS+=usr/lib/clang/9.0.0/include/openmp_wrappers
	OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/emmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/mm_malloc.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/mmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/xmmintrin.h
	OLD_DIRS+=usr/lib/clang/9.0.0/include/ppc_wrappers
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/netbsd_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/9.0.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_device_functions.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_libdevice_declares.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/arm_fp16.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/armintr.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512bf16intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlbf16intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlvp2intersectintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vp2intersectintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/cldemoteintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/enqcmdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/invpcidintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/9.0.0/include/movdirintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/msa.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/opencl-c-base.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/pconfigintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/ptwriteintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/sgxintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/waitpkgintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/wbnoinvdintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/9.0.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/9.0.0/include
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-aarch64.so
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-arm.so
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-armhf.so
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.dd-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.dd-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer_no_main-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer_no_main-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-powerpc.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-powerpc64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.safestack-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-aarch64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-arm.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-armhf.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-x86_64.a
	OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-x86_64.a
	OLD_DIRS+=usr/lib/clang/9.0.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/9.0.0/lib
	OLD_DIRS+=usr/lib/clang/9.0.0
	# 20191214: Removal of sranddev(3)
	OLD_FILES+=usr/share/man/man3/sranddev.3.gz
	# 20191213: remove timeout(9)
	OLD_FILES+=usr/share/man/man9/callout_handle_init.9.gz
	OLD_FILES+=usr/share/man/man9/timeout.9.gz
	OLD_FILES+=usr/share/man/man9/untimeout.9.gz
	# 20191128: Removal of trm(4)
	OLD_FILES+=usr/share/man/man4/trm.4.gz
	# 20191121: Removal of sio(4)
	OLD_FILES+=usr/share/man/man4/sio.4.gz
	# 20191105: picobsd(8), et al, removed.
	OLD_FILES+=usr/share/man/man8/picobsd.8.gz
	# 20191009: new clang import which bumps version from 8.0.1 to 9.0.0.
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/netbsd_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/8.0.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_device_functions.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_libdevice_declares.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/arm_fp16.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/armintr.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/cldemoteintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/invpcidintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/8.0.1/include/movdirintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/msa.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/pconfigintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/ptwriteintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/sgxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/waitpkgintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/wbnoinvdintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/8.0.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/8.0.1/include
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.msan-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-aarch64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/8.0.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/8.0.1/lib
	OLD_DIRS+=usr/lib/clang/8.0.1
	# 20191009: libc++ 9.0.0 removed some experimental files
	OLD_FILES+=usr/include/c++/v1/experimental/any
	OLD_FILES+=usr/include/c++/v1/experimental/chrono
	OLD_FILES+=usr/include/c++/v1/experimental/numeric
	OLD_FILES+=usr/include/c++/v1/experimental/optional
	OLD_FILES+=usr/include/c++/v1/experimental/ratio
	OLD_FILES+=usr/include/c++/v1/experimental/string_view
	OLD_FILES+=usr/include/c++/v1/experimental/system_error
	OLD_FILES+=usr/include/c++/v1/experimental/tuple
	OLD_FILES+=usr/lib/libc++fs.a
	OLD_FILES+=usr/lib32/libc++fs.a
	# 20191003: Remove useless ZFS tests
	OLD_FILES+=usr/tests/sys/cddl/zfs/tests/cli_root/zpool_create/zpool_create_013_neg.ksh
	OLD_FILES+=usr/tests/sys/cddl/zfs/tests/cli_root/zpool_create/zpool_create_014_neg.ksh
	OLD_FILES+=usr/tests/sys/cddl/zfs/tests/cli_root/zpool_create/zpool_create_016_pos.ksh
	# 20190910: mklocale(1) and colldef(1) removed
	OLD_FILES+=usr/bin/mklocale
	OLD_FILES+=usr/share/man/man1/mklocale.1.gz
	OLD_FILES+=usr/bin/colldef
	OLD_FILES+=usr/share/man/man1/colldef.1.gz
	# 20190904: Remove boot1.efifat
	OLD_FILES+=boot/boot1.efifat
	# 20190903: pc-sysinstall(8) removed
	OLD_FILES+=usr/share/examples/pc-sysinstall/README
	OLD_FILES+=usr/share/examples/pc-sysinstall/pc-autoinstall.conf
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.fbsd-netinstall
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.geli
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.gmirror
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.netinstall
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.restore
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.rsync
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.upgrade
	OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.zfs
	OLD_FILES+=usr/share/man/man8/pc-sysinstall.8.gz
	OLD_FILES+=usr/share/pc-sysinstall/backend-partmanager/create-part.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-partmanager/delete-part.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-emulation.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-laptop.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-nics.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/disk-info.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/disk-list.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/disk-part.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/enable-net.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/get-packages.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-components.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-config.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-mirrors.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-packages.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-rsync-backups.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-tzones.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/query-langs.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/send-logs.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/setup-ssh-keys.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/set-mirror.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/sys-mem.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/test-live.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/test-netup.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/update-part-list.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/xkeyboard-layouts.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/xkeyboard-models.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/xkeyboard-variants.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-bsdlabel.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-cleanup.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-disk.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-extractimage.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-ftp.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-installcomponents.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-installpackages.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-localize.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-mountdisk.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-mountoptical.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-networking.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-newfs.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-parse.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-packages.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-runcommands.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-unmount.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-upgrade.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions-users.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/functions.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/installimage.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/parseconfig.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend/startautoinstall.sh
	OLD_FILES+=usr/share/pc-sysinstall/conf/avail-langs
	OLD_FILES+=usr/share/pc-sysinstall/conf/exclude-from-upgrade
	OLD_FILES+=usr/share/pc-sysinstall/conf/license/bsd-en.txt
	OLD_FILES+=usr/share/pc-sysinstall/conf/license/intel-en.txt
	OLD_FILES+=usr/share/pc-sysinstall/conf/license/nvidia-en.txt
	OLD_FILES+=usr/share/pc-sysinstall/conf/pc-sysinstall.conf
	OLD_FILES+=usr/share/pc-sysinstall/doc/help-disk-list
	OLD_FILES+=usr/share/pc-sysinstall/doc/help-disk-size
	OLD_FILES+=usr/share/pc-sysinstall/doc/help-index
	OLD_FILES+=usr/share/pc-sysinstall/doc/help-start-autoinstall
	OLD_FILES+=usr/sbin/pc-sysinstall
	OLD_DIRS+=usr/share/examples/pc-sysinstall
	OLD_DIRS+=usr/share/pc-sysinstall/backend
	OLD_DIRS+=usr/share/pc-sysinstall/backend-partmanager
	OLD_DIRS+=usr/share/pc-sysinstall/backend-query
	OLD_DIRS+=usr/share/pc-sysinstall/conf/license
	OLD_DIRS+=usr/share/pc-sysinstall/conf
	OLD_DIRS+=usr/share/pc-sysinstall/doc
	OLD_DIRS+=usr/share/pc-sysinstall
	# 20190825: zlib 1.0.4 removed from kernel
	OLD_FILES+=usr/include/sys/zlib.h
	OLD_FILES+=usr/include/sys/zutil.h
	# 20190817: pft_ping.py and sniffer.py moved to /usr/tests/sys/netpfil/common
	OLD_FILES+=usr/tests/sys/netpfil/pf/sniffer.py
	OLD_FILES+=usr/tests/sys/netpfil/pf/pft_ping.py
	# 20190816: dir.h removed from POSIX
	OLD_FILES+=usr/include/sys/dir.h
	# 20190729: gzip'ed a.out support removed
	OLD_FILES+=usr/include/sys/inflate.h
	# 20190722: cap_random(3) removed
	OLD_FILES+=lib/casper/libcap_random.so.1
	OLD_FILES+=usr/include/casper/cap_random.h
	OLD_FILES+=usr/share/man/man3/cap_random.3.gz
	OLD_FILES+=usr/share/man/man3/cap_random_buf.3.gz
	# 20190708: vm_page_hold() and _unhold() removed
	OLD_FILES+=usr/share/man/man9/vm_page_hold.9.gz
	# 20190618: sys/capability.h removed (sys/capsicum.h is the one to use)
	OLD_FILES+=usr/include/sys/capability.h
	# 20190615: sys/pwm.h renamed to dev/pwmc.h
	OLD_FILES+=usr/include/sys/pwm.h
	# 20190612: new clang import which bumps version from 8.0.0 to 8.0.1.
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/netbsd_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/8.0.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_device_functions.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_libdevice_declares.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/arm_fp16.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/armintr.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/cldemoteintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/invpcidintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/8.0.0/include/movdirintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/msa.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/pconfigintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/ptwriteintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/sgxintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/waitpkgintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/wbnoinvdintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/8.0.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/8.0.0/include
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.msan-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/8.0.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/8.0.0/lib
	OLD_DIRS+=usr/lib/clang/8.0.0
	# 20190523: Remove obsolete kgzip and support files
	OLD_FILES+=usr/sbin/kgzip
	OLD_FILES+=usr/lib/kgzldr.o
	OLD_FILES+=usr/share/man/man8/kgzip.8.gz
	# 20190517: Remove obsolete 10 and 10/100 ethernet drivers.
	OLD_FILES+=usr/share/man/man4/bm.4.gz
	OLD_FILES+=usr/share/man/man4/cs.4.gz
	OLD_FILES+=usr/share/man/man4/de.4.gz
	OLD_FILES+=usr/share/man/man4/if_de.4.gz
	OLD_FILES+=usr/share/man/man4/ed.4.gz
	OLD_FILES+=usr/share/man/man4/if_ed.4.gz
	OLD_FILES+=usr/share/man/man4/ep.4.gz
	OLD_FILES+=usr/share/man/man4/ex.4.gz
	OLD_FILES+=usr/share/man/man4/fe.4.gz
	OLD_FILES+=usr/share/man/man4/pcn.4.gz
	OLD_FILES+=usr/share/man/man4/if_pcn.4.gz
	OLD_FILES+=usr/share/man/man4/sf.4.gz
	OLD_FILES+=usr/share/man/man4/if_sf.4.gz
	OLD_FILES+=usr/share/man/man4/sn.4.gz
	OLD_FILES+=usr/share/man/man4/if_sn.4.gz
	OLD_FILES+=usr/share/man/man4/tl.4.gz
	OLD_FILES+=usr/share/man/man4/if_tl.4.gz
	OLD_FILES+=usr/share/man/man4/tx.4.gz
	OLD_FILES+=usr/share/man/man4/if_tx.4.gz
	OLD_FILES+=usr/share/man/man4/txp.4.gz
	OLD_FILES+=usr/share/man/man4/if_txp.4.gz
	OLD_FILES+=usr/share/man/man4/vx.4.gz
	OLD_FILES+=usr/share/man/man4/wb.4.gz
	OLD_FILES+=usr/share/man/man4/xe.4.gz
	OLD_FILES+=usr/share/man/man4/if_xe.4.gz
	# 20190513: libcap_sysctl interface change
	OLD_FILES+=lib/casper/libcap_sysctl.so.1
	# 20190509: tests/sys/opencrypto requires the net/py-dpkt package.
	OLD_FILES+=usr/tests/sys/opencrypto/dpkt.py
	OLD_FILES+=usr/tests/sys/opencrypto/dpkt.pyc
	# 20190304: new libc++ import which bumps version from 7.0.1 to 8.0.0.
	OLD_FILES+=usr/include/c++/v1/experimental/dynarray
	# 20190304: new clang import which bumps version from 7.0.1 to 8.0.0.
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/netbsd_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/7.0.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_device_functions.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_libdevice_declares.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/arm_fp16.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/armintr.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/cldemoteintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/invpcidintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/7.0.1/include/movdirintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/msa.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/pconfigintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/ptwriteintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/sgxintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/waitpkgintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/wbnoinvdintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/7.0.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/7.0.1/include
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.msan-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/7.0.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/7.0.1/lib
	OLD_DIRS+=usr/lib/clang/7.0.1
	# 20190227: rename seq.h to seqc.h
	OLD_FILES+=usr/include/sys/seq.h
	# 20190222: libifconfig made INTERNALLIB
	OLD_FILES+=usr/lib/libprivateifconfig.a
	OLD_FILES+=usr/lib/libprivateifconfig_p.a
	OLD_FILES+=usr/lib32/libprivateifconfig.a
	OLD_FILES+=usr/lib32/libprivateifconfig_p.a
	# 20190131: pfil(9) changed
	OLD_FILES+=usr/share/man/man9/pfil_hook_get.9.gz
	OLD_FILES+=usr/share/man/man9/pfil_rlock.9.gz
	OLD_FILES+=usr/share/man/man9/pfil_runlock.9.gz
	OLD_FILES+=usr/share/man/man9/pfil_wlock.9.gz
	OLD_FILES+=usr/share/man/man9/pfil_wunlock.9.gz
	# 20190126: adv(4) / adw(4) removal
	OLD_FILES+=usr/share/man/man4/adv.4.gz
	OLD_FILES+=usr/share/man/man4/adw.4.gz
	# 20190123: nonexistant cred_update_thread(9) removed
	OLD_FILES+=usr/share/man/man9/cred_update_thread.9.gz
	# 20190114: old pbuf allocator removed
	OLD_FILES+=usr/share/man/man9/getpbuf.9.gz
	OLD_FILES+=usr/share/man/man9/pbuf.9.gz
	OLD_FILES+=usr/share/man/man9/relpbuf.9.gz
	OLD_FILES+=usr/share/man/man9/trypbuf.9.gz
	# 20181219: ibcs removal
	OLD_FILES+=usr/share/examples/ibcs2/hello.uu
	OLD_FILES+=usr/share/examples/ibcs2/README
	OLD_DIRS+=usr/share/examples/ibcs2
	# 20181215: Migration of CTM to ports
	OLD_FILES+=usr/sbin/ctm
	OLD_FILES+=usr/sbin/ctm_dequeue
	OLD_FILES+=usr/sbin/ctm_rmail
	OLD_FILES+=usr/sbin/ctm_smail
	OLD_FILES+=usr/share/man/man1/ctm.1.gz
	OLD_FILES+=usr/share/man/man1/ctm_dequeue.1.gz
	OLD_FILES+=usr/share/man/man1/ctm_rmail.1.gz
	OLD_FILES+=usr/share/man/man1/ctm_smail.1.gz
	OLD_FILES+=usr/share/man/man5/ctm.5.gz
	# 20181214: Remove timed files
	OLD_FILES+=etc/rc.d/timed
	OLD_FILES+=usr/sbin/timed
	OLD_FILES+=usr/sbin/timedc
	OLD_FILES+=usr/share/man/man8/timed.8.gz
	OLD_FILES+=usr/share/man/man8/timedc.8.gz
	# 20181211: new clang import which bumps version from 6.0.1 to 7.0.1.
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/6.0.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/armintr.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/6.0.1/include/msa.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/6.0.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/6.0.1/include
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/6.0.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/6.0.1/lib
	OLD_DIRS+=usr/lib/clang/6.0.1
	# 20181116: Rename test file.
	OLD_FILES+=usr/tests/sys/netinet/reuseport_lb
	# 20181113: libufs version bumped to 7.
	OLD_LIBS+=lib/libufs.so.6
	OLD_LIBS+=usr/lib32/libufs.so.6
	# 20181112: Cleanup old libcap_dns.
	OLD_LIBS+=lib/casper/libcap_dns.so.1
	OLD_LIBS+=usr/lib32/libcap_dns.so.1
	# 20181030: malloc_domain(9) KPI change
	OLD_FILES+=usr/share/man/man9/malloc_domain.9.gz
	# 20181026: joy(4) removal
	OLD_FILES+=usr/share/man/man4/joy.4.gz
	# 20181025: OpenSSL libraries version bump to avoid conflict with ports
	OLD_LIBS+=lib/libcrypto.so.9
	OLD_LIBS+=usr/lib/libssl.so.9
	OLD_LIBS+=usr/lib32/libcrypto.so.9
	OLD_LIBS+=usr/lib32/libssl.so.9
	# 20181021: mse(4) removal
	OLD_FILES+=usr/share/man/man4/mse.4.gz
	# 20181015: Stale libcasper(3) files following r329452
	OLD_LIBS+=lib/casper/libcap_sysctl.so.0
	OLD_LIBS+=lib/casper/libcap_grp.so.0
	OLD_LIBS+=lib/casper/libcap_pwd.so.0
	OLD_LIBS+=lib/casper/libcap_random.so.0
	OLD_LIBS+=lib/casper/libcap_dns.so.0
	OLD_LIBS+=lib/casper/libcap_syslog.so.0
	OLD_LIBS+=usr/lib32/libcap_sysctl.so.0
	OLD_LIBS+=usr/lib32/libcap_grp.so.0
	OLD_LIBS+=usr/lib32/libcap_pwd.so.0
	OLD_LIBS+=usr/lib32/libcap_random.so.0
	OLD_LIBS+=usr/lib32/libcap_dns.so.0
	OLD_LIBS+=usr/lib32/libcap_syslog.so.0
	# 20181012: rename of ixlv(4) to iavf(4)
	OLD_FILES+=usr/share/man/man4/ixlv.4.gz
	# 20181009: OpenSSL 1.1.1
	OLD_FILES+=usr/include/openssl/des_old.h
	OLD_FILES+=usr/include/openssl/dso.h
	OLD_FILES+=usr/include/openssl/krb5_asn.h
	OLD_FILES+=usr/include/openssl/kssl.h
	OLD_FILES+=usr/include/openssl/pqueue.h
	OLD_FILES+=usr/include/openssl/ssl23.h
	OLD_FILES+=usr/include/openssl/ui_compat.h
	OLD_FILES+=usr/lib/debug/usr/lib/engines/lib4758cca.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libaep.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libatalla.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libcapi.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libchil.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libcswift.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libgost.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libnuron.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libsureware.so.debug
	OLD_FILES+=usr/lib/debug/usr/lib/engines/libubsec.so.debug
	OLD_FILES+=usr/share/openssl/man/man1/dss1.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/md2.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/md4.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/md5.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/mdc2.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/ripemd160.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/sha.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/sha1.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/sha224.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/sha256.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/sha384.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/sha512.1.gz
	OLD_FILES+=usr/share/openssl/man/man1/x509v3_config.1.gz
	OLD_FILES+=usr/share/openssl/man/man3/ASN1_STRING_length_set.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BIO_get_conn_int_port.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BIO_get_conn_ip.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BIO_set.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BIO_set_conn_int_port.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BIO_set_conn_ip.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_BLINDING_get_thread_id.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_BLINDING_set_thread_id.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_BLINDING_thread_id.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_CTX_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_MONT_CTX_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_RECP_CTX_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BN_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BUF_memdup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BUF_memdup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BUF_strdup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BUF_strlcat.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BUF_strlcpy.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/BUF_strndup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CMS_set1_signer_cert.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_cmp.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_cpy.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_current.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_get_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_hash.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_set_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_destroy_dynlockid.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_get_new_dynlockid.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_lock.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_num_locks.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_dynlock_create_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_dynlock_destroy_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_dynlock_lock_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_locking_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/DES_ede3_cbcm_encrypt.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/DES_enc_read.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/DES_enc_write.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EC_KEY_get_key_method_data.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EC_KEY_insert_key_method_data.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EC_POINT_set_Jprojective_coordinates.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ERR_load_UI_strings.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_CIPHER_CTX_cleanup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_CIPHER_CTX_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_MAX_MD_SIZE.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_cleanup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_create.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_destroy.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEVP_PKEY_CTX_set_app_data.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_CTX_set_rsa_rsa_keygen_bits.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_get_default_digest.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_dss.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_dss1.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_sha.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/HMAC_CTX_cleanup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/HMAC_CTX_init.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/HMAC_cleanup.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/OPENSSL_ia32cap_loc.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/PEM.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/RAND_SSLeay.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/RSA_PKCS1_SSLeay.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/RSA_null_method.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_get_ex_new_index.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_need_tmp_rsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_custom_cli_ext.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_default_read_ahead.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_ecdh_auto.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_tmp_rsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_tmp_rsa_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_SESSION_get_ex_new_index.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_add_session.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_flush_sessions.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_get_accept_state.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_get_ex_new_index.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_get_msg_callback_arg.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_need_tmp_rsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_remove_session.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_set_ecdh_auto.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_set_tmp_rsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSL_set_tmp_rsa_callback.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSLeay.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSLeay_add_ssl_algorithms.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSLeay_version.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSLv2_client_method.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSLv2_method.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/SSLv2_server_method.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/X509_STORE_CTX_set_chain.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/X509_STORE_CTX_trusted_stack.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/blowfish.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_add_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_check_top.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_cmp_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_div_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_dump.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_expand.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_expand2.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_fix_top.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_internal.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_add_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_comba4.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_comba8.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_high.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_low_normal.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_low_recursive.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_normal.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_part_recursive.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_recursive.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_mul_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_print.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_set_high.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_set_low.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_set_max.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_comba4.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_comba8.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_normal.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_recursive.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_sub_words.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/bn_wexpand.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/buffer.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/crypto.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_ECPKParameters_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_ECPKParameters_fp.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_ECPrivate_key.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_Netscape_RSA.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_PKCS8PrivateKey.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_Private_key.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_X509_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_X509_fp.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/des.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/des_read_2passwords.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/des_read_password.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/des_read_pw.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/des_read_pw_string.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/dh.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/dsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ec.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ecdsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/engine.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/err.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/evp.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/hmac.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/i2d_ECPKParameters_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/i2d_ECPKParameters_fp.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/i2d_Netscape_RSA.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/i2d_X509_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/i2d_X509_fp.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_delete.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_doall.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_doall_arg.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_error.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_free.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_insert.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_new.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_node_stats.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_node_stats_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_node_usage_stats.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_node_usage_stats_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_retrieve.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_stats.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lh_stats_bio.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/lhash.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/md5.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/mdc2.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/pem.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/rand.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/rc4.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ripemd.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/rsa.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/sha.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ssl.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/threads.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ui.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/ui_compat.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/x509.3.gz
	OLD_LIBS+=lib/libcrypto.so.8
	OLD_LIBS+=usr/lib/engines/lib4758cca.so
	OLD_LIBS+=usr/lib/engines/libaep.so
	OLD_LIBS+=usr/lib/engines/libatalla.so
	OLD_LIBS+=usr/lib/engines/libcapi.so
	OLD_LIBS+=usr/lib/engines/libchil.so
	OLD_LIBS+=usr/lib/engines/libcswift.so
	OLD_LIBS+=usr/lib/engines/libgost.so
	OLD_LIBS+=usr/lib/engines/libnuron.so
	OLD_LIBS+=usr/lib/engines/libsureware.so
	OLD_LIBS+=usr/lib/engines/libubsec.so
	OLD_LIBS+=usr/lib/libssl.so.8
	OLD_LIBS+=usr/lib32/libcrypto.so.8
	OLD_LIBS+=usr/lib32/lib4758cca.so
	OLD_LIBS+=usr/lib32/libaep.so
	OLD_LIBS+=usr/lib32/libatalla.so
	OLD_LIBS+=usr/lib32/libcapi.so
	OLD_LIBS+=usr/lib32/libchil.so
	OLD_LIBS+=usr/lib32/libcswift.so
	OLD_LIBS+=usr/lib32/libgost.so
	OLD_LIBS+=usr/lib32/libnuron.so
	OLD_LIBS+=usr/lib32/libsureware.so
	OLD_LIBS+=usr/lib32/libubsec.so
	OLD_LIBS+=usr/lib32/libssl.so.8
	# 20180824: libbe(3) SHLIBDIR fixed to reflect correct location
	OLD_LIBS+=usr/lib/libbe.so.1
	# 20180819: Remove deprecated arc4random(3) stir/addrandom interfaces
	OLD_FILES+=usr/share/man/man3/arc4random_addrandom.3.gz
	OLD_FILES+=usr/share/man/man3/arc4random_stir.3.gz
	# 20180819: send-pr(1) placeholder removal
	OLD_FILES+=usr/bin/send-pr
	# 20180725: Cleanup old libcasper.so.0
	OLD_LIBS+=lib/libcasper.so.0
	OLD_LIBS+=usr/lib32/libcasper.so.0
	# 20180722: indent(1) option renamed, test files follow
	OLD_FILES+=usr/bin/indent/tests/nsac.0
	OLD_FILES+=usr/bin/indent/tests/nsac.0.pro
	OLD_FILES+=usr/bin/indent/tests/nsac.0.stdout
	OLD_FILES+=usr/bin/indent/tests/sac.0
	OLD_FILES+=usr/bin/indent/tests/sac.0.pro
	OLD_FILES+=usr/bin/indent/tests/sac.0.stdout
	# 20180721: move of libmlx5.so.1 and libibverbs.so.1
	OLD_LIBS+=usr/lib/libmlx5.so.1
	OLD_LIBS+=usr/lib/libibverbs.so.1
	# 20180710: old numa cleanup
	OLD_FILES+=usr/include/sys/numa.h
	OLD_FILES+=usr/share/man/man2/numa_getaffinity.2.gz
	OLD_FILES+=usr/share/man/man2/numa_setaffinity.2.gz
	OLD_FILES+=usr/share/man/man1/numactl.1.gz
	OLD_FILES+=usr/bin/numactl
	# 20180630: new clang import which bumps version from 6.0.0 to 6.0.1.
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/hwasan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/scudo_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/6.0.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/arm64intr.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/armintr.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512bitalgintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vbmi2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlbitalgintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlvbmi2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlvnniintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vnniintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vpopcntdqvlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/cetintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/clwbintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/gfniintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/6.0.0/include/msa.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/vaesintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/vpclmulqdqintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/6.0.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/6.0.0/include
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/6.0.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/6.0.0/lib
	OLD_DIRS+=usr/lib/clang/6.0.0
	# 20180615: asf(8) removed
	OLD_FILES+=usr/sbin/asf
	OLD_FILES+=usr/share/man/man8/asf.8.gz
	# 20180609: obsolete libc++ files missed from the 5.0.0 import
	OLD_FILES+=usr/include/c++/v1/__refstring
	OLD_FILES+=usr/include/c++/v1/__undef_min_max
	OLD_FILES+=usr/include/c++/v1/tr1/__refstring
	OLD_FILES+=usr/include/c++/v1/tr1/__undef_min_max
	# 20180607: remove nls support from grep
	OLD_FILES+=usr/share/nls/pt_BR.ISO8859-1/grep.cat
	OLD_FILES+=usr/share/nls/hu_HU.ISO8859-2/grep.cat
	OLD_FILES+=usr/share/nls/ja_JP.SJIS/grep.cat
	OLD_FILES+=usr/share/nls/ja_JP.eucJP/grep.cat
	OLD_FILES+=usr/share/nls/gl_ES.ISO8859-1/grep.cat
	OLD_FILES+=usr/share/nls/zh_CN.UTF-8/grep.cat
	OLD_FILES+=usr/share/nls/es_ES.ISO8859-1/grep.cat
	OLD_FILES+=usr/share/nls/ru_RU.KOI8-R/grep.cat
	OLD_FILES+=usr/share/nls/uk_UA.UTF-8/grep.cat
	OLD_FILES+=usr/share/nls/ja_JP.UTF-8/grep.cat
	# 20180528: libpcap update removed header file
	OLD_FILES+=usr/include/pcap/export-defs.h
	# 20180517: retire vxge
	OLD_FILES+=usr/share/man/man4/if_vxge.4.gz
	OLD_FILES+=usr/share/man/man4/vxge.4.gz
	# 20180512: Rename Unbound tools
	OLD_FILES+=usr/sbin/unbound
	OLD_FILES+=usr/sbin/unbound-anchor
	OLD_FILES+=usr/sbin/unbound-checkconf
	OLD_FILES+=usr/sbin/unbound-control
	OLD_FILES+=usr/share/man/man5/unbound.conf.5.gz
	OLD_FILES+=usr/share/man/man8/unbound-anchor.8.gz
	OLD_FILES+=usr/share/man/man8/unbound-checkconf.8.gz
	OLD_FILES+=usr/share/man/man8/unbound-control.8.gz
	OLD_FILES+=usr/share/man/man8/unbound.8.gz
	# 20180508: retire nxge
	OLD_FILES+=usr/share/man/man4/if_nxge.4.gz
	OLD_FILES+=usr/share/man/man4/nxge.4.gz
	# 20180505: rhosts
	OLD_FILES+=usr/share/skel/dot.rhosts
	# 20180502: retire ixgb
	OLD_FILES+=usr/share/man/man4/if_ixgb.4.gz
	OLD_FILES+=usr/share/man/man4/ixgb.4.gz
	# 20180501: retire lmc
	OLD_FILES+=usr/include/dev/lmc/if_lmc.h
	OLD_DIRS+=usr/include/dev/lmc
	OLD_FILES+=usr/sbin/lmcconfig
	OLD_FILES+=usr/share/man/man4/lmc.4.gz
	OLD_FILES+=usr/share/man/man4/if_lmc.4.gz
	OLD_FILES+=usr/share/man/man8/lmcconfig.8.gz
	# 20180417: remove fuswintr and suswintr
	OLD_FILES+=usr/share/man/man9/fuswintr.9.gz
	OLD_FILES+=usr/share/man/man9/suswintr.9.gz
	# 20180413: remove Arcnet support
	OLD_FILES+=usr/include/net/if_arc.h
	OLD_FILES+=usr/share/man/man4/cm.4.gz
	# 20180409: remove FDDI support
	OLD_FILES+=usr/include/net/fddi.h
	OLD_FILES+=usr/share/man/man4/fpa.4.gz
	# 20180319: remove /boot/overlays, replaced by /boot/dtb/overlays
	OLD_DIRS+=boot/overlays
	# 20180311: remove sys/sys/i386/include/pcaudioio.h
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/machine/pcaudioio.h
	.endif
	# 20180310: remove sys/sys/dataacq.h
	OLD_FILES+=usr/include/sys/dataacq.h
	# 20180306: remove DTrace scripts made obsolete by dwatch(1)
	OLD_FILES+=usr/share/dtrace/watch_execve
	OLD_FILES+=usr/share/dtrace/watch_kill
	OLD_FILES+=usr/share/dtrace/watch_vop_remove
	# 20180212: move devmatch
	OLD_FILES+=usr/sbin/devmatch
	# 20180211: remove usb.conf
	OLD_FILES+=etc/devd/usb.conf
	# 20180208: remove c_rehash(1)
	OLD_FILES+=usr/share/openssl/man/man1/c_rehash.1.gz
	# 20180206: remove gdbtui
	OLD_FILES+=usr/bin/gdbtui
	# 20180201: Obsolete forth files
	OLD_FILES+=boot/pcibios.4th
	# 20180114: new clang import which bumps version from 5.0.1 to 6.0.0.
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/5.0.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/armintr.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/5.0.1/include/msa.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/5.0.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/5.0.1/include
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/5.0.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/5.0.1/lib
	OLD_DIRS+=usr/lib/clang/5.0.1
	# 20180109: Remove vestiges of digi(4) driver
	OLD_FILES+=usr/include/sys/digiio.h
	OLD_FILES+=usr/sbin/digictl
	OLD_FILES+=usr/share/man/man8/digictl.8.gz
	# 20180107: Convert remaining geli(8) tests to ATF
	OLD_FILES+=tests/sys/geom/class/eli/nokey_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/readonly_test.sh
	# 20180106: Convert most geli(8) tests to ATF
	OLD_FILES+=tests/sys/geom/class/eli/attach_d_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/configure_b_B_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/detach_l_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/init_B_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/init_J_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/init_a_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/init_alias_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/init_i_P_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/integrity_copy_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/integrity_data_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/integrity_hmac_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/onetime_a_test.sh
	OLD_FILES+=tests/sys/geom/class/eli/onetime_d_test.sh
	# 20171230: Remove /etc/skel from mtree
	OLD_DIRS+=etc/skel
	# 20171208: Remove basename_r(3)
	OLD_FILES+=usr/share/man/man3/basename_r.3.gz
	# 20171204: Move fdformat man page from volume 1 to volume 8.
	OLD_FILES+=usr/share/man/man1/fdformat.1.gz
	# 20171203: libproc version bump
	OLD_LIBS+=usr/lib/libproc.so.4
	OLD_LIBS+=usr/lib32/libproc.so.4
	# 20171203: new clang import which bumps version from 5.0.0 to 5.0.1.
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/tsan_interface.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/5.0.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/armintr.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vpopcntdqintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/clzerointrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/lwpintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/5.0.0/include/msa.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/5.0.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/5.0.0/include
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-armhf.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/5.0.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/5.0.0/lib
	OLD_DIRS+=usr/lib/clang/5.0.0
	# 20171118: Remove old etc casper files
	OLD_FILES+=etc/casper/system.dns
	OLD_FILES+=etc/casper/system.grp
	OLD_FILES+=etc/casper/system.pwd
	OLD_FILES+=etc/casper/system.random
	OLD_FILES+=etc/casper/system.sysctl
	OLD_DIRS+=etc/casper
	# 20171116: lint(1) removal
	OLD_FILES+=usr/bin/lint
	OLD_FILES+=usr/libexec/lint1
	OLD_FILES+=usr/libexec/lint2
	OLD_FILES+=usr/libdata/lint/llib-lposix.ln
	OLD_FILES+=usr/libdata/lint/llib-lstdc.ln
	OLD_FILES+=usr/share/man/man1/lint.1.gz
	OLD_FILES+=usr/share/man/man7/lint.7.gz
	OLD_DIRS+=usr/libdata/lint
	# 20171114: Removal of all fortune datfiles other than freebsd-tips
	OLD_FILES+=usr/share/games/fortune/fortunes
	OLD_FILES+=usr/share/games/fortune/fortunes.dat
	OLD_FILES+=usr/share/games/fortune/gerrold.limerick
	OLD_FILES+=usr/share/games/fortune/gerrold.limerick.dat
	OLD_FILES+=usr/share/games/fortune/limerick
	OLD_FILES+=usr/share/games/fortune/limerick.dat
	OLD_FILES+=usr/share/games/fortune/murphy
	OLD_FILES+=usr/share/games/fortune/murphy-o
	OLD_FILES+=usr/share/games/fortune/murphy-o.dat
	OLD_FILES+=usr/share/games/fortune/murphy.dat
	OLD_FILES+=usr/share/games/fortune/startrek
	OLD_FILES+=usr/share/games/fortune/startrek.dat
	OLD_FILES+=usr/share/games/fortune/zippy
	OLD_FILES+=usr/share/games/fortune/zippy.dat
	# 20171112: Removal of eqnchar definition
	OLD_FILES+=usr/share/misc/eqnchar
	# 20171110: Removal of mailaddr man page
	OLD_FILES+=usr/share/man/man7/mailaddr.7.gz
	# 20171108: badsect(8) removal
	OLD_FILES+=sbin/badsect
	OLD_FILES+=rescue/badsect
	OLD_FILES+=usr/share/man/man8/badsect.8.gz
	# 20171105: fixing lib/libclang_rt CRTARCH for arm:armv[67].
	.if ${MACHINE_ARCH:Marmv[67]*} != "" && \
	(!defined(CPUTYPE) \|\| ${CPUTYPE:Msoft} == "")
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-preinit-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-arm.a
	OLD_LIBS+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-arm.so
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan_cxx-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.safestack-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats_client-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
	OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
	.endif
	# 20171104: libcap_random should be in /lib not in /usr/lib
	OLD_LIBS+=usr/lib/libcap_random.so.0
	# 20171104: Casper can work only as shared library
	OLD_FILES+=usr/lib/libcap_dns.a
	OLD_FILES+=usr/lib/libcap_dns_p.a
	OLD_FILES+=usr/lib/libcap_grp.a
	OLD_FILES+=usr/lib/libcap_grp_p.a
	OLD_FILES+=usr/lib/libcap_pwd.a
	OLD_FILES+=usr/lib/libcap_pwd_p.a
	OLD_FILES+=usr/lib/libcap_random.a
	OLD_FILES+=usr/lib/libcap_random_p.a
	OLD_FILES+=usr/lib/libcap_sysctl.a
	OLD_FILES+=usr/lib/libcap_sysctl_p.a
	OLD_FILES+=usr/lib/libcasper.a
	OLD_FILES+=usr/lib/libcasper_p.a
	OLD_FILES+=usr/lib32/libcap_dns.a
	OLD_FILES+=usr/lib32/libcap_dns_p.a
	OLD_FILES+=usr/lib32/libcap_grp.a
	OLD_FILES+=usr/lib32/libcap_grp_p.a
	OLD_FILES+=usr/lib32/libcap_pwd.a
	OLD_FILES+=usr/lib32/libcap_pwd_p.a
	OLD_FILES+=usr/lib32/libcap_random.a
	OLD_FILES+=usr/lib32/libcap_random_p.a
	OLD_FILES+=usr/lib32/libcap_sysctl.a
	OLD_FILES+=usr/lib32/libcap_sysctl_p.a
	OLD_FILES+=usr/lib32/libcasper.a
	OLD_FILES+=usr/lib32/libcasper_p.a
	# 20171031: Removal of adding_user man page
	OLD_FILES+=usr/share/man/man7/adding_user.7.gz
	# 20171031: Disconnected libpathconv tests
	OLD_DIRS+=usr/tests/lib/libpathconv
	# 20171017: Removal of mbpool(9)
	OLD_FILES+=usr/include/sys/mbpool.h
	OLD_FILES+=usr/share/man/man9/mbpool.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_destroy.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_alloc.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_ext_free.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_count.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_card_free.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_get_keep.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_free.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_get.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_create.9.gz
	OLD_FILES+=usr/share/man/man9/mbp_sync.9.gz
	# 20171010: Remove libstand
	OLD_FILES+=usr/lib/libstand.a
	OLD_FILES+=usr/lib/libstand_p.a
	OLD_FILES+=usr/lib32/libstand.a
	OLD_FILES+=usr/lib32/libstand_p.a
	OLD_FILES+=usr/include/stand.h
	OLD_FILES+=usr/share/man/man3/libstand.3.gz
	# 20171003: remove RCMDS
	OLD_FILES+=bin/rcp
	OLD_FILES+=rescue/rcp
	OLD_FILES+=usr/bin/rlogin
	OLD_FILES+=usr/bin/rsh
	OLD_FILES+=usr/libexec/rlogind
	OLD_FILES+=usr/libexec/rshd
	OLD_FILES+=usr/share/man/man1/rcp.1.gz
	OLD_FILES+=usr/share/man/man1/rlogin.1.gz
	OLD_FILES+=usr/share/man/man1/rsh.1.gz
	OLD_FILES+=usr/share/man/man8/rlogind.8.gz
	OLD_FILES+=usr/share/man/man8/rshd.8.gz
	# 20170927: crshared
	OLD_FILES+=usr/share/man/man9/crshared.9.gz
	# 20170927: procctl
	OLD_FILES+=usr/share/man/man8/procctl.8.gz
	OLD_FILES+=usr/sbin/procctl
	# 20170926: remove unneeded man aliases and locales directory
	OLD_FILES+=usr/share/man/en.ISO8859-1/man1
	OLD_FILES+=usr/share/man/en.ISO8859-1/man2
	OLD_FILES+=usr/share/man/en.ISO8859-1/man3
	OLD_FILES+=usr/share/man/en.ISO8859-1/man4
	OLD_FILES+=usr/share/man/en.ISO8859-1/man5
	OLD_FILES+=usr/share/man/en.ISO8859-1/man6
	OLD_FILES+=usr/share/man/en.ISO8859-1/man7
	OLD_FILES+=usr/share/man/en.ISO8859-1/man8
	OLD_FILES+=usr/share/man/en.ISO8859-1/man9
	OLD_DIRS+=usr/share/man/en.ISO8859-1
	OLD_FILES+=usr/share/man/en.ISO8859-1/mandoc.db
	OLD_FILES+=usr/share/man/en.UTF-8/man1
	OLD_FILES+=usr/share/man/en.UTF-8/man2
	OLD_FILES+=usr/share/man/en.UTF-8/man3
	OLD_FILES+=usr/share/man/en.UTF-8/man4
	OLD_FILES+=usr/share/man/en.UTF-8/man5
	OLD_FILES+=usr/share/man/en.UTF-8/man6
	OLD_FILES+=usr/share/man/en.UTF-8/man7
	OLD_FILES+=usr/share/man/en.UTF-8/man8
	OLD_FILES+=usr/share/man/en.UTF-8/man9
	OLD_FILES+=usr/share/man/en.UTF-8/mandoc.db
	OLD_DIRS+=usr/share/man/en.UTF-8
	OLD_FILES+=usr/share/man/en.ISO8859-15
	OLD_FILES+=usr/share/openssl/man/en.ISO8859-1/man1
	OLD_FILES+=usr/share/openssl/man/en.ISO8859-1/man3
	OLD_FILES+=usr/share/openssl/man/en.ISO8859-1/mandoc.db
	OLD_DIRS+=usr/share/openssl/man/en.ISO8859-1
	OLD_FILES+=usr/share/openssl/man/en.ISO8859-15
	OLD_DIRS+=usr/share/man/ja/man1
	OLD_DIRS+=usr/share/man/ja/man2
	OLD_DIRS+=usr/share/man/ja/man3
	OLD_DIRS+=usr/share/man/ja/man4
	OLD_DIRS+=usr/share/man/ja/man5
	OLD_DIRS+=usr/share/man/ja/man6
	OLD_DIRS+=usr/share/man/ja/man7
	OLD_DIRS+=usr/share/man/ja/man8
	OLD_DIRS+=usr/share/man/ja/man9
	OLD_DIRS+=usr/share/man/ja
	# 20170913: remove unneeded catman utility
	OLD_FILES+=etc/periodic/weekly/330.catman
	OLD_FILES+=usr/bin/catman
	OLD_FILES+=usr/libexec/catman.local
	OLD_FILES+=usr/share/man/man1/catman.1.gz
	OLD_FILES+=usr/share/man/man8/catman.local.8.gz
	OLD_DIRS+=usr/share/man/cat1
	OLD_DIRS+=usr/share/man/cat2
	OLD_DIRS+=usr/share/man/cat3
	OLD_DIRS+=usr/share/man/cat4/amd64
	OLD_DIRS+=usr/share/man/cat4/arm
	OLD_DIRS+=usr/share/man/cat4/i386
	OLD_DIRS+=usr/share/man/cat4/powerpc
	OLD_DIRS+=usr/share/man/cat4/sparc64
	OLD_DIRS+=usr/share/man/cat4
	OLD_DIRS+=usr/share/man/cat5
	OLD_DIRS+=usr/share/man/cat6
	OLD_DIRS+=usr/share/man/cat7
	OLD_DIRS+=usr/share/man/cat8/amd64
	OLD_DIRS+=usr/share/man/cat8/arm
	OLD_DIRS+=usr/share/man/cat8/i386
	OLD_DIRS+=usr/share/man/cat8/powerpc
	OLD_DIRS+=usr/share/man/cat8/sparc64
	OLD_DIRS+=usr/share/man/cat8
	OLD_DIRS+=usr/share/man/cat9
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat1
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat2
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat3
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/amd64
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/arm
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/i386
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/powerpc
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/sparc64
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat5
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat6
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat7
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/amd64
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/arm
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/i386
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/powerpc
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/sparc64
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat9
	OLD_DIRS+=usr/share/man/en.UTF-8/cat1
	OLD_DIRS+=usr/share/man/en.UTF-8/cat2
	OLD_DIRS+=usr/share/man/en.UTF-8/cat3
	OLD_DIRS+=usr/share/man/en.UTF-8/cat4/amd64
	OLD_DIRS+=usr/share/man/en.UTF-8/cat4/arm
	OLD_DIRS+=usr/share/man/en.UTF-8/cat4/i386
	OLD_DIRS+=usr/share/man/en.UTF-8/cat4/powerpc
	OLD_DIRS+=usr/share/man/en.UTF-8/cat4/sparc64
	OLD_DIRS+=usr/share/man/en.UTF-8/cat4
	OLD_DIRS+=usr/share/man/en.UTF-8/cat5
	OLD_DIRS+=usr/share/man/en.UTF-8/cat6
	OLD_DIRS+=usr/share/man/en.UTF-8/cat7
	OLD_DIRS+=usr/share/man/en.UTF-8/cat8/amd64
	OLD_DIRS+=usr/share/man/en.UTF-8/cat8/arm
	OLD_DIRS+=usr/share/man/en.UTF-8/cat8/i386
	OLD_DIRS+=usr/share/man/en.UTF-8/cat8/powerpc
	OLD_DIRS+=usr/share/man/en.UTF-8/cat8/sparc64
	OLD_DIRS+=usr/share/man/en.UTF-8/cat8
	OLD_DIRS+=usr/share/man/en.UTF-8/cat9
	OLD_DIRS+=usr/share/man/ja/cat1
	OLD_DIRS+=usr/share/man/ja/cat2
	OLD_DIRS+=usr/share/man/ja/cat3
	OLD_DIRS+=usr/share/man/ja/cat4/amd64
	OLD_DIRS+=usr/share/man/ja/cat4/arm
	OLD_DIRS+=usr/share/man/ja/cat4/i386
	OLD_DIRS+=usr/share/man/ja/cat4/powerpc
	OLD_DIRS+=usr/share/man/ja/cat4/sparc64
	OLD_DIRS+=usr/share/man/ja/cat4
	OLD_DIRS+=usr/share/man/ja/cat5
	OLD_DIRS+=usr/share/man/ja/cat6
	OLD_DIRS+=usr/share/man/ja/cat7
	OLD_DIRS+=usr/share/man/ja/cat8/amd64
	OLD_DIRS+=usr/share/man/ja/cat8/arm
	OLD_DIRS+=usr/share/man/ja/cat8/powerpc
	OLD_DIRS+=usr/share/man/ja/cat8/sparc64
	OLD_DIRS+=usr/share/man/ja/cat8
	OLD_DIRS+=usr/share/man/ja/cat9
	OLD_DIRS+=usr/share/openssl/man/cat1
	OLD_DIRS+=usr/share/openssl/man/cat3
	OLD_DIRS+=usr/share/openssl/man/en.ISO8859-1/cat1
	OLD_DIRS+=usr/share/openssl/man/en.ISO8859-1/cat3
	# 20170802: ksyms(4) ioctl interface was removed
	OLD_FILES+=usr/include/sys/ksyms.h
	# 20170729: the iicbus/pcf8563 driver is replaced with iicbus/nxprtc
	OLD_FILES+=usr/include/dev/iicbus/pcf8563reg.h
	# 20170722: new clang import which bumps version from 4.0.0 to 5.0.0.
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/4.0.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_complex_builtins.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/armintr.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/4.0.0/include/msa.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/4.0.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/4.0.0/include
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/4.0.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/4.0.0/lib
	OLD_DIRS+=usr/lib/clang/4.0.0
	OLD_FILES+=usr/bin/llvm-pdbdump
	# 20170717: Remove documentation of vaporware
	OLD_FILES+=usr/share/man/man2/pdwait4.2.gz
	# 20170610: chown-f_test replaced by chown_test
	OLD_FILES+=usr/tests/usr.sbin/chown/chown-f_test
	# 20170609: drop obsolete manpage link (if_rtwn.ko -> rtwn.ko)
	OLD_FILES+=usr/share/man/man4/if_rtwn.4.gz
	# 20170531: removal of groff
	OLD_FILES+=usr/bin/addftinfo
	OLD_FILES+=usr/bin/afmtodit
	OLD_FILES+=usr/bin/checknr
	OLD_FILES+=usr/bin/colcrt
	OLD_FILES+=usr/bin/eqn
	OLD_FILES+=usr/bin/grn
	OLD_FILES+=usr/bin/grodvi
	OLD_FILES+=usr/bin/groff
	OLD_FILES+=usr/bin/grog
	OLD_FILES+=usr/bin/grolbp
	OLD_FILES+=usr/bin/grolj4
	OLD_FILES+=usr/bin/grops
	OLD_FILES+=usr/bin/grotty
	OLD_FILES+=usr/bin/hpftodit
	OLD_FILES+=usr/bin/indxbib
	OLD_FILES+=usr/bin/lkbib
	OLD_FILES+=usr/bin/lookbib
	OLD_FILES+=usr/bin/mmroff
	OLD_FILES+=usr/bin/neqn
	OLD_FILES+=usr/bin/nroff
	OLD_FILES+=usr/bin/pfbtops
	OLD_FILES+=usr/bin/pic
	OLD_FILES+=usr/bin/post-grohtml
	OLD_FILES+=usr/bin/pre-grohtml
	OLD_FILES+=usr/bin/psroff
	OLD_FILES+=usr/bin/refer
	OLD_FILES+=usr/bin/tbl
	OLD_FILES+=usr/bin/tfmtodit
	OLD_FILES+=usr/bin/troff
	OLD_FILES+=usr/bin/vgrind
	OLD_FILES+=usr/libexec/vfontedpr
	OLD_FILES+=usr/share/dict/eign
	OLD_FILES+=usr/share/groff_font/devX100-12/CB
	OLD_FILES+=usr/share/groff_font/devX100-12/CBI
	OLD_FILES+=usr/share/groff_font/devX100-12/CI
	OLD_FILES+=usr/share/groff_font/devX100-12/CR
	OLD_FILES+=usr/share/groff_font/devX100-12/DESC
	OLD_FILES+=usr/share/groff_font/devX100-12/HB
	OLD_FILES+=usr/share/groff_font/devX100-12/HBI
	OLD_FILES+=usr/share/groff_font/devX100-12/HI
	OLD_FILES+=usr/share/groff_font/devX100-12/HR
	OLD_FILES+=usr/share/groff_font/devX100-12/NB
	OLD_FILES+=usr/share/groff_font/devX100-12/NBI
	OLD_FILES+=usr/share/groff_font/devX100-12/NI
	OLD_FILES+=usr/share/groff_font/devX100-12/NR
	OLD_FILES+=usr/share/groff_font/devX100-12/S
	OLD_FILES+=usr/share/groff_font/devX100-12/TB
	OLD_FILES+=usr/share/groff_font/devX100-12/TBI
	OLD_FILES+=usr/share/groff_font/devX100-12/TI
	OLD_FILES+=usr/share/groff_font/devX100-12/TR
	OLD_DIRS+=usr/share/groff_font/devX100-12
	OLD_FILES+=usr/share/groff_font/devX100/CB
	OLD_FILES+=usr/share/groff_font/devX100/CBI
	OLD_FILES+=usr/share/groff_font/devX100/CI
	OLD_FILES+=usr/share/groff_font/devX100/CR
	OLD_FILES+=usr/share/groff_font/devX100/DESC
	OLD_FILES+=usr/share/groff_font/devX100/HB
	OLD_FILES+=usr/share/groff_font/devX100/HBI
	OLD_FILES+=usr/share/groff_font/devX100/HI
	OLD_FILES+=usr/share/groff_font/devX100/HR
	OLD_FILES+=usr/share/groff_font/devX100/NB
	OLD_FILES+=usr/share/groff_font/devX100/NBI
	OLD_FILES+=usr/share/groff_font/devX100/NI
	OLD_FILES+=usr/share/groff_font/devX100/NR
	OLD_FILES+=usr/share/groff_font/devX100/S
	OLD_FILES+=usr/share/groff_font/devX100/TB
	OLD_FILES+=usr/share/groff_font/devX100/TBI
	OLD_FILES+=usr/share/groff_font/devX100/TI
	OLD_FILES+=usr/share/groff_font/devX100/TR
	OLD_DIRS+=usr/share/groff_font/devX100
	OLD_FILES+=usr/share/groff_font/devX75-12/CB
	OLD_FILES+=usr/share/groff_font/devX75-12/CBI
	OLD_FILES+=usr/share/groff_font/devX75-12/CI
	OLD_FILES+=usr/share/groff_font/devX75-12/CR
	OLD_FILES+=usr/share/groff_font/devX75-12/DESC
	OLD_FILES+=usr/share/groff_font/devX75-12/HB
	OLD_FILES+=usr/share/groff_font/devX75-12/HBI
	OLD_FILES+=usr/share/groff_font/devX75-12/HI
	OLD_FILES+=usr/share/groff_font/devX75-12/HR
	OLD_FILES+=usr/share/groff_font/devX75-12/NB
	OLD_FILES+=usr/share/groff_font/devX75-12/NBI
	OLD_FILES+=usr/share/groff_font/devX75-12/NI
	OLD_FILES+=usr/share/groff_font/devX75-12/NR
	OLD_FILES+=usr/share/groff_font/devX75-12/S
	OLD_FILES+=usr/share/groff_font/devX75-12/TB
	OLD_FILES+=usr/share/groff_font/devX75-12/TBI
	OLD_FILES+=usr/share/groff_font/devX75-12/TI
	OLD_FILES+=usr/share/groff_font/devX75-12/TR
	OLD_DIRS+=usr/share/groff_font/devX75-12
	OLD_FILES+=usr/share/groff_font/devX75/CB
	OLD_FILES+=usr/share/groff_font/devX75/CBI
	OLD_FILES+=usr/share/groff_font/devX75/CI
	OLD_FILES+=usr/share/groff_font/devX75/CR
	OLD_FILES+=usr/share/groff_font/devX75/DESC
	OLD_FILES+=usr/share/groff_font/devX75/HB
	OLD_FILES+=usr/share/groff_font/devX75/HBI
	OLD_FILES+=usr/share/groff_font/devX75/HI
	OLD_FILES+=usr/share/groff_font/devX75/HR
	OLD_FILES+=usr/share/groff_font/devX75/NB
	OLD_FILES+=usr/share/groff_font/devX75/NBI
	OLD_FILES+=usr/share/groff_font/devX75/NI
	OLD_FILES+=usr/share/groff_font/devX75/NR
	OLD_FILES+=usr/share/groff_font/devX75/S
	OLD_FILES+=usr/share/groff_font/devX75/TB
	OLD_FILES+=usr/share/groff_font/devX75/TBI
	OLD_FILES+=usr/share/groff_font/devX75/TI
	OLD_FILES+=usr/share/groff_font/devX75/TR
	OLD_DIRS+=usr/share/groff_font/devX75
	OLD_FILES+=usr/share/groff_font/devascii/B
	OLD_FILES+=usr/share/groff_font/devascii/BI
	OLD_FILES+=usr/share/groff_font/devascii/CW
	OLD_FILES+=usr/share/groff_font/devascii/DESC
	OLD_FILES+=usr/share/groff_font/devascii/I
	OLD_FILES+=usr/share/groff_font/devascii/L
	OLD_FILES+=usr/share/groff_font/devascii/R
	OLD_FILES+=usr/share/groff_font/devascii/S
	OLD_DIRS+=usr/share/groff_font/devascii
	OLD_FILES+=usr/share/groff_font/devcp1047/B
	OLD_FILES+=usr/share/groff_font/devcp1047/BI
	OLD_FILES+=usr/share/groff_font/devcp1047/CW
	OLD_FILES+=usr/share/groff_font/devcp1047/DESC
	OLD_FILES+=usr/share/groff_font/devcp1047/I
	OLD_FILES+=usr/share/groff_font/devcp1047/L
	OLD_FILES+=usr/share/groff_font/devcp1047/R
	OLD_FILES+=usr/share/groff_font/devcp1047/S
	OLD_DIRS+=usr/share/groff_font/devcp1047
	OLD_FILES+=usr/share/groff_font/devdvi/CW
	OLD_FILES+=usr/share/groff_font/devdvi/CWEC
	OLD_FILES+=usr/share/groff_font/devdvi/CWI
	OLD_FILES+=usr/share/groff_font/devdvi/CWIEC
	OLD_FILES+=usr/share/groff_font/devdvi/CWITC
	OLD_FILES+=usr/share/groff_font/devdvi/CWTC
	OLD_FILES+=usr/share/groff_font/devdvi/CompileFonts
	OLD_FILES+=usr/share/groff_font/devdvi/DESC
	OLD_FILES+=usr/share/groff_font/devdvi/EX
	OLD_FILES+=usr/share/groff_font/devdvi/HB
	OLD_FILES+=usr/share/groff_font/devdvi/HBEC
	OLD_FILES+=usr/share/groff_font/devdvi/HBI
	OLD_FILES+=usr/share/groff_font/devdvi/HBIEC
	OLD_FILES+=usr/share/groff_font/devdvi/HBITC
	OLD_FILES+=usr/share/groff_font/devdvi/HBTC
	OLD_FILES+=usr/share/groff_font/devdvi/HI
	OLD_FILES+=usr/share/groff_font/devdvi/HIEC
	OLD_FILES+=usr/share/groff_font/devdvi/HITC
	OLD_FILES+=usr/share/groff_font/devdvi/HR
	OLD_FILES+=usr/share/groff_font/devdvi/HREC
	OLD_FILES+=usr/share/groff_font/devdvi/HRTC
	OLD_FILES+=usr/share/groff_font/devdvi/MI
	OLD_FILES+=usr/share/groff_font/devdvi/Makefile
	OLD_FILES+=usr/share/groff_font/devdvi/S
	OLD_FILES+=usr/share/groff_font/devdvi/SA
	OLD_FILES+=usr/share/groff_font/devdvi/SB
	OLD_FILES+=usr/share/groff_font/devdvi/SC
	OLD_FILES+=usr/share/groff_font/devdvi/TB
	OLD_FILES+=usr/share/groff_font/devdvi/TBEC
	OLD_FILES+=usr/share/groff_font/devdvi/TBI
	OLD_FILES+=usr/share/groff_font/devdvi/TBIEC
	OLD_FILES+=usr/share/groff_font/devdvi/TBITC
	OLD_FILES+=usr/share/groff_font/devdvi/TBTC
	OLD_FILES+=usr/share/groff_font/devdvi/TI
	OLD_FILES+=usr/share/groff_font/devdvi/TIEC
	OLD_FILES+=usr/share/groff_font/devdvi/TITC
	OLD_FILES+=usr/share/groff_font/devdvi/TR
	OLD_FILES+=usr/share/groff_font/devdvi/TREC
	OLD_FILES+=usr/share/groff_font/devdvi/TRTC
	OLD_FILES+=usr/share/groff_font/devdvi/ec.map
	OLD_FILES+=usr/share/groff_font/devdvi/msam.map
	OLD_FILES+=usr/share/groff_font/devdvi/msbm.map
	OLD_FILES+=usr/share/groff_font/devdvi/tc.map
	OLD_FILES+=usr/share/groff_font/devdvi/texb.map
	OLD_FILES+=usr/share/groff_font/devdvi/texex.map
	OLD_FILES+=usr/share/groff_font/devdvi/texi.map
	OLD_FILES+=usr/share/groff_font/devdvi/texmi.map
	OLD_FILES+=usr/share/groff_font/devdvi/texr.map
	OLD_FILES+=usr/share/groff_font/devdvi/texsy.map
	OLD_FILES+=usr/share/groff_font/devdvi/textex.map
	OLD_FILES+=usr/share/groff_font/devdvi/textt.map
	OLD_DIRS+=usr/share/groff_font/devdvi
	OLD_FILES+=usr/share/groff_font/devhtml/B
	OLD_FILES+=usr/share/groff_font/devhtml/BI
	OLD_FILES+=usr/share/groff_font/devhtml/CB
	OLD_FILES+=usr/share/groff_font/devhtml/CBI
	OLD_FILES+=usr/share/groff_font/devhtml/CI
	OLD_FILES+=usr/share/groff_font/devhtml/CR
	OLD_FILES+=usr/share/groff_font/devhtml/DESC
	OLD_FILES+=usr/share/groff_font/devhtml/I
	OLD_FILES+=usr/share/groff_font/devhtml/R
	OLD_FILES+=usr/share/groff_font/devhtml/S
	OLD_DIRS+=usr/share/groff_font/devhtml
	OLD_FILES+=usr/share/groff_font/devkoi8-r/B
	OLD_FILES+=usr/share/groff_font/devkoi8-r/BI
	OLD_FILES+=usr/share/groff_font/devkoi8-r/CW
	OLD_FILES+=usr/share/groff_font/devkoi8-r/DESC
	OLD_FILES+=usr/share/groff_font/devkoi8-r/I
	OLD_FILES+=usr/share/groff_font/devkoi8-r/L
	OLD_FILES+=usr/share/groff_font/devkoi8-r/R
	OLD_FILES+=usr/share/groff_font/devkoi8-r/S
	OLD_DIRS+=usr/share/groff_font/devkoi8-r
	OLD_FILES+=usr/share/groff_font/devlatin1/B
	OLD_FILES+=usr/share/groff_font/devlatin1/BI
	OLD_FILES+=usr/share/groff_font/devlatin1/CW
	OLD_FILES+=usr/share/groff_font/devlatin1/DESC
	OLD_FILES+=usr/share/groff_font/devlatin1/I
	OLD_FILES+=usr/share/groff_font/devlatin1/L
	OLD_FILES+=usr/share/groff_font/devlatin1/R
	OLD_FILES+=usr/share/groff_font/devlatin1/S
	OLD_DIRS+=usr/share/groff_font/devlatin1
	OLD_FILES+=usr/share/groff_font/devlbp/CB
	OLD_FILES+=usr/share/groff_font/devlbp/CI
	OLD_FILES+=usr/share/groff_font/devlbp/CR
	OLD_FILES+=usr/share/groff_font/devlbp/DESC
	OLD_FILES+=usr/share/groff_font/devlbp/EB
	OLD_FILES+=usr/share/groff_font/devlbp/EI
	OLD_FILES+=usr/share/groff_font/devlbp/ER
	OLD_FILES+=usr/share/groff_font/devlbp/HB
	OLD_FILES+=usr/share/groff_font/devlbp/HBI
	OLD_FILES+=usr/share/groff_font/devlbp/HI
	OLD_FILES+=usr/share/groff_font/devlbp/HNB
	OLD_FILES+=usr/share/groff_font/devlbp/HNBI
	OLD_FILES+=usr/share/groff_font/devlbp/HNI
	OLD_FILES+=usr/share/groff_font/devlbp/HNR
	OLD_FILES+=usr/share/groff_font/devlbp/HR
	OLD_FILES+=usr/share/groff_font/devlbp/TB
	OLD_FILES+=usr/share/groff_font/devlbp/TBI
	OLD_FILES+=usr/share/groff_font/devlbp/TI
	OLD_FILES+=usr/share/groff_font/devlbp/TR
	OLD_DIRS+=usr/share/groff_font/devlbp
	OLD_FILES+=usr/share/groff_font/devlj4/AB
	OLD_FILES+=usr/share/groff_font/devlj4/ABI
	OLD_FILES+=usr/share/groff_font/devlj4/AI
	OLD_FILES+=usr/share/groff_font/devlj4/ALBB
	OLD_FILES+=usr/share/groff_font/devlj4/ALBR
	OLD_FILES+=usr/share/groff_font/devlj4/AOB
	OLD_FILES+=usr/share/groff_font/devlj4/AOI
	OLD_FILES+=usr/share/groff_font/devlj4/AOR
	OLD_FILES+=usr/share/groff_font/devlj4/AR
	OLD_FILES+=usr/share/groff_font/devlj4/CB
	OLD_FILES+=usr/share/groff_font/devlj4/CBI
	OLD_FILES+=usr/share/groff_font/devlj4/CI
	OLD_FILES+=usr/share/groff_font/devlj4/CLARENDON
	OLD_FILES+=usr/share/groff_font/devlj4/CORONET
	OLD_FILES+=usr/share/groff_font/devlj4/CR
	OLD_FILES+=usr/share/groff_font/devlj4/DESC
	OLD_FILES+=usr/share/groff_font/devlj4/GB
	OLD_FILES+=usr/share/groff_font/devlj4/GBI
	OLD_FILES+=usr/share/groff_font/devlj4/GI
	OLD_FILES+=usr/share/groff_font/devlj4/GR
	OLD_FILES+=usr/share/groff_font/devlj4/LGB
	OLD_FILES+=usr/share/groff_font/devlj4/LGI
	OLD_FILES+=usr/share/groff_font/devlj4/LGR
	OLD_FILES+=usr/share/groff_font/devlj4/MARIGOLD
	OLD_FILES+=usr/share/groff_font/devlj4/OB
	OLD_FILES+=usr/share/groff_font/devlj4/OBI
	OLD_FILES+=usr/share/groff_font/devlj4/OI
	OLD_FILES+=usr/share/groff_font/devlj4/OR
	OLD_FILES+=usr/share/groff_font/devlj4/S
	OLD_FILES+=usr/share/groff_font/devlj4/SYMBOL
	OLD_FILES+=usr/share/groff_font/devlj4/TB
	OLD_FILES+=usr/share/groff_font/devlj4/TBI
	OLD_FILES+=usr/share/groff_font/devlj4/TI
	OLD_FILES+=usr/share/groff_font/devlj4/TNRB
	OLD_FILES+=usr/share/groff_font/devlj4/TNRBI
	OLD_FILES+=usr/share/groff_font/devlj4/TNRI
	OLD_FILES+=usr/share/groff_font/devlj4/TNRR
	OLD_FILES+=usr/share/groff_font/devlj4/TR
	OLD_FILES+=usr/share/groff_font/devlj4/UB
	OLD_FILES+=usr/share/groff_font/devlj4/UBI
	OLD_FILES+=usr/share/groff_font/devlj4/UCB
	OLD_FILES+=usr/share/groff_font/devlj4/UCBI
	OLD_FILES+=usr/share/groff_font/devlj4/UCI
	OLD_FILES+=usr/share/groff_font/devlj4/UCR
	OLD_FILES+=usr/share/groff_font/devlj4/UI
	OLD_FILES+=usr/share/groff_font/devlj4/UR
	OLD_FILES+=usr/share/groff_font/devlj4/WINGDINGS
	OLD_DIRS+=usr/share/groff_font/devlj4
	OLD_FILES+=usr/share/groff_font/devps/AB
	OLD_FILES+=usr/share/groff_font/devps/ABI
	OLD_FILES+=usr/share/groff_font/devps/AI
	OLD_FILES+=usr/share/groff_font/devps/AR
	OLD_FILES+=usr/share/groff_font/devps/BMB
	OLD_FILES+=usr/share/groff_font/devps/BMBI
	OLD_FILES+=usr/share/groff_font/devps/BMI
	OLD_FILES+=usr/share/groff_font/devps/BMR
	OLD_FILES+=usr/share/groff_font/devps/CB
	OLD_FILES+=usr/share/groff_font/devps/CBI
	OLD_FILES+=usr/share/groff_font/devps/CI
	OLD_FILES+=usr/share/groff_font/devps/CR
	OLD_FILES+=usr/share/groff_font/devps/DESC
	OLD_FILES+=usr/share/groff_font/devps/EURO
	OLD_FILES+=usr/share/groff_font/devps/HB
	OLD_FILES+=usr/share/groff_font/devps/HBI
	OLD_FILES+=usr/share/groff_font/devps/HI
	OLD_FILES+=usr/share/groff_font/devps/HNB
	OLD_FILES+=usr/share/groff_font/devps/HNBI
	OLD_FILES+=usr/share/groff_font/devps/HNI
	OLD_FILES+=usr/share/groff_font/devps/HNR
	OLD_FILES+=usr/share/groff_font/devps/HR
	OLD_FILES+=usr/share/groff_font/devps/Makefile
	OLD_FILES+=usr/share/groff_font/devps/NB
	OLD_FILES+=usr/share/groff_font/devps/NBI
	OLD_FILES+=usr/share/groff_font/devps/NI
	OLD_FILES+=usr/share/groff_font/devps/NR
	OLD_FILES+=usr/share/groff_font/devps/PB
	OLD_FILES+=usr/share/groff_font/devps/PBI
	OLD_FILES+=usr/share/groff_font/devps/PI
	OLD_FILES+=usr/share/groff_font/devps/PR
	OLD_FILES+=usr/share/groff_font/devps/S
	OLD_FILES+=usr/share/groff_font/devps/SS
	OLD_FILES+=usr/share/groff_font/devps/TB
	OLD_FILES+=usr/share/groff_font/devps/TBI
	OLD_FILES+=usr/share/groff_font/devps/TI
	OLD_FILES+=usr/share/groff_font/devps/TR
	OLD_FILES+=usr/share/groff_font/devps/ZCMI
	OLD_FILES+=usr/share/groff_font/devps/ZD
	OLD_FILES+=usr/share/groff_font/devps/ZDR
	OLD_FILES+=usr/share/groff_font/devps/afmname
	OLD_FILES+=usr/share/groff_font/devps/dingbats.map
	OLD_FILES+=usr/share/groff_font/devps/dingbats.rmap
	OLD_FILES+=usr/share/groff_font/devps/download
	OLD_FILES+=usr/share/groff_font/devps/freeeuro.pfa
	OLD_FILES+=usr/share/groff_font/devps/lgreekmap
	OLD_FILES+=usr/share/groff_font/devps/prologue
	OLD_FILES+=usr/share/groff_font/devps/symbol.sed
	OLD_FILES+=usr/share/groff_font/devps/symbolchars
	OLD_FILES+=usr/share/groff_font/devps/symbolsl.afm
	OLD_FILES+=usr/share/groff_font/devps/symbolsl.pfa
	OLD_FILES+=usr/share/groff_font/devps/text.enc
	OLD_FILES+=usr/share/groff_font/devps/textmap
	OLD_FILES+=usr/share/groff_font/devps/zapfdr.pfa
	OLD_DIRS+=usr/share/groff_font/devps
	OLD_FILES+=usr/share/groff_font/devutf8/B
	OLD_FILES+=usr/share/groff_font/devutf8/BI
	OLD_FILES+=usr/share/groff_font/devutf8/CW
	OLD_FILES+=usr/share/groff_font/devutf8/DESC
	OLD_FILES+=usr/share/groff_font/devutf8/I
	OLD_FILES+=usr/share/groff_font/devutf8/L
	OLD_FILES+=usr/share/groff_font/devutf8/R
	OLD_FILES+=usr/share/groff_font/devutf8/S
	OLD_DIRS+=usr/share/groff_font/devutf8
	OLD_DIRS+=usr/share/groff_font
	OLD_FILES+=usr/share/man/man1/addftinfo.1.gz
	OLD_FILES+=usr/share/man/man1/afmtodit.1.gz
	OLD_FILES+=usr/share/man/man1/checknr.1.gz
	OLD_FILES+=usr/share/man/man1/colcrt.1.gz
	OLD_FILES+=usr/share/man/man1/eqn.1.gz
	OLD_FILES+=usr/share/man/man1/grn.1.gz
	OLD_FILES+=usr/share/man/man1/grodvi.1.gz
	OLD_FILES+=usr/share/man/man1/groff.1.gz
	OLD_FILES+=usr/share/man/man1/grog.1.gz
	OLD_FILES+=usr/share/man/man1/grolbp.1.gz
	OLD_FILES+=usr/share/man/man1/grolj4.1.gz
	OLD_FILES+=usr/share/man/man1/grops.1.gz
	OLD_FILES+=usr/share/man/man1/grotty.1.gz
	OLD_FILES+=usr/share/man/man1/hpftodit.1.gz
	OLD_FILES+=usr/share/man/man1/indxbib.1.gz
	OLD_FILES+=usr/share/man/man1/lkbib.1.gz
	OLD_FILES+=usr/share/man/man1/lookbib.1.gz
	OLD_FILES+=usr/share/man/man1/mmroff.1.gz
	OLD_FILES+=usr/share/man/man1/neqn.1.gz
	OLD_FILES+=usr/share/man/man1/nroff.1.gz
	OLD_FILES+=usr/share/man/man1/pfbtops.1.gz
	OLD_FILES+=usr/share/man/man1/pic.1.gz
	OLD_FILES+=usr/share/man/man1/psroff.1.gz
	OLD_FILES+=usr/share/man/man1/refer.1.gz
	OLD_FILES+=usr/share/man/man1/tbl.1.gz
	OLD_FILES+=usr/share/man/man1/tfmtodit.1.gz
	OLD_FILES+=usr/share/man/man1/troff.1.gz
	OLD_FILES+=usr/share/man/man1/vgrind.1.gz
	OLD_FILES+=usr/share/man/man5/groff_font.5.gz
	OLD_FILES+=usr/share/man/man5/groff_out.5.gz
	OLD_FILES+=usr/share/man/man5/groff_tmac.5.gz
	OLD_FILES+=usr/share/man/man5/lj4_font.5.gz
	OLD_FILES+=usr/share/man/man5/tmac.5.gz
	OLD_FILES+=usr/share/man/man5/vgrindefs.5.gz
	OLD_FILES+=usr/share/man/man7/ditroff.7.gz
	OLD_FILES+=usr/share/man/man7/groff.7.gz
	OLD_FILES+=usr/share/man/man7/groff_char.7.gz
	OLD_FILES+=usr/share/man/man7/groff_diff.7.gz
	OLD_FILES+=usr/share/man/man7/groff_man.7.gz
	OLD_FILES+=usr/share/man/man7/groff_mdoc.7.gz
	OLD_FILES+=usr/share/man/man7/groff_me.7.gz
	OLD_FILES+=usr/share/man/man7/groff_mm.7.gz
	OLD_FILES+=usr/share/man/man7/groff_mmse.7.gz
	OLD_FILES+=usr/share/man/man7/groff_ms.7.gz
	OLD_FILES+=usr/share/man/man7/groff_trace.7.gz
	OLD_FILES+=usr/share/man/man7/groff_www.7.gz
	OLD_FILES+=usr/share/man/man7/mdoc.samples.7.gz
	OLD_FILES+=usr/share/man/man7/me.7.gz
	OLD_FILES+=usr/share/man/man7/mm.7.gz
	OLD_FILES+=usr/share/man/man7/mmse.7.gz
	OLD_FILES+=usr/share/man/man7/ms.7.gz
	OLD_FILES+=usr/share/man/man7/orig_me.7.gz
	OLD_FILES+=usr/share/me/acm.me
	OLD_FILES+=usr/share/me/chars.me
	OLD_FILES+=usr/share/me/deltext.me
	OLD_FILES+=usr/share/me/eqn.me
	OLD_FILES+=usr/share/me/float.me
	OLD_FILES+=usr/share/me/footnote.me
	OLD_FILES+=usr/share/me/index.me
	OLD_FILES+=usr/share/me/letterhead.me
	OLD_FILES+=usr/share/me/local.me
	OLD_FILES+=usr/share/me/null.me
	OLD_FILES+=usr/share/me/refer.me
	OLD_FILES+=usr/share/me/revisions
	OLD_FILES+=usr/share/me/sh.me
	OLD_FILES+=usr/share/me/tbl.me
	OLD_FILES+=usr/share/me/thesis.me
	OLD_DIRS+=usr/share/me
	OLD_FILES+=usr/share/misc/vgrindefs
	OLD_FILES+=usr/share/misc/vgrindefs.db
	OLD_FILES+=usr/share/tmac/X.tmac
	OLD_FILES+=usr/share/tmac/Xps.tmac
	OLD_FILES+=usr/share/tmac/a4.tmac
	OLD_FILES+=usr/share/tmac/an-old.tmac
	OLD_FILES+=usr/share/tmac/an.tmac
	OLD_FILES+=usr/share/tmac/andoc.tmac
	OLD_FILES+=usr/share/tmac/composite.tmac
	OLD_FILES+=usr/share/tmac/cp1047.tmac
	OLD_FILES+=usr/share/tmac/devtag.tmac
	OLD_FILES+=usr/share/tmac/doc.tmac
	OLD_FILES+=usr/share/tmac/dvi.tmac
	OLD_FILES+=usr/share/tmac/e.tmac
	OLD_FILES+=usr/share/tmac/ec.tmac
	OLD_FILES+=usr/share/tmac/eqnrc
	OLD_FILES+=usr/share/tmac/europs.tmac
	OLD_FILES+=usr/share/tmac/html-end.tmac
	OLD_FILES+=usr/share/tmac/html.tmac
	OLD_FILES+=usr/share/tmac/hyphen.ru
	OLD_FILES+=usr/share/tmac/hyphen.us
	OLD_FILES+=usr/share/tmac/hyphenex.us
	OLD_FILES+=usr/share/tmac/koi8-r.tmac
	OLD_FILES+=usr/share/tmac/latin1.tmac
	OLD_FILES+=usr/share/tmac/latin2.tmac
	OLD_FILES+=usr/share/tmac/latin9.tmac
	OLD_FILES+=usr/share/tmac/lbp.tmac
	OLD_FILES+=usr/share/tmac/lj4.tmac
	OLD_FILES+=usr/share/tmac/m.tmac
	OLD_FILES+=usr/share/tmac/man.local
	OLD_FILES+=usr/share/tmac/man.tmac
	OLD_FILES+=usr/share/tmac/mandoc.tmac
	OLD_FILES+=usr/share/tmac/mdoc.local
	OLD_FILES+=usr/share/tmac/mdoc.tmac
	OLD_FILES+=usr/share/tmac/mdoc/doc-common
	OLD_FILES+=usr/share/tmac/mdoc/doc-ditroff
	OLD_FILES+=usr/share/tmac/mdoc/doc-nroff
	OLD_FILES+=usr/share/tmac/mdoc/doc-syms
	OLD_FILES+=usr/share/tmac/mdoc/fr.ISO8859-1
	OLD_FILES+=usr/share/tmac/mdoc/ru.KOI8-R
	OLD_DIRS+=usr/share/tmac/mdoc
	OLD_FILES+=usr/share/tmac/me.tmac
	OLD_FILES+=usr/share/tmac/mm/0.MT
	OLD_FILES+=usr/share/tmac/mm/4.MT
	OLD_FILES+=usr/share/tmac/mm/5.MT
	OLD_FILES+=usr/share/tmac/mm/locale
	OLD_FILES+=usr/share/tmac/mm/mm.tmac
	OLD_FILES+=usr/share/tmac/mm/mmse.tmac
	OLD_FILES+=usr/share/tmac/mm/ms.cov
	OLD_FILES+=usr/share/tmac/mm/se_locale
	OLD_FILES+=usr/share/tmac/mm/se_ms.cov
	OLD_DIRS+=usr/share/tmac/mm
	OLD_FILES+=usr/share/tmac/ms.tmac
	OLD_FILES+=usr/share/tmac/mse.tmac
	OLD_FILES+=usr/share/tmac/papersize.tmac
	OLD_FILES+=usr/share/tmac/pic.tmac
	OLD_FILES+=usr/share/tmac/ps.tmac
	OLD_FILES+=usr/share/tmac/psatk.tmac
	OLD_FILES+=usr/share/tmac/psold.tmac
	OLD_FILES+=usr/share/tmac/pspic.tmac
	OLD_FILES+=usr/share/tmac/s.tmac
	OLD_FILES+=usr/share/tmac/safer.tmac
	OLD_FILES+=usr/share/tmac/tmac.orig_me
	OLD_FILES+=usr/share/tmac/tmac.vgrind
	OLD_FILES+=usr/share/tmac/trace.tmac
	OLD_FILES+=usr/share/tmac/troffrc
	OLD_FILES+=usr/share/tmac/troffrc-end
	OLD_FILES+=usr/share/tmac/tty-char.tmac
	OLD_FILES+=usr/share/tmac/tty.tmac
	OLD_FILES+=usr/share/tmac/unicode.tmac
	OLD_FILES+=usr/share/tmac/www.tmac
	OLD_DIRS+=usr/share/tmac
	# 20170607: remove incorrect atf_check(1) manpage link
	OLD_FILES+=usr/share/man/man1/atf_check.1.gz
	# 20170601: remove stale manpage
	OLD_FILES+=usr/share/man/man2/cap_rights_get.2.gz
	# 20170601: old libifconfig and libifc
	OLD_FILES+=usr/lib/libifc.a
	OLD_FILES+=usr/lib/libifc_p.a
	OLD_FILES+=usr/lib/libifconfig.a
	OLD_FILES+=usr/lib/libifconfig_p.a
	OLD_FILES+=usr/lib32/libifc.a
	OLD_FILES+=usr/lib32/libifc_p.a
	OLD_FILES+=usr/lib32/libifconfig.a
	OLD_FILES+=usr/lib32/libifconfig_p.a
	# 20170529: mount.conf(8) -> mount.conf(5)
	OLD_FILES+=usr/share/man/man8/mount.conf.8.gz
	# 20170525: remove misleading template
	OLD_FILES+=usr/share/misc/man.template
	# 20170525: disconnect the roff docs from the build
	OLD_FILES+=usr/share/doc/papers/beyond43.ascii.gz
	OLD_FILES+=usr/share/doc/papers/bio.ascii.gz
	OLD_FILES+=usr/share/doc/papers/contents.ascii.gz
	OLD_FILES+=usr/share/doc/papers/devfs.ascii.gz
	OLD_FILES+=usr/share/doc/papers/diskperf.ascii.gz
	OLD_FILES+=usr/share/doc/papers/fsinterface.ascii.gz
	OLD_FILES+=usr/share/doc/papers/hwpmc.ascii.gz
	OLD_FILES+=usr/share/doc/papers/jail.ascii.gz
	OLD_FILES+=usr/share/doc/papers/kernmalloc.ascii.gz
	OLD_FILES+=usr/share/doc/papers/kerntune.ascii.gz
	OLD_FILES+=usr/share/doc/papers/malloc.ascii.gz
	OLD_FILES+=usr/share/doc/papers/newvm.ascii.gz
	OLD_FILES+=usr/share/doc/papers/releng.ascii.gz
	OLD_FILES+=usr/share/doc/papers/sysperf.ascii.gz
	OLD_FILES+=usr/share/doc/papers/timecounter.ascii.gz
	OLD_DIRS+=usr/share/doc/papers
	OLD_FILES+=usr/share/doc/psd/01.cacm/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/01.cacm
	OLD_FILES+=usr/share/doc/psd/02.implement/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/02.implement
	OLD_FILES+=usr/share/doc/psd/03.iosys/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/03.iosys
	OLD_FILES+=usr/share/doc/psd/04.uprog/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/04.uprog
	OLD_FILES+=usr/share/doc/psd/05.sysman/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/05.sysman
	OLD_FILES+=usr/share/doc/psd/06.Clang/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/06.Clang
	OLD_FILES+=usr/share/doc/psd/12.make/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/12.make
	OLD_FILES+=usr/share/doc/psd/13.rcs/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/13.rcs
	OLD_FILES+=usr/share/doc/psd/13.rcs/rcs_func.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/13.rcs
	OLD_FILES+=usr/share/doc/psd/15.yacc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/15.yacc
	OLD_FILES+=usr/share/doc/psd/16.lex/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/16.lex
	OLD_FILES+=usr/share/doc/psd/17.m4/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/17.m4
	OLD_FILES+=usr/share/doc/psd/18.gprof/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/18.gprof
	OLD_FILES+=usr/share/doc/psd/20.ipctut/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/20.ipctut
	OLD_FILES+=usr/share/doc/psd/21.ipc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/21.ipc
	OLD_FILES+=usr/share/doc/psd/22.rpcgen/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/22.rpcgen
	OLD_FILES+=usr/share/doc/psd/23.rpc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/23.rpc
	OLD_FILES+=usr/share/doc/psd/24.xdr/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/24.xdr
	OLD_FILES+=usr/share/doc/psd/25.xdrrfc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/25.xdrrfc
	OLD_FILES+=usr/share/doc/psd/26.rpcrfc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/26.rpcrfc
	OLD_FILES+=usr/share/doc/psd/27.nfsrfc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/27.nfsrfc
	OLD_FILES+=usr/share/doc/psd/Title.ascii.gz
	OLD_FILES+=usr/share/doc/psd/contents.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/
	OLD_FILES+=usr/share/doc/smm/01.setup/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/01.setup
	OLD_FILES+=usr/share/doc/smm/02.config/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/02.config
	OLD_FILES+=usr/share/doc/smm/03.fsck/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/03.fsck
	OLD_FILES+=usr/share/doc/smm/04.quotas/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/04.quotas
	OLD_FILES+=usr/share/doc/smm/05.fastfs/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/05.fastfs
	OLD_FILES+=usr/share/doc/smm/06.nfs/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/06.nfs
	OLD_FILES+=usr/share/doc/smm/07.lpd/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/07.lpd
	OLD_FILES+=usr/share/doc/smm/08.sendmailop/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/08.sendmailop
	OLD_FILES+=usr/share/doc/smm/11.timedop/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/11.timedop
	OLD_FILES+=usr/share/doc/smm/12.timed/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/12.timed
	OLD_FILES+=usr/share/doc/smm/18.net/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/smm/18.net
	OLD_FILES+=usr/share/doc/smm/Title.ascii.gz
	OLD_FILES+=usr/share/doc/smm/contents.ascii.gz
	OLD_DIRS+=usr/share/doc/smm
	OLD_FILES+=usr/share/doc/usd/04.csh/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/04.csh
	OLD_FILES+=usr/share/doc/usd/05.dc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/05.dc
	OLD_FILES+=usr/share/doc/usd/06.bc/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/06.bc
	OLD_FILES+=usr/share/doc/usd/07.mail/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/07.mail
	OLD_FILES+=usr/share/doc/usd/10.exref/paper.ascii.gz
	OLD_FILES+=usr/share/doc/usd/10.exref/summary.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/10.exref
	OLD_FILES+=usr/share/doc/usd/11.edit/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/11.edit
	OLD_FILES+=usr/share/doc/usd/12.vi/paper.ascii.gz
	OLD_FILES+=usr/share/doc/usd/12.vi/summary.ascii.gz
	OLD_FILES+=usr/share/doc/usd/12.vi/viapwh.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/12.vi
	OLD_FILES+=usr/share/doc/usd/13.viref/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/13.viref
	OLD_FILES+=usr/share/doc/usd/18.msdiffs/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/18.msdiffs
	OLD_FILES+=usr/share/doc/usd/19.memacros/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/19.memacros
	OLD_FILES+=usr/share/doc/usd/20.meref/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/20.meref
	OLD_FILES+=usr/share/doc/usd/21.troff/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/21.troff
	OLD_FILES+=usr/share/doc/usd/22.trofftut/paper.ascii.gz
	OLD_DIRS+=usr/share/doc/usd/22.trofftut
	OLD_FILES+=usr/share/doc/usd/Title.ascii.gz
	OLD_FILES+=usr/share/doc/usd/contents.ascii.gz
	OLD_DIRS+=usr/share/doc/usd
	# 20170523: 64-bit inode support, library version bumps
	OLD_LIBS+=lib/libzfs.so.2
	OLD_LIBS+=usr/lib/libarchive.so.6
	OLD_LIBS+=usr/lib/libmilter.so.5
	OLD_LIBS+=usr/lib32/libzfs.so.2
	OLD_LIBS+=usr/lib32/libarchive.so.6
	OLD_LIBS+=usr/lib32/libmilter.so.5
	# 20170427: NATM configuration support removed
	OLD_FILES+=etc/rc.d/atm1
	OLD_FILES+=etc/rc.d/atm2
	OLD_FILES+=etc/rc.d/atm3
	# 20170424: NATM support removed
	OLD_FILES+=rescue/atmconfig
	OLD_FILES+=sbin/atmconfig
	OLD_FILES+=usr/include/bsnmp/snmp_atm.h
	OLD_FILES+=usr/include/dev/utopia/idtphy.h
	OLD_FILES+=usr/include/dev/utopia/suni.h
	OLD_FILES+=usr/include/dev/utopia/utopia.h
	OLD_FILES+=usr/include/dev/utopia/utopia_priv.h
	OLD_DIRS+=usr/include/dev/utopia
	OLD_FILES+=usr/include/net/if_atm.h
	OLD_FILES+=usr/include/netgraph/atm/ng_atm.h
	OLD_FILES+=usr/include/netinet/if_atm.h
	OLD_FILES+=usr/include/netnatm/natm.h
	OLD_FILES+=usr/lib/debug/sbin/atmconfig.debug
	OLD_FILES+=usr/lib/debug/usr/lib/snmp_atm.so.6.debug
	OLD_FILES+=usr/lib/snmp_atm.so
	OLD_FILES+=usr/lib/snmp_atm.so.6
	OLD_FILES+=usr/share/doc/atm/atmconfig.help
	OLD_FILES+=usr/share/doc/atm/atmconfig_device.help
	OLD_DIRS+=usr/share/doc/atm
	OLD_FILES+=usr/share/man/man3/snmp_atm.3.gz
	OLD_FILES+=usr/share/man/man4/en.4.gz
	OLD_FILES+=usr/share/man/man4/fatm.4.gz
	OLD_FILES+=usr/share/man/man4/hatm.4.gz
	OLD_FILES+=usr/share/man/man4/if_en.4.gz
	OLD_FILES+=usr/share/man/man4/if_fatm.4.gz
	OLD_FILES+=usr/share/man/man4/if_hatm.4.gz
	OLD_FILES+=usr/share/man/man4/if_patm.4.gz
	OLD_FILES+=usr/share/man/man4/natm.4.gz
	OLD_FILES+=usr/share/man/man4/natmip.4.gz
	OLD_FILES+=usr/share/man/man4/ng_atm.4.gz
	OLD_FILES+=usr/share/man/man4/patm.4.gz
	OLD_FILES+=usr/share/man/man4/utopia.4.gz
	OLD_FILES+=usr/share/man/man8/atmconfig.8.gz
	OLD_FILES+=usr/share/man/man9/utopia.9.gz
	OLD_FILES+=usr/share/snmp/defs/atm_freebsd.def
	OLD_FILES+=usr/share/snmp/defs/atm_tree.def
	OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-ATM-FREEBSD-MIB.txt
	OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-ATM.txt
	# 20170420: remove GNU diff
	OLD_FILES+=usr/share/man/man7/diff.7.gz
	# 20170322: rename <x> to <x>_test to match the FreeBSD test suite name scheme
	OLD_FILES+=usr/tests/usr.bin/col/col
	OLD_FILES+=usr/tests/usr.bin/diff/diff
	OLD_FILES+=usr/tests/usr.bin/ident/ident
	OLD_FILES+=usr/tests/usr.bin/mkimg/mkimg
	OLD_FILES+=usr/tests/usr.bin/sdiff/sdiff
	OLD_FILES+=usr/tests/usr.bin/soelim/soelim
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_config
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_etcdir
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_groupadd
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_groupdel
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_groupmod
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_lock
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_useradd
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_userdel
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_usermod
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_usernext
	# 20170319: io_test requires zh_TW.Big5 locale.
	OLD_FILES+=usr/tests/lib/libc/locale/io_test
	# 20170319: remove nls for non supported Big5* locales
	OLD_DIRS+=usr/share/nls/zh_HK.Big5HKSCS
	OLD_DIRS+=usr/share/nls/zh_TW.Big5
	# 20170313: move .../sys/geom/eli/... to .../sys/geom/class/eli/...
	OLD_FILES+=usr/tests/sys/geom/eli/pbkdf2/pbkdf2
	OLD_FILES+=usr/tests/sys/geom/eli/pbkdf2/Kyuafile
	OLD_FILES+=usr/tests/sys/geom/eli/Kyuafile
	OLD_DIRS+=usr/tests/sys/geom/eli/pbkdf2
	OLD_DIRS+=usr/tests/sys/geom/eli
	# 20170313: sbin/ipftest and ipresend temporarily disconnected.
	OLD_FILES+=sbin/ipftest
	OLD_FILES+=sbin/ipresend
	# 20170311: Remove WITHOUT_MANDOCDB option
	OLD_FILES+=usr/share/man/man1/makewhatis.1.gz
	# 20170308: rename some tests
	OLD_FILES+=usr/tests/bin/pwait/pwait
	OLD_FILES+=usr/tests/usr.bin/timeout/timeout
	# 20170307: remove pcap-int.h
	OLD_FILES+=usr/include/pcap-int.h
	# 20170302: new libc++ import which bumps version from 3.9.1 to 4.0.0.
	OLD_FILES+=usr/include/c++/v1/__undef___deallocate
	OLD_FILES+=usr/include/c++/v1/tr1/__undef___deallocate
	# 20170302: new clang import which bumps version from 3.9.1 to 4.0.0.
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/3.9.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.9.1/include/msa.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/3.9.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/3.9.1/include
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.9.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.9.1/lib
	OLD_DIRS+=usr/lib/clang/3.9.1
	# 20170226: SVR4 compatibility removed
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/share/man/man4/streams.4
	OLD_FILES+=usr/share/man/man4/svr4.4
	.endif
	# 20170219: OpenPAM RADULA upgrade removed the libpam tests
	OLD_FILES+=usr/tests/lib/libpam/Kyuafile
	OLD_FILES+=usr/tests/lib/libpam/t_openpam_ctype
	OLD_FILES+=usr/tests/lib/libpam/t_openpam_readlinev
	OLD_FILES+=usr/tests/lib/libpam/t_openpam_readword
	OLD_DIRS+=usr/test/lib/libpam
	# 20170206: remove bdes(1)
	OLD_FILES+=usr/bin/bdes
	OLD_FILES+=usr/lib/debug/usr/bin/bdes.debug
	OLD_FILES+=usr/share/man/man1/bdes.1.gz
	# 20170206: merged projects/ipsec
	OLD_FILES+=usr/include/netinet/ip_ipsec.h
	OLD_FILES+=usr/include/netinet6/ip6_ipsec.h
	# 20170128: remove pc98 support
	OLD_FILES+=usr/include/dev/ic/i8251.h
	OLD_FILES+=usr/include/dev/ic/i8255.h
	OLD_FILES+=usr/include/dev/ic/rsa.h
	OLD_FILES+=usr/include/dev/ic/wd33c93reg.h
	OLD_FILES+=usr/include/sys/disk/pc98.h
	OLD_FILES+=usr/include/sys/diskpc98.h
	OLD_FILES+=usr/share/man/man4/i386/ct.4.gz
	OLD_FILES+=usr/share/man/man4/i386/snc.4.gz
	OLD_FILES+=usr/share/syscons/keymaps/jp.pc98.iso.kbd
	OLD_FILES+=usr/share/syscons/keymaps/jp.pc98.kbd
	OLD_FILES+=usr/share/vt/keymaps/jp.pc98.iso.kbd
	OLD_FILES+=usr/share/vt/keymaps/jp.pc98.kbd
	# 20170110: Four files from ggate tests consolidated into one
	OLD_FILES+=usr/tests/sys/geom/class/gate/1_test
	OLD_FILES+=usr/tests/sys/geom/class/gate/2_test
	OLD_FILES+=usr/tests/sys/geom/class/gate/3_test
	OLD_FILES+=usr/tests/sys/geom/class/gate/conf.sh
	# 20170103: libbsnmptools.so made into an INTERNALLIB
	OLD_FILES+=usr/lib/libbsnmptools.a
	OLD_FILES+=usr/lib/libbsnmptools_p.a
	OLD_LIBS+=usr/lib/libbsnmptools.so.0
	OLD_LIBS+=usr/lib/libbsnmptools.so
	# 20170102: sysdecode_getfsstat_flags() renamed to sysdecode_getfsstat_mode()
	OLD_FILES+=usr/share/man/man3/sysdecode_getfsstat_flags.3.gz
	# 20161230: libarchive ACL pax test renamed to test_acl_pax_posix1e.tar.uu
	OLD_FILES+=usr/tests/lib/libarchive/test_acl_pax.tar.uu
	# 20161229: Three files from gnop tests consolidated into one
	OLD_FILES+=usr/tests/sys/geom/class/nop/1_test
	OLD_FILES+=usr/tests/sys/geom/class/nop/2_test
	OLD_FILES+=usr/tests/sys/geom/class/nop/conf.sh
	# 20161217: new clang import which bumps version from 3.9.0 to 3.9.1.
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/esan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/3.9.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_cmath.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_intrinsics.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_math_forward_declares.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512ifmaintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512ifmavlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512pfintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vbmiintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vbmivlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vlcdintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/clflushoptintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.9.0/include/mwaitxintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/opencl-c.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/3.9.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/3.9.0/include
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats_client-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.9.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.9.0/lib
	OLD_DIRS+=usr/lib/clang/3.9.0
	# 20161205: libproc version bump
	OLD_LIBS+=usr/lib/libproc.so.3
	OLD_LIBS+=usr/lib32/libproc.so.3
	# 20161127: Remove vm_page_cache(9)
	OLD_FILES+=usr/share/man/man9/vm_page_cache.9.gz
	# 20161124: new clang import which bumps version from 3.8.0 to 3.9.0.
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/3.8.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/3.8.0/include/__clang_cuda_runtime_wrapper.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.8.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/pkuintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xsavecintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xsaveintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xsaveoptintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xsavesintrin.h
	OLD_FILES+=usr/lib/clang/3.8.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/3.8.0/include
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-i386.so
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-x86_64.so
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.8.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.8.0/lib
	OLD_DIRS+=usr/lib/clang/3.8.0
	# 20161121: Hyper-V manuals only apply to amd64 and i386.
	.if ${TARGET_ARCH} != "amd64" && ${TARGET_ARCH} != "i386"
	OLD_FILES+=usr/share/man/man4/hv_kvp.4.gz
	OLD_FILES+=usr/share/man/man4/hv_netvsc.4.gz
	OLD_FILES+=usr/share/man/man4/hv_storvsc.4.gz
	OLD_FILES+=usr/share/man/man4/hv_utils.4.gz
	OLD_FILES+=usr/share/man/man4/hv_vmbus.4.gz
	OLD_FILES+=usr/share/man/man4/hv_vss.4.gz
	.endif
	# 20161118: Remove hv_ata_pci_disengage(4)
	OLD_FILES+=usr/share/man/man4/hv_ata_pci_disengage.4.gz
	# 20161017: urtwn(4) was merged into rtwn(4)
	OLD_FILES+=usr/share/man/man4/urtwn.4.gz
	OLD_FILES+=usr/share/man/man4/urtwnfw.4.gz
	# 20161015: Remove GNU rcs
	OLD_FILES+=usr/bin/ci
	OLD_FILES+=usr/bin/co
	OLD_FILES+=usr/bin/merge
	OLD_FILES+=usr/bin/rcs
	OLD_FILES+=usr/bin/rcsclean
	OLD_FILES+=usr/bin/rcsdiff
	OLD_FILES+=usr/bin/rcsfreeze
	OLD_FILES+=usr/bin/rcsmerge
	OLD_FILES+=usr/bin/rlog
	OLD_FILES+=usr/share/doc/psd/13.rcs/paper.ascii.gz
	OLD_FILES+=usr/share/doc/psd/13.rcs/rcs_func.ascii.gz
	OLD_DIRS+=usr/share/doc/psd/13.rcs
	OLD_FILES+=usr/share/man/man1/ci.1.gz
	OLD_FILES+=usr/share/man/man1/co.1.gz
	OLD_FILES+=usr/share/man/man1/merge.1.gz
	OLD_FILES+=usr/share/man/man1/rcs.1.gz
	OLD_FILES+=usr/share/man/man1/rcsclean.1.gz
	OLD_FILES+=usr/share/man/man1/rcsdiff.1.gz
	OLD_FILES+=usr/share/man/man1/rcsfreeze.1.gz
	OLD_FILES+=usr/share/man/man1/rcsintro.1.gz
	OLD_FILES+=usr/share/man/man1/rcsmerge.1.gz
	OLD_FILES+=usr/share/man/man1/rlog.1.gz
	OLD_FILES+=usr/share/man/man5/rcsfile.5.gz
	# 20161010: remove link to removed m_getclr(9) macro
	OLD_FILES+=usr/share/man/man9/m_getclr.9.gz
	# 20161003: MK_ELFCOPY_AS_OBJCOPY option retired
	OLD_FILES+=usr/bin/elfcopy
	OLD_FILES+=usr/share/man/man1/elfcopy.1.gz
	# 20160906: libkqueue tests moved to /usr/tests/sys/kqueue/libkqueue
	OLD_FILES+=usr/tests/sys/kqueue/kqtest
	OLD_FILES+=usr/tests/sys/kqueue/kqueue_test
	# 20160903: idle page zeroing support removed
	OLD_FILES+=usr/share/man/man9/pmap_zero_idle.9.gz
	# 20160901: Remove digi(4)
	OLD_FILES+=usr/share/man/man4/digi.4.gz
	# 20160819: Remove ie(4)
	OLD_FILES+=usr/share/man/man4/i386/ie.4.gz
	# 20160819: Remove spic(4)
	OLD_FILES+=usr/share/man/man4/spic.4.gz
	# 20160819: Remove wl(4) and wlconfig(8)
	OLD_FILES+=usr/share/man/man4/i386/wl.4.gz
	OLD_FILES+=usr/sbin/wlconfig
	OLD_FILES+=usr/share/man/man8/i386/wlconfig.8.gz
	# 20160819: Remove si(4) and sicontrol(8)
	OLD_FILES+=usr/share/man/man4/si.4.gz
	OLD_FILES+=usr/sbin/sicontrol
	OLD_FILES+=usr/share/man/man8/sicontrol.8.gz
	# 20160819: Remove scd(4)
	OLD_FILES+=usr/share/man/man4/scd.4.gz
	# 20160815: Remove mcd(4)
	OLD_FILES+=usr/share/man/man4/mcd.4.gz
	# 20160805: lockmgr_waiters(9) removed
	OLD_FILES+=usr/share/man/man9/lockmgr_waiters.9.gz
	# 20160703: POSIXify locales with variants
	OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hant_TW.UTF-8
	OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hant_TW.Big5
	OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hant_HK.UTF-8
	OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hans_CN.eucCN
	OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hans_CN.UTF-8
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hans_CN.GBK
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hans_CN.GB2312
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hans_CN.GB18030
	OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/sr_Latn_RS.UTF-8
	OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_MESSAGES
	OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_MONETARY
	OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_TIME
	OLD_DIRS+=usr/share/locale/sr_Latn_RS.ISO8859-2
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/sr_Cyrl_RS.UTF-8
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_MESSAGES
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_MONETARY
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_TIME
	OLD_DIRS+=usr/share/locale/sr_Cyrl_RS.ISO8859-5
	OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/mn_Cyrl_MN.UTF-8
	OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/kk_Cyrl_KZ.UTF-8
	# 20160608: removed pam_verbose_error
	OLD_LIBS+=usr/lib/libpam.so.5
	OLD_LIBS+=usr/lib/pam_chroot.so.5
	OLD_LIBS+=usr/lib/pam_deny.so.5
	OLD_LIBS+=usr/lib/pam_echo.so.5
	OLD_LIBS+=usr/lib/pam_exec.so.5
	OLD_LIBS+=usr/lib/pam_ftpusers.so.5
	OLD_LIBS+=usr/lib/pam_group.so.5
	OLD_LIBS+=usr/lib/pam_guest.so.5
	OLD_LIBS+=usr/lib/pam_krb5.so.5
	OLD_LIBS+=usr/lib/pam_ksu.so.5
	OLD_LIBS+=usr/lib/pam_lastlog.so.5
	OLD_LIBS+=usr/lib/pam_login_access.so.5
	OLD_LIBS+=usr/lib/pam_nologin.so.5
	OLD_LIBS+=usr/lib/pam_opie.so.5
	OLD_LIBS+=usr/lib/pam_opieaccess.so.5
	OLD_LIBS+=usr/lib/pam_passwdqc.so.5
	OLD_LIBS+=usr/lib/pam_permit.so.5
	OLD_LIBS+=usr/lib/pam_radius.so.5
	OLD_LIBS+=usr/lib/pam_rhosts.so.5
	OLD_LIBS+=usr/lib/pam_rootok.so.5
	OLD_LIBS+=usr/lib/pam_securetty.so.5
	OLD_LIBS+=usr/lib/pam_self.so.5
	OLD_LIBS+=usr/lib/pam_ssh.so.5
	OLD_LIBS+=usr/lib/pam_tacplus.so.5
	OLD_LIBS+=usr/lib/pam_unix.so.5
	OLD_LIBS+=usr/lib32/libpam.so.5
	OLD_LIBS+=usr/lib32/pam_chroot.so.5
	OLD_LIBS+=usr/lib32/pam_deny.so.5
	OLD_LIBS+=usr/lib32/pam_echo.so.5
	OLD_LIBS+=usr/lib32/pam_exec.so.5
	OLD_LIBS+=usr/lib32/pam_ftpusers.so.5
	OLD_LIBS+=usr/lib32/pam_group.so.5
	OLD_LIBS+=usr/lib32/pam_guest.so.5
	OLD_LIBS+=usr/lib32/pam_krb5.so.5
	OLD_LIBS+=usr/lib32/pam_ksu.so.5
	OLD_LIBS+=usr/lib32/pam_lastlog.so.5
	OLD_LIBS+=usr/lib32/pam_login_access.so.5
	OLD_LIBS+=usr/lib32/pam_nologin.so.5
	OLD_LIBS+=usr/lib32/pam_opie.so.5
	OLD_LIBS+=usr/lib32/pam_opieaccess.so.5
	OLD_LIBS+=usr/lib32/pam_passwdqc.so.5
	OLD_LIBS+=usr/lib32/pam_permit.so.5
	OLD_LIBS+=usr/lib32/pam_radius.so.5
	OLD_LIBS+=usr/lib32/pam_rhosts.so.5
	OLD_LIBS+=usr/lib32/pam_rootok.so.5
	OLD_LIBS+=usr/lib32/pam_securetty.so.5
	OLD_LIBS+=usr/lib32/pam_self.so.5
	OLD_LIBS+=usr/lib32/pam_ssh.so.5
	OLD_LIBS+=usr/lib32/pam_tacplus.so.5
	OLD_LIBS+=usr/lib32/pam_unix.so.5
	# 20160523: remove extranous ALTQ files
	OLD_FILES+=usr/include/altq/altq_codel.h
	OLD_FILES+=usr/include/altq/altq_fairq.h
	# 20160519: remove DTrace Toolkit from base
	OLD_FILES+=usr/sbin/dtruss
	OLD_FILES+=usr/share/dtrace/toolkit/execsnoop
	OLD_FILES+=usr/share/dtrace/toolkit/hotkernel
	OLD_FILES+=usr/share/dtrace/toolkit/hotuser
	OLD_FILES+=usr/share/dtrace/toolkit/opensnoop
	OLD_FILES+=usr/share/dtrace/toolkit/procsystime
	OLD_DIRS+=usr/share/dtrace/toolkit
	OLD_FILES+=usr/share/man/man1/dtruss.1.gz
	# 20160519: stale MLINK removed
	OLD_FILES+=usr/share/man/man9/rman_await_resource.9.gz
	# 20160517: ReiserFS removed
	OLD_FILES+=usr/share/man/man5/reiserfs.5.gz
	# 20160504: tests rework
	OLD_FILES+=usr/tests/lib/libc/regex/data/README
	# 20160430: kvm_getfiles(3) removed from kvm(3)
	OLD_LIBS+=lib/libkvm.so.6
	OLD_LIBS+=usr/lib32/libkvm.so.6
	OLD_FILES+=usr/share/man/man3/kvm_getfiles.3.gz
	# 20160423: remove mroute6d
	OLD_FILES+=etc/rc.d/mroute6d
	# 20160419: rename units.lib -> definitions.units
	OLD_FILES+=usr/share/misc/units.lib
	# 20160419: remove Big5HKSCS locales
	OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_HK.Big5HKSCS
	OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_COLLATE
	OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_CTYPE
	OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_MESSAGES
	OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_MONETARY
	OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_NUMERIC
	OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_TIME
	OLD_DIRS+=usr/share/locale/zh_Hant_HK.Big5HKSCS
	# 20160317: rman_res_t size bump to uintmax_t
	OLD_LIBS+=usr/lib/libdevinfo.so.5
	OLD_LIBS+=usr/lib32/libdevinfo.so.5
	# 20160305: new clang import which bumps version from 3.7.1 to 3.8.0.
	OLD_FILES+=usr/bin/macho-dump
	OLD_FILES+=usr/bin/tblgen
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/3.7.1/include/sanitizer
	OLD_FILES+=usr/lib/clang/3.7.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.7.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/vadefs.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/3.7.1/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/3.7.1/include
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.7.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.7.1/lib
	OLD_DIRS+=usr/lib/clang/3.7.1
	# 20160301: Remove taskqueue_enqueue_fast
	OLD_FILES+=usr/share/man/man9/taskqueue_enqueue_fast.9.gz
	# 20160225: Remove casperd and libcapsicum.
	OLD_FILES+=sbin/casperd
	OLD_FILES+=etc/rc.d/casperd
	OLD_FILES+=usr/share/man/man8/casperd.8.gz
	OLD_FILES+=usr/include/libcapsicum.h
	OLD_FILES+=usr/include/libcapsicum_service.h
	OLD_FILES+=usr/include/libcapsicum.h
	OLD_FILES+=usr/share/man/man3/libcapsicum.3.gz
	OLD_FILES+=usr/include/libcapsicum_dns.h
	OLD_FILES+=usr/include/libcapsicum_grp.h
	OLD_FILES+=usr/include/libcapsicum_impl.h
	OLD_FILES+=usr/include/libcapsicum_pwd.h
	OLD_FILES+=usr/include/libcapsicum_random.h
	OLD_FILES+=usr/include/libcapsicum_sysctl.h
	OLD_FILES+=libexec/casper/dns
	OLD_FILES+=libexec/casper/grp
	OLD_FILES+=libexec/casper/pwd
	OLD_FILES+=libexec/casper/random
	OLD_FILES+=libexec/casper/sysctl
	OLD_FILES+=libexec/casper/.debug/random.debug
	OLD_FILES+=libexec/casper/.debug/dns.debug
	OLD_FILES+=libexec/casper/.debug/sysctl.debug
	OLD_FILES+=libexec/casper/.debug/pwd.debug
	OLD_FILES+=libexec/casper/.debug/grp.debug
	OLD_DIRS+=libexec/casper/.debug
	OLD_DIRS+=libexec/casper
	OLD_FILES+=usr/lib/libcapsicum.a
	OLD_FILES+=usr/lib/libcapsicum.so
	OLD_LIBS+=lib/libcapsicum.so.0
	OLD_FILES+=usr/lib/libcapsicum_p.a
	OLD_FILES+=usr/lib32/libcapsicum.a
	OLD_FILES+=usr/lib32/libcapsicum.so
	OLD_LIBS+=usr/lib32/libcapsicum.so.0
	OLD_FILES+=usr/lib32/libcapsicum_p.a
	# 20160223: functionality from mkulzma(1) merged into mkuzip(1)
	OLD_FILES+=usr/bin/mkulzma
	OLD_FILES+=usr/share/man/man4/geom_uncompress.4.gz
	OLD_FILES+=usr/share/man/man8/mkulzma.8.gz
	# 20160211: Remove obsolete unbound-control-setup
	OLD_FILES+=usr/sbin/unbound-control-setup
	# 20160121: cc.h moved
	OLD_FILES+=usr/include/netinet/cc.h
	# 20160116: Update mandoc to cvs snapshot 20160116
	OLD_FILES+=usr/share/mdocml/example.style.css
	OLD_FILES+=usr/share/mdocml/style.css
	OLD_DIRS+=usr/share/mdocml
	# 20160114: SA-16:06.snmpd
	OLD_FILES+=usr/share/examples/etc/snmpd.config
	# 20160107: GNU ld installed as ld.bfd and linked as ld
	OLD_FILES+=usr/lib/debug/usr/bin/ld.debug
	# 20151225: new clang import which bumps version from 3.7.0 to 3.7.1.
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/allocator_interface.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/asan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/common_interface_defs.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/coverage_interface.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/dfsan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/linux_syscall_hooks.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/lsan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/msan_interface.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/tsan_interface_atomic.h
	OLD_DIRS+=usr/lib/clang/3.7.0/include/sanitizer
	OLD_FILES+=usr/lib/clang/3.7.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512cdintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512dqintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512vldqintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/cuda_builtin_vars.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/fxsrintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/htmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/htmxlintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.7.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/s390intrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/vadefs.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/vecintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/xopintrin.h
	OLD_FILES+=usr/lib/clang/3.7.0/include/xtestintrin.h
	OLD_DIRS+=usr/lib/clang/3.7.0/include
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.safestack-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.safestack-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.7.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.7.0/lib
	OLD_DIRS+=usr/lib/clang/3.7.0
	# 20151130: libelf moved from /usr/lib to /lib (libkvm dependency in r291406)
	OLD_LIBS+=usr/lib/libelf.so.2
	# 20151115: Fox bad upgrade scheme
	OLD_FILES+=usr/share/locale/zh_CN.GB18030/zh_Hans_CN.GB18030
	OLD_FILES+=usr/share/locale/zh_CN.GB2312/zh_Hans_CN.GB2312
	OLD_FILES+=usr/share/locale/zh_CN.GBK/zh_Hans_CN.GBK
	OLD_FILES+=usr/share/locale/zh_CN.UTF-8/zh_Hans_CN.UTF-8
	OLD_FILES+=usr/share/locale/zh_CN.eucCN/zh_Hans_CN.eucCN
	OLD_FILES+=usr/share/locale/zh_TW.Big5/zh_Hant_TW.Big5
	OLD_FILES+=usr/share/locale/zh_TW.UTF-8/zh_Hant_TW.UTF-8
	# 20151107: String collation improvements
	OLD_FILES+=usr/share/locale/UTF-8/LC_CTYPE
	OLD_DIRS+=usr/share/locale/UTF-8
	OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_COLLATE
	OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_CTYPE
	OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_MESSAGES
	OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_MONETARY
	OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_NUMERIC
	OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_TIME
	OLD_DIRS+=usr/share/locale/kk_KZ.PT154/
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-1/LC_COLLATE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-1/LC_CTYPE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-1/LC_TIME
	OLD_DIRS+=usr/share/locale/la_LN.ISO8859-1
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-13/LC_COLLATE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-13/LC_CTYPE
	OLD_DIRS+=usr/share/locale/la_LN.ISO8859-13
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-15/LC_COLLATE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-15/LC_CTYPE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-15/LC_TIME
	OLD_DIRS+=usr/share/locale/la_LN.ISO8859-15
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-2/LC_COLLATE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-2/LC_CTYPE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-2/LC_TIME
	OLD_DIRS+=usr/share/locale/la_LN.ISO8859-2
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-4/LC_COLLATE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-4/LC_CTYPE
	OLD_FILES+=usr/share/locale/la_LN.ISO8859-4/LC_TIME
	OLD_DIRS+=usr/share/locale/la_LN.ISO8859-4
	OLD_FILES+=usr/share/locale/la_LN.US-ASCII/LC_COLLATE
	OLD_FILES+=usr/share/locale/la_LN.US-ASCII/LC_CTYPE
	OLD_FILES+=usr/share/locale/la_LN.US-ASCII/LC_TIME
	OLD_DIRS+=usr/share/locale/la_LN.US-ASCII
	OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_MESSAGES
	OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_TIME
	OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_COLLATE
	OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_MONETARY
	OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_CTYPE
	OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_NUMERIC
	OLD_DIRS+=usr/share/locale/lt_LT.ISO8859-4
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_COLLATE
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_CTYPE
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_MESSAGES
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_MONETARY
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_NUMERIC
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_TIME
	OLD_DIRS+=usr/share/locale/no_NO.ISO8859-1
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_COLLATE
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_CTYPE
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_MESSAGES
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_MONETARY
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_NUMERIC
	OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_TIME
	OLD_DIRS+=usr/share/locale/no_NO.ISO8859-15
	OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_MESSAGES
	OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_TIME
	OLD_DIRS+=usr/share/locale/no_NO.UTF-8
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_TIME
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_MESSAGES
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_MONETARY
	OLD_DIRS+=usr/share/locale/sr_YU.ISO8859-2
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_MONETARY
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_TIME
	OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_MESSAGES
	OLD_DIRS+=usr/share/locale/sr_YU.ISO8859-5
	OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_COLLATE
	OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_MONETARY
	OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_CTYPE
	OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_TIME
	OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_NUMERIC
	OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_MESSAGES
	OLD_DIRS+=usr/share/locale/sr_YU.UTF-8
	# 20151101: added missing _test suffix on multiple tests in lib/libc
	OLD_FILES+=usr/tests/lib/libc/c063/faccessat
	OLD_FILES+=usr/tests/lib/libc/c063/fchmodat
	OLD_FILES+=usr/tests/lib/libc/c063/fchownat
	OLD_FILES+=usr/tests/lib/libc/c063/fexecve
	OLD_FILES+=usr/tests/lib/libc/c063/fstatat
	OLD_FILES+=usr/tests/lib/libc/c063/linkat
	OLD_FILES+=usr/tests/lib/libc/c063/mkdirat
	OLD_FILES+=usr/tests/lib/libc/c063/mkfifoat
	OLD_FILES+=usr/tests/lib/libc/c063/mknodat
	OLD_FILES+=usr/tests/lib/libc/c063/openat
	OLD_FILES+=usr/tests/lib/libc/c063/readlinkat
	OLD_FILES+=usr/tests/lib/libc/c063/renameat
	OLD_FILES+=usr/tests/lib/libc/c063/symlinkat
	OLD_FILES+=usr/tests/lib/libc/c063/unlinkat
	OLD_FILES+=usr/tests/lib/libc/c063/utimensat
	OLD_FILES+=usr/tests/lib/libc/string/memchr
	OLD_FILES+=usr/tests/lib/libc/string/memcpy
	OLD_FILES+=usr/tests/lib/libc/string/memmem
	OLD_FILES+=usr/tests/lib/libc/string/memset
	OLD_FILES+=usr/tests/lib/libc/string/strcat
	OLD_FILES+=usr/tests/lib/libc/string/strchr
	OLD_FILES+=usr/tests/lib/libc/string/strcmp
	OLD_FILES+=usr/tests/lib/libc/string/strcpy
	OLD_FILES+=usr/tests/lib/libc/string/strcspn
	OLD_FILES+=usr/tests/lib/libc/string/strerror
	OLD_FILES+=usr/tests/lib/libc/string/strlen
	OLD_FILES+=usr/tests/lib/libc/string/strpbrk
	OLD_FILES+=usr/tests/lib/libc/string/strrchr
	OLD_FILES+=usr/tests/lib/libc/string/strspn
	OLD_FILES+=usr/tests/lib/libc/string/swab
	# 20151101: 430.status-rwho was renamed to 430.status-uptime
	OLD_FILES+=etc/periodic/daily/430.status-rwho
	# 20151030: OpenSSL 1.0.2d import
	OLD_FILES+=usr/share/openssl/man/man3/CMS_set1_signer_certs.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_ctrl.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_ctrl_str.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/d2i_509_CRL_fp.3.gz
	OLD_LIBS+=lib/libcrypto.so.7
	OLD_LIBS+=usr/lib/libssl.so.7
	OLD_LIBS+=usr/lib32/libcrypto.so.7
	OLD_LIBS+=usr/lib32/libssl.so.7
	# 20151029: LinuxKPI moved to sys/compat/linuxkpi
	OLD_FILES+=usr/include/dev/usb/usb_compat_linux.h
	# 20151015: test symbols moved to /usr/lib/debug
	OLD_DIRS+=usr/tests/lib/atf/libatf-c++/.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/atf_c++_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/build_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/check_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/config_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/macros_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/tests_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/utils_test.debug
	OLD_DIRS+=usr/tests/lib/atf/libatf-c++/detail/.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/application_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/env_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/exceptions_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/fs_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/process_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/sanity_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/text_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/version_helper.debug
	OLD_DIRS+=usr/tests/lib/atf/libatf-c/.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/atf_c_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/build_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/check_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/config_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/error_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/macros_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/tc_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/tp_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/utils_test.debug
	OLD_DIRS+=usr/tests/lib/atf/libatf-c/detail/.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/dynstr_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/env_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/fs_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/list_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/map_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/process_helpers.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/process_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/sanity_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/text_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/user_test.debug
	OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/version_helper.debug
	OLD_DIRS+=usr/tests/lib/atf/test-programs/.debug
	OLD_FILES+=usr/tests/lib/atf/test-programs/.debug/c_helpers.debug
	OLD_FILES+=usr/tests/lib/atf/test-programs/.debug/cpp_helpers.debug
	OLD_DIRS+=usr/tests/lib/libc/c063/.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/faccessat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/fchmodat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/fchownat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/fexecve.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/fstatat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/linkat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/mkdirat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/mkfifoat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/mknodat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/openat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/readlinkat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/renameat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/symlinkat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/unlinkat.debug
	OLD_FILES+=usr/tests/lib/libc/c063/.debug/utimensat.debug
	OLD_DIRS+=usr/tests/lib/libc/db/.debug
	OLD_FILES+=usr/tests/lib/libc/db/.debug/h_db.debug
	OLD_DIRS+=usr/tests/lib/libc/gen/.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/alarm_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/arc4random_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/assert_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/basedirname_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/dir_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/floatunditf_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/fnmatch_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpclassify2_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpclassify_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpsetmask_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpsetround_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/ftok_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/getcwd_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/getgrent_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/glob_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/humanize_number_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/isnan_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/nice_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/pause_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/raise_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/realpath_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/setdomainname_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/sethostname_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/sleep_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/syslog_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/time_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/ttyname_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/.debug/vis_test.debug
	OLD_DIRS+=usr/tests/lib/libc/gen/execve/.debug
	OLD_FILES+=usr/tests/lib/libc/gen/execve/.debug/execve_test.debug
	OLD_DIRS+=usr/tests/lib/libc/gen/posix_spawn/.debug
	OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/fileactions_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_fileactions.debug
	OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_spawn.debug
	OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_spawnattr.debug
	OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/spawn_test.debug
	OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/spawnattr_test.debug
	OLD_DIRS+=usr/tests/lib/libc/hash/.debug
	OLD_FILES+=usr/tests/lib/libc/hash/.debug/h_hash.debug
	OLD_FILES+=usr/tests/lib/libc/hash/.debug/sha2_test.debug
	OLD_DIRS+=usr/tests/lib/libc/inet/.debug
	OLD_FILES+=usr/tests/lib/libc/inet/.debug/inet_network_test.debug
	OLD_DIRS+=usr/tests/lib/libc/locale/.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/io_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbrtowc_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbsnrtowcs_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbstowcs_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbtowc_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcscspn_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcspbrk_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcsspn_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcstod_test.debug
	OLD_FILES+=usr/tests/lib/libc/locale/.debug/wctomb_test.debug
	OLD_DIRS+=usr/tests/lib/libc/net/.debug
	OLD_FILES+=usr/tests/lib/libc/net/.debug/ether_aton_test.debug
	OLD_FILES+=usr/tests/lib/libc/net/.debug/getprotoent_test.debug
	OLD_FILES+=usr/tests/lib/libc/net/.debug/h_dns_server.debug
	OLD_FILES+=usr/tests/lib/libc/net/.debug/h_nsd_recurse.debug
	OLD_FILES+=usr/tests/lib/libc/net/.debug/h_protoent.debug
	OLD_FILES+=usr/tests/lib/libc/net/.debug/h_servent.debug
	OLD_DIRS+=usr/tests/lib/libc/regex/.debug
	OLD_FILES+=usr/tests/lib/libc/regex/.debug/exhaust_test.debug
	OLD_FILES+=usr/tests/lib/libc/regex/.debug/h_regex.debug
	OLD_FILES+=usr/tests/lib/libc/regex/.debug/regex_att_test.debug
	OLD_DIRS+=usr/tests/lib/libc/ssp/.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_fgets.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_getcwd.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_gets.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memcpy.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memmove.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memset.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_raw.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_read.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_readlink.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_snprintf.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_sprintf.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_stpcpy.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_stpncpy.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strcat.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strcpy.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strncat.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strncpy.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_vsnprintf.debug
	OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_vsprintf.debug
	OLD_DIRS+=usr/tests/lib/libc/stdio/.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/clearerr_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fflush_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fmemopen2_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fmemopen_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fopen_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fputc_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/mktemp_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/popen_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/printf_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdio/.debug/scanf_test.debug
	OLD_DIRS+=usr/tests/lib/libc/stdlib/.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/abs_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/atoi_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/div_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/exit_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/getenv_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/h_getopt.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/h_getopt_long.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/hsearch_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/posix_memalign_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/random_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/strtod_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/strtol_test.debug
	OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/system_test.debug
	OLD_DIRS+=usr/tests/lib/libc/string/.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/memchr.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/memcpy.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/memmem.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/memset.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strcat.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strchr.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strcmp.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strcpy.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strcspn.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strerror.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strlen.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strpbrk.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strrchr.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/strspn.debug
	OLD_FILES+=usr/tests/lib/libc/string/.debug/swab.debug
	OLD_DIRS+=usr/tests/lib/libc/sys/.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/access_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/chroot_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/clock_gettime_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/connect_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/dup_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/fsync_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getcontext_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getgroups_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getitimer_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getlogin_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getpid_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getrusage_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/getsid_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/gettimeofday_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/issetugid_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/kevent_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/kill_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/link_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/listen_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mincore_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mkdir_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mkfifo_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mknod_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mlock_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mmap_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/mprotect_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgctl_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgget_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgrcv_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgsnd_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/msync_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/nanosleep_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/pipe2_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/pipe_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/poll_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/revoke_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/select_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/setrlimit_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/setuid_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigaction_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigqueue_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigtimedwait_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/socketpair_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/stat_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/timer_create_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/truncate_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/ucontext_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/umask_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/unlink_test.debug
	OLD_FILES+=usr/tests/lib/libc/sys/.debug/write_test.debug
	OLD_DIRS+=usr/tests/lib/libc/termios/.debug
	OLD_FILES+=usr/tests/lib/libc/termios/.debug/tcsetpgrp_test.debug
	OLD_DIRS+=usr/tests/lib/libc/tls/.debug
	OLD_FILES+=usr/tests/lib/libc/tls/.debug/h_tls_dlopen.so.debug
	OLD_FILES+=usr/tests/lib/libc/tls/.debug/libh_tls_dynamic.so.1.debug
	OLD_FILES+=usr/tests/lib/libc/tls/.debug/tls_dlopen_test.debug
	OLD_FILES+=usr/tests/lib/libc/tls/.debug/tls_dynamic_test.debug
	OLD_DIRS+=usr/tests/lib/libc/ttyio/.debug
	OLD_FILES+=usr/tests/lib/libc/ttyio/.debug/ttyio_test.debug
	OLD_DIRS+=usr/tests/lib/libcrypt/.debug
	OLD_FILES+=usr/tests/lib/libcrypt/.debug/crypt_tests.debug
	OLD_DIRS+=usr/tests/lib/libmp/.debug
	OLD_FILES+=usr/tests/lib/libmp/.debug/legacy_test.debug
	OLD_DIRS+=usr/tests/lib/libnv/.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/dnv_tests.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nv_array_tests.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nv_tests.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_add_test.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_exists_test.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_free_test.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_get_test.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_move_test.debug
	OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_send_recv_test.debug
	OLD_DIRS+=usr/tests/lib/libpam/.debug
	OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_ctype.debug
	OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_readlinev.debug
	OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_readword.debug
	OLD_DIRS+=usr/tests/lib/libproc/.debug
	OLD_FILES+=usr/tests/lib/libproc/.debug/proc_test.debug
	OLD_FILES+=usr/tests/lib/libproc/.debug/target_prog.debug
	OLD_DIRS+=usr/tests/lib/librt/.debug
	OLD_FILES+=usr/tests/lib/librt/.debug/sched_test.debug
	OLD_FILES+=usr/tests/lib/librt/.debug/sem_test.debug
	OLD_DIRS+=usr/tests/lib/libthr/.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/barrier_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/cond_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/condwait_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/detach_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/equal_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/fork_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/fpu_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/h_atexit.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/h_cancel.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/h_exit.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/h_resolv.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/join_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/kill_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/mutex_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/once_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/preempt_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/rwlock_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/sem_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/siglongjmp_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/sigmask_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/sigsuspend_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/sleep_test.debug
	OLD_FILES+=usr/tests/lib/libthr/.debug/swapcontext_test.debug
	OLD_DIRS+=usr/tests/lib/libthr/dlopen/.debug
	OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/dlopen_test.debug
	OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/h_pthread_dlopen.so.1.debug
	OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/main_pthread_create_test.debug
	OLD_DIRS+=usr/tests/lib/libutil/.debug
	OLD_FILES+=usr/tests/lib/libutil/.debug/flopen_test.debug
	OLD_FILES+=usr/tests/lib/libutil/.debug/grp_test.debug
	OLD_FILES+=usr/tests/lib/libutil/.debug/humanize_number_test.debug
	OLD_FILES+=usr/tests/lib/libutil/.debug/pidfile_test.debug
	OLD_FILES+=usr/tests/lib/libutil/.debug/trimdomain-nodomain_test.debug
	OLD_FILES+=usr/tests/lib/libutil/.debug/trimdomain_test.debug
	OLD_DIRS+=usr/tests/lib/libxo/.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/libenc_test.so.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_01.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_02.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_03.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_04.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_05.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_06.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_07.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_08.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_09.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_10.debug
	OLD_FILES+=usr/tests/lib/libxo/.debug/test_11.debug
	OLD_DIRS+=usr/tests/lib/msun/.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/acos_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/asin_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/atan_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/cbrt_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/ceil_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/cos_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/cosh_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/erf_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/exp_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/fmod_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/infinity_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/ldexp_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/log_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/pow_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/precision_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/round_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/scalbn_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/sin_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/sinh_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/sqrt_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/tan_test.debug
	OLD_FILES+=usr/tests/lib/msun/.debug/tanh_test.debug
	OLD_DIRS+=usr/tests/libexec/rtld-elf/.debug
	OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/ld_library_pathfds.debug
	OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/libpythagoras.so.0.debug
	OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/target.debug
	OLD_DIRS+=usr/tests/sbin/devd/.debug
	OLD_FILES+=usr/tests/sbin/devd/.debug/client_test.debug
	OLD_DIRS+=usr/tests/sbin/dhclient/.debug
	OLD_FILES+=usr/tests/sbin/dhclient/.debug/option-domain-search_test.debug
	OLD_DIRS+=usr/tests/share/examples/tests/atf/.debug
	OLD_FILES+=usr/tests/share/examples/tests/atf/.debug/printf_test.debug
	OLD_DIRS+=usr/tests/share/examples/tests/plain/.debug
	OLD_FILES+=usr/tests/share/examples/tests/plain/.debug/printf_test.debug
	OLD_DIRS+=usr/tests/sys/aio/.debug
	OLD_FILES+=usr/tests/sys/aio/.debug/aio_kqueue_test.debug
	OLD_FILES+=usr/tests/sys/aio/.debug/aio_test.debug
	OLD_FILES+=usr/tests/sys/aio/.debug/lio_kqueue_test.debug
	OLD_DIRS+=usr/tests/sys/fifo/.debug
	OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_create.debug
	OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_io.debug
	OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_misc.debug
	OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_open.debug
	OLD_DIRS+=usr/tests/sys/file/.debug
	OLD_FILES+=usr/tests/sys/file/.debug/closefrom_test.debug
	OLD_FILES+=usr/tests/sys/file/.debug/dup_test.debug
	OLD_FILES+=usr/tests/sys/file/.debug/fcntlflags_test.debug
	OLD_FILES+=usr/tests/sys/file/.debug/flock_helper.debug
	OLD_FILES+=usr/tests/sys/file/.debug/ftruncate_test.debug
	OLD_FILES+=usr/tests/sys/file/.debug/newfileops_on_fork_test.debug
	OLD_DIRS+=usr/tests/sys/kern/.debug
	OLD_FILES+=usr/tests/sys/kern/.debug/kern_descrip_test.debug
	OLD_FILES+=usr/tests/sys/kern/.debug/ptrace_test.debug
	OLD_FILES+=usr/tests/sys/kern/.debug/unix_seqpacket_test.debug
	OLD_DIRS+=usr/tests/sys/kern/execve/.debug
	OLD_FILES+=usr/tests/sys/kern/execve/.debug/execve_helper.debug
	OLD_FILES+=usr/tests/sys/kern/execve/.debug/good_aout.debug
	OLD_DIRS+=usr/tests/sys/kqueue/.debug
	OLD_FILES+=usr/tests/sys/kqueue/.debug/kqtest.debug
	OLD_DIRS+=usr/tests/sys/mqueue/.debug
	OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest1.debug
	OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest2.debug
	OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest3.debug
	OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest4.debug
	OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest5.debug
	OLD_DIRS+=usr/tests/sys/netinet/.debug
	OLD_FILES+=usr/tests/sys/netinet/.debug/udp_dontroute.debug
	OLD_DIRS+=usr/tests/sys/pjdfstest/.debug
	OLD_FILES+=usr/tests/sys/pjdfstest/.debug/pjdfstest.debug
	OLD_DIRS+=usr/tests/sys/vm/.debug
	OLD_FILES+=usr/tests/sys/vm/.debug/mmap_test.debug
	# 20151015: Rename files due to file-installed-as-dir bug
	OLD_FILES+=usr/share/doc/legal/realtek
	OLD_FILES+=usr/share/doc/legal/realtek/LICENSE
	OLD_DIRS+=usr/share/doc/legal/realtek
	OLD_DIRS+=usr/share/doc/legal/intel_ipw
	OLD_FILES+=usr/share/doc/legal/intel_ipw/LICENSE
	OLD_FILES+=usr/share/doc/legal/intel_iwn
	OLD_FILES+=usr/share/doc/legal/intel_iwn/LICENSE
	OLD_DIRS+=usr/share/doc/legal/intel_iwn
	OLD_DIRS+=usr/share/doc/legal/intel_iwi
	OLD_FILES+=usr/share/doc/legal/intel_iwi/LICENSE
	OLD_DIRS+=usr/share/doc/legal/intel_wpi
	OLD_FILES+=usr/share/doc/legal/intel_wpi/LICENSE
	# 20151006: new libc++ import
	OLD_FILES+=usr/include/c++/__tuple_03
	OLD_FILES+=usr/include/c++/v1/__tuple_03
	OLD_FILES+=usr/include/c++/v1/tr1/__tuple_03
	# 20151006: new clang import which bumps version from 3.6.1 to 3.7.0.
	OLD_FILES+=usr/lib/clang/3.6.1/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.6.1/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.1/include/xopintrin.h
	OLD_DIRS+=usr/lib/clang/3.6.1/include
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.san-i386.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.san-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan-i386.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.6.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.6.1/lib
	OLD_DIRS+=usr/lib/clang/3.6.1
	# 20150928: unused sgsmsg utility is removed
	OLD_FILES+=usr/bin/sgsmsg
	# 20150926: remove links to removed/unimplemented mbuf(9) macros
	OLD_FILES+=usr/share/man/man9/MEXT_ADD_REF.9.gz
	OLD_FILES+=usr/share/man/man9/MEXTFREE.9.gz
	OLD_FILES+=usr/share/man/man9/MEXT_IS_REF.9.gz
	OLD_FILES+=usr/share/man/man9/MEXT_REM_REF.9.gz
	OLD_FILES+=usr/share/man/man9/MFREE.9.gz
	# 20150818: *allocm() are gone in jemalloc 4.0.0
	OLD_FILES+=usr/share/man/man3/allocm.3.gz
	OLD_FILES+=usr/share/man/man3/dallocm.3.gz
	OLD_FILES+=usr/share/man/man3/nallocm.3.gz
	OLD_FILES+=usr/share/man/man3/rallocm.3.gz
	OLD_FILES+=usr/share/man/man3/sallocm.3.gz
	# 20150802: Remove netbsd's test on pw(8)
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_test
	# 20150719: Remove libarchive.pc
	OLD_FILES+=usr/libdata/pkgconfig/libarchive.pc
	# 20150705: Rename DTrace provider man pages.
	OLD_FILES+=usr/share/man/man4/dtrace-io.4.gz
	OLD_FILES+=usr/share/man/man4/dtrace-ip.4.gz
	OLD_FILES+=usr/share/man/man4/dtrace-proc.4.gz
	OLD_FILES+=usr/share/man/man4/dtrace-sched.4.gz
	OLD_FILES+=usr/share/man/man4/dtrace-tcp.4.gz
	OLD_FILES+=usr/share/man/man4/dtrace-udp.4.gz
	# 20150704: nvlist private headers no longer installed
	OLD_FILES+=usr/include/sys/nv_impl.h
	OLD_FILES+=usr/include/sys/nvlist_impl.h
	OLD_FILES+=usr/include/sys/nvpair_impl.h
	# 20150624
	OLD_LIBS+=usr/lib/libugidfw.so.4
	OLD_LIBS+=usr/lib32/libugidfw.so.4
	# 20150604: Move nvlist man pages to section 9.
	OLD_FILES+=usr/share/man/man3/libnv.3.gz
	OLD_FILES+=usr/share/man/man3/nv.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_stringf.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_add_stringv.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_clone.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_create.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_destroy.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_dump.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_empty.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_error.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_exists_type.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_fdump.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_flags.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_free_type.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_parent.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_get_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_move_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_move_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_move_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_move_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_next.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_pack.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_recv.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_send.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_set_error.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_size.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_take_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_take_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_take_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_take_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_take_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_take_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_unpack.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_xfer.3.gz
	# 20150702: Remove duplicated nvlist includes.
	OLD_FILES+=usr/include/dnv.h
	OLD_FILES+=usr/include/nv.h
	# 20150528: PCI IOV device driver methods moved to a separate kobj interface.
	OLD_FILES+=usr/share/man/man9/PCI_ADD_VF.9.gz
	OLD_FILES+=usr/share/man/man9/PCI_INIT_IOV.9.gz
	OLD_FILES+=usr/share/man/man9/PCI_UNINIT_IOV.9.gz
	# 20150525: new clang import which bumps version from 3.6.0 to 3.6.1.
	OLD_FILES+=usr/lib/clang/3.6.0/include/__stddef_max_align_t.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/__wmmintrin_aes.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/__wmmintrin_pclmul.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/adxintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/altivec.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/ammintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/arm_acle.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/arm_neon.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avx2intrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avx512bwintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avx512erintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avx512fintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avx512vlbwintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avx512vlintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/avxintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/bmi2intrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/bmiintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/cpuid.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/emmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/f16cintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/fma4intrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/fmaintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/ia32intrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/immintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/lzcntintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/mm3dnow.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/mm_malloc.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/mmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/module.modulemap
	OLD_FILES+=usr/lib/clang/3.6.0/include/nmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/pmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/popcntintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/prfchwintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/rdseedintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/rtmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/shaintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/smmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/tbmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/tmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/wmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/x86intrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/xmmintrin.h
	OLD_FILES+=usr/lib/clang/3.6.0/include/xopintrin.h
	OLD_DIRS+=usr/lib/clang/3.6.0/include
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.san-i386.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.san-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan-i386.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.6.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.6.0/lib
	OLD_DIRS+=usr/lib/clang/3.6.0
	# 20150521
	OLD_FILES+=usr/bin/demandoc
	OLD_FILES+=usr/share/man/man1/demandoc.1.gz
	OLD_FILES+=usr/share/man/man3/mandoc.3.gz
	OLD_FILES+=usr/share/man/man3/mandoc_headers.3.gz
	# 20150520
	OLD_FILES+=usr/lib/libheimsqlite.a
	OLD_FILES+=usr/lib/libheimsqlite.so
	OLD_LIBS+=usr/lib/libheimsqlite.so.11
	OLD_FILES+=usr/lib/libheimsqlite_p.a
	OLD_FILES+=usr/lib32/libheimsqlite.a
	OLD_FILES+=usr/lib32/libheimsqlite.so
	OLD_LIBS+=usr/lib32/libheimsqlite.so.11
	OLD_FILES+=usr/lib32/libheimsqlite_p.a
	# 20150506
	OLD_FILES+=usr/share/man/man9/NDHASGIANT.9.gz
	# 20150504
	OLD_FILES+=usr/share/examples/etc/libmap32.conf
	OLD_FILES+=usr/include/bsdstat.h
	OLD_LIBS+=usr/lib32/private/libatf-c++.so.2
	OLD_LIBS+=usr/lib32/private/libbsdstat.so.1
	OLD_LIBS+=usr/lib32/private/libheimipcs.so.11
	OLD_LIBS+=usr/lib32/private/libsqlite3.so.0
	OLD_LIBS+=usr/lib32/private/libunbound.so.5
	OLD_LIBS+=usr/lib32/private/libatf-c.so.1
	OLD_LIBS+=usr/lib32/private/libheimipcc.so.11
	OLD_LIBS+=usr/lib32/private/libldns.so.5
	OLD_LIBS+=usr/lib32/private/libssh.so.5
	OLD_LIBS+=usr/lib32/private/libucl.so.1
	OLD_DIRS+=usr/lib32/private
	OLD_LIBS+=usr/lib/private/libatf-c++.so.2
	OLD_LIBS+=usr/lib/private/libbsdstat.so.1
	OLD_LIBS+=usr/lib/private/libheimipcs.so.11
	OLD_LIBS+=usr/lib/private/libsqlite3.so.0
	OLD_LIBS+=usr/lib/private/libunbound.so.5
	OLD_LIBS+=usr/lib/private/libatf-c.so.1
	OLD_LIBS+=usr/lib/private/libheimipcc.so.11
	OLD_LIBS+=usr/lib/private/libldns.so.5
	OLD_LIBS+=usr/lib/private/libssh.so.5
	OLD_LIBS+=usr/lib/private/libucl.so.1
	OLD_DIRS+=usr/lib/private
	# 20150501
	OLD_FILES+=usr/bin/soeliminate
	OLD_FILES+=usr/share/man/man1/soeliminate.1.gz
	# 20150501: Remove the nvlist_.*[vf] functions manpages.
	OLD_FILES+=usr/share/man/man3/nvlist_addf_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addf_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addf_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addf_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addf_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addf_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addf_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_addv_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsf_type.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_existsv_type.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freef_type.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_null.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_freev_type.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getf_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getf_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getf_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getf_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getf_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getf_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getv_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getv_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getv_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getv_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getv_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_getv_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movef_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movef_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movef_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movef_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movev_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movev_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movev_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_movev_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takef_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takef_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takef_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takef_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takef_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takef_string.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takev_binary.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takev_bool.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takev_descriptor.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takev_number.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takev_nvlist.3.gz
	OLD_FILES+=usr/share/man/man3/nvlist_takev_string.3.gz
	# 20150429: remove never written documentation
	OLD_FILES+=usr/share/doc/papers/hwpmc.ascii.gz
	# 20150427: test/sys/kern/mmap_test moved to test/sys/vm/mmap_test
	OLD_FILES+=usr/tests/sys/kern/mmap_test
	# 20150422: zlib.c moved from net to libkern
	OLD_FILES+=usr/include/net/zlib.h
	OLD_FILES+=usr/include/net/zutil.h
	# 20150418
	OLD_FILES+=sbin/mount_oldnfs
	OLD_FILES+=usr/share/man/man8/mount_oldnfs.8.gz
	# 20150416: ALTQ moved to net/altq
	OLD_FILES+=usr/include/altq/altq_rmclass_debug.h
	OLD_FILES+=usr/include/altq/altq.h
	OLD_FILES+=usr/include/altq/altq_cdnr.h
	OLD_FILES+=usr/include/altq/altq_hfsc.h
	OLD_FILES+=usr/include/altq/altq_priq.h
	OLD_FILES+=usr/include/altq/altqconf.h
	OLD_FILES+=usr/include/altq/altq_classq.h
	OLD_FILES+=usr/include/altq/altq_red.h
	OLD_FILES+=usr/include/altq/if_altq.h
	OLD_FILES+=usr/include/altq/altq_var.h
	OLD_FILES+=usr/include/altq/altq_rmclass.h
	OLD_FILES+=usr/include/altq/altq_cbq.h
	OLD_FILES+=usr/include/altq/altq_rio.h
	OLD_DIRS+=usr/include/altq
	# 20150330: ntp 4.2.8p1
	OLD_FILES+=usr/share/doc/ntp/driver1.html
	OLD_FILES+=usr/share/doc/ntp/driver10.html
	OLD_FILES+=usr/share/doc/ntp/driver11.html
	OLD_FILES+=usr/share/doc/ntp/driver12.html
	OLD_FILES+=usr/share/doc/ntp/driver16.html
	OLD_FILES+=usr/share/doc/ntp/driver18.html
	OLD_FILES+=usr/share/doc/ntp/driver19.html
	OLD_FILES+=usr/share/doc/ntp/driver2.html
	OLD_FILES+=usr/share/doc/ntp/driver20.html
	OLD_FILES+=usr/share/doc/ntp/driver22.html
	OLD_FILES+=usr/share/doc/ntp/driver26.html
	OLD_FILES+=usr/share/doc/ntp/driver27.html
	OLD_FILES+=usr/share/doc/ntp/driver28.html
	OLD_FILES+=usr/share/doc/ntp/driver29.html
	OLD_FILES+=usr/share/doc/ntp/driver3.html
	OLD_FILES+=usr/share/doc/ntp/driver30.html
	OLD_FILES+=usr/share/doc/ntp/driver32.html
	OLD_FILES+=usr/share/doc/ntp/driver33.html
	OLD_FILES+=usr/share/doc/ntp/driver34.html
	OLD_FILES+=usr/share/doc/ntp/driver35.html
	OLD_FILES+=usr/share/doc/ntp/driver36.html
	OLD_FILES+=usr/share/doc/ntp/driver37.html
	OLD_FILES+=usr/share/doc/ntp/driver4.html
	OLD_FILES+=usr/share/doc/ntp/driver5.html
	OLD_FILES+=usr/share/doc/ntp/driver6.html
	OLD_FILES+=usr/share/doc/ntp/driver7.html
	OLD_FILES+=usr/share/doc/ntp/driver8.html
	OLD_FILES+=usr/share/doc/ntp/driver9.html
	OLD_FILES+=usr/share/doc/ntp/ldisc.html
	OLD_FILES+=usr/share/doc/ntp/measure.html
	OLD_FILES+=usr/share/doc/ntp/mx4200data.html
	OLD_FILES+=usr/share/doc/ntp/notes.html
	OLD_FILES+=usr/share/doc/ntp/patches.html
	OLD_FILES+=usr/share/doc/ntp/porting.html
	OLD_FILES+=usr/share/man/man1/sntp.1.gz
	# 20150329
	.if ${TARGET_ARCH} == "arm"
	OLD_FILES+=usr/include/bootconfig.h
	.endif
	# 20150326
	OLD_FILES+=usr/share/man/man1/pmcstudy.1.gz
	# 20150315: new clang import which bumps version from 3.5.1 to 3.6.0.
	OLD_FILES+=usr/include/clang/3.5.1/__wmmintrin_aes.h
	OLD_FILES+=usr/include/clang/3.5.1/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/clang/3.5.1/altivec.h
	OLD_FILES+=usr/include/clang/3.5.1/ammintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/arm_acle.h
	OLD_FILES+=usr/include/clang/3.5.1/arm_neon.h
	OLD_FILES+=usr/include/clang/3.5.1/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.5.1/avxintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.5.1/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/cpuid.h
	OLD_FILES+=usr/include/clang/3.5.1/emmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/f16cintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.5.1/fmaintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/ia32intrin.h
	OLD_FILES+=usr/include/clang/3.5.1/immintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.5.1/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.5.1/mmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/module.modulemap
	OLD_FILES+=usr/include/clang/3.5.1/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/prfchwintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/rdseedintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/rtmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/shaintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/smmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/tbmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/x86intrin.h
	OLD_FILES+=usr/include/clang/3.5.1/xmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.1/xopintrin.h
	OLD_DIRS+=usr/include/clang/3.5.1
	OLD_DIRS+=usr/include/clang
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.san-i386.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.san-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan-i386.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.5.1/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.5.1/lib
	OLD_DIRS+=usr/lib/clang/3.5.1
	# 20150302: binutils documentation distributed as a manpage
	OLD_FILES+=usr/share/doc/binutils/as.txt
	OLD_FILES+=usr/share/doc/binutils/ld.txt
	OLD_DIRS+=usr/share/doc/binutils
	# 20150222: Removed bcd(6) and ppt(6)
	OLD_FILES+=usr/bin/bcd
	OLD_FILES+=usr/bin/ppt
	OLD_FILES+=usr/share/man/man6/bcd.6.gz
	OLD_FILES+=usr/share/man/man6/ppt.6.gz
	# 20150217: Removed remnants of ar(4) driver
	OLD_FILES+=usr/include/dev/ic/hd64570.h
	# 20150212: /usr/games moving into /usr/bin
	OLD_FILES+=usr/games/bcd
	OLD_FILES+=usr/games/caesar
	OLD_FILES+=usr/games/factor
	OLD_FILES+=usr/games/fortune
	OLD_FILES+=usr/games/grdc
	OLD_FILES+=usr/games/morse
	OLD_FILES+=usr/games/number
	OLD_FILES+=usr/games/pom
	OLD_FILES+=usr/games/ppt
	OLD_FILES+=usr/games/primes
	OLD_FILES+=usr/games/random
	OLD_FILES+=usr/games/rot13
	OLD_FILES+=usr/games/strfile
	OLD_FILES+=usr/games/unstr
	OLD_DIRS+=usr/games
	# 20150209: liblzma header
	OLD_FILES+=usr/include/lzma/lzma.h
	# 20150124: spl.9 and friends
	OLD_FILES+=usr/share/man/man9/spl.9.gz
	OLD_FILES+=usr/share/man/man9/spl0.9.gz
	OLD_FILES+=usr/share/man/man9/splbio.9.gz
	OLD_FILES+=usr/share/man/man9/splclock.9.gz
	OLD_FILES+=usr/share/man/man9/splhigh.9.gz
	OLD_FILES+=usr/share/man/man9/splimp.9.gz
	OLD_FILES+=usr/share/man/man9/splnet.9.gz
	OLD_FILES+=usr/share/man/man9/splsoftclock.9.gz
	OLD_FILES+=usr/share/man/man9/splsofttty.9.gz
	OLD_FILES+=usr/share/man/man9/splstatclock.9.gz
	OLD_FILES+=usr/share/man/man9/spltty.9.gz
	OLD_FILES+=usr/share/man/man9/splvm.9.gz
	OLD_FILES+=usr/share/man/man9/splx.9.gz
	# 20150118: toeplitz.c moved from netinet to net
	OLD_FILES+=usr/include/netinet/toeplitz.h
	# 20150118: new clang import which bumps version from 3.5.0 to 3.5.1.
	OLD_FILES+=usr/include/clang/3.5.0/__wmmintrin_aes.h
	OLD_FILES+=usr/include/clang/3.5.0/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/clang/3.5.0/altivec.h
	OLD_FILES+=usr/include/clang/3.5.0/ammintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/arm_acle.h
	OLD_FILES+=usr/include/clang/3.5.0/arm_neon.h
	OLD_FILES+=usr/include/clang/3.5.0/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.5.0/avxintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.5.0/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/cpuid.h
	OLD_FILES+=usr/include/clang/3.5.0/emmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/f16cintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.5.0/fmaintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/ia32intrin.h
	OLD_FILES+=usr/include/clang/3.5.0/immintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.5.0/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.5.0/mmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/module.modulemap
	OLD_FILES+=usr/include/clang/3.5.0/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/prfchwintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/rdseedintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/rtmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/shaintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/smmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/tbmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/x86intrin.h
	OLD_FILES+=usr/include/clang/3.5.0/xmmintrin.h
	OLD_FILES+=usr/include/clang/3.5.0/xopintrin.h
	OLD_DIRS+=usr/include/clang/3.5.0
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan-i386.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.profile-arm.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.profile-i386.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.profile-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.san-i386.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.san-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan-i386.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan-x86_64.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
	OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
	OLD_DIRS+=usr/lib/clang/3.5.0/lib/freebsd
	OLD_DIRS+=usr/lib/clang/3.5.0/lib
	OLD_DIRS+=usr/lib/clang/3.5.0
	# 20150102: removal of asr(4)
	OLD_FILES+=usr/share/man/man4/asr.4.gz
	# 20150102: removal of texinfo
	OLD_FILES+=usr/bin/info
	OLD_FILES+=usr/bin/infokey
	OLD_FILES+=usr/bin/install-info
	OLD_FILES+=usr/bin/makeinfo
	OLD_FILES+=usr/bin/texindex
	OLD_FILES+=usr/share/info/am-utils.info.gz
	OLD_FILES+=usr/share/info/as.info.gz
	OLD_FILES+=usr/share/info/binutils.info.gz
	OLD_FILES+=usr/share/info/com_err.info.gz
	OLD_FILES+=usr/share/info/cpp.info.gz
	OLD_FILES+=usr/share/info/cppinternals.info.gz
	OLD_FILES+=usr/share/info/diff.info.gz
	OLD_FILES+=usr/share/info/dir
	OLD_FILES+=usr/share/info/gcc.info.gz
	OLD_FILES+=usr/share/info/gccint.info.gz
	OLD_FILES+=usr/share/info/gdb.info.gz
	OLD_FILES+=usr/share/info/gdbint.info.gz
	OLD_FILES+=usr/share/info/gperf.info.gz
	OLD_FILES+=usr/share/info/grep.info.gz
	OLD_FILES+=usr/share/info/groff.info.gz
	OLD_FILES+=usr/share/info/heimdal.info.gz
	OLD_FILES+=usr/share/info/history.info.gz
	OLD_FILES+=usr/share/info/info-stnd.info.gz
	OLD_FILES+=usr/share/info/info.info.gz
	OLD_FILES+=usr/share/info/ld.info.gz
	OLD_FILES+=usr/share/info/regex.info.gz
	OLD_FILES+=usr/share/info/rluserman.info.gz
	OLD_FILES+=usr/share/info/stabs.info.gz
	OLD_FILES+=usr/share/info/texinfo.info.gz
	OLD_FILES+=usr/share/man/man1/info.1.gz
	OLD_FILES+=usr/share/man/man1/infokey.1.gz
	OLD_FILES+=usr/share/man/man1/install-info.1.gz
	OLD_FILES+=usr/share/man/man1/makeinfo.1.gz
	OLD_FILES+=usr/share/man/man1/texindex.1.gz
	OLD_FILES+=usr/share/man/man5/info.5.gz
	OLD_FILES+=usr/share/man/man5/texinfo.5.gz
	OLD_DIRS+=usr/share/info
	# 20141231: new clang import which bumps version from 3.4.1 to 3.5.0.
	OLD_FILES+=usr/include/clang/3.4.1/__wmmintrin_aes.h
	OLD_FILES+=usr/include/clang/3.4.1/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/clang/3.4.1/altivec.h
	OLD_FILES+=usr/include/clang/3.4.1/ammintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/arm_neon.h
	OLD_FILES+=usr/include/clang/3.4.1/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.4.1/avxintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.4.1/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/cpuid.h
	OLD_FILES+=usr/include/clang/3.4.1/emmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/f16cintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.4.1/fmaintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/immintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.4.1/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.4.1/mmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/module.map
	OLD_FILES+=usr/include/clang/3.4.1/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/prfchwintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/rdseedintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/rtmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/shaintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/smmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/tbmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/x86intrin.h
	OLD_FILES+=usr/include/clang/3.4.1/xmmintrin.h
	OLD_FILES+=usr/include/clang/3.4.1/xopintrin.h
	OLD_DIRS+=usr/include/clang/3.4.1
	# 20141225: Remove gpib/ieee488
	OLD_FILES+=usr/include/dev/ieee488/ibfoo_int.h
	OLD_FILES+=usr/include/dev/ieee488/tnt4882.h
	OLD_FILES+=usr/include/dev/ieee488/ugpib.h
	OLD_FILES+=usr/include/dev/ieee488/upd7210.h
	OLD_DIRS+=usr/include/dev/ieee488
	OLD_FILES+=usr/include/gpib/gpib.h
	OLD_DIRS+=usr/include/gpib
	OLD_FILES+=usr/lib/libgpib.a
	OLD_FILES+=usr/lib/libgpib_p.a
	OLD_FILES+=usr/lib/libgpib.so
	OLD_LIBS+=usr/lib/libgpib.so.3
	OLD_FILES+=usr/lib/libgpib_p.a
	OLD_FILES+=usr/lib32/libgpib.a
	OLD_FILES+=usr/lib32/libgpib_p.a
	OLD_FILES+=usr/lib32/libgpib.so
	OLD_LIBS+=usr/lib32/libgpib.so.3
	OLD_FILES+=usr/share/man/man3/gpib.3.gz
	OLD_FILES+=usr/share/man/man3/ibclr.3.gz
	OLD_FILES+=usr/share/man/man3/ibdev.3.gz
	OLD_FILES+=usr/share/man/man3/ibdma.3.gz
	OLD_FILES+=usr/share/man/man3/ibeos.3.gz
	OLD_FILES+=usr/share/man/man3/ibeot.3.gz
	OLD_FILES+=usr/share/man/man3/ibloc.3.gz
	OLD_FILES+=usr/share/man/man3/ibonl.3.gz
	OLD_FILES+=usr/share/man/man3/ibpad.3.gz
	OLD_FILES+=usr/share/man/man3/ibrd.3.gz
	OLD_FILES+=usr/share/man/man3/ibsad.3.gz
	OLD_FILES+=usr/share/man/man3/ibsic.3.gz
	OLD_FILES+=usr/share/man/man3/ibtmo.3.gz
	OLD_FILES+=usr/share/man/man3/ibtrg.3.gz
	OLD_FILES+=usr/share/man/man3/ibwrt.3.gz
	OLD_FILES+=usr/share/man/man4/gpib.4.gz
	OLD_FILES+=usr/share/man/man4/pcii.4.gz
	OLD_FILES+=usr/share/man/man4/tnt4882.4.gz
	# 20141224: libxo moved to /lib
	OLD_LIBS+=usr/lib/libxo.so.0
	# 20141223: remove in6_gif.h, in_gif.h and if_stf.h
	OLD_FILES+=usr/include/net/if_stf.h
	OLD_FILES+=usr/include/netinet/in_gif.h
	OLD_FILES+=usr/include/netinet6/in6_gif.h
	# 20141209: pw tests broken into a file per command
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_delete
	OLD_FILES+=usr/tests/usr.sbin/pw/pw_modify
	# 20141202: update to mandoc CVS 20141201
	OLD_FILES+=usr.bin/preconv
	OLD_FILES+=share/man/man1/preconv.1.gz
	# 20141129: mrouted rc.d scripts removed from base
	OLD_FILES+=etc/rc.d/mrouted
	# 20141126: convert sbin/mdconfig/tests to ATF format tests
	OLD_FILES+=usr/tests/sbin/mdconfig/legacy_test
	OLD_FILES+=usr/tests/sbin/mdconfig/mdconfig.test
	OLD_FILES+=usr/tests/sbin/mdconfig/run.pl
	# 20141126: remove xform_ipip decapsulation fallback
	OLD_FILES+=usr/include/netipsec/ipip_var.h
	# 20141122: mandoc updated to 1.13.1
	OLD_FILES+=usr/share/mdocml/external.png
	# 20141111: SF_KQUEUE code removed
	OLD_FILES+=usr/include/sys/sf_base.h
	OLD_FILES+=usr/include/sys/sf_sync.h
	# 20141109: faith/faithd removal
	OLD_FILES+=etc/rc.d/faith
	OLD_FILES+=usr/share/man/man4/faith.4.gz
	OLD_FILES+=usr/share/man/man4/if_faith.4.gz
	OLD_FILES+=usr/sbin/faithd
	OLD_FILES+=usr/share/man/man8/faithd.8.gz
	# 20141107: overhaul if_gre(4)
	OLD_FILES+=usr/include/netinet/ip_gre.h
	# 20141102: postrandom obsoleted by new /dev/random code
	OLD_FILES+=etc/rc.d/postrandom
	# 20141031: initrandom obsoleted by new /dev/random code
	OLD_FILES+=etc/rc.d/initrandom
	# 20141030: atf 0.21 import
	OLD_FILES+=usr/share/man/man3/atf-c++-api.3.gz
	# 20141028: debug files accidentally installed as directory name
	OLD_FILES+=usr/lib/debug/usr/lib/i18n
	OLD_FILES+=usr/lib/debug/usr/lib/private
	OLD_FILES+=usr/lib/debug/usr/lib32/i18n
	OLD_FILES+=usr/lib/debug/usr/lib32/private
	# 20141015: OpenSSL 1.0.1j import
	OLD_FILES+=usr/share/openssl/man/man3/CMS_sign_add1_signer.3.gz
	# 20141003: libproc version bump
	OLD_LIBS+=usr/lib/libproc.so.2
	OLD_LIBS+=usr/lib32/libproc.so.2
	# 20140922: sleepq_calc_signal_retval.9 and sleepq_catch_signals.9 removed
	OLD_FILES+=usr/share/man/man9/sleepq_calc_signal_retval.9.gz
	OLD_FILES+=usr/share/man/man9/sleepq_catch_signals.9.gz
	# 20140917: hv_kvpd rc.d script removed in favor of devd configuration
	OLD_FILES+=etc/rc.d/hv_kvpd
	# 20140917: libnv was accidentally being installed to /usr/lib instead of /lib
	OLD_LIBS+=usr/lib/libnv.so.0
	# 20140829: rc.d/kerberos removed
	OLD_FILES+=etc/rc.d/kerberos
	# 20140814: libopie version bump
	OLD_LIBS+=usr/lib/libopie.so.7
	OLD_LIBS+=usr/lib32/libopie.so.7
	# 20140811: otp-sha renamed to otp-sha1
	OLD_FILES+=usr/bin/otp-sha
	OLD_FILES+=usr/share/man/man1/otp-sha.1.gz
	# 20140807: Remove private lib files that should not be installed.
	OLD_FILES+=usr/lib32/private/libatf-c.a
	OLD_FILES+=usr/lib32/private/libatf-c.so
	OLD_FILES+=usr/lib32/private/libatf-c_p.a
	OLD_FILES+=usr/lib32/private/libatf-c++.a
	OLD_FILES+=usr/lib32/private/libatf-c++.so
	OLD_FILES+=usr/lib32/private/libatf-c++_p.a
	OLD_FILES+=usr/lib32/private/libbsdstat.a
	OLD_FILES+=usr/lib32/private/libbsdstat.so
	OLD_FILES+=usr/lib32/private/libbsdstat_p.a
	OLD_FILES+=usr/lib32/private/libheimipcc.a
	OLD_FILES+=usr/lib32/private/libheimipcc.so
	OLD_FILES+=usr/lib32/private/libheimipcc_p.a
	OLD_FILES+=usr/lib32/private/libheimipcs.a
	OLD_FILES+=usr/lib32/private/libheimipcs.so
	OLD_FILES+=usr/lib32/private/libheimipcs_p.a
	OLD_FILES+=usr/lib32/private/libldns.a
	OLD_FILES+=usr/lib32/private/libldns.so
	OLD_FILES+=usr/lib32/private/libldns_p.a
	OLD_FILES+=usr/lib32/private/libssh.a
	OLD_FILES+=usr/lib32/private/libssh.so
	OLD_FILES+=usr/lib32/private/libssh_p.a
	OLD_FILES+=usr/lib32/private/libunbound.a
	OLD_FILES+=usr/lib32/private/libunbound.so
	OLD_FILES+=usr/lib32/private/libunbound_p.a
	OLD_FILES+=usr/lib32/private/libucl.a
	OLD_FILES+=usr/lib32/private/libucl.so
	OLD_FILES+=usr/lib32/private/libucl_p.a
	OLD_FILES+=usr/lib/private/libatf-c.a
	OLD_FILES+=usr/lib/private/libatf-c.so
	OLD_FILES+=usr/lib/private/libatf-c_p.a
	OLD_FILES+=usr/lib/private/libatf-c++.a
	OLD_FILES+=usr/lib/private/libatf-c++.so
	OLD_FILES+=usr/lib/private/libatf-c++_p.a
	OLD_FILES+=usr/lib/private/libbsdstat.a
	OLD_FILES+=usr/lib/private/libbsdstat.so
	OLD_FILES+=usr/lib/private/libbsdstat_p.a
	OLD_FILES+=usr/lib/private/libheimipcc.a
	OLD_FILES+=usr/lib/private/libheimipcc.so
	OLD_FILES+=usr/lib/private/libheimipcc_p.a
	OLD_FILES+=usr/lib/private/libheimipcs.a
	OLD_FILES+=usr/lib/private/libheimipcs.so
	OLD_FILES+=usr/lib/private/libheimipcs_p.a
	OLD_FILES+=usr/lib/private/libldns.a
	OLD_FILES+=usr/lib/private/libldns.so
	OLD_FILES+=usr/lib/private/libldns_p.a
	OLD_FILES+=usr/lib/private/libssh.a
	OLD_FILES+=usr/lib/private/libssh.so
	OLD_FILES+=usr/lib/private/libssh_p.a
	OLD_FILES+=usr/lib/private/libunbound.a
	OLD_FILES+=usr/lib/private/libunbound.so
	OLD_FILES+=usr/lib/private/libunbound_p.a
	OLD_FILES+=usr/lib/private/libucl.a
	OLD_FILES+=usr/lib/private/libucl.so
	OLD_FILES+=usr/lib/private/libucl_p.a
	# 20140803: Remove an obsolete man page
	OLD_FILES+=usr/share/man/man9/pmap_change_wiring.9.gz
	# 20140731
	OLD_FILES+=usr/share/man/man9/SYSCTL_ADD_OID.9.gz
	# 20140728: libsbuf restored to old version.
	OLD_LIBS+=lib/libsbuf.so.7
	OLD_LIBS+=usr/lib32/libsbuf.so.7
	# 20140728: Remove an obsolete man page
	OLD_FILES+=usr/share/man/man9/VOP_GETVOBJECT.9.gz
	OLD_FILES+=usr/share/man/man9/VOP_CREATEVOBJECT.9.gz
	OLD_FILES+=usr/share/man/man9/VOP_DESTROYVOBJECT.9.gz
	# 20140723: renamed to PCBGROUP.9
	OLD_FILES+=usr/share/man/man9/PCBGROUPS.9.gz
	# 20140722: browse_packages_ftp.sh removed
	OLD_FILES+=usr/share/examples/bsdconfig/browse_packages_ftp.sh
	# 20140718: Remove obsolete man pages
	OLD_FILES+=usr/share/man/man9/zero_copy.9.gz
	OLD_FILES+=usr/share/man/man9/zero_copy_sockets.9.gz
	# 20140718: Remove an obsolete man page
	OLD_FILES+=usr/share/man/man9/pmap_page_protect.9.gz
	# 20140717: Remove an obsolete man page
	OLD_FILES+=usr/share/man/man9/pmap_clear_reference.9.gz
	# 20140716: Remove an incorrectly named man page
	OLD_FILES+=usr/share/man/man9/pmap_ts_modified.9.gz
	# 20140712: Removal of bsd.dtrace.mk
	OLD_FILES+=usr/share/mk/bsd.dtrace.mk
	# 20140705: turn libreadline into an internal lib
	OLD_LIBS+=lib/libreadline.so.8
	OLD_FILES+=usr/lib/libreadline.a
	OLD_FILES+=usr/lib/libreadline_p.a
	OLD_FILES+=usr/lib/libreadline.so
	OLD_FILES+=usr/lib/libhistory.a
	OLD_FILES+=usr/lib/libhistory_p.a
	OLD_FILES+=usr/lib/libhistory.so
	OLD_LIBS+=usr/lib/libhistory.so.8
	OLD_FILES+=usr/lib32/libhistory.a
	OLD_FILES+=usr/lib32/libhistory.so
	OLD_LIBS+=usr/lib32/libhistory.so.8
	OLD_FILES+=usr/lib32/libhistory_p.a
	OLD_FILES+=usr/lib32/libreadline.a
	OLD_FILES+=usr/lib32/libreadline.so
	OLD_LIBS+=usr/lib32/libreadline.so.8
	OLD_FILES+=usr/lib32/libreadline_p.a
	OLD_FILES+=usr/include/readline/chardefs.h
	OLD_FILES+=usr/include/readline/history.h
	OLD_FILES+=usr/include/readline/keymaps.h
	OLD_FILES+=usr/include/readline/readline.h
	OLD_FILES+=usr/include/readline/tilde.h
	OLD_FILES+=usr/include/readline/rlconf.h
	OLD_FILES+=usr/include/readline/rlstdc.h
	OLD_FILES+=usr/include/readline/rltypedefs.h
	OLD_FILES+=usr/include/readline/rltypedefs.h
	OLD_DIRS+=usr/include/readline
	OLD_FILES+=usr/share/info/readline.info.gz
	OLD_FILES+=usr/share/man/man3/readline.3.gz
	OLD_FILES+=usr/share/man/man3/rlhistory.3.gz
	# 20140625: csup removal
	OLD_FILES+=usr/bin/csup
	OLD_FILES+=usr/bin/cpasswd
	OLD_FILES+=usr/share/man/man1/csup.1.gz
	OLD_FILES+=usr/share/man/man1/cpasswd.1.gz
	OLD_FILES+=usr/share/examples/cvsup/README
	OLD_FILES+=usr/share/examples/cvsup/cvs-supfile
	OLD_FILES+=usr/share/examples/cvsup/stable-supfile
	OLD_FILES+=usr/share/examples/cvsup/standard-supfile
	OLD_DIRS+=usr/share/examples/cvsup
	# 20140614: send-pr removal
	OLD_FILES+=usr/bin/sendbug
	OLD_FILES+=usr/share/info/send-pr.info.gz
	OLD_FILES+=usr/share/man/man1/send-pr.1.gz
	OLD_FILES+=usr/share/man/man1/sendbug.1.gz
	OLD_FILES+=etc/gnats/freefall
	OLD_DIRS+=etc/gnats
	# 20140512: new clang import which bumps version from 3.4 to 3.4.1.
	OLD_FILES+=usr/include/clang/3.4/__wmmintrin_aes.h
	OLD_FILES+=usr/include/clang/3.4/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/clang/3.4/altivec.h
	OLD_FILES+=usr/include/clang/3.4/ammintrin.h
	OLD_FILES+=usr/include/clang/3.4/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.4/avxintrin.h
	OLD_FILES+=usr/include/clang/3.4/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.4/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.4/cpuid.h
	OLD_FILES+=usr/include/clang/3.4/emmintrin.h
	OLD_FILES+=usr/include/clang/3.4/f16cintrin.h
	OLD_FILES+=usr/include/clang/3.4/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.4/fmaintrin.h
	OLD_FILES+=usr/include/clang/3.4/immintrin.h
	OLD_FILES+=usr/include/clang/3.4/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.4/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.4/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.4/mmintrin.h
	OLD_FILES+=usr/include/clang/3.4/module.map
	OLD_FILES+=usr/include/clang/3.4/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.4/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.4/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.4/prfchwintrin.h
	OLD_FILES+=usr/include/clang/3.4/rdseedintrin.h
	OLD_FILES+=usr/include/clang/3.4/rtmintrin.h
	OLD_FILES+=usr/include/clang/3.4/shaintrin.h
	OLD_FILES+=usr/include/clang/3.4/smmintrin.h
	OLD_FILES+=usr/include/clang/3.4/tbmintrin.h
	OLD_FILES+=usr/include/clang/3.4/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.4/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.4/x86intrin.h
	OLD_FILES+=usr/include/clang/3.4/xmmintrin.h
	OLD_FILES+=usr/include/clang/3.4/xopintrin.h
	OLD_FILES+=usr/include/clang/3.4/arm_neon.h
	OLD_FILES+=usr/include/clang/3.4/module.map
	OLD_DIRS+=usr/include/clang/3.4
	# 20140505: Bogusly installing src.opts.mk
	OLD_FILES+=usr/share/mk/src.opts.mk
	# 20140505: Reject PR kern/187551
	OLD_FILES+=usr/tests/sbin/ifconfig/fibs_test
	# 20140502: Removal of lindev(4)
	OLD_FILES+=usr/share/man/man4/lindev.4.gz
	# 20140425
	OLD_FILES+=usr/lib/libssp_p.a
	OLD_FILES+=usr/lib/libstand_p.a
	OLD_FILES+=usr/lib32/libssp_p.a
	OLD_FILES+=usr/lib32/libstand_p.a
	# 20140314: AppleTalk
	OLD_DIRS+=usr/include/netatalk
	OLD_FILES+=usr/include/netatalk/aarp.h
	OLD_FILES+=usr/include/netatalk/at.h
	OLD_FILES+=usr/include/netatalk/at_extern.h
	OLD_FILES+=usr/include/netatalk/at_var.h
	OLD_FILES+=usr/include/netatalk/ddp.h
	OLD_FILES+=usr/include/netatalk/ddp_pcb.h
	OLD_FILES+=usr/include/netatalk/ddp_var.h
	OLD_FILES+=usr/include/netatalk/endian.h
	OLD_FILES+=usr/include/netatalk/phase2.h
	# 20140314: Remove IPX/SPX
	OLD_LIBS+=lib/libipx.so.5
	OLD_FILES+=usr/include/netipx/ipx.h
	OLD_FILES+=usr/include/netipx/ipx_if.h
	OLD_FILES+=usr/include/netipx/ipx_pcb.h
	OLD_FILES+=usr/include/netipx/ipx_var.h
	OLD_FILES+=usr/include/netipx/spx.h
	OLD_FILES+=usr/include/netipx/spx_debug.h
	OLD_FILES+=usr/include/netipx/spx_timer.h
	OLD_FILES+=usr/include/netipx/spx_var.h
	OLD_DIRS+=usr/include/netipx
	OLD_FILES+=usr/lib/libipx.a
	OLD_FILES+=usr/lib/libipx.so
	OLD_FILES+=usr/lib/libipx_p.a
	OLD_FILES+=usr/lib32/libipx.a
	OLD_FILES+=usr/lib32/libipx.so
	OLD_LIBS+=usr/lib32/libipx.so.5
	OLD_FILES+=usr/lib32/libipx_p.a
	OLD_FILES+=usr/sbin/IPXrouted
	OLD_FILES+=usr/share/man/man3/ipx.3.gz
	OLD_FILES+=usr/share/man/man3/ipx_addr.3.gz
	OLD_FILES+=usr/share/man/man3/ipx_ntoa.3.gz
	OLD_FILES+=usr/share/man/man4/ef.4.gz
	OLD_FILES+=usr/share/man/man4/if_ef.4.gz
	OLD_FILES+=usr/share/man/man8/IPXrouted.8.gz
	# 20140314: bsdconfig usermgmt rewrite
	OLD_FILES+=usr/libexec/bsdconfig/070.usermgmt/userinput
	# 20140307: bsdconfig groupmgmt rewrite
	OLD_FILES+=usr/libexec/bsdconfig/070.usermgmt/groupinput
	# 20140223: Remove libyaml
	OLD_FILES+=usr/lib/private/libyaml.a
	OLD_FILES+=usr/lib/private/libyaml.so
	OLD_LIBS+=usr/lib/private/libyaml.so.1
	OLD_FILES+=usr/lib/private/libyaml_p.a
	OLD_FILES+=usr/lib32/private/libyaml.a
	OLD_FILES+=usr/lib32/private/libyaml.so
	OLD_LIBS+=usr/lib32/private/libyaml.so.1
	OLD_FILES+=usr/lib32/private/libyaml_p.a
	# 20140216: new clang import which bumps version from 3.3 to 3.4.
	OLD_FILES+=usr/bin/llvm-prof
	OLD_FILES+=usr/include/clang/3.3/__wmmintrin_aes.h
	OLD_FILES+=usr/include/clang/3.3/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/clang/3.3/altivec.h
	OLD_FILES+=usr/include/clang/3.3/ammintrin.h
	OLD_FILES+=usr/include/clang/3.3/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.3/avxintrin.h
	OLD_FILES+=usr/include/clang/3.3/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.3/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.3/cpuid.h
	OLD_FILES+=usr/include/clang/3.3/emmintrin.h
	OLD_FILES+=usr/include/clang/3.3/f16cintrin.h
	OLD_FILES+=usr/include/clang/3.3/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.3/fmaintrin.h
	OLD_FILES+=usr/include/clang/3.3/immintrin.h
	OLD_FILES+=usr/include/clang/3.3/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.3/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.3/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.3/mmintrin.h
	OLD_FILES+=usr/include/clang/3.3/module.map
	OLD_FILES+=usr/include/clang/3.3/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.3/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.3/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.3/prfchwintrin.h
	OLD_FILES+=usr/include/clang/3.3/rdseedintrin.h
	OLD_FILES+=usr/include/clang/3.3/rtmintrin.h
	OLD_FILES+=usr/include/clang/3.3/smmintrin.h
	OLD_FILES+=usr/include/clang/3.3/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.3/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.3/x86intrin.h
	OLD_FILES+=usr/include/clang/3.3/xmmintrin.h
	OLD_FILES+=usr/include/clang/3.3/xopintrin.h
	OLD_FILES+=usr/share/man/man1/llvm-prof.1.gz
	OLD_DIRS+=usr/include/clang/3.3
	# 20140216: nve(4) removed
	OLD_FILES+=usr/share/man/man4/if_nve.4.gz
	OLD_FILES+=usr/share/man/man4/nve.4.gz
	# 20140205: Open Firmware device moved
	OLD_FILES+=usr/include/dev/ofw/ofw_nexus.h
	# 20140128: libelf and libdwarf import
	OLD_LIBS+=usr/lib/libelf.so.1
	OLD_LIBS+=usr/lib32/libelf.so.1
	OLD_LIBS+=usr/lib/libdwarf.so.3
	OLD_LIBS+=usr/lib32/libdwarf.so.3
	# 20140123: apicvar header moved to x86
	OLD_FILES+=usr/include/machine/apicvar.h
	# 20131215: libcam version bumped
	OLD_LIBS+=lib/libcam.so.6 usr/lib32/libcam.so.6
	# 20131202: libcapsicum and libcasper moved to /lib/
	OLD_LIBS+=usr/lib/libcapsicum.so.0
	OLD_LIBS+=usr/lib/libcasper.so.0
	# 20131109: extattr(2) mlinks fixed
	OLD_FILES+=usr/share/man/man2/extattr_delete_list.2.gz
	OLD_FILES+=usr/share/man/man2/extattr_get_list.2.gz
	# 20131107: example files removed
	OLD_FILES+=usr/share/examples/libusb20/aux.c
	OLD_FILES+=usr/share/examples/libusb20/aux.h
	# 20131103: WITH_LIBICONV_COMPAT removal
	OLD_FILES+=usr/include/_libiconv_compat.h
	OLD_FILES+=usr/lib/libiconv.a
	OLD_FILES+=usr/lib/libiconv.so
	OLD_FILES+=usr/lib/libiconv.so.3
	OLD_FILES+=usr/lib/libiconv_p.a
	OLD_FILES+=usr/lib32/libiconv.a
	OLD_FILES+=usr/lib32/libiconv.so
	OLD_FILES+=usr/lib32/libiconv.so.3
	OLD_FILES+=usr/lib32/libiconv_p.a
	# 20131103: removal of utxrm(8), use 'utx rm' instead.
	OLD_FILES+=usr/sbin/utxrm
	OLD_FILES+=usr/share/man/man8/utxrm.8.gz
	# 20131031: pkg_install has been removed
	OLD_FILES+=etc/periodic/daily/220.backup-pkgdb
	OLD_FILES+=etc/periodic/daily/490.status-pkg-changes
	OLD_FILES+=etc/periodic/security/460.chkportsum
	OLD_FILES+=etc/periodic/weekly/400.status-pkg
	OLD_FILES+=usr/sbin/pkg_add
	OLD_FILES+=usr/sbin/pkg_create
	OLD_FILES+=usr/sbin/pkg_delete
	OLD_FILES+=usr/sbin/pkg_info
	OLD_FILES+=usr/sbin/pkg_updating
	OLD_FILES+=usr/sbin/pkg_version
	OLD_FILES+=usr/share/man/man1/pkg_add.1.gz
	OLD_FILES+=usr/share/man/man1/pkg_create.1.gz
	OLD_FILES+=usr/share/man/man1/pkg_delete.1.gz
	OLD_FILES+=usr/share/man/man1/pkg_info.1.gz
	OLD_FILES+=usr/share/man/man1/pkg_updating.1.gz
	OLD_FILES+=usr/share/man/man1/pkg_version.1.gz
	# 20131030: /etc/keys moved to /usr/share/keys
	OLD_DIRS+=etc/keys
	OLD_DIRS+=etc/keys/pkg
	OLD_DIRS+=etc/keys/pkg/revoked
	OLD_DIRS+=etc/keys/pkg/trusted
	OLD_FILES+=etc/keys/pkg/trusted/pkg.freebsd.org.2013102301
	# 20131028: ng_fec(4) removed
	OLD_FILES+=usr/include/netgraph/ng_fec.h
	OLD_FILES+=usr/share/man/man4/ng_fec.4.gz
	# 20131027: header moved
	OLD_FILES+=usr/include/net/pf_mtag.h
	# 20131023: remove never used iscsi directory
	OLD_DIRS+=usr/share/examples/iscsi
	# 20131021: isf(4) removed
	OLD_FILES+=usr/sbin/isfctl
	OLD_FILES+=usr/share/man/man4/isf.4.gz
	OLD_FILES+=usr/share/man/man8/isfctl.8.gz
	# 20131014: libbsdyml becomes private
	OLD_FILES+=usr/lib/libbsdyml.a
	OLD_FILES+=usr/lib/libbsdyml.so
	OLD_LIBS+=usr/lib/libbsdyml.so.0
	OLD_FILES+=usr/lib/libbsdyml_p.a
	OLD_FILES+=usr/lib32/libbsdyml.a
	OLD_FILES+=usr/lib32/libbsdyml.so
	OLD_LIBS+=usr/lib32/libbsdyml.so.0
	OLD_FILES+=usr/lib32/libbsdyml_p.a
	OLD_FILES+=usr/share/man/man3/libbsdyml.3.gz
	OLD_FILES+=usr/include/bsdyml.h
	# 20131013: Removal of the ATF tools
	OLD_FILES+=etc/atf/FreeBSD.conf
	OLD_FILES+=etc/atf/atf-run.hooks
	OLD_FILES+=etc/atf/common.conf
	OLD_FILES+=usr/bin/atf-config
	OLD_FILES+=usr/bin/atf-report
	OLD_FILES+=usr/bin/atf-run
	OLD_FILES+=usr/bin/atf-version
	OLD_FILES+=usr/share/atf/atf-run.hooks
	OLD_FILES+=usr/share/examples/atf/atf-run.hooks
	OLD_FILES+=usr/share/examples/atf/tests-results.css
	OLD_FILES+=usr/share/man/man1/atf-config.1.gz
	OLD_FILES+=usr/share/man/man1/atf-report.1.gz
	OLD_FILES+=usr/share/man/man1/atf-run.1.gz
	OLD_FILES+=usr/share/man/man1/atf-version.1.gz
	OLD_FILES+=usr/share/man/man5/atf-formats.5.gz
	OLD_FILES+=usr/share/xml/atf/tests-results.dtd
	OLD_FILES+=usr/share/xsl/atf/tests-results.xsl
	OLD_DIRS+=etc/atf
	OLD_DIRS+=usr/share/examples/atf
	OLD_DIRS+=usr/share/xml/atf
	OLD_DIRS+=usr/share/xml
	OLD_DIRS+=usr/share/xsl/atf
	OLD_DIRS+=usr/share/xsl
	# 20131009: freebsd-version moved from /libexec to /bin
	OLD_FILES+=libexec/freebsd-version
	# 20131001: ar and ranlib from binutils not used
	OLD_FILES+=usr/bin/gnu-ar
	OLD_FILES+=usr/bin/gnu-ranlib
	OLD_FILES+=usr/share/man/man1/gnu-ar.1.gz
	OLD_FILES+=usr/share/man/man1/gnu-ranlib.1.gz
	# 20130930: BIND removed from base
	OLD_FILES+=etc/mtree/BIND.chroot.dist
	OLD_FILES+=etc/namedb
	OLD_FILES+=etc/periodic/daily/470.status-named
	OLD_FILES+=usr/bin/dig
	OLD_FILES+=usr/bin/nslookup
	OLD_FILES+=usr/bin/nsupdate
	OLD_DIRS+=usr/include/lwres
	OLD_FILES+=usr/include/lwres/context.h
	OLD_FILES+=usr/include/lwres/int.h
	OLD_FILES+=usr/include/lwres/ipv6.h
	OLD_FILES+=usr/include/lwres/lang.h
	OLD_FILES+=usr/include/lwres/list.h
	OLD_FILES+=usr/include/lwres/lwbuffer.h
	OLD_FILES+=usr/include/lwres/lwpacket.h
	OLD_FILES+=usr/include/lwres/lwres.h
	OLD_FILES+=usr/include/lwres/net.h
	OLD_FILES+=usr/include/lwres/netdb.h
	OLD_FILES+=usr/include/lwres/platform.h
	OLD_FILES+=usr/include/lwres/result.h
	OLD_FILES+=usr/include/lwres/string.h
	OLD_FILES+=usr/include/lwres/version.h
	OLD_FILES+=usr/lib/liblwres.a
	OLD_FILES+=usr/lib/liblwres.so
	OLD_LIBS+=usr/lib/liblwres.so.90
	OLD_FILES+=usr/lib/liblwres_p.a
	OLD_FILES+=usr/sbin/arpaname
	OLD_FILES+=usr/sbin/ddns-confgen
	OLD_FILES+=usr/sbin/dnssec-dsfromkey
	OLD_FILES+=usr/sbin/dnssec-keyfromlabel
	OLD_FILES+=usr/sbin/dnssec-keygen
	OLD_FILES+=usr/sbin/dnssec-revoke
	OLD_FILES+=usr/sbin/dnssec-settime
	OLD_FILES+=usr/sbin/dnssec-signzone
	OLD_FILES+=usr/sbin/dnssec-verify
	OLD_FILES+=usr/sbin/genrandom
	OLD_FILES+=usr/sbin/isc-hmac-fixup
	OLD_FILES+=usr/sbin/lwresd
	OLD_FILES+=usr/sbin/named
	OLD_FILES+=usr/sbin/named-checkconf
	OLD_FILES+=usr/sbin/named-checkzone
	OLD_FILES+=usr/sbin/named-compilezone
	OLD_FILES+=usr/sbin/named-journalprint
	OLD_FILES+=usr/sbin/named.reconfig
	OLD_FILES+=usr/sbin/named.reload
	OLD_FILES+=usr/sbin/nsec3hash
	OLD_FILES+=usr/sbin/rndc
	OLD_FILES+=usr/sbin/rndc-confgen
	OLD_DIRS+=usr/share/doc/bind9
	OLD_FILES+=usr/share/doc/bind9/CHANGES
	OLD_FILES+=usr/share/doc/bind9/COPYRIGHT
	OLD_FILES+=usr/share/doc/bind9/FAQ
	OLD_FILES+=usr/share/doc/bind9/HISTORY
	OLD_FILES+=usr/share/doc/bind9/README
	OLD_DIRS+=usr/share/doc/bind9/arm
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch01.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch02.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch03.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch04.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch05.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch06.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch07.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch08.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch09.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch10.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.html
	OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.pdf
	OLD_FILES+=usr/share/doc/bind9/arm/man.arpaname.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.ddns-confgen.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dig.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-dsfromkey.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-keyfromlabel.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-keygen.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-revoke.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-settime.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-signzone.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-verify.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.genrandom.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.host.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.isc-hmac-fixup.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.named-checkconf.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.named-checkzone.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.named-journalprint.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.named.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.nsec3hash.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.nsupdate.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.rndc-confgen.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.rndc.conf.html
	OLD_FILES+=usr/share/doc/bind9/arm/man.rndc.html
	OLD_DIRS+=usr/share/doc/bind9/misc
	OLD_FILES+=usr/share/doc/bind9/misc/dnssec
	OLD_FILES+=usr/share/doc/bind9/misc/format-options.pl
	OLD_FILES+=usr/share/doc/bind9/misc/ipv6
	OLD_FILES+=usr/share/doc/bind9/misc/migration
	OLD_FILES+=usr/share/doc/bind9/misc/migration-4to9
	OLD_FILES+=usr/share/doc/bind9/misc/options
	OLD_FILES+=usr/share/doc/bind9/misc/rfc-compliance
	OLD_FILES+=usr/share/doc/bind9/misc/roadmap
	OLD_FILES+=usr/share/doc/bind9/misc/sdb
	OLD_FILES+=usr/share/doc/bind9/misc/sort-options.pl
	OLD_FILES+=usr/share/man/man1/arpaname.1.gz
	OLD_FILES+=usr/share/man/man1/dig.1.gz
	OLD_FILES+=usr/share/man/man1/nslookup.1.gz
	OLD_FILES+=usr/share/man/man1/nsupdate.1.gz
	OLD_FILES+=usr/share/man/man3/lwres.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_addr_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_add.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_back.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_clear.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_first.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_forward.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_getmem.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_getuint16.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_getuint32.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_getuint8.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_init.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_invalidate.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_putmem.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_putuint16.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_putuint32.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_putuint8.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_buffer_subtract.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_conf_clear.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_conf_get.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_conf_init.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_conf_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_conf_print.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_config.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_allocmem.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_create.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_destroy.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_freemem.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_initserial.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_nextserial.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_context_sendrecv.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_endhostent.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_endhostent_r.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_freeaddrinfo.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_freehostent.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabn.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabnrequest_free.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabnrequest_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabnrequest_render.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabnresponse_free.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabnresponse_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gabnresponse_render.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gai_strerror.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getaddrinfo.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getaddrsbyname.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostbyaddr.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostbyaddr_r.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostbyname.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostbyname2.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostbyname_r.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostent.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gethostent_r.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getipnode.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getipnodebyaddr.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getipnodebyname.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getnamebyaddr.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getnameinfo.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_getrrsetbyname.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnba.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnbarequest_free.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnbarequest_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnbarequest_render.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnbaresponse_free.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnbaresponse_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_gnbaresponse_render.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_herror.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_hstrerror.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_inetntop.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_lwpacket_parseheader.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_lwpacket_renderheader.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_net_ntop.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_noop.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_nooprequest_free.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_nooprequest_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_nooprequest_render.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_noopresponse_free.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_noopresponse_parse.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_noopresponse_render.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_packet.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_resutil.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_sethostent.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_sethostent_r.3.gz
	OLD_FILES+=usr/share/man/man3/lwres_string_parse.3.gz
	OLD_FILES+=usr/share/man/man5/named.conf.5.gz
	OLD_FILES+=usr/share/man/man5/rndc.conf.5.gz
	OLD_FILES+=usr/share/man/man8/ddns-confgen.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-dsfromkey.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-keyfromlabel.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-keygen.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-revoke.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-settime.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-signzone.8.gz
	OLD_FILES+=usr/share/man/man8/dnssec-verify.8.gz
	OLD_FILES+=usr/share/man/man8/genrandom.8.gz
	OLD_FILES+=usr/share/man/man8/isc-hmac-fixup.8.gz
	OLD_FILES+=usr/share/man/man8/lwresd.8.gz
	OLD_FILES+=usr/share/man/man8/named-checkconf.8.gz
	OLD_FILES+=usr/share/man/man8/named-checkzone.8.gz
	OLD_FILES+=usr/share/man/man8/named-compilezone.8.gz
	OLD_FILES+=usr/share/man/man8/named-journalprint.8.gz
	OLD_FILES+=usr/share/man/man8/named.8.gz
	OLD_FILES+=usr/share/man/man8/named.reconfig.8.gz
	OLD_FILES+=usr/share/man/man8/named.reload.8.gz
	OLD_FILES+=usr/share/man/man8/nsec3hash.8.gz
	OLD_FILES+=usr/share/man/man8/rndc-confgen.8.gz
	OLD_FILES+=usr/share/man/man8/rndc.8.gz
	OLD_DIRS+=var/named/dev
	OLD_DIRS+=var/named/etc
	OLD_DIRS+=var/named/etc/namedb
	OLD_FILES+=var/named/etc/namedb/PROTO.localhost-v6.rev
	OLD_FILES+=var/named/etc/namedb/PROTO.localhost.rev
	OLD_DIRS+=var/named/etc/namedb/dynamic
	OLD_FILES+=var/named/etc/namedb/make-localhost
	OLD_DIRS+=var/named/etc/namedb/master
	OLD_FILES+=var/named/etc/namedb/master/empty.db
	OLD_FILES+=var/named/etc/namedb/master/localhost-forward.db
	OLD_FILES+=var/named/etc/namedb/master/localhost-reverse.db
	#OLD_FILES+=var/named/etc/namedb/named.conf # intentionally left out
	OLD_FILES+=var/named/etc/namedb/named.root
	OLD_DIRS+=var/named/etc/namedb/working
	OLD_DIRS+=var/named/etc/namedb/slave
	OLD_DIRS+=var/named/var
	OLD_DIRS+=var/named/var/dump
	OLD_DIRS+=var/named/var/log
	OLD_DIRS+=var/named/var/run
	OLD_DIRS+=var/named/var/run/named
	OLD_DIRS+=var/named/var/stats
	OLD_DIRS+=var/run/named
	# 20130923: example moved
	OLD_FILES+=usr/share/examples/bsdconfig/browse_packages.sh
	# 20130908: libssh becomes private
	OLD_FILES+=usr/lib/libssh.a
	OLD_FILES+=usr/lib/libssh.so
	OLD_LIBS+=usr/lib/libssh.so.5
	OLD_FILES+=usr/lib/libssh_p.a
	OLD_FILES+=usr/lib32/libssh.a
	OLD_FILES+=usr/lib32/libssh.so
	OLD_LIBS+=usr/lib32/libssh.so.5
	OLD_FILES+=usr/lib32/libssh_p.a
	# 20130903: gnupatch is no more
	OLD_FILES+=usr/bin/gnupatch
	OLD_FILES+=usr/share/man/man1/gnupatch.1.gz
	# 20130829: bsdpatch is patch unconditionally
	OLD_FILES+=usr/bin/bsdpatch
	OLD_FILES+=usr/share/man/man1/bsdpatch.1.gz
	# 20130822: bind 9.9.3-P2 import
	OLD_LIBS+=usr/lib/liblwres.so.80
	# 20130814: vm_page_busy(9)
	OLD_FILES+=usr/share/man/man9/vm_page_flash.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_io.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_io_finish.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_io_start.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_wakeup.9.gz
	# 20130710: libkvm version bump
	OLD_LIBS+=lib/libkvm.so.5
	OLD_LIBS+=usr/lib32/libkvm.so.5
	# 20130623: dialog update from 1.1 to 1.2
	OLD_LIBS+=usr/lib/libdialog.so.7
	OLD_LIBS+=usr/lib32/libdialog.so.7
	# 20130616: vfs_mount.9 removed
	OLD_FILES+=usr/share/man/man9/vfs_mount.9.gz
	# 20130614: remove CVS from base
	OLD_FILES+=usr/bin/cvs
	OLD_FILES+=usr/bin/cvsbug
	OLD_FILES+=usr/share/doc/psd/28.cvs/paper.ascii.gz
	OLD_FILES+=usr/share/doc/psd/28.cvs/paper.ps.gz
	OLD_DIRS+=usr/share/doc/psd/28.cvs
	OLD_FILES+=usr/share/examples/cvs/contrib/README
	OLD_FILES+=usr/share/examples/cvs/contrib/clmerge
	OLD_FILES+=usr/share/examples/cvs/contrib/cln_hist
	OLD_FILES+=usr/share/examples/cvs/contrib/commit_prep
	OLD_FILES+=usr/share/examples/cvs/contrib/cvs2vendor
	OLD_FILES+=usr/share/examples/cvs/contrib/cvs_acls
	OLD_FILES+=usr/share/examples/cvs/contrib/cvscheck
	OLD_FILES+=usr/share/examples/cvs/contrib/cvscheck.man
	OLD_FILES+=usr/share/examples/cvs/contrib/cvshelp.man
	OLD_FILES+=usr/share/examples/cvs/contrib/descend.man
	OLD_FILES+=usr/share/examples/cvs/contrib/easy-import
	OLD_FILES+=usr/share/examples/cvs/contrib/intro.doc
	OLD_FILES+=usr/share/examples/cvs/contrib/log
	OLD_FILES+=usr/share/examples/cvs/contrib/log_accum
	OLD_FILES+=usr/share/examples/cvs/contrib/mfpipe
	OLD_FILES+=usr/share/examples/cvs/contrib/rcs-to-cvs
	OLD_FILES+=usr/share/examples/cvs/contrib/rcs2log
	OLD_FILES+=usr/share/examples/cvs/contrib/rcslock
	OLD_FILES+=usr/share/examples/cvs/contrib/sccs2rcs
	OLD_DIRS+=usr/share/examples/cvs/contrib
	OLD_DIRS+=usr/share/examples/cvs
	OLD_FILES+=usr/share/info/cvs.info.gz
	OLD_FILES+=usr/share/info/cvsclient.info.gz
	OLD_FILES+=usr/share/man/man1/cvs.1.gz
	OLD_FILES+=usr/share/man/man5/cvs.5.gz
	OLD_FILES+=usr/share/man/man8/cvsbug.8.gz
	# 20130607: WITH_DEBUG_FILES added
	OLD_FILES+=lib/libufs.so.6.symbols
	OLD_FILES+=usr/lib32/libufs.so.6.symbols
	# 20130417: nfs fha moved from nfsserver to nfs
	OLD_FILES+=usr/include/nfsserver/nfs_fha.h
	# 20130411: new clang import which bumps version from 3.2 to 3.3.
	OLD_FILES+=usr/include/clang/3.2/__wmmintrin_aes.h
	OLD_FILES+=usr/include/clang/3.2/__wmmintrin_pclmul.h
	OLD_FILES+=usr/include/clang/3.2/altivec.h
	OLD_FILES+=usr/include/clang/3.2/ammintrin.h
	OLD_FILES+=usr/include/clang/3.2/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.2/avxintrin.h
	OLD_FILES+=usr/include/clang/3.2/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.2/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.2/cpuid.h
	OLD_FILES+=usr/include/clang/3.2/emmintrin.h
	OLD_FILES+=usr/include/clang/3.2/f16cintrin.h
	OLD_FILES+=usr/include/clang/3.2/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.2/fmaintrin.h
	OLD_FILES+=usr/include/clang/3.2/immintrin.h
	OLD_FILES+=usr/include/clang/3.2/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.2/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.2/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.2/mmintrin.h
	OLD_FILES+=usr/include/clang/3.2/module.map
	OLD_FILES+=usr/include/clang/3.2/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.2/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.2/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.2/rtmintrin.h
	OLD_FILES+=usr/include/clang/3.2/smmintrin.h
	OLD_FILES+=usr/include/clang/3.2/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.2/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.2/x86intrin.h
	OLD_FILES+=usr/include/clang/3.2/xmmintrin.h
	OLD_FILES+=usr/include/clang/3.2/xopintrin.h
	OLD_DIRS+=usr/include/clang/3.2
	# 20130404: legacy ATA stack removed
	OLD_FILES+=etc/periodic/daily/405.status-ata-raid
	OLD_FILES+=rescue/atacontrol
	OLD_FILES+=sbin/atacontrol
	OLD_FILES+=usr/share/man/man8/atacontrol.8.gz
	OLD_FILES+=usr/share/man/man4/atapicam.4.gz
	OLD_FILES+=usr/share/man/man4/ataraid.4.gz
	OLD_FILES+=usr/sbin/burncd
	OLD_FILES+=usr/share/man/man8/burncd.8.gz
	# 20130316: vinum.4 removed
	OLD_FILES+=usr/share/man/man4/vinum.4.gz
	# 20130312: fortunes-o removed
	OLD_FILES+=usr/share/games/fortune/fortunes-o
	OLD_FILES+=usr/share/games/fortune/fortunes-o.dat
	# 20130311: Ports are no more available via cvsup
	OLD_FILES+=usr/share/examples/cvsup/ports-supfile
	OLD_FILES+=usr/share/examples/cvsup/refuse
	OLD_FILES+=usr/share/examples/cvsup/refuse.README
	# 20130309: NWFS and NCP supports removed
	OLD_FILES+=usr/bin/ncplist
	OLD_FILES+=usr/bin/ncplogin
	OLD_FILES+=usr/bin/ncplogout
	OLD_FILES+=usr/include/fs/nwfs/nwfs.h
	OLD_FILES+=usr/include/fs/nwfs/nwfs_mount.h
	OLD_FILES+=usr/include/fs/nwfs/nwfs_node.h
	OLD_FILES+=usr/include/fs/nwfs/nwfs_subr.h
	OLD_DIRS+=usr/include/fs/nwfs
	OLD_FILES+=usr/include/netncp/ncp.h
	OLD_FILES+=usr/include/netncp/ncp_cfg.h
	OLD_FILES+=usr/include/netncp/ncp_conn.h
	OLD_FILES+=usr/include/netncp/ncp_file.h
	OLD_FILES+=usr/include/netncp/ncp_lib.h
	OLD_FILES+=usr/include/netncp/ncp_ncp.h
	OLD_FILES+=usr/include/netncp/ncp_nls.h
	OLD_FILES+=usr/include/netncp/ncp_rcfile.h
	OLD_FILES+=usr/include/netncp/ncp_rq.h
	OLD_FILES+=usr/include/netncp/ncp_sock.h
	OLD_FILES+=usr/include/netncp/ncp_subr.h
	OLD_FILES+=usr/include/netncp/ncp_user.h
	OLD_FILES+=usr/include/netncp/ncpio.h
	OLD_FILES+=usr/include/netncp/nwerror.h
	OLD_DIRS+=usr/include/netncp
	OLD_FILES+=usr/lib/libncp.a
	OLD_FILES+=usr/lib/libncp.so
	OLD_LIBS+=usr/lib/libncp.so.4
	OLD_FILES+=usr/lib/libncp_p.a
	OLD_FILES+=usr/lib32/libncp.a
	OLD_FILES+=usr/lib32/libncp.so
	OLD_LIBS+=usr/lib32/libncp.so.4
	OLD_FILES+=usr/lib32/libncp_p.a
	OLD_FILES+=usr/sbin/mount_nwfs
	OLD_FILES+=usr/share/examples/nwclient/dot.nwfsrc
	OLD_FILES+=usr/share/examples/nwclient/nwfs.sh.sample
	OLD_DIRS+=usr/share/examples/nwclient
	OLD_FILES+=usr/share/man/man1/ncplist.1.gz
	OLD_FILES+=usr/share/man/man1/ncplogin.1.gz
	OLD_FILES+=usr/share/man/man1/ncplogout.1.gz
	OLD_FILES+=usr/share/man/man8/mount_nwfs.8.gz
	# 20130302: NTFS support removed
	OLD_FILES+=rescue/mount_ntfs
	OLD_FILES+=sbin/mount_ntfs
	OLD_FILES+=usr/include/fs/ntfs/ntfs.h
	OLD_FILES+=usr/include/fs/ntfs/ntfs_compr.h
	OLD_FILES+=usr/include/fs/ntfs/ntfs_ihash.h
	OLD_FILES+=usr/include/fs/ntfs/ntfs_inode.h
	OLD_FILES+=usr/include/fs/ntfs/ntfs_subr.h
	OLD_FILES+=usr/include/fs/ntfs/ntfs_vfsops.h
	OLD_FILES+=usr/include/fs/ntfs/ntfsmount.h
	OLD_DIRS+=usr/include/fs/ntfs
	OLD_FILES+=usr/share/man/man8/mount_ntfs.8.gz
	# 20130302: PORTALFS support removed
	OLD_FILES+=usr/include/fs/portalfs/portal.h
	OLD_DIRS+=usr/include/fs/portalfs
	OLD_FILES+=usr/sbin/mount_portalfs
	OLD_FILES+=usr/share/examples/portal/README
	OLD_FILES+=usr/share/examples/portal/portal.conf
	OLD_DIRS+=usr/share/examples/portal
	OLD_FILES+=usr/share/man/man8/mount_portalfs.8.gz
	# 20130302: CODAFS support removed
	OLD_FILES+=usr/share/man/man4/coda.4.gz
	# 20130302: XFS support removed
	OLD_FILES+=usr/share/man/man5/xfs.5.gz
	# 20130302: Capsicum overhaul
	OLD_FILES+=usr/share/man/man2/cap_getrights.2.gz
	OLD_FILES+=usr/share/man/man2/cap_new.2.gz
	# 20130213: OpenSSL 1.0.1e import
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_verifyrecover.3.gz
	OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_verifyrecover_init.3.gz
	# 20130116: removed long unused directories for .1aout section manpages
	OLD_FILES+=usr/share/man/en.ISO8859-1/man1aout
	OLD_FILES+=usr/share/man/en.UTF-8/man1aout
	OLD_DIRS+=usr/share/man/man1aout
	OLD_DIRS+=usr/share/man/cat1aout
	OLD_DIRS+=usr/share/man/en.ISO8859-1/cat1aout
	OLD_DIRS+=usr/share/man/en.UTF-8/cat1aout
	# 20130103: gnats-supfile removed
	OLD_FILES+=usr/share/examples/cvsup/gnats-supfile
	# 20121230: libdisk removed
	OLD_FILES+=usr/share/man/man3/libdisk.3.gz usr/include/libdisk.h
	OLD_FILES+=usr/lib/libdisk.a usr/lib32/libdisk.a
	# 20121230: remove wrongly created directories for auditdistd
	OLD_DIRS+=var/dist
	OLD_DIRS+=var/remote
	# 20121114: zpool-features manual page moved from section 5 to 7
	OLD_FILES+=usr/share/man/man5/zpool-features.5.gz
	# 20121022: remove harp, hfa and idt man page
	OLD_FILES+=usr/share/man/man4/harp.4.gz
	OLD_FILES+=usr/share/man/man4/hfa.4.gz
	OLD_FILES+=usr/share/man/man4/idt.4.gz
	OLD_FILES+=usr/share/man/man4/if_idt.4.gz
	# 20121022: VFS_LOCK_GIANT elimination
	OLD_FILES+=usr/share/man/man9/VFS_LOCK_GIANT.9.gz
	OLD_FILES+=usr/share/man/man9/VFS_UNLOCK_GIANT.9.gz
	# 20121004: remove incomplete unwind.h
	OLD_FILES+=usr/include/clang/3.2/unwind.h
	# 20120910: NetBSD compat shims removed
	OLD_FILES+=usr/include/cam/scsi/scsi_low_pisa.h
	OLD_FILES+=usr/include/sys/device_port.h
	# 20120909: doc and www supfiles removed
	OLD_FILES+=usr/share/examples/cvsup/doc-supfile
	OLD_FILES+=usr/share/examples/cvsup/www-supfile
	# 20120908: pf cleanup
	OLD_FILES+=usr/include/net/if_pflow.h
	# 20120816: new clang import which bumps version from 3.1 to 3.2
	OLD_FILES+=usr/bin/llvm-ld
	OLD_FILES+=usr/bin/llvm-stub
	OLD_FILES+=usr/include/clang/3.1/altivec.h
	OLD_FILES+=usr/include/clang/3.1/avx2intrin.h
	OLD_FILES+=usr/include/clang/3.1/avxintrin.h
	OLD_FILES+=usr/include/clang/3.1/bmi2intrin.h
	OLD_FILES+=usr/include/clang/3.1/bmiintrin.h
	OLD_FILES+=usr/include/clang/3.1/cpuid.h
	OLD_FILES+=usr/include/clang/3.1/emmintrin.h
	OLD_FILES+=usr/include/clang/3.1/fma4intrin.h
	OLD_FILES+=usr/include/clang/3.1/immintrin.h
	OLD_FILES+=usr/include/clang/3.1/lzcntintrin.h
	OLD_FILES+=usr/include/clang/3.1/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.1/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.1/mmintrin.h
	OLD_FILES+=usr/include/clang/3.1/module.map
	OLD_FILES+=usr/include/clang/3.1/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.1/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.1/popcntintrin.h
	OLD_FILES+=usr/include/clang/3.1/smmintrin.h
	OLD_FILES+=usr/include/clang/3.1/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.1/unwind.h
	OLD_FILES+=usr/include/clang/3.1/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.1/x86intrin.h
	OLD_FILES+=usr/include/clang/3.1/xmmintrin.h
	OLD_DIRS+=usr/include/clang/3.1
	OLD_FILES+=usr/share/man/man1/llvm-ld.1.gz
	# 20120712: OpenSSL 1.0.1c import
	OLD_LIBS+=lib/libcrypto.so.6
	OLD_LIBS+=usr/lib/libssl.so.6
	OLD_LIBS+=usr/lib32/libcrypto.so.6
	OLD_LIBS+=usr/lib32/libssl.so.6
	OLD_FILES+=usr/include/openssl/aes_locl.h
	OLD_FILES+=usr/include/openssl/bio_lcl.h
	OLD_FILES+=usr/include/openssl/e_os.h
	OLD_FILES+=usr/include/openssl/fips.h
	OLD_FILES+=usr/include/openssl/fips_rand.h
	OLD_FILES+=usr/include/openssl/pq_compat.h
	OLD_FILES+=usr/include/openssl/tmdiff.h
	OLD_FILES+=usr/include/openssl/ui_locl.h
	OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_id_callback.3.gz
	# 20120621: remove old man page
	OLD_FILES+=usr/share/man/man8/vnconfig.8.gz
	# 20120619: TOE support updated
	OLD_FILES+=usr/include/netinet/toedev.h
	# 20120613: auth.conf removed
	OLD_FILES+=etc/auth.conf
	OLD_FILES+=usr/share/examples/etc/auth.conf
	OLD_FILES+=usr/share/man/man3/auth.3.gz
	OLD_FILES+=usr/share/man/man3/auth_getval.3.gz
	OLD_FILES+=usr/share/man/man5/auth.conf.5.gz
	# 20120530: kde pam lives now in ports
	OLD_FILES+=etc/pam.d/kde
	# 20120521: byacc import
	OLD_FILES+=usr/bin/yyfix
	OLD_FILES+=usr/share/man/man1/yyfix.1.gz
	# 20120505: new clang import installed a redundant internal header
	OLD_FILES+=usr/include/clang/3.1/stdalign.h
	# 20120428: MD2 removed from libmd
	OLD_LIBS+=lib/libmd.so.5
	OLD_FILES+=usr/include/md2.h
	OLD_LIBS+=usr/lib32/libmd.so.5
	OLD_FILES+=usr/share/man/man3/MD2Data.3.gz
	OLD_FILES+=usr/share/man/man3/MD2End.3.gz
	OLD_FILES+=usr/share/man/man3/MD2File.3.gz
	OLD_FILES+=usr/share/man/man3/MD2FileChunk.3.gz
	OLD_FILES+=usr/share/man/man3/MD2Final.3.gz
	OLD_FILES+=usr/share/man/man3/MD2Init.3.gz
	OLD_FILES+=usr/share/man/man3/MD2Update.3.gz
	OLD_FILES+=usr/share/man/man3/md2.3.gz
	# 20120425: libusb version bump (r234684)
	OLD_LIBS+=usr/lib/libusb.so.2
	OLD_LIBS+=usr/lib32/libusb.so.2
	OLD_FILES+=usr/share/man/man3/libsub_get_active_config_descriptor.3.gz
	# 20120415: new clang import which bumps version from 3.0 to 3.1
	OLD_FILES+=usr/include/clang/3.0/altivec.h
	OLD_FILES+=usr/include/clang/3.0/avxintrin.h
	OLD_FILES+=usr/include/clang/3.0/emmintrin.h
	OLD_FILES+=usr/include/clang/3.0/immintrin.h
	OLD_FILES+=usr/include/clang/3.0/mm3dnow.h
	OLD_FILES+=usr/include/clang/3.0/mm_malloc.h
	OLD_FILES+=usr/include/clang/3.0/mmintrin.h
	OLD_FILES+=usr/include/clang/3.0/nmmintrin.h
	OLD_FILES+=usr/include/clang/3.0/pmmintrin.h
	OLD_FILES+=usr/include/clang/3.0/smmintrin.h
	OLD_FILES+=usr/include/clang/3.0/tmmintrin.h
	OLD_FILES+=usr/include/clang/3.0/wmmintrin.h
	OLD_FILES+=usr/include/clang/3.0/x86intrin.h
	OLD_FILES+=usr/include/clang/3.0/xmmintrin.h
	OLD_DIRS+=usr/include/clang/3.0
	# 20120412: BIND 9.8.1 release notes removed
	OLD_FILES+=usr/share/doc/bind9/RELEASE-NOTES-BIND-9.8.1.pdf
	OLD_FILES+=usr/share/doc/bind9/RELEASE-NOTES-BIND-9.8.1.txt
	OLD_FILES+=usr/share/doc/bind9/RELEASE-NOTES-BIND-9.8.1.html
	OLD_FILES+=usr/share/doc/bind9/release-notes.css
	# 20120330: legacy(4) moved to x86
	OLD_FILES+=usr/include/machine/legacyvar.h
	# 20120324: MPI headers updated
	OLD_FILES+=usr/include/dev/mpt/mpilib/mpi_inb.h
	# 20120322: hwpmc_mips24k.h removed
	OLD_FILES+=usr/include/dev/hwpmc/hwpmc_mips24k.h
	# 20120322: Update heimdal to 1.5.1.
	OLD_FILES+=usr/include/krb5-v4compat.h \
	usr/include/krb_err.h \
	usr/include/hdb-private.h \
	usr/share/man/man3/krb5_addresses.3.gz \
	usr/share/man/man3/krb5_cc_cursor.3.gz \
	usr/share/man/man3/krb5_cc_ops.3.gz \
	usr/share/man/man3/krb5_config.3.gz \
	usr/share/man/man3/krb5_config_get_int_default.3.gz \
	usr/share/man/man3/krb5_context.3.gz \
	usr/share/man/man3/krb5_data.3.gz \
	usr/share/man/man3/krb5_err.3.gz \
	usr/share/man/man3/krb5_errx.3.gz \
	usr/share/man/man3/krb5_keyblock.3.gz \
	usr/share/man/man3/krb5_keytab_entry.3.gz \
	usr/share/man/man3/krb5_kt_cursor.3.gz \
	usr/share/man/man3/krb5_kt_ops.3.gz \
	usr/share/man/man3/krb5_set_warn_dest.3.gz \
	usr/share/man/man3/krb5_verr.3.gz \
	usr/share/man/man3/krb5_verrx.3.gz \
	usr/share/man/man3/krb5_vwarnx.3.gz \
	usr/share/man/man3/krb5_warn.3.gz \
	usr/share/man/man3/krb5_warnx.3.gz
	OLD_LIBS+=usr/lib/libasn1.so.10 \
	usr/lib/libhdb.so.10 \
	usr/lib/libheimntlm.so.10 \
	usr/lib/libhx509.so.10 \
	usr/lib/libkadm5clnt.so.10 \
	usr/lib/libkadm5srv.so.10 \
	usr/lib/libkafs5.so.10 \
	usr/lib/libkrb5.so.10 \
	usr/lib/libroken.so.10 \
	usr/lib32/libasn1.so.10 \
	usr/lib32/libhdb.so.10 \
	usr/lib32/libheimntlm.so.10 \
	usr/lib32/libhx509.so.10 \
	usr/lib32/libkadm5clnt.so.10 \
	usr/lib32/libkadm5srv.so.10 \
	usr/lib32/libkafs5.so.10 \
	usr/lib32/libkrb5.so.10 \
	usr/lib32/libroken.so.10
	# 20120309: Remove fifofs header files.
	OLD_FILES+=usr/include/fs/fifofs/fifo.h
	OLD_DIRS+=usr/include/fs/fifofs
	# 20120304: xlocale cleanup
	OLD_FILES+=usr/include/_xlocale_ctype.h
	# 20120225: libarchive 3.0.3
	OLD_FILES+=usr/share/man/man3/archive_read_data_into_buffer.3.gz \
	usr/share/man/man3/archive_read_support_compression_all.3.gz \
	usr/share/man/man3/archive_read_support_compression_bzip2.3.gz \
	usr/share/man/man3/archive_read_support_compression_compress.3.gz \
	usr/share/man/man3/archive_read_support_compression_gzip.3.gz \
	usr/share/man/man3/archive_read_support_compression_lzma.3.gz \
	usr/share/man/man3/archive_read_support_compression_none.3.gz \
	usr/share/man/man3/archive_read_support_compression_program.3.gz \
	usr/share/man/man3/archive_read_support_compression_program_signature.3.gz \
	usr/share/man/man3/archive_read_support_compression_xz.3.gz \
	usr/share/man/man3/archive_write_set_callbacks.3.gz \
	usr/share/man/man3/archive_write_set_compression_bzip2.3.gz \
	usr/share/man/man3/archive_write_set_compression_compress.3.gz \
	usr/share/man/man3/archive_write_set_compression_gzip.3.gz \
	usr/share/man/man3/archive_write_set_compression_none.3.gz \
	usr/share/man/man3/archive_write_set_compression_program.3.gz
	OLD_LIBS+=usr/lib/libarchive.so.5
	OLD_LIBS+=usr/lib32/libarchive.so.5
	# 20120113: removal of wtmpcvt(1)
	OLD_FILES+=usr/bin/wtmpcvt
	OLD_FILES+=usr/share/man/man1/wtmpcvt.1.gz
	# 20111214: eventtimers(7) moved to eventtimers(4)
	OLD_FILES+=usr/share/man/man7/eventtimers.7.gz
	# 20111125: amd(4) removed
	OLD_FILES+=usr/share/man/man4/amd.4.gz
	# 20111125: libodialog removed
	OLD_FILES+=usr/lib/libodialog.a
	OLD_FILES+=usr/lib/libodialog.so
	OLD_LIBS+=usr/lib/libodialog.so.7
	OLD_FILES+=usr/lib/libodialog_p.a
	OLD_FILES+=usr/lib32/libodialog.a
	OLD_FILES+=usr/lib32/libodialog.so
	OLD_LIBS+=usr/lib32/libodialog.so.7
	OLD_FILES+=usr/lib32/libodialog_p.a
	# 20110930: sysinstall removed
	OLD_FILES+=usr/sbin/sysinstall
	OLD_FILES+=usr/share/man/man8/sysinstall.8.gz
	OLD_FILES+=usr/lib/libftpio.a
	OLD_FILES+=usr/lib/libftpio.so
	OLD_LIBS+=usr/lib/libftpio.so.8
	OLD_FILES+=usr/lib/libftpio_p.a
	OLD_FILES+=usr/lib32/libftpio.a
	OLD_FILES+=usr/lib32/libftpio.so
	OLD_LIBS+=usr/lib32/libftpio.so.8
	OLD_FILES+=usr/lib32/libftpio_p.a
	OLD_FILES+=usr/include/ftpio.h
	OLD_FILES+=usr/share/man/man3/ftpio.3.gz
	# 20110915: rename congestion control manpages
	OLD_FILES+=usr/share/man/man9/cc.9.gz
	# 20110831: atomic page flags operations
	OLD_FILES+=usr/share/man/man9/vm_page_flag.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_flag_clear.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_flag_set.9.gz
	# 20110828: library version bump for 9.0
	OLD_LIBS+=lib/libcam.so.5
	OLD_LIBS+=lib/libpcap.so.7
	OLD_LIBS+=lib/libufs.so.5
	OLD_LIBS+=usr/lib/libbsnmp.so.5
	OLD_LIBS+=usr/lib/libdwarf.so.2
	OLD_LIBS+=usr/lib/libopie.so.6
	OLD_LIBS+=usr/lib/librtld_db.so.1
	OLD_LIBS+=usr/lib/libtacplus.so.4
	OLD_LIBS+=usr/lib32/libcam.so.5
	OLD_LIBS+=usr/lib32/libpcap.so.7
	OLD_LIBS+=usr/lib32/libufs.so.5
	OLD_LIBS+=usr/lib32/libbsnmp.so.5
	OLD_LIBS+=usr/lib32/libdwarf.so.2
	OLD_LIBS+=usr/lib32/libopie.so.6
	OLD_LIBS+=usr/lib32/librtld_db.so.1
	OLD_LIBS+=usr/lib32/libtacplus.so.4
	# 20110817: no more acd.4, ad.4, afd.4 and ast.4
	OLD_FILES+=usr/share/man/man4/acd.4.gz
	OLD_FILES+=usr/share/man/man4/ad.4.gz
	OLD_FILES+=usr/share/man/man4/afd.4.gz
	OLD_FILES+=usr/share/man/man4/ast.4.gz
	# 20110718: no longer useful in the age of rc.d
	OLD_FILES+=usr/sbin/named.reconfig
	OLD_FILES+=usr/sbin/named.reload
	OLD_FILES+=usr/share/man/man8/named.reconfig.8.gz
	OLD_FILES+=usr/share/man/man8/named.reload.8.gz
	# 20110716: bind 9.8.0 import
	OLD_LIBS+=usr/lib/liblwres.so.50
	OLD_FILES+=usr/share/doc/bind9/KNOWN-DEFECTS
	OLD_FILES+=usr/share/doc/bind9/NSEC3-NOTES
	OLD_FILES+=usr/share/doc/bind9/README.idnkit
	OLD_FILES+=usr/share/doc/bind9/README.pkcs11
	# 20110709: vm_map_clean.9 -> vm_map_sync.9
	OLD_FILES+=usr/share/man/man9/vm_map_clean.9.gz
	# 20110709: Catch up with removal of these functions.
	OLD_FILES+=usr/share/man/man9/vm_page_copy.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_protect.9.gz
	OLD_FILES+=usr/share/man/man9/vm_page_zero_fill.9.gz
	# 20110707: script no longer needed by /etc/rc.d/nfsd
	OLD_FILES+=etc/rc.d/nfsserver
	# 20110705: files moved so both NFS clients can share them
	OLD_FILES+=usr/include/nfsclient/krpc.h
	OLD_FILES+=usr/include/nfsclient/nfsdiskless.h
	# 20110705: the switch of default NFS client to the new one
	OLD_FILES+=sbin/mount_newnfs
	OLD_FILES+=usr/share/man/man8/mount_newnfs.8.gz
	OLD_FILES+=usr/include/nfsclient/nfs_kdtrace.h
	# 20110628: calendar.msk removed
	OLD_FILES+=usr/share/calendar/ru_RU.KOI8-R/calendar.msk
	# 20110517: libpkg removed
	OLD_FILES+=usr/include/pkg.h
	OLD_FILES+=usr/lib/libpkg.a
	OLD_FILES+=usr/lib/libpkg.so
	OLD_LIBS+=usr/lib/libpkg.so.0
	OLD_FILES+=usr/lib/libpkg_p.a
	OLD_FILES+=usr/lib32/libpkg.a
	OLD_FILES+=usr/lib32/libpkg.so
	OLD_LIBS+=usr/lib32/libpkg.so.0
	OLD_FILES+=usr/lib32/libpkg_p.a
	# 20110517: libsbuf version bump
	OLD_LIBS+=lib/libsbuf.so.5
	OLD_LIBS+=usr/lib32/libsbuf.so.5
	# 20110502: new clang import which bumps version from 2.9 to 3.0
	OLD_FILES+=usr/include/clang/2.9/emmintrin.h
	OLD_FILES+=usr/include/clang/2.9/mm_malloc.h
	OLD_FILES+=usr/include/clang/2.9/mmintrin.h
	OLD_FILES+=usr/include/clang/2.9/pmmintrin.h
	OLD_FILES+=usr/include/clang/2.9/tmmintrin.h
	OLD_FILES+=usr/include/clang/2.9/xmmintrin.h
	OLD_DIRS+=usr/include/clang/2.9
	# 20110417: removal of Objective-C support
	OLD_FILES+=usr/include/objc/encoding.h
	OLD_FILES+=usr/include/objc/hash.h
	OLD_FILES+=usr/include/objc/NXConstStr.h
	OLD_FILES+=usr/include/objc/objc-api.h
	OLD_FILES+=usr/include/objc/objc-decls.h
	OLD_FILES+=usr/include/objc/objc-list.h
	OLD_FILES+=usr/include/objc/objc.h
	OLD_FILES+=usr/include/objc/Object.h
	OLD_FILES+=usr/include/objc/Protocol.h
	OLD_FILES+=usr/include/objc/runtime.h
	OLD_FILES+=usr/include/objc/sarray.h
	OLD_FILES+=usr/include/objc/thr.h
	OLD_FILES+=usr/include/objc/typedstream.h
	OLD_FILES+=usr/lib/libobjc.a
	OLD_FILES+=usr/lib/libobjc.so
	OLD_FILES+=usr/lib/libobjc_p.a
	OLD_FILES+=usr/libexec/cc1obj
	OLD_LIBS+=usr/lib/libobjc.so.4
	OLD_DIRS+=usr/include/objc
	OLD_FILES+=usr/lib32/libobjc.a
	OLD_FILES+=usr/lib32/libobjc.so
	OLD_FILES+=usr/lib32/libobjc_p.a
	OLD_LIBS+=usr/lib32/libobjc.so.4
	# 20110331: firmware.img created at build time
	OLD_FILES+=usr/share/examples/kld/firmware/fwimage/firmware.img
	# 20110224: sticky.8 -> sticky.7
	OLD_FILES+=usr/share/man/man8/sticky.8.gz
	# 20110220: new clang import which bumps version from 2.8 to 2.9
	OLD_FILES+=usr/include/clang/2.8/emmintrin.h
	OLD_FILES+=usr/include/clang/2.8/mm_malloc.h
	OLD_FILES+=usr/include/clang/2.8/mmintrin.h
	OLD_FILES+=usr/include/clang/2.8/pmmintrin.h
	OLD_FILES+=usr/include/clang/2.8/tmmintrin.h
	OLD_FILES+=usr/include/clang/2.8/xmmintrin.h
	OLD_DIRS+=usr/include/clang/2.8
	# 20110119: netinet/sctp_cc_functions.h removed
	OLD_FILES+=usr/include/netinet/sctp_cc_functions.h
	# 20110119: Remove SYSCTL_X sysctl additions.
	OLD_FILES+=usr/share/man/man9/SYSCTL_XINT.9.gz \
	usr/share/man/man9/SYSCTL_XLONG.9.gz

	# 20110112: Update dialog to new version, rename old libdialog to libodialog,
	# removing associated man pages and header files.
	OLD_FILES+=usr/share/man/man3/draw_shadow.3.gz \
	usr/share/man/man3/draw_box.3.gz usr/share/man/man3/line_edit.3.gz \
	usr/share/man/man3/strheight.3.gz usr/share/man/man3/strwidth.3.gz \
	usr/share/man/man3/dialog_create_rc.3.gz \
	usr/share/man/man3/dialog_yesno.3.gz usr/share/man/man3/dialog_noyes.3.gz \
	usr/share/man/man3/dialog_prgbox.3.gz \
	usr/share/man/man3/dialog_textbox.3.gz usr/share/man/man3/dialog_menu.3.gz \
	usr/share/man/man3/dialog_checklist.3.gz \
	usr/share/man/man3/dialog_radiolist.3.gz \
	usr/share/man/man3/dialog_inputbox.3.gz \
	usr/share/man/man3/dialog_clear_norefresh.3.gz \
	usr/share/man/man3/dialog_clear.3.gz usr/share/man/man3/dialog_update.3.gz \
	usr/share/man/man3/dialog_fselect.3.gz \
	usr/share/man/man3/dialog_notify.3.gz \
	usr/share/man/man3/dialog_mesgbox.3.gz \
	usr/share/man/man3/dialog_gauge.3.gz usr/share/man/man3/init_dialog.3.gz \
	usr/share/man/man3/end_dialog.3.gz usr/share/man/man3/use_helpfile.3.gz \
	usr/share/man/man3/use_helpline.3.gz usr/share/man/man3/get_helpline.3.gz \
	usr/share/man/man3/restore_helpline.3.gz \
	usr/share/man/man3/dialog_msgbox.3.gz \
	usr/share/man/man3/dialog_ftree.3.gz usr/share/man/man3/dialog_tree.3.gz \
	usr/share/examples/dialog/README usr/share/examples/dialog/checklist \
	usr/share/examples/dialog/ftreebox usr/share/examples/dialog/infobox \
	usr/share/examples/dialog/inputbox usr/share/examples/dialog/menubox \
	usr/share/examples/dialog/msgbox usr/share/examples/dialog/prgbox \
	usr/share/examples/dialog/radiolist usr/share/examples/dialog/textbox \
	usr/share/examples/dialog/treebox usr/share/examples/dialog/yesno \
	usr/share/examples/libdialog/Makefile usr/share/examples/libdialog/check1.c\
	usr/share/examples/libdialog/check2.c usr/share/examples/libdialog/check3.c\
	usr/share/examples/libdialog/dselect.c \
	usr/share/examples/libdialog/fselect.c \
	usr/share/examples/libdialog/ftree1.c \
	usr/share/examples/libdialog/ftree1.test \
	usr/share/examples/libdialog/ftree2.c \
	usr/share/examples/libdialog/ftree2.test \
	usr/share/examples/libdialog/gauge.c usr/share/examples/libdialog/input1.c \
	usr/share/examples/libdialog/input2.c usr/share/examples/libdialog/menu1.c \
	usr/share/examples/libdialog/menu2.c usr/share/examples/libdialog/menu3.c \
	usr/share/examples/libdialog/msg.c usr/share/examples/libdialog/prgbox.c \
	usr/share/examples/libdialog/radio1.c usr/share/examples/libdialog/radio2.c\
	usr/share/examples/libdialog/radio3.c usr/share/examples/libdialog/text.c \
	usr/share/examples/libdialog/tree.c usr/share/examples/libdialog/yesno.c
	OLD_DIRS+=usr/share/examples/libdialog usr/share/examples/dialog
	# 20101114: Remove long-obsolete MAKEDEV.8
	OLD_FILES+=usr/share/man/man8/MAKEDEV.8.gz
	# 20101112: vgonel(9) has gone to private API a while ago
	OLD_FILES+=usr/share/man/man9/vgonel.9.gz
	# 20101112: removed gasp.info
	OLD_FILES+=usr/share/info/gasp.info.gz
	# 20101109: machine/mutex.h removed
	OLD_FILES+=usr/include/machine/mutex.h
	# 20101109: headers moved from machine/ to x86/
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/machine/mptable.h
	.endif
	# 20101101: headers moved from machine/ to x86/
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/machine/apicreg.h
	OLD_FILES+=usr/include/machine/mca.h
	.endif
	# 20101020: catch up with vm_page_sleep_if_busy rename
	OLD_FILES+=usr/share/man/man9/vm_page_sleep_busy.9.gz
	# 20101018: taskqueue(9) updates
	OLD_FILES+=usr/share/man/man9/taskqueue_find.9.gz
	# 20101011: removed subblock.h from liblzma
	OLD_FILES+=usr/include/lzma/subblock.h
	# 20101002: removed manpath.config
	OLD_FILES+=etc/manpath.config
	OLD_FILES+=usr/share/examples/etc/manpath.config
	# 20100910: renamed sbuf_overflowed to sbuf_error
	OLD_FILES+=usr/share/man/man9/sbuf_overflowed.9.gz
	# 20100815: retired last traces of chooseproc(9)
	OLD_FILES+=usr/share/man/man9/chooseproc.9.gz
	# 20100806: removal of unused libcompat routines
	OLD_FILES+=usr/share/man/man3/ascftime.3.gz
	OLD_FILES+=usr/share/man/man3/cfree.3.gz
	OLD_FILES+=usr/share/man/man3/cftime.3.gz
	OLD_FILES+=usr/share/man/man3/getpw.3.gz
	# 20100725: acpi_aiboost(4) removal.
	OLD_FILES+=usr/share/man/man4/acpi_aiboost.4.gz
	# 20100724: nfsclient/nfs_lock.h moved to nfs/nfs_lock.h
	OLD_FILES+=usr/include/nfsclient/nfs_lock.h
	# 20100720: new clang import which bumps version from 2.0 to 2.8
	OLD_FILES+=usr/include/clang/2.0/emmintrin.h
	OLD_FILES+=usr/include/clang/2.0/mm_malloc.h
	OLD_FILES+=usr/include/clang/2.0/mmintrin.h
	OLD_FILES+=usr/include/clang/2.0/pmmintrin.h
	OLD_FILES+=usr/include/clang/2.0/tmmintrin.h
	OLD_FILES+=usr/include/clang/2.0/xmmintrin.h
	OLD_DIRS+=usr/include/clang/2.0
	# 20100706: removed pc-sysinstall's detect-vmware.sh
	OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-vmware.sh
	# 20100701: [powerpc] removed <machine/intr.h>
	.if ${TARGET_ARCH} == "powerpc"
	OLD_FILES+=usr/include/machine/intr.h
	.endif
	# 20100514: library version bump for versioned symbols for liblzma
	OLD_LIBS+=usr/lib/liblzma.so.0
	OLD_LIBS+=usr/lib32/liblzma.so.0
	# 20100511: move GCC-specific headers to /usr/include/gcc
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/emmintrin.h
	OLD_FILES+=usr/include/mm_malloc.h
	OLD_FILES+=usr/include/pmmintrin.h
	OLD_FILES+=usr/include/xmmintrin.h
	.endif
	.if ${TARGET_ARCH} == "amd64" \|\| ${TARGET_ARCH} == "i386" \|\| ${TARGET_ARCH} == "arm"
	OLD_FILES+=usr/include/mmintrin.h
	.endif
	.if ${TARGET_ARCH} == "powerpc"
	OLD_FILES+=usr/include/altivec.h
	OLD_FILES+=usr/include/ppc-asm.h
	OLD_FILES+=usr/include/spe.h
	.endif
	# 20100416: [mips] removed <machine/psl.h>
	.if ${TARGET_ARCH} == "mips"
	OLD_FILES+=usr/include/machine/psl.h
	.endif
	# 20100415: [mips] removed unused headers
	.if ${TARGET_ARCH} == "mips"
	OLD_FILES+=usr/include/machine/archtype.h
	OLD_FILES+=usr/include/machine/segments.h
	OLD_FILES+=usr/include/machine/rm7000.h
	OLD_FILES+=usr/include/machine/defs.h
	OLD_FILES+=usr/include/machine/queue.h
	.endif
	# 20100326: gcpio removal
	OLD_FILES+=usr/bin/gcpio
	OLD_FILES+=usr/share/info/cpio.info.gz
	OLD_FILES+=usr/share/man/man1/gcpio.1.gz
	# 20100322: libz update
	OLD_LIBS+=lib/libz.so.5
	OLD_LIBS+=usr/lib32/libz.so.5
	# 20100314: removal of regexp.h
	OLD_FILES+=usr/include/regexp.h
	OLD_FILES+=usr/share/man/man3/regexp.3.gz
	OLD_FILES+=usr/share/man/man3/regsub.3.gz
	# 20100303: actual removal of utmp.h
	OLD_FILES+=usr/include/utmp.h
	# 20100208: man pages moved
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/share/man/man4/i386/alpm.4.gz
	OLD_FILES+=usr/share/man/man4/i386/amdpm.4.gz
	OLD_FILES+=usr/share/man/man4/i386/mcd.4.gz
	OLD_FILES+=usr/share/man/man4/i386/padlock.4.gz
	OLD_FILES+=usr/share/man/man4/i386/pcf.4.gz
	OLD_FILES+=usr/share/man/man4/i386/scd.4.gz
	OLD_FILES+=usr/share/man/man4/i386/viapm.4.gz
	.endif
	# 20100122: move BSDL bc/dc USD documents to /usr/share/doc/usd
	OLD_FILES+=usr/share/doc/papers/bc.ascii.gz
	OLD_FILES+=usr/share/doc/papers/dc.ascii.gz
	# 20100120: replacing GNU bc/dc with BSDL versions
	OLD_FILES+=usr/share/examples/bc/ckbook.b
	OLD_FILES+=usr/share/examples/bc/pi.b
	OLD_FILES+=usr/share/examples/bc/primes.b
	OLD_FILES+=usr/share/examples/bc/twins.b
	OLD_FILES+=usr/share/info/dc.info.gz
	OLD_DIRS+=usr/share/examples/bc
	# 20100114: removal of ttyslot(3)
	OLD_FILES+=usr/share/man/man3/ttyslot.3.gz
	# 20100113: remove utmp.h, replace it by utmpx.h
	OLD_FILES+=usr/share/man/man3/login.3.gz
	OLD_FILES+=usr/share/man/man3/logout.3.gz
	OLD_FILES+=usr/share/man/man3/logwtmp.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_endutxent.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_getutxent.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_getutxline.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_getutxuser.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_pututxline.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_setutxent.3.gz
	OLD_FILES+=usr/share/man/man3/ulog_setutxfile.3.gz
	OLD_FILES+=usr/share/man/man5/lastlog.5.gz
	OLD_FILES+=usr/share/man/man5/utmp.5.gz
	OLD_FILES+=usr/share/man/man5/wtmp.5.gz
	OLD_LIBS+=lib/libutil.so.8
	OLD_LIBS+=usr/lib32/libutil.so.8
	# 20100105: new userland semaphore implementation
	OLD_FILES+=usr/include/sys/semaphore.h
	# 20100103: ntptrace(8) removed
	OLD_FILES+=usr/sbin/ntptrace
	OLD_FILES+=usr/share/man/man8/ntptrace.8.gz
	# 20091229: remove no longer relevant examples
	OLD_FILES+=usr/share/examples/pppd/auth-down.sample
	OLD_FILES+=usr/share/examples/pppd/auth-up.sample
	OLD_FILES+=usr/share/examples/pppd/chap-secrets.sample
	OLD_FILES+=usr/share/examples/pppd/chat.sh.sample
	OLD_FILES+=usr/share/examples/pppd/ip-down.sample
	OLD_FILES+=usr/share/examples/pppd/ip-up.sample
	OLD_FILES+=usr/share/examples/pppd/options.sample
	OLD_FILES+=usr/share/examples/pppd/pap-secrets.sample
	OLD_FILES+=usr/share/examples/pppd/ppp.deny.sample
	OLD_FILES+=usr/share/examples/pppd/ppp.shells.sample
	OLD_DIRS+=usr/share/examples/pppd
	OLD_FILES+=usr/share/examples/slattach/unit-command.sh
	OLD_DIRS+=usr/share/examples/slattach
	OLD_FILES+=usr/share/examples/sliplogin/slip.hosts
	OLD_FILES+=usr/share/examples/sliplogin/slip.login
	OLD_FILES+=usr/share/examples/sliplogin/slip.logout
	OLD_FILES+=usr/share/examples/sliplogin/slip.slparms
	OLD_DIRS+=usr/share/examples/sliplogin
	OLD_FILES+=usr/share/examples/startslip/sldown.sh
	OLD_FILES+=usr/share/examples/startslip/slip.sh
	OLD_FILES+=usr/share/examples/startslip/slup.sh
	OLD_DIRS+=usr/share/examples/startslip
	# 20091202: unify rc.firewall and rc.firewall6.
	OLD_FILES+=etc/rc.d/ip6fw
	OLD_FILES+=etc/rc.firewall6
	OLD_FILES+=usr/share/examples/etc/rc.firewall6
	# 20091117: removal of rc.early(8) link
	OLD_FILES+=usr/share/man/man8/rc.early.8.gz
	# 20091117: usr/share/zoneinfo/GMT link removed
	OLD_FILES+=usr/share/zoneinfo/GMT
	# 20091027: pselect.3 implemented as syscall
	OLD_FILES+=usr/share/man/man3/pselect.3.gz
	# 20091005: fusword.9 and susword.9 removed
	OLD_FILES+=usr/share/man/man9/fusword.9.gz
	OLD_FILES+=usr/share/man/man9/susword.9.gz
	# 20090909: vesa and dpms promoted to be i386/amd64 common
	OLD_FILES+=usr/include/machine/pc/vesa.h
	OLD_FILES+=usr/share/man/man4/i386/dpms.4.gz
	# 20090904: remove lukemftpd
	OLD_FILES+=usr/libexec/lukemftpd
	OLD_FILES+=usr/share/man/man5/ftpd.conf.5.gz
	OLD_FILES+=usr/share/man/man5/ftpusers.5.gz
	OLD_FILES+=usr/share/man/man8/lukemftpd.8.gz
	# 20090902: BSD.{x11,x11-4}.dist are dead and BSD.local.dist lives in ports/
	OLD_FILES+=etc/mtree/BSD.local.dist
	OLD_FILES+=etc/mtree/BSD.x11.dist
	OLD_FILES+=etc/mtree/BSD.x11-4.dist
	# 20090812: net80211 documentation overhaul
	OLD_FILES+=usr/share/man/man9/ieee80211_add_rates.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_add_xrates.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_alloc_node.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_attach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_begin_scan.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_cfgget.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_cfgset.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_chan2ieee.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_chan2mode.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_create_ibss.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_crypto_attach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_crypto_detach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_decap.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_dump_pkt.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_dup_bss.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_encap.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_end_scan.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_find_node.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_fix_rate.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_free_allnodes.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_ieee2mhz.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_ioctl.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_lookup_node.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_media2rate.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_media_change.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_media_init.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_media_status.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_mhz2ieee.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_next_scan.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_node_attach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_node_detach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_node_lateattach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_print_essid.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_proto_attach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_proto_detach.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_rate2media.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_recv_mgmt.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_send_mgmt.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_setmode.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_timeout_nodes.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_watchdog.9.gz
	OLD_FILES+=usr/share/man/man9/ieee80211_wep_crypt.9.gz
	# 20090801: vimage.h removed in favour of vnet.h
	OLD_FILES+=usr/include/sys/vimage.h
	# 20101208: libbsnmp was moved to usr/lib
	OLD_LIBS+=lib/libbsnmp.so.5
	# 20090719: library version bump for 8.0
	OLD_LIBS+=lib/libalias.so.6
	OLD_LIBS+=lib/libavl.so.1
	OLD_LIBS+=lib/libbegemot.so.3
	OLD_LIBS+=lib/libbsdxml.so.3
	OLD_LIBS+=lib/libbsnmp.so.4
	OLD_LIBS+=lib/libcam.so.4
	OLD_LIBS+=lib/libcrypt.so.4
	OLD_LIBS+=lib/libcrypto.so.5
	OLD_LIBS+=lib/libctf.so.1
	OLD_LIBS+=lib/libdevstat.so.6
	OLD_LIBS+=lib/libdtrace.so.1
	OLD_LIBS+=lib/libedit.so.6
	OLD_LIBS+=lib/libgeom.so.4
	OLD_LIBS+=lib/libipsec.so.3
	OLD_LIBS+=lib/libipx.so.4
	OLD_LIBS+=lib/libkiconv.so.3
	OLD_LIBS+=lib/libkvm.so.4
	OLD_LIBS+=lib/libmd.so.4
	OLD_LIBS+=lib/libncurses.so.7
	OLD_LIBS+=lib/libncursesw.so.7
	OLD_LIBS+=lib/libnvpair.so.1
	OLD_LIBS+=lib/libpcap.so.6
	OLD_LIBS+=lib/libreadline.so.7
	OLD_LIBS+=lib/libsbuf.so.4
	OLD_LIBS+=lib/libufs.so.4
	OLD_LIBS+=lib/libumem.so.1
	OLD_LIBS+=lib/libutil.so.7
	OLD_LIBS+=lib/libuutil.so.1
	OLD_LIBS+=lib/libz.so.4
	OLD_LIBS+=lib/libzfs.so.1
	OLD_LIBS+=lib/libzpool.so.1
	OLD_LIBS+=usr/lib/libarchive.so.4
	OLD_LIBS+=usr/lib/libauditd.so.4
	OLD_LIBS+=usr/lib/libbluetooth.so.3
	OLD_LIBS+=usr/lib/libbsm.so.2
	OLD_LIBS+=usr/lib/libbz2.so.3
	OLD_LIBS+=usr/lib/libcalendar.so.4
	OLD_LIBS+=usr/lib/libcom_err.so.4
	OLD_LIBS+=usr/lib/libdevinfo.so.4
	OLD_LIBS+=usr/lib/libdialog.so.6
	OLD_LIBS+=usr/lib/libdwarf.so.1
	OLD_LIBS+=usr/lib/libfetch.so.5
	OLD_LIBS+=usr/lib/libform.so.4
	OLD_LIBS+=usr/lib/libformw.so.4
	OLD_LIBS+=usr/lib/libftpio.so.7
	OLD_LIBS+=usr/lib/libgnuregex.so.4
	OLD_LIBS+=usr/lib/libgpib.so.2
	OLD_LIBS+=usr/lib/libhistory.so.7
	OLD_LIBS+=usr/lib/libmagic.so.3
	OLD_LIBS+=usr/lib/libmemstat.so.2
	OLD_LIBS+=usr/lib/libmenu.so.4
	OLD_LIBS+=usr/lib/libmenuw.so.4
	OLD_LIBS+=usr/lib/libmilter.so.4
	OLD_LIBS+=usr/lib/libncp.so.3
	OLD_LIBS+=usr/lib/libnetgraph.so.3
	OLD_LIBS+=usr/lib/libngatm.so.3
	OLD_LIBS+=usr/lib/libobjc.so.3
	OLD_LIBS+=usr/lib/libopie.so.5
	OLD_LIBS+=usr/lib/libpam.so.4
	OLD_LIBS+=usr/lib/libpanel.so.4
	OLD_LIBS+=usr/lib/libpanelw.so.4
	OLD_LIBS+=usr/lib/libpmc.so.4
	OLD_LIBS+=usr/lib/libproc.so.1
	OLD_LIBS+=usr/lib/libradius.so.3
	OLD_LIBS+=usr/lib/librpcsvc.so.4
	OLD_LIBS+=usr/lib/libsdp.so.3
	OLD_LIBS+=usr/lib/libsmb.so.3
	OLD_LIBS+=usr/lib/libssh.so.4
	OLD_LIBS+=usr/lib/libssl.so.5
	OLD_LIBS+=usr/lib/libtacplus.so.3
	OLD_LIBS+=usr/lib/libugidfw.so.3
	OLD_LIBS+=usr/lib/libusb.so.1
	OLD_LIBS+=usr/lib/libusbhid.so.3
	OLD_LIBS+=usr/lib/libvgl.so.5
	OLD_LIBS+=usr/lib/libwrap.so.5
	OLD_LIBS+=usr/lib/libypclnt.so.3
	OLD_LIBS+=usr/lib/pam_chroot.so.4
	OLD_LIBS+=usr/lib/pam_deny.so.4
	OLD_LIBS+=usr/lib/pam_echo.so.4
	OLD_LIBS+=usr/lib/pam_exec.so.4
	OLD_LIBS+=usr/lib/pam_ftpusers.so.4
	OLD_LIBS+=usr/lib/pam_group.so.4
	OLD_LIBS+=usr/lib/pam_guest.so.4
	OLD_LIBS+=usr/lib/pam_krb5.so.4
	OLD_LIBS+=usr/lib/pam_ksu.so.4
	OLD_LIBS+=usr/lib/pam_lastlog.so.4
	OLD_LIBS+=usr/lib/pam_login_access.so.4
	OLD_LIBS+=usr/lib/pam_nologin.so.4
	OLD_LIBS+=usr/lib/pam_opie.so.4
	OLD_LIBS+=usr/lib/pam_opieaccess.so.4
	OLD_LIBS+=usr/lib/pam_passwdqc.so.4
	OLD_LIBS+=usr/lib/pam_permit.so.4
	OLD_LIBS+=usr/lib/pam_radius.so.4
	OLD_LIBS+=usr/lib/pam_rhosts.so.4
	OLD_LIBS+=usr/lib/pam_rootok.so.4
	OLD_LIBS+=usr/lib/pam_securetty.so.4
	OLD_LIBS+=usr/lib/pam_self.so.4
	OLD_LIBS+=usr/lib/pam_ssh.so.4
	OLD_LIBS+=usr/lib/pam_tacplus.so.4
	OLD_LIBS+=usr/lib/pam_unix.so.4
	OLD_LIBS+=usr/lib/snmp_atm.so.5
	OLD_LIBS+=usr/lib/snmp_bridge.so.5
	OLD_LIBS+=usr/lib/snmp_hostres.so.5
	OLD_LIBS+=usr/lib/snmp_mibII.so.5
	OLD_LIBS+=usr/lib/snmp_netgraph.so.5
	OLD_LIBS+=usr/lib/snmp_pf.so.5
	OLD_LIBS+=usr/lib32/libalias.so.6
	OLD_LIBS+=usr/lib32/libarchive.so.4
	OLD_LIBS+=usr/lib32/libauditd.so.4
	OLD_LIBS+=usr/lib32/libavl.so.1
	OLD_LIBS+=usr/lib32/libbegemot.so.3
	OLD_LIBS+=usr/lib32/libbluetooth.so.3
	OLD_LIBS+=usr/lib32/libbsdxml.so.3
	OLD_LIBS+=usr/lib32/libbsm.so.2
	OLD_LIBS+=usr/lib32/libbsnmp.so.4
	OLD_LIBS+=usr/lib32/libbz2.so.3
	OLD_LIBS+=usr/lib32/libcalendar.so.4
	OLD_LIBS+=usr/lib32/libcam.so.4
	OLD_LIBS+=usr/lib32/libcom_err.so.4
	OLD_LIBS+=usr/lib32/libcrypt.so.4
	OLD_LIBS+=usr/lib32/libcrypto.so.5
	OLD_LIBS+=usr/lib32/libctf.so.1
	OLD_LIBS+=usr/lib32/libdevinfo.so.4
	OLD_LIBS+=usr/lib32/libdevstat.so.6
	OLD_LIBS+=usr/lib32/libdialog.so.6
	OLD_LIBS+=usr/lib32/libdtrace.so.1
	OLD_LIBS+=usr/lib32/libdwarf.so.1
	OLD_LIBS+=usr/lib32/libedit.so.6
	OLD_LIBS+=usr/lib32/libfetch.so.5
	OLD_LIBS+=usr/lib32/libform.so.4
	OLD_LIBS+=usr/lib32/libformw.so.4
	OLD_LIBS+=usr/lib32/libftpio.so.7
	OLD_LIBS+=usr/lib32/libgeom.so.4
	OLD_LIBS+=usr/lib32/libgnuregex.so.4
	OLD_LIBS+=usr/lib32/libgpib.so.2
	OLD_LIBS+=usr/lib32/libhistory.so.7
	OLD_LIBS+=usr/lib32/libipsec.so.3
	OLD_LIBS+=usr/lib32/libipx.so.4
	OLD_LIBS+=usr/lib32/libkiconv.so.3
	OLD_LIBS+=usr/lib32/libkvm.so.4
	OLD_LIBS+=usr/lib32/libmagic.so.3
	OLD_LIBS+=usr/lib32/libmd.so.4
	OLD_LIBS+=usr/lib32/libmemstat.so.2
	OLD_LIBS+=usr/lib32/libmenu.so.4
	OLD_LIBS+=usr/lib32/libmenuw.so.4
	OLD_LIBS+=usr/lib32/libmilter.so.4
	OLD_LIBS+=usr/lib32/libncp.so.3
	OLD_LIBS+=usr/lib32/libncurses.so.7
	OLD_LIBS+=usr/lib32/libncursesw.so.7
	OLD_LIBS+=usr/lib32/libnetgraph.so.3
	OLD_LIBS+=usr/lib32/libngatm.so.3
	OLD_LIBS+=usr/lib32/libnvpair.so.1
	OLD_LIBS+=usr/lib32/libobjc.so.3
	OLD_LIBS+=usr/lib32/libopie.so.5
	OLD_LIBS+=usr/lib32/libpam.so.4
	OLD_LIBS+=usr/lib32/libpanel.so.4
	OLD_LIBS+=usr/lib32/libpanelw.so.4
	OLD_LIBS+=usr/lib32/libpcap.so.6
	OLD_LIBS+=usr/lib32/libpmc.so.4
	OLD_LIBS+=usr/lib32/libproc.so.1
	OLD_LIBS+=usr/lib32/libradius.so.3
	OLD_LIBS+=usr/lib32/libreadline.so.7
	OLD_LIBS+=usr/lib32/librpcsvc.so.4
	OLD_LIBS+=usr/lib32/libsbuf.so.4
	OLD_LIBS+=usr/lib32/libsdp.so.3
	OLD_LIBS+=usr/lib32/libsmb.so.3
	OLD_LIBS+=usr/lib32/libssh.so.4
	OLD_LIBS+=usr/lib32/libssl.so.5
	OLD_LIBS+=usr/lib32/libtacplus.so.3
	OLD_LIBS+=usr/lib32/libufs.so.4
	OLD_LIBS+=usr/lib32/libugidfw.so.3
	OLD_LIBS+=usr/lib32/libumem.so.1
	OLD_LIBS+=usr/lib32/libusb.so.1
	OLD_LIBS+=usr/lib32/libusbhid.so.3
	OLD_LIBS+=usr/lib32/libutil.so.7
	OLD_LIBS+=usr/lib32/libuutil.so.1
	OLD_LIBS+=usr/lib32/libvgl.so.5
	OLD_LIBS+=usr/lib32/libwrap.so.5
	OLD_LIBS+=usr/lib32/libypclnt.so.3
	OLD_LIBS+=usr/lib32/libz.so.4
	OLD_LIBS+=usr/lib32/libzfs.so.1
	OLD_LIBS+=usr/lib32/libzpool.so.1
	OLD_LIBS+=usr/lib32/pam_chroot.so.4
	OLD_LIBS+=usr/lib32/pam_deny.so.4
	OLD_LIBS+=usr/lib32/pam_echo.so.4
	OLD_LIBS+=usr/lib32/pam_exec.so.4
	OLD_LIBS+=usr/lib32/pam_ftpusers.so.4
	OLD_LIBS+=usr/lib32/pam_group.so.4
	OLD_LIBS+=usr/lib32/pam_guest.so.4
	OLD_LIBS+=usr/lib32/pam_krb5.so.4
	OLD_LIBS+=usr/lib32/pam_ksu.so.4
	OLD_LIBS+=usr/lib32/pam_lastlog.so.4
	OLD_LIBS+=usr/lib32/pam_login_access.so.4
	OLD_LIBS+=usr/lib32/pam_nologin.so.4
	OLD_LIBS+=usr/lib32/pam_opie.so.4
	OLD_LIBS+=usr/lib32/pam_opieaccess.so.4
	OLD_LIBS+=usr/lib32/pam_passwdqc.so.4
	OLD_LIBS+=usr/lib32/pam_permit.so.4
	OLD_LIBS+=usr/lib32/pam_radius.so.4
	OLD_LIBS+=usr/lib32/pam_rhosts.so.4
	OLD_LIBS+=usr/lib32/pam_rootok.so.4
	OLD_LIBS+=usr/lib32/pam_securetty.so.4
	OLD_LIBS+=usr/lib32/pam_self.so.4
	OLD_LIBS+=usr/lib32/pam_ssh.so.4
	OLD_LIBS+=usr/lib32/pam_tacplus.so.4
	OLD_LIBS+=usr/lib32/pam_unix.so.4
	# 20090718: the gdm pam.d file is no longer required.
	OLD_FILES+=etc/pam.d/gdm
	# 20090714: net_add_domain(9) renamed to domain_add(9)
	OLD_FILES+=usr/share/man/man9/net_add_domain.9.gz
	# 20090713: vimage container structs removed.
	OLD_FILES+=usr/include/netinet/vinet.h
	OLD_FILES+=usr/include/netinet6/vinet6.h
	OLD_FILES+=usr/include/netipsec/vipsec.h
	# 20090712: ieee80211.4 -> net80211.4
	OLD_FILES+=usr/share/man/man4/ieee80211.4.gz
	# 20090711: typo fixed, kproc_resume,.9 -> kproc_resume.9
	OLD_FILES+=usr/share/man/man9/kproc_resume,.9.gz
	# 20090709: msgctl.3 msgget.3 msgrcv.3 msgsnd.3 manual pages moved
	OLD_FILES+=usr/share/man/man3/msgctl.3.gz
	OLD_FILES+=usr/share/man/man3/msgget.3.gz
	OLD_FILES+=usr/share/man/man3/msgrcv.3.gz
	OLD_FILES+=usr/share/man/man3/msgsnd.3.gz
	# 20090630: old kernel RPC implementation removal
	OLD_FILES+=usr/include/nfs/rpcv2.h
	# 20090624: update usbdi(9)
	OLD_FILES+=usr/share/man/man9/usbd_abort_default_pipe.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_abort_pipe.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_alloc_buffer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_alloc_xfer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_clear_endpoint_stall.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_clear_endpoint_stall_async.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_clear_endpoint_toggle.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_close_pipe.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_device2interface_handle.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_do_request_async.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_do_request_flags_pipe.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_endpoint_count.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_find_edesc.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_find_idesc.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_free_buffer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_free_xfer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_buffer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_config.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_config_desc.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_config_desc_full.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_config_descriptor.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_device_descriptor.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_endpoint_descriptor.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_interface_altindex.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_interface_descriptor.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_no_alts.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_quirks.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_speed.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_string.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_string_desc.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_get_xfer_status.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_interface2device_handle.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_interface2endpoint_descriptor.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_interface_count.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_open_pipe.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_open_pipe_intr.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_pipe2device_handle.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_set_config_index.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_set_config_no.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_set_interface.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_setup_default_xfer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_setup_isoc_xfer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_setup_xfer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_sync_transfer.9.gz
	OLD_FILES+=usr/share/man/man9/usbd_transfer.9.gz
	OLD_FILES+=usr/share/man/man9/usb_find_desc.9.gz
	# 20090623: number of headers needed for a usb driver reduced
	OLD_FILES+=usr/include/dev/usb/usb_defs.h
	OLD_FILES+=usr/include/dev/usb/usb_error.h
	OLD_FILES+=usr/include/dev/usb/usb_handle_request.h
	OLD_FILES+=usr/include/dev/usb/usb_hid.h
	OLD_FILES+=usr/include/dev/usb/usb_lookup.h
	OLD_FILES+=usr/include/dev/usb/usb_mfunc.h
	OLD_FILES+=usr/include/dev/usb/usb_parse.h
	OLD_FILES+=usr/include/dev/usb/usb_revision.h
	# 20090609: devclass_add_driver is no longer public
	OLD_FILES+=usr/share/man/man9/devclass_add_driver.9.gz
	OLD_FILES+=usr/share/man/man9/devclass_delete_driver.9.gz
	OLD_FILES+=usr/share/man/man9/devclass_find_driver.9.gz
	# 20090605: removal of clists
	OLD_FILES+=usr/include/sys/clist.h
	# 20090602: removal of window(1)
	OLD_FILES+=usr/bin/window
	OLD_FILES+=usr/share/man/man1/window.1.gz
	# 20090531: bind 9.6.1rc1 import
	OLD_LIBS+=usr/lib/liblwres.so.30
	# 20090530: removal of early.sh
	OLD_FILES+=etc/rc.d/early.sh
	# 20090527: renaming of S{LIST,TAILQ}_REMOVE_NEXT() to _REMOVE_AFTER()
	OLD_FILES+=usr/share/man/man3/SLIST_REMOVE_NEXT.3.gz
	OLD_FILES+=usr/share/man/man3/STAILQ_REMOVE_NEXT.3.gz
	# 20090527: removal of legacy USB stack
	OLD_FILES+=usr/include/legacy/dev/usb/dsbr100io.h
	OLD_FILES+=usr/include/legacy/dev/usb/ehcireg.h
	OLD_FILES+=usr/include/legacy/dev/usb/ehcivar.h
	OLD_FILES+=usr/include/legacy/dev/usb/hid.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_urtwreg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_urtwvar.h
	OLD_FILES+=usr/include/legacy/dev/usb/ohcireg.h
	OLD_FILES+=usr/include/legacy/dev/usb/ohcivar.h
	OLD_FILES+=usr/include/legacy/dev/usb/rio500_usb.h
	OLD_FILES+=usr/include/legacy/dev/usb/rt2573_ucode.h
	OLD_FILES+=usr/include/legacy/dev/usb/sl811hsreg.h
	OLD_FILES+=usr/include/legacy/dev/usb/sl811hsvar.h
	OLD_FILES+=usr/include/legacy/dev/usb/ubser.h
	OLD_FILES+=usr/include/legacy/dev/usb/ucomvar.h
	OLD_FILES+=usr/include/legacy/dev/usb/udbp.h
	OLD_FILES+=usr/include/legacy/dev/usb/uftdireg.h
	OLD_FILES+=usr/include/legacy/dev/usb/ugraphire_rdesc.h
	OLD_FILES+=usr/include/legacy/dev/usb/uhcireg.h
	OLD_FILES+=usr/include/legacy/dev/usb/uhcivar.h
	OLD_FILES+=usr/include/legacy/dev/usb/usb.h
	OLD_FILES+=usr/include/legacy/dev/usb/usb_mem.h
	OLD_FILES+=usr/include/legacy/dev/usb/usb_port.h
	OLD_FILES+=usr/include/legacy/dev/usb/usb_quirks.h
	OLD_FILES+=usr/include/legacy/dev/usb/usbcdc.h
	OLD_FILES+=usr/include/legacy/dev/usb/usbdi.h
	OLD_FILES+=usr/include/legacy/dev/usb/usbdi_util.h
	OLD_FILES+=usr/include/legacy/dev/usb/usbdivar.h
	OLD_FILES+=usr/include/legacy/dev/usb/usbhid.h
	OLD_FILES+=usr/include/legacy/dev/usb/uxb360gp_rdesc.h
	OLD_DIRS+=usr/include/legacy/dev/usb
	OLD_DIRS+=usr/include/legacy/dev
	OLD_DIRS+=usr/include/legacy
	# 20090526: removal of makekey(8)
	OLD_FILES+=usr/libexec/makekey
	OLD_FILES+=usr/share/man/man8/makekey.8.gz
	# 20090522: removal of University of Michigan NFSv4 client
	OLD_FILES+=etc/rc.d/idmapd
	OLD_FILES+=sbin/idmapd
	OLD_FILES+=sbin/mount_nfs4
	OLD_FILES+=usr/share/man/man8/idmapd.8.gz
	OLD_FILES+=usr/share/man/man8/mount_nfs4.8.gz
	# 20090513: removal of legacy versions of USB network interface drivers
	OLD_FILES+=usr/include/legacy/dev/usb/if_upgtvar.h
	OLD_FILES+=usr/include/legacy/dev/usb/usb_ethersubr.h
	# 20090417: removal of legacy versions of USB network interface drivers
	OLD_FILES+=usr/include/legacy/dev/usb/if_auereg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_axereg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_cdcereg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_cuereg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_kuereg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_ruereg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_rumreg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_rumvar.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_udavreg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_uralreg.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_uralvar.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_zydfw.h
	OLD_FILES+=usr/include/legacy/dev/usb/if_zydreg.h
	OLD_FILES+=usr/include/legacy/dev/usb/kue_fw.h
	# 20090416: removal of ar(4), ray(4), sr(4), raycontrol(8)
	OLD_FILES+=usr/sbin/raycontrol
	OLD_FILES+=usr/share/man/man4/i386/ar.4.gz
	OLD_FILES+=usr/share/man/man4/i386/ray.4.gz
	OLD_FILES+=usr/share/man/man4/i386/sr.4.gz
	OLD_FILES+=usr/share/man/man8/raycontrol.8.gz
	# 20090410: VOP_LEASE.9 removed
	OLD_FILES+=usr/share/man/man9/VOP_LEASE.9.gz
	# 20090406: usb_sw_transfer.h removed
	OLD_FILES+=usr/include/dev/usb/usb_sw_transfer.h
	# 20090405: removal of if_ppp(4) and if_sl(4)
	OLD_FILES+=sbin/slattach rescue/slattach
	OLD_FILES+=sbin/startslip rescue/startslip
	OLD_FILES+=usr/include/net/if_ppp.h
	OLD_FILES+=usr/include/net/if_pppvar.h
	OLD_FILES+=usr/include/net/if_slvar.h
	OLD_FILES+=usr/include/net/ppp_comp.h
	OLD_FILES+=usr/include/net/slip.h
	OLD_FILES+=usr/sbin/sliplogin
	OLD_FILES+=usr/sbin/slstat
	OLD_FILES+=usr/sbin/pppd
	OLD_FILES+=usr/sbin/pppstats
	OLD_FILES+=usr/share/man/man1/startslip.1.gz
	OLD_FILES+=usr/share/man/man4/if_ppp.4.gz
	OLD_FILES+=usr/share/man/man4/if_sl.4.gz
	OLD_FILES+=usr/share/man/man4/ppp.4.gz
	OLD_FILES+=usr/share/man/man4/sl.4.gz
	OLD_FILES+=usr/share/man/man8/pppd.8.gz
	OLD_FILES+=usr/share/man/man8/pppstats.8.gz
	OLD_FILES+=usr/share/man/man8/slattach.8.gz
	OLD_FILES+=usr/share/man/man8/slip.8.gz
	OLD_FILES+=usr/share/man/man8/sliplogin.8.gz
	OLD_FILES+=usr/share/man/man8/slstat.8.gz
	# 20090321: libpcap upgraded to 1.0.0
	OLD_LIBS+=lib/libpcap.so.5
	OLD_LIBS+=usr/lib32/libpcap.so.5
	# 20090319: uscanner(4) has been removed
	OLD_FILES+=usr/share/man/man4/uscanner.4.gz
	# 20090313: k8temp(4) renamed to amdtemp(4)
	OLD_FILES+=usr/share/man/man4/k8temp.4.gz
	# 20090308: libusb.so.1 renamed
	OLD_LIBS+=usr/lib/libusb20.so.1
	OLD_FILES+=usr/lib/libusb20.a
	OLD_FILES+=usr/lib/libusb20.so
	OLD_FILES+=usr/lib/libusb20_p.a
	OLD_FILES+=usr/include/libusb20_compat01.h
	OLD_FILES+=usr/include/libusb20_compat10.h
	OLD_LIBS+=usr/lib32/libusb20.so.1
	OLD_FILES+=usr/lib32/libusb20.a
	OLD_FILES+=usr/lib32/libusb20.so
	OLD_FILES+=usr/lib32/libusb20_p.a
	# 20090226: libmp(3) functions renamed
	OLD_LIBS+=usr/lib/libmp.so.6
	OLD_LIBS+=usr/lib32/libmp.so.6
	# 20090223: changeover of USB stacks
	OLD_FILES+=usr/include/dev/usb2/include/ufm2_ioctl.h
	OLD_FILES+=usr/include/dev/usb2/include/urio2_ioctl.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_cdc.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_defs.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_devid.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_devtable.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_endian.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_error.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_hid.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_ioctl.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_mfunc.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_revision.h
	OLD_FILES+=usr/include/dev/usb2/include/usb2_standard.h
	OLD_DIRS+=usr/include/dev/usb2/include
	OLD_DIRS+=usr/include/dev/usb2
	OLD_FILES+=usr/include/dev/usb/dsbr100io.h
	OLD_FILES+=usr/include/dev/usb/ehcireg.h
	OLD_FILES+=usr/include/dev/usb/ehcivar.h
	OLD_FILES+=usr/include/dev/usb/hid.h
	OLD_FILES+=usr/include/dev/usb/if_auereg.h
	OLD_FILES+=usr/include/dev/usb/if_axereg.h
	OLD_FILES+=usr/include/dev/usb/if_cdcereg.h
	OLD_FILES+=usr/include/dev/usb/if_cuereg.h
	OLD_FILES+=usr/include/dev/usb/if_kuereg.h
	OLD_FILES+=usr/include/dev/usb/if_ruereg.h
	OLD_FILES+=usr/include/dev/usb/if_rumreg.h
	OLD_FILES+=usr/include/dev/usb/if_rumvar.h
	OLD_FILES+=usr/include/dev/usb/if_udavreg.h
	OLD_FILES+=usr/include/dev/usb/if_upgtvar.h
	OLD_FILES+=usr/include/dev/usb/if_uralreg.h
	OLD_FILES+=usr/include/dev/usb/if_uralvar.h
	OLD_FILES+=usr/include/dev/usb/if_urtwreg.h
	OLD_FILES+=usr/include/dev/usb/if_urtwvar.h
	OLD_FILES+=usr/include/dev/usb/if_zydfw.h
	OLD_FILES+=usr/include/dev/usb/if_zydreg.h
	OLD_FILES+=usr/include/dev/usb/kue_fw.h
	OLD_FILES+=usr/include/dev/usb/ohcireg.h
	OLD_FILES+=usr/include/dev/usb/ohcivar.h
	OLD_FILES+=usr/include/dev/usb/rio500_usb.h
	OLD_FILES+=usr/include/dev/usb/rt2573_ucode.h
	OLD_FILES+=usr/include/dev/usb/sl811hsreg.h
	OLD_FILES+=usr/include/dev/usb/sl811hsvar.h
	OLD_FILES+=usr/include/dev/usb/ubser.h
	OLD_FILES+=usr/include/dev/usb/ucomvar.h
	OLD_FILES+=usr/include/dev/usb/udbp.h
	OLD_FILES+=usr/include/dev/usb/uftdireg.h
	OLD_FILES+=usr/include/dev/usb/ugraphire_rdesc.h
	OLD_FILES+=usr/include/dev/usb/uhcireg.h
	OLD_FILES+=usr/include/dev/usb/uhcivar.h
	OLD_FILES+=usr/include/dev/usb/usb_ethersubr.h
	OLD_FILES+=usr/include/dev/usb/usb_mem.h
	OLD_FILES+=usr/include/dev/usb/usb_port.h
	OLD_FILES+=usr/include/dev/usb/usb_quirks.h
	OLD_FILES+=usr/include/dev/usb/usbcdc.h
	OLD_FILES+=usr/include/dev/usb/usbdivar.h
	OLD_FILES+=usr/include/dev/usb/uxb360gp_rdesc.h
	OLD_FILES+=usr/sbin/usbdevs
	OLD_FILES+=usr/share/man/man8/usbdevs.8.gz
	# 20090203: removal of pccard header files
	OLD_FILES+=usr/include/pccard/cardinfo.h
	OLD_FILES+=usr/include/pccard/cis.h
	OLD_DIRS+=usr/include/pccard
	# 20090203: adding_user.8 moved to adding_user.7
	OLD_FILES+=usr/share/man/man8/adding_user.8.gz
	# 20090102: file 4.26 import
	OLD_FILES+=usr/share/misc/magic.mime
	OLD_FILES+=usr/share/misc/magic.mime.mgc
	# 20081223: bind 9.4.3 import, nsupdate.8 moved to nsupdate.1
	OLD_FILES+=usr/share/man/man8/nsupdate.8.gz
	# 20081223: ipprotosw.h removed
	OLD_FILES+=usr/include/netinet/ipprotosw.h
	# 20081123: vfs_mountedon.9 removed
	OLD_FILES+=usr/share/man/man9/vfs_mountedon.9.gz
	# 20081023: FREE.9 and MALLOC.9 removed
	OLD_FILES+=usr/share/man/man9/FREE.9.gz
	OLD_FILES+=usr/share/man/man9/MALLOC.9.gz
	# 20080928: removal of inaccurate device_ids(9) manual page
	OLD_FILES+=usr/share/man/man9/device_ids.9.gz
	OLD_FILES+=usr/share/man/man9/major.9.gz
	OLD_FILES+=usr/share/man/man9/minor.9.gz
	OLD_FILES+=usr/share/man/man9/umajor.9.gz
	OLD_FILES+=usr/share/man/man9/uminor.9.gz
	# 20080917: removal of manpage for axed kernel primitive suser(9)
	OLD_FILES+=usr/share/man/man9/suser.9.gz
	OLD_FILES+=usr/share/man/man9/suser_cred.9.gz
	# 20080913: pax removed from rescue
	OLD_FILES+=rescue/pax
	# 20080823: removal of unneeded pt_chown, to implement grantpt(3)
	OLD_FILES+=usr/libexec/pt_chown
	# 20080822: ntp 4.2.4p5 import
	OLD_FILES+=usr/share/doc/ntp/driver23.html
	OLD_FILES+=usr/share/doc/ntp/driver24.html
	# 20080821: several man pages moved from man4.i386 to man4
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/share/man/man4/i386/acpi_aiboost.4.gz
	OLD_FILES+=usr/share/man/man4/i386/acpi_asus.4.gz
	OLD_FILES+=usr/share/man/man4/i386/acpi_fujitsu.4.gz
	OLD_FILES+=usr/share/man/man4/i386/acpi_ibm.4.gz
	OLD_FILES+=usr/share/man/man4/i386/acpi_panasonic.4.gz
	OLD_FILES+=usr/share/man/man4/i386/acpi_sony.4.gz
	OLD_FILES+=usr/share/man/man4/i386/acpi_toshiba.4.gz
	OLD_FILES+=usr/share/man/man4/i386/ichwd.4.gz
	OLD_FILES+=usr/share/man/man4/i386/if_ndis.4.gz
	OLD_FILES+=usr/share/man/man4/i386/io.4.gz
	OLD_FILES+=usr/share/man/man4/i386/linux.4.gz
	OLD_FILES+=usr/share/man/man4/i386/ndis.4.gz
	.endif
	# 20080820: MPSAFE TTY layer integrated
	OLD_FILES+=usr/include/sys/linedisc.h
	OLD_FILES+=usr/share/man/man3/posix_openpt.3.gz
	# 20080725: sgtty.h removed
	OLD_FILES+=usr/include/sgtty.h
	# 20080706: bsdlabel(8) removed on powerpc
	.if ${TARGET_ARCH} == "powerpc"
	OLD_FILES+=sbin/bsdlabel
	OLD_FILES+=usr/share/man/man8/bsdlabel.8.gz
	.endif
	# 20080704: sbsh(4) removed
	OLD_FILES+=usr/share/man/man4/if_sbsh.4.gz
	OLD_FILES+=usr/share/man/man4/sbsh.4.gz
	# 20080704: cnw(4) removed
	OLD_FILES+=usr/share/man/man4/if_cnw.4.gz
	OLD_FILES+=usr/share/man/man4/cnw.4.gz
	# 20080704: oltr(4) removed
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/share/man/man4/i386/if_oltr.4.gz
	OLD_FILES+=usr/share/man/man4/i386/oltr.4.gz
	.endif
	# 20080704: arl(4) removed
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/sbin/arlcontrol
	OLD_FILES+=usr/share/man/man4/i386/arl.4.gz
	OLD_FILES+=usr/share/man/man8/arlcontrol.8.gz
	.endif
	# 20080703: sunlabel only for sparc64
	.if ${TARGET_ARCH} != "sparc64"
	OLD_FILES+=sbin/sunlabel
	OLD_FILES+=usr/share/man/man8/sunlabel.8.gz
	.endif
	# 20080701: wpa_supplicant.conf moved to share/examples/etc/
	OLD_FILES+=usr/share/examples/wpa_supplicant/wpa_supplicant.conf
	OLD_DIRS+=usr/share/examples/wpa_supplicant
	# 20080614: pecoff image activator removed
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/machine/pecoff_machdep.h
	.endif
	# 20080614: sgtty removed
	OLD_FILES+=usr/include/sys/ttychars.h
	OLD_FILES+=usr/include/sys/ttydev.h
	OLD_FILES+=usr/share/man/man3/gtty.3.gz
	OLD_FILES+=usr/share/man/man3/stty.3.gz
	# 20080609: gpt(8) removed
	OLD_FILES+=sbin/gpt
	OLD_FILES+=usr/share/man/man8/gpt.8.gz
	# 20080525: I4B removed
	OLD_FILES+=etc/isdn/answer
	OLD_FILES+=etc/isdn/isdntel
	OLD_FILES+=etc/isdn/record
	OLD_FILES+=etc/isdn/tell
	OLD_FILES+=etc/isdn/tell-record
	OLD_FILES+=etc/isdn/unknown_incoming
	OLD_FILES+=etc/isdn/holidays.D
	OLD_FILES+=etc/isdn/isdnd.rates.A
	OLD_FILES+=etc/isdn/isdnd.rates.D
	OLD_FILES+=etc/isdn/isdnd.rates.F
	OLD_FILES+=etc/isdn/isdnd.rates.L
	OLD_FILES+=etc/isdn/isdnd.rates.UK.BT
	OLD_FILES+=etc/isdn/isdnd.rc.sample
	OLD_FILES+=etc/isdn/isdntel.alias.sample
	OLD_DIRS+=etc/isdn
	OLD_FILES+=etc/rc.d/isdnd
	OLD_FILES+=usr/include/i4b/i4b_cause.h
	OLD_FILES+=usr/include/i4b/i4b_debug.h
	OLD_FILES+=usr/include/i4b/i4b_ioctl.h
	OLD_FILES+=usr/include/i4b/i4b_rbch_ioctl.h
	OLD_FILES+=usr/include/i4b/i4b_tel_ioctl.h
	OLD_FILES+=usr/include/i4b/i4b_trace.h
	OLD_DIRS+=usr/include/i4b
	OLD_FILES+=usr/sbin/dtmfdecode
	OLD_FILES+=usr/sbin/g711conv
	OLD_FILES+=usr/sbin/isdnd
	OLD_FILES+=usr/sbin/isdndebug
	OLD_FILES+=usr/sbin/isdndecode
	OLD_FILES+=usr/sbin/isdnmonitor
	OLD_FILES+=usr/sbin/isdnphone
	OLD_FILES+=usr/sbin/isdntel
	OLD_FILES+=usr/sbin/isdntelctl
	OLD_FILES+=usr/sbin/isdntrace
	OLD_FILES+=usr/share/isdn/0.al
	OLD_FILES+=usr/share/isdn/1.al
	OLD_FILES+=usr/share/isdn/2.al
	OLD_FILES+=usr/share/isdn/3.al
	OLD_FILES+=usr/share/isdn/4.al
	OLD_FILES+=usr/share/isdn/5.al
	OLD_FILES+=usr/share/isdn/6.al
	OLD_FILES+=usr/share/isdn/7.al
	OLD_FILES+=usr/share/isdn/8.al
	OLD_FILES+=usr/share/isdn/9.al
	OLD_FILES+=usr/share/isdn/beep.al
	OLD_FILES+=usr/share/isdn/msg.al
	OLD_DIRS+=usr/share/isdn
	OLD_FILES+=usr/share/man/man1/dtmfdecode.1.gz
	OLD_FILES+=usr/share/man/man1/g711conv.1.gz
	OLD_FILES+=usr/share/man/man4/i4b.4.gz
	OLD_FILES+=usr/share/man/man4/i4bcapi.4.gz
	OLD_FILES+=usr/share/man/man4/i4bctl.4.gz
	OLD_FILES+=usr/share/man/man4/i4bing.4.gz
	OLD_FILES+=usr/share/man/man4/i4bipr.4.gz
	OLD_FILES+=usr/share/man/man4/i4bisppp.4.gz
	OLD_FILES+=usr/share/man/man4/i4bq921.4.gz
	OLD_FILES+=usr/share/man/man4/i4bq931.4.gz
	OLD_FILES+=usr/share/man/man4/i4brbch.4.gz
	OLD_FILES+=usr/share/man/man4/i4btel.4.gz
	OLD_FILES+=usr/share/man/man4/i4btrc.4.gz
	OLD_FILES+=usr/share/man/man4/iavc.4.gz
	OLD_FILES+=usr/share/man/man4/isic.4.gz
	OLD_FILES+=usr/share/man/man4/ifpi.4.gz
	OLD_FILES+=usr/share/man/man4/ifpi2.4.gz
	OLD_FILES+=usr/share/man/man4/ifpnp.4.gz
	OLD_FILES+=usr/share/man/man4/ihfc.4.gz
	OLD_FILES+=usr/share/man/man4/itjc.4.gz
	OLD_FILES+=usr/share/man/man4/iwic.4.gz
	OLD_FILES+=usr/share/man/man5/isdnd.rc.5.gz
	OLD_FILES+=usr/share/man/man5/isdnd.rates.5.gz
	OLD_FILES+=usr/share/man/man5/isdnd.acct.5.gz
	OLD_FILES+=usr/share/man/man8/isdnd.8.gz
	OLD_FILES+=usr/share/man/man8/isdndebug.8.gz
	OLD_FILES+=usr/share/man/man8/isdndecode.8.gz
	OLD_FILES+=usr/share/man/man8/isdnmonitor.8.gz
	OLD_FILES+=usr/share/man/man8/isdnphone.8.gz
	OLD_FILES+=usr/share/man/man8/isdntel.8.gz
	OLD_FILES+=usr/share/man/man8/isdntelctl.8.gz
	OLD_FILES+=usr/share/man/man8/isdntrace.8.gz
	OLD_FILES+=usr/share/examples/isdn/contrib/README
	OLD_FILES+=usr/share/examples/isdn/contrib/anleitung.ppp
	OLD_FILES+=usr/share/examples/isdn/contrib/answer.c
	OLD_FILES+=usr/share/examples/isdn/contrib/answer.sh
	OLD_FILES+=usr/share/examples/isdn/contrib/convert.sh
	OLD_FILES+=usr/share/examples/isdn/contrib/hplay.c
	OLD_FILES+=usr/share/examples/isdn/contrib/i4b-ppp-newbie.txt
	OLD_FILES+=usr/share/examples/isdn/contrib/isdnctl
	OLD_FILES+=usr/share/examples/isdn/contrib/isdnd_acct
	OLD_FILES+=usr/share/examples/isdn/contrib/isdnd_acct.pl
	OLD_FILES+=usr/share/examples/isdn/contrib/isdntelmux.c
	OLD_FILES+=usr/share/examples/isdn/contrib/mrtg-isp0.sh
	OLD_FILES+=usr/share/examples/isdn/i4brunppp/Makefile
	OLD_FILES+=usr/share/examples/isdn/i4brunppp/README
	OLD_FILES+=usr/share/examples/isdn/i4brunppp/i4brunppp-isdnd.rc
	OLD_FILES+=usr/share/examples/isdn/i4brunppp/i4brunppp.8
	OLD_FILES+=usr/share/examples/isdn/i4brunppp/i4brunppp.c
	OLD_FILES+=usr/share/examples/isdn/v21/Makefile
	OLD_FILES+=usr/share/examples/isdn/v21/README
	OLD_FILES+=usr/share/examples/isdn/v21/v21modem.c
	OLD_FILES+=usr/share/examples/isdn/FAQ
	OLD_FILES+=usr/share/examples/isdn/KERNEL
	OLD_FILES+=usr/share/examples/isdn/Overview
	OLD_FILES+=usr/share/examples/isdn/README
	OLD_FILES+=usr/share/examples/isdn/ROADMAP
	OLD_FILES+=usr/share/examples/isdn/ReleaseNotes
	OLD_FILES+=usr/share/examples/isdn/Resources
	OLD_FILES+=usr/share/examples/isdn/SupportedCards
	OLD_FILES+=usr/share/examples/isdn/ThankYou
	OLD_DIRS+=usr/share/examples/isdn/contrib
	OLD_DIRS+=usr/share/examples/isdn/i4brunppp
	OLD_DIRS+=usr/share/examples/isdn/v21
	OLD_DIRS+=usr/share/examples/isdn
	OLD_FILES+=usr/share/examples/ppp/isdnd.rc
	OLD_FILES+=usr/share/examples/ppp/ppp.conf.isdn
	# 20080525: ng_atmpif removed
	OLD_FILES+=usr/include/netgraph/atm/ng_atmpif.h
	OLD_FILES+=usr/share/man/man4/ng_atmpif.4.gz
	# 20080522: pmap_addr_hint removed
	OLD_FILES+=usr/share/man/man9/pmap_addr_hint.9.gz
	# 20080517: ipsec_osdep.h removed
	OLD_FILES+=usr/include/netipsec/ipsec_osdep.h
	# 20080507: heimdal 1.1 import
	OLD_LIBS+=usr/lib/libasn1.so.9
	OLD_LIBS+=usr/lib/libgssapi.so.9
	OLD_LIBS+=usr/lib/libgssapi_krb5.so.9
	OLD_LIBS+=usr/lib/libhdb.so.9
	OLD_LIBS+=usr/lib/libkadm5clnt.so.9
	OLD_LIBS+=usr/lib/libkadm5srv.so.9
	OLD_LIBS+=usr/lib/libkafs5.so.9
	OLD_LIBS+=usr/lib/libkrb5.so.9
	OLD_LIBS+=usr/lib/libroken.so.9
	OLD_LIBS+=usr/lib32/libgssapi.so.9
	# 20080420: Symbol card support dropped
	OLD_FILES+=usr/include/dev/wi/spectrum24t_cf.h
	# 20080420: awi removal
	OLD_FILES+=usr/share/man/man4/awi.4.gz
	OLD_FILES+=usr/share/man/man4/if_awi.4.gz
	# 20080331: pkg_sign has been removed
	OLD_FILES+=usr/sbin/pkg_check
	OLD_FILES+=usr/sbin/pkg_sign
	OLD_FILES+=usr/share/man/man1/pkg_check.1.gz
	OLD_FILES+=usr/share/man/man1/pkg_sign.1.gz
	# 20080314: stack_print(9) mlink fixed
	OLD_FILES+=usr/share/man/man9/stack_printf.9.gz
	# 20080312: libkse removal
	OLD_FILES+=usr/include/sys/kse.h
	OLD_FILES+=usr/lib/libkse.so
	OLD_LIBS+=usr/lib/libkse.so.3
	OLD_FILES+=usr/share/man/man2/kse.2.gz
	OLD_FILES+=usr/share/man/man2/kse_create.2.gz
	OLD_FILES+=usr/share/man/man2/kse_exit.2.gz
	OLD_FILES+=usr/share/man/man2/kse_release.2.gz
	OLD_FILES+=usr/share/man/man2/kse_switchin.2.gz
	OLD_FILES+=usr/share/man/man2/kse_thr_interrupt.2.gz
	OLD_FILES+=usr/share/man/man2/kse_wakeup.2.gz
	OLD_FILES+=usr/lib32/libkse.so
	OLD_LIBS+=usr/lib32/libkse.so.3
	# 20080225: bsdar/bsdranlib rename to ar/ranlib
	OLD_FILES+=usr/bin/bsdar
	OLD_FILES+=usr/bin/bsdranlib
	OLD_FILES+=usr/share/man/man1/bsdar.1.gz
	OLD_FILES+=usr/share/man/man1/bsdranlib.1.gz
	# 20080220: geom_lvm rename to geom_linux_lvm
	OLD_FILES+=usr/share/man/man4/geom_lvm.4.gz
	# 20080126: oldcard.4 removal
	OLD_FILES+=usr/share/man/man4/card.4.gz
	OLD_FILES+=usr/share/man/man4/oldcard.4.gz
	# 20080122: Removed from the tree
	OLD_FILES+=usr/share/man/man9/BUF_REFCNT.9.gz
	# 20080108: Moved to section 2
	OLD_FILES+=usr/share/man/man3/shm_open.3.gz
	OLD_FILES+=usr/share/man/man3/shm_unlink.3.gz
	# 20071207: Merged with fortunes-o.real
	OLD_FILES+=usr/share/games/fortune/fortunes2-o
	OLD_FILES+=usr/share/games/fortune/fortunes2-o.dat
	# 20071201: Removal of XRPU driver
	OLD_FILES+=usr/include/sys/xrpuio.h
	# 20071129: Disabled static versions of libkse by default
	OLD_FILES+=usr/lib/libkse.a
	OLD_FILES+=usr/lib/libkse_p.a
	OLD_FILES+=usr/lib/libkse_pic.a
	OLD_FILES+=usr/lib32/libkse.a
	OLD_FILES+=usr/lib32/libkse_p.a
	OLD_FILES+=usr/lib32/libkse_pic.a
	# 20071129: Removed a Solaris compatibility header
	OLD_FILES+=usr/include/sys/_elf_solaris.h
	# 20071125: Renamed to pmc_get_msr()
	OLD_FILES+=usr/share/man/man3/pmc_x86_get_msr.3.gz
	# 20071108: Removed very crunch OLDCARD support file
	OLD_FILES+=etc/defaults/pccard.conf
	# 20071025: rc.d/nfslocking superseded by rc.d/lockd and rc.d/statd
	OLD_FILES+=etc/rc.d/nfslocking
	# 20070930: rename of cached to nscd
	OLD_FILES+=etc/cached.conf
	OLD_FILES+=etc/rc.d/cached
	OLD_FILES+=usr/sbin/cached
	OLD_FILES+=usr/share/man/man5/cached.conf.5.gz
	OLD_FILES+=usr/share/man/man8/cached.8.gz
	# 20070807: removal of PowerPC specific header file.
	.if ${TARGET_ARCH} == "powerpc"
	OLD_FILES+=usr/include/machine/interruptvar.h
	.endif
	# 20070801: fast_ipsec.4 gone
	OLD_FILES+=usr/share/man/man4/fast_ipsec.4.gz
	# 20070715: netatm temporarily disconnected (removed 20080525)
	OLD_FILES+=rescue/atm
	OLD_FILES+=rescue/fore_dnld
	OLD_FILES+=rescue/ilmid
	OLD_FILES+=sbin/atm
	OLD_FILES+=sbin/fore_dnld
	OLD_FILES+=sbin/ilmid
	OLD_FILES+=usr/include/libatm.h
	OLD_FILES+=usr/include/netatm/atm.h
	OLD_FILES+=usr/include/netatm/atm_cm.h
	OLD_FILES+=usr/include/netatm/atm_if.h
	OLD_FILES+=usr/include/netatm/atm_ioctl.h
	OLD_FILES+=usr/include/netatm/atm_pcb.h
	OLD_FILES+=usr/include/netatm/atm_sap.h
	OLD_FILES+=usr/include/netatm/atm_sigmgr.h
	OLD_FILES+=usr/include/netatm/atm_stack.h
	OLD_FILES+=usr/include/netatm/atm_sys.h
	OLD_FILES+=usr/include/netatm/atm_var.h
	OLD_FILES+=usr/include/netatm/atm_vc.h
	OLD_FILES+=usr/include/netatm/ipatm/ipatm.h
	OLD_FILES+=usr/include/netatm/ipatm/ipatm_serv.h
	OLD_FILES+=usr/include/netatm/ipatm/ipatm_var.h
	OLD_FILES+=usr/include/netatm/port.h
	OLD_FILES+=usr/include/netatm/queue.h
	OLD_FILES+=usr/include/netatm/sigpvc/sigpvc_var.h
	OLD_FILES+=usr/include/netatm/spans/spans_cls.h
	OLD_FILES+=usr/include/netatm/spans/spans_kxdr.h
	OLD_FILES+=usr/include/netatm/spans/spans_var.h
	OLD_FILES+=usr/include/netatm/uni/sscf_uni.h
	OLD_FILES+=usr/include/netatm/uni/sscf_uni_var.h
	OLD_FILES+=usr/include/netatm/uni/sscop.h
	OLD_FILES+=usr/include/netatm/uni/sscop_misc.h
	OLD_FILES+=usr/include/netatm/uni/sscop_pdu.h
	OLD_FILES+=usr/include/netatm/uni/sscop_var.h
	OLD_FILES+=usr/include/netatm/uni/uni.h
	OLD_FILES+=usr/include/netatm/uni/uniip_var.h
	OLD_FILES+=usr/include/netatm/uni/unisig.h
	OLD_FILES+=usr/include/netatm/uni/unisig_decode.h
	OLD_FILES+=usr/include/netatm/uni/unisig_mbuf.h
	OLD_FILES+=usr/include/netatm/uni/unisig_msg.h
	OLD_FILES+=usr/include/netatm/uni/unisig_print.h
	OLD_FILES+=usr/include/netatm/uni/unisig_var.h
	OLD_FILES+=usr/lib/libatm.a
	OLD_FILES+=usr/lib/libatm_p.a
	OLD_FILES+=usr/sbin/atmarpd
	OLD_FILES+=usr/sbin/scspd
	OLD_FILES+=usr/share/man/en.ISO8859-1/man8/atm.8.gz
	OLD_FILES+=usr/share/man/en.ISO8859-1/man8/atmarpd.8.gz
	OLD_FILES+=usr/share/man/en.ISO8859-1/man8/fore_dnld.8.gz
	OLD_FILES+=usr/share/man/en.ISO8859-1/man8/ilmid.8.gz
	OLD_FILES+=usr/share/man/en.ISO8859-1/man8/scspd.8.gz
	OLD_FILES+=usr/share/man/man8/atm.8.gz
	OLD_FILES+=usr/share/man/man8/atmarpd.8.gz
	OLD_FILES+=usr/share/man/man8/fore_dnld.8.gz
	OLD_FILES+=usr/share/man/man8/ilmid.8.gz
	OLD_FILES+=usr/share/man/man8/scspd.8.gz
	OLD_FILES+=usr/share/examples/atm/NOTES
	OLD_FILES+=usr/share/examples/atm/README
	OLD_FILES+=usr/share/examples/atm/Startup
	OLD_FILES+=usr/share/examples/atm/atm-config.sh
	OLD_FILES+=usr/share/examples/atm/atm-sockets.txt
	OLD_FILES+=usr/share/examples/atm/cpcs-design.txt
	OLD_FILES+=usr/share/examples/atm/fore-microcode.txt
	OLD_FILES+=usr/share/examples/atm/sscf-design.txt
	OLD_FILES+=usr/share/examples/atm/sscop-design.txt
	OLD_LIBS+=lib/libatm.so.5
	OLD_LIBS+=usr/lib/libatm.so
	OLD_DIRS+=usr/include/netatm/sigpvc
	OLD_DIRS+=usr/include/netatm/spans
	OLD_DIRS+=usr/include/netatm/ipatm
	OLD_DIRS+=usr/include/netatm/uni
	OLD_DIRS+=usr/include/netatm
	OLD_DIRS+=usr/share/examples/atm
	OLD_FILES+=usr/lib32/libatm.a
	OLD_FILES+=usr/lib32/libatm.so
	OLD_LIBS+=usr/lib32/libatm.so.5
	OLD_FILES+=usr/lib32/libatm_p.a
	# 20070705: I4B headers repo-copied to include/i4b/
	.if ${TARGET_ARCH} == "i386"
	OLD_FILES+=usr/include/machine/i4b_cause.h
	OLD_FILES+=usr/include/machine/i4b_debug.h
	OLD_FILES+=usr/include/machine/i4b_ioctl.h
	OLD_FILES+=usr/include/machine/i4b_rbch_ioctl.h
	OLD_FILES+=usr/include/machine/i4b_tel_ioctl.h
	OLD_FILES+=usr/include/machine/i4b_trace.h
	.endif
	# 20070703: pf 4.1 import
	OLD_FILES+=usr/libexec/ftp-proxy
	# 20070701: KAME IPSec removal
	OLD_FILES+=usr/include/netinet6/ah.h
	OLD_FILES+=usr/include/netinet6/ah6.h
	OLD_FILES+=usr/include/netinet6/ah_aesxcbcmac.h
	OLD_FILES+=usr/include/netinet6/esp.h
	OLD_FILES+=usr/include/netinet6/esp6.h
	OLD_FILES+=usr/include/netinet6/esp_aesctr.h
	OLD_FILES+=usr/include/netinet6/esp_camellia.h
	OLD_FILES+=usr/include/netinet6/esp_rijndael.h
	OLD_FILES+=usr/include/netinet6/ipsec.h
	OLD_FILES+=usr/include/netinet6/ipsec6.h
	OLD_FILES+=usr/include/netinet6/ipcomp.h
	OLD_FILES+=usr/include/netinet6/ipcomp6.h
	OLD_FILES+=usr/include/netkey/key.h
	OLD_FILES+=usr/include/netkey/key_debug.h
	OLD_FILES+=usr/include/netkey/key_var.h
	OLD_FILES+=usr/include/netkey/keydb.h
	OLD_FILES+=usr/include/netkey/keysock.h
	OLD_DIRS+=usr/include/netkey
	# 20070701: remove wicontrol
	OLD_FILES+=usr/sbin/wicontrol
	OLD_FILES+=usr/share/man/man8/wicontrol.8.gz
	# 20070625: umapfs removal
	OLD_FILES+=rescue/mount_umapfs
	OLD_FILES+=sbin/mount_umapfs
	OLD_FILES+=usr/include/fs/umapfs/umap.h
	OLD_FILES+=usr/share/man/man8/mount_umapfs.8.gz
	OLD_DIRS+=usr/include/fs/umapfs
	# 20070618: Removal of the PROTO.localhost* files
	OLD_FILES+=etc/namedb/PROTO.localhost-v6.rev
	OLD_FILES+=etc/namedb/PROTO.localhost.rev
	OLD_FILES+=etc/namedb/make-localhost
	# 20070618: shared library version bump
	OLD_LIBS+=lib/libalias.so.5
	OLD_LIBS+=lib/libbsnmp.so.3
	OLD_LIBS+=lib/libncurses.so.6
	OLD_LIBS+=lib/libncursesw.so.6
	OLD_LIBS+=lib/libreadline.so.6
	OLD_LIBS+=usr/lib/libdialog.so.5
	OLD_LIBS+=usr/lib/libgnuregex.so.3
	OLD_LIBS+=usr/lib/libhistory.so.6
	OLD_LIBS+=usr/lib/libpam.so.3
	OLD_LIBS+=usr/lib/libssh.so.3
	OLD_LIBS+=usr/lib/pam_chroot.so.3
	OLD_LIBS+=usr/lib/pam_deny.so.3
	OLD_LIBS+=usr/lib/pam_echo.so.3
	OLD_LIBS+=usr/lib/pam_exec.so.3
	OLD_LIBS+=usr/lib/pam_ftpusers.so.3
	OLD_LIBS+=usr/lib/pam_group.so.3
	OLD_LIBS+=usr/lib/pam_guest.so.3
	OLD_LIBS+=usr/lib/pam_krb5.so.3
	OLD_LIBS+=usr/lib/pam_ksu.so.3
	OLD_LIBS+=usr/lib/pam_lastlog.so.3
	OLD_LIBS+=usr/lib/pam_login_access.so.3
	OLD_LIBS+=usr/lib/pam_nologin.so.3
	OLD_LIBS+=usr/lib/pam_opie.so.3
	OLD_LIBS+=usr/lib/pam_opieaccess.so.3
	OLD_LIBS+=usr/lib/pam_passwdqc.so.3
	OLD_LIBS+=usr/lib/pam_permit.so.3
	OLD_LIBS+=usr/lib/pam_radius.so.3
	OLD_LIBS+=usr/lib/pam_rhosts.so.3
	OLD_LIBS+=usr/lib/pam_rootok.so.3
	OLD_LIBS+=usr/lib/pam_securetty.so.3
	OLD_LIBS+=usr/lib/pam_self.so.3
	OLD_LIBS+=usr/lib/pam_ssh.so.3
	OLD_LIBS+=usr/lib/pam_tacplus.so.3
	OLD_LIBS+=usr/lib/pam_unix.so.3
	OLD_LIBS+=usr/lib/snmp_atm.so.4
	OLD_LIBS+=usr/lib/snmp_bridge.so.4
	OLD_LIBS+=usr/lib/snmp_hostres.so.4
	OLD_LIBS+=usr/lib/snmp_mibII.so.4
	OLD_LIBS+=usr/lib/snmp_netgraph.so.4
	OLD_LIBS+=usr/lib/snmp_pf.so.4
	OLD_LIBS+=usr/lib32/libalias.so.5
	OLD_LIBS+=usr/lib32/libbsnmp.so.3
	OLD_LIBS+=usr/lib32/libdialog.so.5
	OLD_LIBS+=usr/lib32/libgnuregex.so.3
	OLD_LIBS+=usr/lib32/libhistory.so.6
	OLD_LIBS+=usr/lib32/libncurses.so.6
	OLD_LIBS+=usr/lib32/libncursesw.so.6
	OLD_LIBS+=usr/lib32/libpam.so.3
	OLD_LIBS+=usr/lib32/libreadline.so.6
	OLD_LIBS+=usr/lib32/libssh.so.3
	OLD_LIBS+=usr/lib32/pam_chroot.so.3
	OLD_LIBS+=usr/lib32/pam_deny.so.3
	OLD_LIBS+=usr/lib32/pam_echo.so.3
	OLD_LIBS+=usr/lib32/pam_exec.so.3
	OLD_LIBS+=usr/lib32/pam_ftpusers.so.3
	OLD_LIBS+=usr/lib32/pam_group.so.3
	OLD_LIBS+=usr/lib32/pam_guest.so.3
	OLD_LIBS+=usr/lib32/pam_krb5.so.3
	OLD_LIBS+=usr/lib32/pam_ksu.so.3
	OLD_LIBS+=usr/lib32/pam_lastlog.so.3
	OLD_LIBS+=usr/lib32/pam_login_access.so.3
	OLD_LIBS+=usr/lib32/pam_nologin.so.3
	OLD_LIBS+=usr/lib32/pam_opie.so.3
	OLD_LIBS+=usr/lib32/pam_opieaccess.so.3
	OLD_LIBS+=usr/lib32/pam_passwdqc.so.3
	OLD_LIBS+=usr/lib32/pam_permit.so.3
	OLD_LIBS+=usr/lib32/pam_radius.so.3
	OLD_LIBS+=usr/lib32/pam_rhosts.so.3
	OLD_LIBS+=usr/lib32/pam_rootok.so.3
	OLD_LIBS+=usr/lib32/pam_securetty.so.3
	OLD_LIBS+=usr/lib32/pam_self.so.3
	OLD_LIBS+=usr/lib32/pam_ssh.so.3
	OLD_LIBS+=usr/lib32/pam_tacplus.so.3
	OLD_LIBS+=usr/lib32/pam_unix.so.3
	# 20070613: IPX over IP tunnel removal
	OLD_FILES+=usr/include/netipx/ipx_ip.h
	# 20070605: sched_core removal
	OLD_FILES+=usr/share/man/man4/sched_core.4.gz
	# 20070603: BIND 9.4.1 import
	OLD_LIBS+=usr/lib/liblwres.so.10
	# 20070521: shared library version bump
	OLD_LIBS+=lib/libatm.so.4
	OLD_LIBS+=lib/libbegemot.so.2
	OLD_LIBS+=lib/libbsdxml.so.2
	OLD_LIBS+=lib/libcam.so.3
	OLD_LIBS+=lib/libcrypt.so.3
	OLD_LIBS+=lib/libdevstat.so.5
	OLD_LIBS+=lib/libedit.so.5
	OLD_LIBS+=lib/libgeom.so.3
	OLD_LIBS+=lib/libipsec.so.2
	OLD_LIBS+=lib/libipx.so.3
	OLD_LIBS+=lib/libkiconv.so.2
	OLD_LIBS+=lib/libkse.so.2
	OLD_LIBS+=lib/libkvm.so.3
	OLD_LIBS+=lib/libm.so.4
	OLD_LIBS+=lib/libmd.so.3
	OLD_LIBS+=lib/libpcap.so.4
	OLD_LIBS+=lib/libpthread.so.2
	OLD_LIBS+=lib/libsbuf.so.3
	OLD_LIBS+=lib/libthr.so.2
	OLD_LIBS+=lib/libufs.so.3
	OLD_LIBS+=lib/libutil.so.6
	OLD_LIBS+=lib/libz.so.3
	OLD_LIBS+=usr/lib/libbluetooth.so.2
	OLD_LIBS+=usr/lib/libbsm.so.1
	OLD_LIBS+=usr/lib/libbz2.so.2
	OLD_LIBS+=usr/lib/libcalendar.so.3
	OLD_LIBS+=usr/lib/libcom_err.so.3
	OLD_LIBS+=usr/lib/libdevinfo.so.3
	OLD_LIBS+=usr/lib/libfetch.so.4
	OLD_LIBS+=usr/lib/libform.so.3
	OLD_LIBS+=usr/lib/libformw.so.3
	OLD_LIBS+=usr/lib/libftpio.so.6
	OLD_LIBS+=usr/lib/libgpib.so.1
	OLD_LIBS+=usr/lib/libkse.so.2
	OLD_LIBS+=usr/lib/libmagic.so.2
	OLD_LIBS+=usr/lib/libmemstat.so.1
	OLD_LIBS+=usr/lib/libmenu.so.3
	OLD_LIBS+=usr/lib/libmenuw.so.3
	OLD_LIBS+=usr/lib/libmilter.so.3
	OLD_LIBS+=usr/lib/libmp.so.5
	OLD_LIBS+=usr/lib/libncp.so.2
	OLD_LIBS+=usr/lib/libnetgraph.so.2
	OLD_LIBS+=usr/lib/libngatm.so.2
	OLD_LIBS+=usr/lib/libopie.so.4
	OLD_LIBS+=usr/lib/libpanel.so.3
	OLD_LIBS+=usr/lib/libpanelw.so.3
	OLD_LIBS+=usr/lib/libpmc.so.3
	OLD_LIBS+=usr/lib/libradius.so.2
	OLD_LIBS+=usr/lib/librpcsvc.so.3
	OLD_LIBS+=usr/lib/libsdp.so.2
	OLD_LIBS+=usr/lib/libsmb.so.2
	OLD_LIBS+=usr/lib/libstdc++.so.5
	OLD_LIBS+=usr/lib/libtacplus.so.2
	OLD_LIBS+=usr/lib/libthr.so.2
	OLD_LIBS+=usr/lib/libthread_db.so.2
	OLD_LIBS+=usr/lib/libugidfw.so.2
	OLD_LIBS+=usr/lib/libusbhid.so.2
	OLD_LIBS+=usr/lib/libvgl.so.4
	OLD_LIBS+=usr/lib/libwrap.so.4
	OLD_LIBS+=usr/lib/libypclnt.so.2
	OLD_LIBS+=usr/lib/snmp_bridge.so.3
	OLD_LIBS+=usr/lib/snmp_hostres.so.3
	OLD_LIBS+=usr/lib32/libatm.so.4
	OLD_LIBS+=usr/lib32/libbegemot.so.2
	OLD_LIBS+=usr/lib32/libbluetooth.so.2
	OLD_LIBS+=usr/lib32/libbsdxml.so.2
	OLD_LIBS+=usr/lib32/libbsm.so.1
	OLD_LIBS+=usr/lib32/libbz2.so.2
	OLD_LIBS+=usr/lib32/libcalendar.so.3
	OLD_LIBS+=usr/lib32/libcam.so.3
	OLD_LIBS+=usr/lib32/libcom_err.so.3
	OLD_LIBS+=usr/lib32/libcrypt.so.3
	OLD_LIBS+=usr/lib32/libdevinfo.so.3
	OLD_LIBS+=usr/lib32/libdevstat.so.5
	OLD_LIBS+=usr/lib32/libedit.so.5
	OLD_LIBS+=usr/lib32/libfetch.so.4
	OLD_LIBS+=usr/lib32/libform.so.3
	OLD_LIBS+=usr/lib32/libformw.so.3
	OLD_LIBS+=usr/lib32/libftpio.so.6
	OLD_LIBS+=usr/lib32/libgeom.so.3
	OLD_LIBS+=usr/lib32/libgpib.so.1
	OLD_LIBS+=usr/lib32/libipsec.so.2
	OLD_LIBS+=usr/lib32/libipx.so.3
	OLD_LIBS+=usr/lib32/libkiconv.so.2
	OLD_LIBS+=usr/lib32/libkse.so.2
	OLD_LIBS+=usr/lib32/libkvm.so.3
	OLD_LIBS+=usr/lib32/libm.so.4
	OLD_LIBS+=usr/lib32/libmagic.so.2
	OLD_LIBS+=usr/lib32/libmd.so.3
	OLD_LIBS+=usr/lib32/libmemstat.so.1
	OLD_LIBS+=usr/lib32/libmenu.so.3
	OLD_LIBS+=usr/lib32/libmenuw.so.3
	OLD_LIBS+=usr/lib32/libmilter.so.3
	OLD_LIBS+=usr/lib32/libmp.so.5
	OLD_LIBS+=usr/lib32/libncp.so.2
	OLD_LIBS+=usr/lib32/libnetgraph.so.2
	OLD_LIBS+=usr/lib32/libngatm.so.2
	OLD_LIBS+=usr/lib32/libopie.so.4
	OLD_LIBS+=usr/lib32/libpanel.so.3
	OLD_LIBS+=usr/lib32/libpanelw.so.3
	OLD_LIBS+=usr/lib32/libpcap.so.4
	OLD_LIBS+=usr/lib32/libpmc.so.3
	OLD_LIBS+=usr/lib32/libpthread.so.2
	OLD_LIBS+=usr/lib32/libradius.so.2
	OLD_LIBS+=usr/lib32/librpcsvc.so.3
	OLD_LIBS+=usr/lib32/libsbuf.so.3
	OLD_LIBS+=usr/lib32/libsdp.so.2
	OLD_LIBS+=usr/lib32/libsmb.so.2
	OLD_LIBS+=usr/lib32/libstdc++.so.5
	OLD_LIBS+=usr/lib32/libtacplus.so.2
	OLD_LIBS+=usr/lib32/libthr.so.2
	OLD_LIBS+=usr/lib32/libthread_db.so.2
	OLD_LIBS+=usr/lib32/libufs.so.3
	OLD_LIBS+=usr/lib32/libugidfw.so.2
	OLD_LIBS+=usr/lib32/libusbhid.so.2
	OLD_LIBS+=usr/lib32/libutil.so.6
	OLD_LIBS+=usr/lib32/libvgl.so.4
	OLD_LIBS+=usr/lib32/libwrap.so.4
	OLD_LIBS+=usr/lib32/libypclnt.so.2
	OLD_LIBS+=usr/lib32/libz.so.3
	# 20070519: GCC 4.2
	OLD_FILES+=usr/bin/f77
	OLD_FILES+=usr/bin/protoize
	OLD_FILES+=usr/include/g2c.h
	OLD_FILES+=usr/libexec/f771
	OLD_FILES+=usr/share/info/g77.info.gz
	OLD_FILES+=usr/share/man/man1/f77.1.gz
	OLD_FILES+=usr/include/c++/3.4/algorithm
	OLD_FILES+=usr/include/c++/3.4/backward/algo.h
	OLD_FILES+=usr/include/c++/3.4/backward/algobase.h
	OLD_FILES+=usr/include/c++/3.4/backward/alloc.h
	OLD_FILES+=usr/include/c++/3.4/backward/backward_warning.h
	OLD_FILES+=usr/include/c++/3.4/backward/bvector.h
	OLD_FILES+=usr/include/c++/3.4/backward/complex.h
	OLD_FILES+=usr/include/c++/3.4/backward/defalloc.h
	OLD_FILES+=usr/include/c++/3.4/backward/deque.h
	OLD_FILES+=usr/include/c++/3.4/backward/fstream.h
	OLD_FILES+=usr/include/c++/3.4/backward/function.h
	OLD_FILES+=usr/include/c++/3.4/backward/hash_map.h
	OLD_FILES+=usr/include/c++/3.4/backward/hash_set.h
	OLD_FILES+=usr/include/c++/3.4/backward/hashtable.h
	OLD_FILES+=usr/include/c++/3.4/backward/heap.h
	OLD_FILES+=usr/include/c++/3.4/backward/iomanip.h
	OLD_FILES+=usr/include/c++/3.4/backward/iostream.h
	OLD_FILES+=usr/include/c++/3.4/backward/istream.h
	OLD_FILES+=usr/include/c++/3.4/backward/iterator.h
	OLD_FILES+=usr/include/c++/3.4/backward/list.h
	OLD_FILES+=usr/include/c++/3.4/backward/map.h
	OLD_FILES+=usr/include/c++/3.4/backward/multimap.h
	OLD_FILES+=usr/include/c++/3.4/backward/multiset.h
	OLD_FILES+=usr/include/c++/3.4/backward/new.h
	OLD_FILES+=usr/include/c++/3.4/backward/ostream.h
	OLD_FILES+=usr/include/c++/3.4/backward/pair.h
	OLD_FILES+=usr/include/c++/3.4/backward/queue.h
	OLD_FILES+=usr/include/c++/3.4/backward/rope.h
	OLD_FILES+=usr/include/c++/3.4/backward/set.h
	OLD_FILES+=usr/include/c++/3.4/backward/slist.h
	OLD_FILES+=usr/include/c++/3.4/backward/stack.h
	OLD_FILES+=usr/include/c++/3.4/backward/stream.h
	OLD_FILES+=usr/include/c++/3.4/backward/streambuf.h
	OLD_FILES+=usr/include/c++/3.4/backward/strstream
	OLD_FILES+=usr/include/c++/3.4/backward/tempbuf.h
	OLD_FILES+=usr/include/c++/3.4/backward/tree.h
	OLD_FILES+=usr/include/c++/3.4/backward/vector.h
	OLD_FILES+=usr/include/c++/3.4/bits/allocator.h
	OLD_FILES+=usr/include/c++/3.4/bits/atomic_word.h
	OLD_FILES+=usr/include/c++/3.4/bits/atomicity.h
	OLD_FILES+=usr/include/c++/3.4/bits/basic_file.h
	OLD_FILES+=usr/include/c++/3.4/bits/basic_ios.h
	OLD_FILES+=usr/include/c++/3.4/bits/basic_ios.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/basic_string.h
	OLD_FILES+=usr/include/c++/3.4/bits/basic_string.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/boost_concept_check.h
	OLD_FILES+=usr/include/c++/3.4/bits/c++allocator.h
	OLD_FILES+=usr/include/c++/3.4/bits/c++config.h
	OLD_FILES+=usr/include/c++/3.4/bits/c++io.h
	OLD_FILES+=usr/include/c++/3.4/bits/c++locale.h
	OLD_FILES+=usr/include/c++/3.4/bits/c++locale_internal.h
	OLD_FILES+=usr/include/c++/3.4/bits/char_traits.h
	OLD_FILES+=usr/include/c++/3.4/bits/cmath.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/codecvt.h
	OLD_FILES+=usr/include/c++/3.4/bits/codecvt_specializations.h
	OLD_FILES+=usr/include/c++/3.4/bits/concept_check.h
	OLD_FILES+=usr/include/c++/3.4/bits/concurrence.h
	OLD_FILES+=usr/include/c++/3.4/bits/cpp_type_traits.h
	OLD_FILES+=usr/include/c++/3.4/bits/ctype_base.h
	OLD_FILES+=usr/include/c++/3.4/bits/ctype_inline.h
	OLD_FILES+=usr/include/c++/3.4/bits/ctype_noninline.h
	OLD_FILES+=usr/include/c++/3.4/bits/deque.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/fstream.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/functexcept.h
	OLD_FILES+=usr/include/c++/3.4/bits/gslice.h
	OLD_FILES+=usr/include/c++/3.4/bits/gslice_array.h
	OLD_FILES+=usr/include/c++/3.4/bits/gthr-default.h
	OLD_FILES+=usr/include/c++/3.4/bits/gthr-posix.h
	OLD_FILES+=usr/include/c++/3.4/bits/gthr-single.h
	OLD_FILES+=usr/include/c++/3.4/bits/gthr.h
	OLD_FILES+=usr/include/c++/3.4/bits/indirect_array.h
	OLD_FILES+=usr/include/c++/3.4/bits/ios_base.h
	OLD_FILES+=usr/include/c++/3.4/bits/istream.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/list.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/locale_classes.h
	OLD_FILES+=usr/include/c++/3.4/bits/locale_facets.h
	OLD_FILES+=usr/include/c++/3.4/bits/locale_facets.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/localefwd.h
	OLD_FILES+=usr/include/c++/3.4/bits/mask_array.h
	OLD_FILES+=usr/include/c++/3.4/bits/messages_members.h
	OLD_FILES+=usr/include/c++/3.4/bits/os_defines.h
	OLD_FILES+=usr/include/c++/3.4/bits/ostream.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/postypes.h
	OLD_FILES+=usr/include/c++/3.4/bits/slice_array.h
	OLD_FILES+=usr/include/c++/3.4/bits/sstream.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/stl_algo.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_algobase.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_bvector.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_construct.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_deque.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_function.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_heap.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_iterator.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_iterator_base_funcs.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_iterator_base_types.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_list.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_map.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_multimap.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_multiset.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_numeric.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_pair.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_queue.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_raw_storage_iter.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_relops.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_set.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_stack.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_tempbuf.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_threads.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_tree.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_uninitialized.h
	OLD_FILES+=usr/include/c++/3.4/bits/stl_vector.h
	OLD_FILES+=usr/include/c++/3.4/bits/stream_iterator.h
	OLD_FILES+=usr/include/c++/3.4/bits/streambuf.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/streambuf_iterator.h
	OLD_FILES+=usr/include/c++/3.4/bits/stringfwd.h
	OLD_FILES+=usr/include/c++/3.4/bits/time_members.h
	OLD_FILES+=usr/include/c++/3.4/bits/type_traits.h
	OLD_FILES+=usr/include/c++/3.4/bits/valarray_after.h
	OLD_FILES+=usr/include/c++/3.4/bits/valarray_array.h
	OLD_FILES+=usr/include/c++/3.4/bits/valarray_array.tcc
	OLD_FILES+=usr/include/c++/3.4/bits/valarray_before.h
	OLD_FILES+=usr/include/c++/3.4/bits/vector.tcc
	OLD_FILES+=usr/include/c++/3.4/bitset
	OLD_FILES+=usr/include/c++/3.4/cassert
	OLD_FILES+=usr/include/c++/3.4/cctype
	OLD_FILES+=usr/include/c++/3.4/cerrno
	OLD_FILES+=usr/include/c++/3.4/cfloat
	OLD_FILES+=usr/include/c++/3.4/ciso646
	OLD_FILES+=usr/include/c++/3.4/climits
	OLD_FILES+=usr/include/c++/3.4/clocale
	OLD_FILES+=usr/include/c++/3.4/cmath
	OLD_FILES+=usr/include/c++/3.4/complex
	OLD_FILES+=usr/include/c++/3.4/csetjmp
	OLD_FILES+=usr/include/c++/3.4/csignal
	OLD_FILES+=usr/include/c++/3.4/cstdarg
	OLD_FILES+=usr/include/c++/3.4/cstddef
	OLD_FILES+=usr/include/c++/3.4/cstdio
	OLD_FILES+=usr/include/c++/3.4/cstdlib
	OLD_FILES+=usr/include/c++/3.4/cstring
	OLD_FILES+=usr/include/c++/3.4/ctime
	OLD_FILES+=usr/include/c++/3.4/cwchar
	OLD_FILES+=usr/include/c++/3.4/cwctype
	OLD_FILES+=usr/include/c++/3.4/cxxabi.h
	OLD_FILES+=usr/include/c++/3.4/debug/bitset
	OLD_FILES+=usr/include/c++/3.4/debug/debug.h
	OLD_FILES+=usr/include/c++/3.4/debug/deque
	OLD_FILES+=usr/include/c++/3.4/debug/formatter.h
	OLD_FILES+=usr/include/c++/3.4/debug/hash_map
	OLD_FILES+=usr/include/c++/3.4/debug/hash_map.h
	OLD_FILES+=usr/include/c++/3.4/debug/hash_multimap.h
	OLD_FILES+=usr/include/c++/3.4/debug/hash_multiset.h
	OLD_FILES+=usr/include/c++/3.4/debug/hash_set
	OLD_FILES+=usr/include/c++/3.4/debug/hash_set.h
	OLD_FILES+=usr/include/c++/3.4/debug/list
	OLD_FILES+=usr/include/c++/3.4/debug/map
	OLD_FILES+=usr/include/c++/3.4/debug/map.h
	OLD_FILES+=usr/include/c++/3.4/debug/multimap.h
	OLD_FILES+=usr/include/c++/3.4/debug/multiset.h
	OLD_FILES+=usr/include/c++/3.4/debug/safe_base.h
	OLD_FILES+=usr/include/c++/3.4/debug/safe_iterator.h
	OLD_FILES+=usr/include/c++/3.4/debug/safe_iterator.tcc
	OLD_FILES+=usr/include/c++/3.4/debug/safe_sequence.h
	OLD_FILES+=usr/include/c++/3.4/debug/set
	OLD_FILES+=usr/include/c++/3.4/debug/set.h
	OLD_FILES+=usr/include/c++/3.4/debug/string
	OLD_FILES+=usr/include/c++/3.4/debug/vector
	OLD_FILES+=usr/include/c++/3.4/deque
	OLD_FILES+=usr/include/c++/3.4/exception
	OLD_FILES+=usr/include/c++/3.4/exception_defines.h
	OLD_FILES+=usr/include/c++/3.4/ext/algorithm
	OLD_FILES+=usr/include/c++/3.4/ext/bitmap_allocator.h
	OLD_FILES+=usr/include/c++/3.4/ext/debug_allocator.h
	OLD_FILES+=usr/include/c++/3.4/ext/enc_filebuf.h
	OLD_FILES+=usr/include/c++/3.4/ext/functional
	OLD_FILES+=usr/include/c++/3.4/ext/hash_fun.h
	OLD_FILES+=usr/include/c++/3.4/ext/hash_map
	OLD_FILES+=usr/include/c++/3.4/ext/hash_set
	OLD_FILES+=usr/include/c++/3.4/ext/hashtable.h
	OLD_FILES+=usr/include/c++/3.4/ext/iterator
	OLD_FILES+=usr/include/c++/3.4/ext/malloc_allocator.h
	OLD_FILES+=usr/include/c++/3.4/ext/memory
	OLD_FILES+=usr/include/c++/3.4/ext/mt_allocator.h
	OLD_FILES+=usr/include/c++/3.4/ext/new_allocator.h
	OLD_FILES+=usr/include/c++/3.4/ext/numeric
	OLD_FILES+=usr/include/c++/3.4/ext/pod_char_traits.h
	OLD_FILES+=usr/include/c++/3.4/ext/pool_allocator.h
	OLD_FILES+=usr/include/c++/3.4/ext/rb_tree
	OLD_FILES+=usr/include/c++/3.4/ext/rope
	OLD_FILES+=usr/include/c++/3.4/ext/ropeimpl.h
	OLD_FILES+=usr/include/c++/3.4/ext/slist
	OLD_FILES+=usr/include/c++/3.4/ext/stdio_filebuf.h
	OLD_FILES+=usr/include/c++/3.4/ext/stdio_sync_filebuf.h
	OLD_FILES+=usr/include/c++/3.4/fstream
	OLD_FILES+=usr/include/c++/3.4/functional
	OLD_FILES+=usr/include/c++/3.4/iomanip
	OLD_FILES+=usr/include/c++/3.4/ios
	OLD_FILES+=usr/include/c++/3.4/iosfwd
	OLD_FILES+=usr/include/c++/3.4/iostream
	OLD_FILES+=usr/include/c++/3.4/istream
	OLD_FILES+=usr/include/c++/3.4/iterator
	OLD_FILES+=usr/include/c++/3.4/limits
	OLD_FILES+=usr/include/c++/3.4/list
	OLD_FILES+=usr/include/c++/3.4/locale
	OLD_FILES+=usr/include/c++/3.4/map
	OLD_FILES+=usr/include/c++/3.4/memory
	OLD_FILES+=usr/include/c++/3.4/new
	OLD_FILES+=usr/include/c++/3.4/numeric
	OLD_FILES+=usr/include/c++/3.4/ostream
	OLD_FILES+=usr/include/c++/3.4/queue
	OLD_FILES+=usr/include/c++/3.4/set
	OLD_FILES+=usr/include/c++/3.4/sstream
	OLD_FILES+=usr/include/c++/3.4/stack
	OLD_FILES+=usr/include/c++/3.4/stdexcept
	OLD_FILES+=usr/include/c++/3.4/streambuf
	OLD_FILES+=usr/include/c++/3.4/string
	OLD_FILES+=usr/include/c++/3.4/typeinfo
	OLD_FILES+=usr/include/c++/3.4/utility
	OLD_FILES+=usr/include/c++/3.4/valarray
	OLD_FILES+=usr/include/c++/3.4/vector
	OLD_DIRS+=usr/include/c++/3.4/backward
	OLD_DIRS+=usr/include/c++/3.4/bits
	OLD_DIRS+=usr/include/c++/3.4/debug
	OLD_DIRS+=usr/include/c++/3.4/ext
	OLD_DIRS+=usr/include/c++/3.4
	# 20070510: zpool/zfs moved to /sbin
	OLD_FILES+=usr/sbin/zfs
	OLD_FILES+=usr/sbin/zpool
	# 20070423: rc.bluetooth (examples) removed
	OLD_FILES+=usr/share/examples/netgraph/bluetooth/rc.bluetooth
	OLD_DIRS+=usr/share/examples/netgraph/bluetooth
	# 20070421: worm.4 removed
	OLD_FILES+=usr/share/man/man4/worm.4.gz
	# 20070417: trunk(4) renamed to lagg(4)
	OLD_FILES+=usr/include/net/if_trunk.h
	# 20070409: uuidgen moved to /bin/
	OLD_FILES+=usr/bin/uuidgen
	# 20070328: bzip2 1.0.4
	OLD_FILES+=usr/share/info/bzip2.info.gz
	# 20070303: libarchive 2.0
	OLD_LIBS+=usr/lib/libarchive.so.3
	OLD_LIBS+=usr/lib32/libarchive.so.3
	# 20070301: remove addr2ascii and ascii2addr
	OLD_FILES+=usr/share/man/man3/addr2ascii.3.gz
	OLD_FILES+=usr/share/man/man3/ascii2addr.3.gz
	# 20070225: vm_page_unmanage() removed
	OLD_FILES+=usr/share/man/man9/vm_page_unmanage.9.gz
	# 20070216: VFS_VPTOFH(9) -> VOP_VPTOFH(9)
	OLD_FILES+=usr/share/man/man9/VFS_VPTOFH.9.gz
	# 20070212: kame.4 removed
	OLD_FILES+=usr/share/man/man4/kame.4.gz
	# 20070201: remove libmytinfo link
	OLD_FILES+=usr/lib/libmytinfo.a
	OLD_FILES+=usr/lib/libmytinfo.so
	OLD_FILES+=usr/lib/libmytinfo_p.a
	OLD_FILES+=usr/lib/libmytinfow.a
	OLD_FILES+=usr/lib/libmytinfow.so
	OLD_FILES+=usr/lib/libmytinfow_p.a
	OLD_FILES+=usr/lib32/libmytinfo.a
	OLD_FILES+=usr/lib32/libmytinfo.so
	OLD_FILES+=usr/lib32/libmytinfo_p.a
	OLD_FILES+=usr/lib32/libmytinfow.a
	OLD_FILES+=usr/lib32/libmytinfow.so
	OLD_FILES+=usr/lib32/libmytinfow_p.a
	# 20070128: remove vnconfig
	OLD_FILES+=usr/sbin/vnconfig
	# 20070127: remove bpf_compat.h
	OLD_FILES+=usr/include/net/bpf_compat.h
	# 20070125: objformat bites the dust
	OLD_FILES+=usr/bin/objformat
	OLD_FILES+=usr/share/man/man1/objformat.1.gz
	OLD_FILES+=usr/include/objformat.h
	OLD_FILES+=usr/share/man/man3/getobjformat.3.gz
	# 20061201: remove symlink to *.so.4 libalias modules
	OLD_FILES+=usr/lib/libalias_cuseeme.so
	OLD_FILES+=usr/lib/libalias_dummy.so
	OLD_FILES+=usr/lib/libalias_ftp.so
	OLD_FILES+=usr/lib/libalias_irc.so
	OLD_FILES+=usr/lib/libalias_nbt.so
	OLD_FILES+=usr/lib/libalias_pptp.so
	OLD_FILES+=usr/lib/libalias_skinny.so
	OLD_FILES+=usr/lib/libalias_smedia.so
	# 20061201: remove old *.so.4 libalias modules
	OLD_FILES+=lib/libalias_cuseeme.so.4
	OLD_FILES+=lib/libalias_dummy.so.4
	OLD_FILES+=lib/libalias_ftp.so.4
	OLD_FILES+=lib/libalias_irc.so.4
	OLD_FILES+=lib/libalias_nbt.so.4
	OLD_FILES+=lib/libalias_pptp.so.4
	OLD_FILES+=lib/libalias_skinny.so.4
	OLD_FILES+=lib/libalias_smedia.so.4
	# 20061126: remove old man page
	OLD_FILES+=usr/share/man/man3/archive_read_set_bytes_per_block.3.gz
	# 20061125: remove old man page
	OLD_FILES+=usr/share/man/man9/devsw.9.gz
	# 20061122: remove obsolete mount programs
	OLD_FILES+=sbin/mount_devfs
	OLD_FILES+=sbin/mount_ext2fs
	OLD_FILES+=sbin/mount_fdescfs
	OLD_FILES+=sbin/mount_linprocfs
	OLD_FILES+=sbin/mount_procfs
	OLD_FILES+=sbin/mount_std
	OLD_FILES+=rescue/mount_devfs
	OLD_FILES+=rescue/mount_ext2fs
	OLD_FILES+=rescue/mount_fdescfs
	OLD_FILES+=rescue/mount_linprocfs
	OLD_FILES+=rescue/mount_procfs
	OLD_FILES+=rescue/mount_std
	OLD_FILES+=usr/share/man/man8/mount_devfs.8.gz
	OLD_FILES+=usr/share/man/man8/mount_ext2fs.8.gz
	OLD_FILES+=usr/share/man/man8/mount_fdescfs.8.gz
	OLD_FILES+=usr/share/man/man8/mount_linprocfs.8.gz
	OLD_FILES+=usr/share/man/man8/mount_procfs.8.gz
	OLD_FILES+=usr/share/man/man8/mount_std.8.gz
	# 20061116: uhidev.4 removed
	OLD_FILES+=usr/share/man/man4/uhidev.4.gz
	# 20061106: archive_write_prepare.3 removed
	OLD_FILES+=usr/share/man/man3/archive_write_prepare.3.gz
	# 20061018: pccardc removed
	OLD_FILES+=usr/sbin/pccardc usr/share/man/man8/pccardc.8.gz
	# 20060930: demangle.h from contrib/libstdc++/include/ext/
	OLD_FILES+=usr/include/c++/3.4/ext/demangle.h
	# 20060929: mrouted removed
	OLD_FILES+=usr/sbin/map-mbone
	OLD_FILES+=usr/sbin/mrinfo
	OLD_FILES+=usr/sbin/mrouted
	OLD_FILES+=usr/sbin/mtrace
	OLD_FILES+=usr/share/man/man8/map-mbone.8.gz
	OLD_FILES+=usr/share/man/man8/mrinfo.8.gz
	OLD_FILES+=usr/share/man/man8/mrouted.8.gz
	OLD_FILES+=usr/share/man/man8/mtrace.8.gz
	# 20060924: tcpslice removed
	OLD_FILES+=usr/sbin/tcpslice
	OLD_FILES+=usr/share/man/man1/tcpslice.1.gz
	# 20060829: kvmdb cleanup script removed
	OLD_FILES+=etc/periodic/weekly/120.clean-kvmdb
	# 20060822: ramdisk{,-own} have been replaced by mdconfig{,2}
	OLD_FILES+=etc/rc.d/ramdisk
	OLD_FILES+=etc/rc.d/ramdisk-own
	# 20060729: OpenSSL 0.9.7e -> 0.9.8b upgrade
	OLD_FILES+=usr/include/openssl/eng_int.h
	OLD_FILES+=usr/include/openssl/hw_4758_cca_err.h
	OLD_FILES+=usr/include/openssl/hw_aep_err.h
	OLD_FILES+=usr/include/openssl/hw_atalla_err.h
	OLD_FILES+=usr/include/openssl/hw_cswift_err.h
	OLD_FILES+=usr/include/openssl/hw_ncipher_err.h
	OLD_FILES+=usr/include/openssl/hw_nuron_err.h
	OLD_FILES+=usr/include/openssl/hw_sureware_err.h
	OLD_FILES+=usr/include/openssl/hw_ubsec_err.h
	# 20060713: mount_linsysfs(8) never existed in 7.x
	OLD_FILES+=sbin/mount_linsysfs
	OLD_FILES+=usr/share/man/man8/mount_linsysfs.8.gz
	# 20060704: KAME compat file net_osdep.h removed
	OLD_FILES+=usr/include/net/net_osdep.h
	# 20060605: man page links removed by OpenBSM 1.0 alpha 6 import
	OLD_FILES+=usr/share/man/man3/au_to_socket.3.gz
	OLD_FILES+=usr/share/man/man3/au_to_socket_ex_128.3.gz
	OLD_FILES+=usr/share/man/man3/au_to_socket_ex_32.3.gz
	# 20060517: pcvt removed
	OLD_FILES+=usr/share/pcvt/README.FIRST
	OLD_FILES+=usr/share/pcvt/Etc/xmodmap-german
	OLD_FILES+=usr/share/pcvt/Etc/pcvt.sh
	OLD_FILES+=usr/share/pcvt/Etc/pcvt.el
	OLD_FILES+=usr/share/pcvt/Etc/Terminfo
	OLD_FILES+=usr/share/pcvt/Etc/Termcap
	OLD_DIRS+=usr/share/pcvt/Etc
	OLD_FILES+=usr/share/pcvt/Doc/NotesAndHints
	OLD_FILES+=usr/share/pcvt/Doc/Keyboard.VT
	OLD_FILES+=usr/share/pcvt/Doc/Keyboard.HP
	OLD_FILES+=usr/share/pcvt/Doc/EscapeSequences
	OLD_FILES+=usr/share/pcvt/Doc/Charsets
	OLD_FILES+=usr/share/pcvt/Doc/CharGen
	OLD_FILES+=usr/share/pcvt/Doc/Bibliography
	OLD_FILES+=usr/share/pcvt/Doc/Acknowledgements
	OLD_DIRS+=usr/share/pcvt/Doc
	OLD_DIRS+=usr/share/pcvt
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.816
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.814
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.810
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.808
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.816
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.814
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.810
	OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.808
	OLD_DIRS+=usr/share/misc/pcvtfonts
	OLD_FILES+=usr/share/misc/keycap.pcvt
	OLD_FILES+=usr/share/man/man8/ispcvt.8.gz
	OLD_FILES+=usr/share/man/man5/keycap.5.gz
	OLD_FILES+=usr/share/man/man4/pcvt.4.gz
	OLD_FILES+=usr/share/man/man3/kgetstr.3.gz
	OLD_FILES+=usr/share/man/man3/kgetnum.3.gz
	OLD_FILES+=usr/share/man/man3/kgetflag.3.gz
	OLD_FILES+=usr/share/man/man3/kgetent.3.gz
	OLD_FILES+=usr/share/man/man3/keycap.3.gz
	OLD_FILES+=usr/share/man/man1/vt220keys.1.gz
	OLD_FILES+=usr/share/man/man1/scon.1.gz
	OLD_FILES+=usr/share/man/man1/loadfont.1.gz
	OLD_FILES+=usr/share/man/man1/kcon.1.gz
	OLD_FILES+=usr/share/man/man1/fontedit.1.gz
	OLD_FILES+=usr/share/man/man1/cursor.1.gz
	OLD_FILES+=usr/sbin/vt220keys
	OLD_FILES+=usr/sbin/scon
	OLD_FILES+=usr/sbin/loadfont
	OLD_FILES+=usr/sbin/kcon
	OLD_FILES+=usr/sbin/ispcvt
	OLD_FILES+=usr/sbin/fontedit
	OLD_FILES+=usr/sbin/cursor
	OLD_FILES+=usr/lib/libkeycap_p.a
	OLD_FILES+=usr/lib/libkeycap.a
	OLD_FILES+=usr/include/machine/pcvt_ioctl.h
	# 20060514: lnc(4) replaced by le(4)
	OLD_FILES+=usr/share/man/man4/i386/lnc.4.gz
	# 20060512: remove ip6fw
	OLD_FILES+=etc/periodic/security/600.ip6fwdenied
	OLD_FILES+=etc/periodic/security/650.ip6fwlimit
	OLD_FILES+=sbin/ip6fw
	OLD_FILES+=usr/include/netinet6/ip6_fw.h
	OLD_FILES+=usr/share/man/man8/ip6fw.8.gz
	# 20060424: sab(4) removed
	OLD_FILES+=usr/share/man/man4/sab.4.gz
	# 20060328: remove redundant rc.d script
	OLD_FILES+=etc/rc.d/ike
	# 20060127: revert libdisk to static-only
	OLD_FILES+=usr/lib/libdisk.so
	# 20060115: sys/pccard includes cleanup
	OLD_FILES+=usr/include/pccard/driver.h
	OLD_FILES+=usr/include/pccard/i82365.h
	OLD_FILES+=usr/include/pccard/meciareg.h
	OLD_FILES+=usr/include/pccard/pccard_nbk.h
	OLD_FILES+=usr/include/pccard/pcic_pci.h
	OLD_FILES+=usr/include/pccard/pcicvar.h
	OLD_FILES+=usr/include/pccard/slot.h
	# 20051215: rescue/nextboot.sh renamed to rescue/nextboot
	OLD_FILES+=rescue/nextboot.sh
	# 20051214: usbd(8) removed
	OLD_FILES+=etc/rc.d/usbd
	OLD_FILES+=etc/usbd.conf
	OLD_FILES+=usr/sbin/usbd
	OLD_FILES+=usr/share/man/man8/usbd.8.gz
	# 20051029: rc.d/ppp-user renamed to rc.d/ppp for convenience
	OLD_FILES+=etc/rc.d/ppp-user
	# 20051012: setkey(8) moved to /sbin/
	OLD_FILES+=usr/sbin/setkey
	# 20050930: pccardd(8) removed
	OLD_FILES+=usr/sbin/pccardd
	OLD_FILES+=usr/share/man/man5/pccard.conf.5.gz
	OLD_FILES+=usr/share/man/man8/pccardd.8.gz
	# 20050927: bridge(4) replaced by if_bridge(4)
	OLD_FILES+=usr/include/net/bridge.h
	# 20050831: not implemented
	OLD_FILES+=usr/share/man/man3/getino.3.gz
	OLD_FILES+=usr/share/man/man3/putino.3.gz
	# 20050825: T/TCP retired several months ago
	OLD_FILES+=usr/share/man/man4/ttcp.4.gz
	# 20050805 tn3270 retired long ago
	OLD_FILES+=usr/share/misc/map3270
	# 20050801: too old to be interesting here
	OLD_FILES+=usr/share/doc/papers/px.ps.gz
	# 20050721: moved to ports
	OLD_FILES+=usr/sbin/vttest
	OLD_FILES+=usr/share/man/man1/vttest.1.gz
	# 20050617: wpa man pages moved to section 8
	OLD_FILES+=usr/share/man/man1/hostapd.1.gz
	OLD_FILES+=usr/share/man/man1/hostapd_cli.1.gz
	OLD_FILES+=usr/share/man/man1/wpa_cli.1.gz
	OLD_FILES+=usr/share/man/man1/wpa_supplicant.1.gz
	# 20050610: rexecd (insecure by design)
	OLD_FILES+=etc/pam.d/rexecd
	OLD_FILES+=usr/share/man/man8/rexecd.8.gz
	OLD_FILES+=usr/libexec/rexecd
	# 20050606: OpenBSD dhclient replaces ISC one
	OLD_FILES+=bin/omshell
	OLD_FILES+=sbin/omshell
	OLD_FILES+=usr/share/man/man1/omshell.1.gz
	OLD_FILES+=usr/share/man/man5/dhcp-eval.5.gz
	# 200504XX: ipf tools moved from /usr to /
	OLD_FILES+=rescue/ipfs
	OLD_FILES+=rescue/ipfstat
	OLD_FILES+=rescue/ipmon
	OLD_FILES+=rescue/ipnat
	OLD_FILES+=usr/sbin/ipftest
	OLD_FILES+=usr/sbin/ipresend
	OLD_FILES+=usr/sbin/ipsend
	OLD_FILES+=usr/sbin/iptest
	OLD_FILES+=usr/share/man/man1/ipnat.1.gz
	OLD_FILES+=usr/share/man/man1/ipsend.1.gz
	OLD_FILES+=usr/share/man/man1/iptest.1.gz
	OLD_FILES+=usr/share/man/man5/ipsend.5.gz
	# 200503XX: bsdtar takes over gtar
	OLD_FILES+=usr/bin/gtar
	OLD_FILES+=usr/share/man/man1/gtar.1.gz
	# 200503XX
	OLD_FILES+=usr/share/man/man3/exp10.3.gz
	OLD_FILES+=usr/share/man/man3/exp10f.3.gz
	OLD_FILES+=usr/share/man/man3/fpsetsticky.3.gz
	# 20050324: updated release infrastructure
	OLD_FILES+=usr/share/man/man5/drivers.conf.5.gz
	# 20050317: removed from BIND 9 distribution
	OLD_FILES+=usr/share/doc/bind9/KNOWN_DEFECTS
	# 2005XXXX:
	OLD_FILES+=sbin/mount_autofs
	OLD_FILES+=usr/lib/libautofs.a
	OLD_FILES+=usr/lib/libautofs.so
	OLD_FILES+=usr/share/man/man8/mount_autofs.8.gz
	# 20050203: Merged with fortunes
	OLD_FILES+=usr/share/games/fortune/fortunes2
	OLD_FILES+=usr/share/games/fortune/fortunes2.dat
	# 200501XX:
	OLD_FILES+=usr/libexec/getNAME
	# 200411XX: gvinum replaces vinum
	OLD_FILES+=bin/vinum
	OLD_FILES+=rescue/vinum
	OLD_FILES+=sbin/vinum
	OLD_FILES+=usr/share/man/man8/vinum.8.gz
	# 200411XX: libxpg4 removal
	OLD_FILES+=usr/lib/libxpg4.a
	OLD_FILES+=usr/lib/libxpg4.so
	OLD_FILES+=usr/lib/libxpg4_p.a
	# 20041109: replaced by em(4)
	OLD_FILES+=usr/share/man/man4/gx.4.gz
	OLD_FILES+=usr/share/man/man4/if_gx.4.gz
	# 20041017: rune interface removed
	OLD_FILES+=usr/include/rune.h
	OLD_FILES+=usr/share/man/man3/fgetrune.3.gz
	OLD_FILES+=usr/share/man/man3/fputrune.3.gz
	OLD_FILES+=usr/share/man/man3/fungetrune.3.gz
	OLD_FILES+=usr/share/man/man3/mbrrune.3.gz
	OLD_FILES+=usr/share/man/man3/mbrune.3.gz
	OLD_FILES+=usr/share/man/man3/rune.3.gz
	OLD_FILES+=usr/share/man/man3/setinvalidrune.3.gz
	OLD_FILES+=usr/share/man/man3/sgetrune.3.gz
	OLD_FILES+=usr/share/man/man3/sputrune.3.gz
	# 20040925: bind9 import
	OLD_FILES+=usr/bin/dnskeygen
	OLD_FILES+=usr/bin/dnsquery
	OLD_FILES+=usr/lib/libisc.a
	OLD_FILES+=usr/lib/libisc.so
	OLD_FILES+=usr/lib/libisc_p.a
	OLD_FILES+=usr/libexec/named-xfer
	OLD_FILES+=usr/sbin/named.restart
	OLD_FILES+=usr/sbin/ndc
	OLD_FILES+=usr/sbin/nslookup
	OLD_FILES+=usr/sbin/nsupdate
	OLD_FILES+=usr/share/doc/bind/html/acl.html
	OLD_FILES+=usr/share/doc/bind/html/address_list.html
	OLD_FILES+=usr/share/doc/bind/html/comments.html
	OLD_FILES+=usr/share/doc/bind/html/config.html
	OLD_FILES+=usr/share/doc/bind/html/controls.html
	OLD_FILES+=usr/share/doc/bind/html/docdef.html
	OLD_FILES+=usr/share/doc/bind/html/example.html
	OLD_FILES+=usr/share/doc/bind/html/include.html
	OLD_FILES+=usr/share/doc/bind/html/index.html
	OLD_FILES+=usr/share/doc/bind/html/key.html
	OLD_FILES+=usr/share/doc/bind/html/logging.html
	OLD_FILES+=usr/share/doc/bind/html/master.html
	OLD_FILES+=usr/share/doc/bind/html/options.html
	OLD_FILES+=usr/share/doc/bind/html/server.html
	OLD_FILES+=usr/share/doc/bind/html/trusted-keys.html
	OLD_FILES+=usr/share/doc/bind/html/zone.html
	OLD_FILES+=usr/share/doc/bind/misc/DynamicUpdate
	OLD_FILES+=usr/share/doc/bind/misc/FAQ.1of2
	OLD_FILES+=usr/share/doc/bind/misc/FAQ.2of2
	OLD_FILES+=usr/share/doc/bind/misc/rfc2317-notes.txt
	OLD_FILES+=usr/share/doc/bind/misc/style.txt
	OLD_FILES+=usr/share/man/man1/dnskeygen.1.gz
	OLD_FILES+=usr/share/man/man1/dnsquery.1.gz
	OLD_FILES+=usr/share/man/man8/named-bootconf.8.gz
	OLD_FILES+=usr/share/man/man8/named-xfer.8.gz
	OLD_FILES+=usr/share/man/man8/named.restart.8.gz
	OLD_FILES+=usr/share/man/man8/ndc.8.gz
	OLD_FILES+=usr/share/man/man8/nslookup.8.gz
	# 200409XX
	OLD_FILES+=usr/share/man/man3/ENSURE.3.gz
	OLD_FILES+=usr/share/man/man3/ENSURE_ERR.3.gz
	OLD_FILES+=usr/share/man/man3/INSIST.3.gz
	OLD_FILES+=usr/share/man/man3/INSIST_ERR.3.gz
	OLD_FILES+=usr/share/man/man3/INVARIANT.3.gz
	OLD_FILES+=usr/share/man/man3/INVARIANT_ERR.3.gz
	OLD_FILES+=usr/share/man/man3/REQUIRE.3.gz
	OLD_FILES+=usr/share/man/man3/REQUIRE_ERR.3.gz
	OLD_FILES+=usr/share/man/man3/assertion_type_to_text.3.gz
	OLD_FILES+=usr/share/man/man3/assertions.3.gz
	OLD_FILES+=usr/share/man/man3/bitncmp.3.gz
	OLD_FILES+=usr/share/man/man3/evAddTime.3.gz
	OLD_FILES+=usr/share/man/man3/evCancelConn.3.gz
	OLD_FILES+=usr/share/man/man3/evCancelRW.3.gz
	OLD_FILES+=usr/share/man/man3/evClearIdleTimer.3.gz
	OLD_FILES+=usr/share/man/man3/evClearTimer.3.gz
	OLD_FILES+=usr/share/man/man3/evCmpTime.3.gz
	OLD_FILES+=usr/share/man/man3/evConnFunc.3.gz
	OLD_FILES+=usr/share/man/man3/evConnect.3.gz
	OLD_FILES+=usr/share/man/man3/evConsIovec.3.gz
	OLD_FILES+=usr/share/man/man3/evConsTime.3.gz
	OLD_FILES+=usr/share/man/man3/evCreate.3.gz
	OLD_FILES+=usr/share/man/man3/evDefer.3.gz
	OLD_FILES+=usr/share/man/man3/evDeselectFD.3.gz
	OLD_FILES+=usr/share/man/man3/evDestroy.3.gz
	OLD_FILES+=usr/share/man/man3/evDispatch.3.gz
	OLD_FILES+=usr/share/man/man3/evDo.3.gz
	OLD_FILES+=usr/share/man/man3/evDrop.3.gz
	OLD_FILES+=usr/share/man/man3/evFileFunc.3.gz
	OLD_FILES+=usr/share/man/man3/evGetNext.3.gz
	OLD_FILES+=usr/share/man/man3/evHold.3.gz
	OLD_FILES+=usr/share/man/man3/evInitID.3.gz
	OLD_FILES+=usr/share/man/man3/evLastEventTime.3.gz
	OLD_FILES+=usr/share/man/man3/evListen.3.gz
	OLD_FILES+=usr/share/man/man3/evMainLoop.3.gz
	OLD_FILES+=usr/share/man/man3/evNowTime.3.gz
	OLD_FILES+=usr/share/man/man3/evPrintf.3.gz
	OLD_FILES+=usr/share/man/man3/evRead.3.gz
	OLD_FILES+=usr/share/man/man3/evResetTimer.3.gz
	OLD_FILES+=usr/share/man/man3/evSelectFD.3.gz
	OLD_FILES+=usr/share/man/man3/evSetDebug.3.gz
	OLD_FILES+=usr/share/man/man3/evSetIdleTimer.3.gz
	OLD_FILES+=usr/share/man/man3/evSetTimer.3.gz
	OLD_FILES+=usr/share/man/man3/evStreamFunc.3.gz
	OLD_FILES+=usr/share/man/man3/evSubTime.3.gz
	OLD_FILES+=usr/share/man/man3/evTestID.3.gz
	OLD_FILES+=usr/share/man/man3/evTimeRW.3.gz
	OLD_FILES+=usr/share/man/man3/evTimeSpec.3.gz
	OLD_FILES+=usr/share/man/man3/evTimeVal.3.gz
	OLD_FILES+=usr/share/man/man3/evTimerFunc.3.gz
	OLD_FILES+=usr/share/man/man3/evTouchIdleTimer.3.gz
	OLD_FILES+=usr/share/man/man3/evTryAccept.3.gz
	OLD_FILES+=usr/share/man/man3/evUnhold.3.gz
	OLD_FILES+=usr/share/man/man3/evUntimeRW.3.gz
	OLD_FILES+=usr/share/man/man3/evUnwait.3.gz
	OLD_FILES+=usr/share/man/man3/evWaitFor.3.gz
	OLD_FILES+=usr/share/man/man3/evWaitFunc.3.gz
	OLD_FILES+=usr/share/man/man3/evWrite.3.gz
	OLD_FILES+=usr/share/man/man3/eventlib.3.gz
	OLD_FILES+=usr/share/man/man3/heap.3.gz
	OLD_FILES+=usr/share/man/man3/heap_decreased.3.gz
	OLD_FILES+=usr/share/man/man3/heap_delete.3.gz
	OLD_FILES+=usr/share/man/man3/heap_element.3.gz
	OLD_FILES+=usr/share/man/man3/heap_for_each.3.gz
	OLD_FILES+=usr/share/man/man3/heap_free.3.gz
	OLD_FILES+=usr/share/man/man3/heap_increased.3.gz
	OLD_FILES+=usr/share/man/man3/heap_insert.3.gz
	OLD_FILES+=usr/share/man/man3/heap_new.3.gz
	OLD_FILES+=usr/share/man/man3/log_add_channel.3.gz
	OLD_FILES+=usr/share/man/man3/log_category_is_active.3.gz
	OLD_FILES+=usr/share/man/man3/log_close_stream.3.gz
	OLD_FILES+=usr/share/man/man3/log_dec_references.3.gz
	OLD_FILES+=usr/share/man/man3/log_free_channel.3.gz
	OLD_FILES+=usr/share/man/man3/log_free_context.3.gz
	OLD_FILES+=usr/share/man/man3/log_get_filename.3.gz
	OLD_FILES+=usr/share/man/man3/log_get_stream.3.gz
	OLD_FILES+=usr/share/man/man3/log_inc_references.3.gz
	OLD_FILES+=usr/share/man/man3/log_new_context.3.gz
	OLD_FILES+=usr/share/man/man3/log_new_file_channel.3.gz
	OLD_FILES+=usr/share/man/man3/log_new_null_channel.3.gz
	OLD_FILES+=usr/share/man/man3/log_new_syslog_channel.3.gz
	OLD_FILES+=usr/share/man/man3/log_open_stream.3.gz
	OLD_FILES+=usr/share/man/man3/log_option.3.gz
	OLD_FILES+=usr/share/man/man3/log_remove_channel.3.gz
	OLD_FILES+=usr/share/man/man3/log_set_file_owner.3.gz
	OLD_FILES+=usr/share/man/man3/log_vwrite.3.gz
	OLD_FILES+=usr/share/man/man3/log_write.3.gz
	OLD_FILES+=usr/share/man/man3/logging.3.gz
	OLD_FILES+=usr/share/man/man3/memcluster.3.gz
	OLD_FILES+=usr/share/man/man3/memget.3.gz
	OLD_FILES+=usr/share/man/man3/memput.3.gz
	OLD_FILES+=usr/share/man/man3/memstats.3.gz
	OLD_FILES+=usr/share/man/man3/set_assertion_failure_callback.3.
	OLD_FILES+=usr/share/man/man3/sigwait.3.gz
	OLD_FILES+=usr/share/man/man3/tree_add.3.gz
	OLD_FILES+=usr/share/man/man3/tree_delete.3.gz
	OLD_FILES+=usr/share/man/man3/tree_init.3.gz
	OLD_FILES+=usr/share/man/man3/tree_mung.3.gz
	OLD_FILES+=usr/share/man/man3/tree_srch.3.gz
	OLD_FILES+=usr/share/man/man3/tree_trav.3.gz
	# 2004XXYY: OS internal libs, no ports use them, no need to use OLD_LIBS
	OLD_FILES+=lib/geom/geom_concat.so.1
	OLD_FILES+=lib/geom/geom_label.so.1
	OLD_FILES+=lib/geom/geom_nop.so.1
	OLD_FILES+=lib/geom/geom_stripe.so.1
	# 20040728: GCC 3.4.2
	OLD_DIRS+=usr/include/c++/3.3
	OLD_FILES+=usr/include/c++/3.3/FlexLexer.h
	OLD_FILES+=usr/include/c++/3.3/algorithm
	OLD_FILES+=usr/include/c++/3.3/backward/algo.h
	OLD_FILES+=usr/include/c++/3.3/backward/algobase.h
	OLD_FILES+=usr/include/c++/3.3/backward/alloc.h
	OLD_FILES+=usr/include/c++/3.3/backward/backward_warning.h
	OLD_FILES+=usr/include/c++/3.3/backward/bvector.h
	OLD_FILES+=usr/include/c++/3.3/backward/complex.h
	OLD_FILES+=usr/include/c++/3.3/backward/defalloc.h
	OLD_FILES+=usr/include/c++/3.3/backward/deque.h
	OLD_FILES+=usr/include/c++/3.3/backward/fstream.h
	OLD_FILES+=usr/include/c++/3.3/backward/function.h
	OLD_FILES+=usr/include/c++/3.3/backward/hash_map.h
	OLD_FILES+=usr/include/c++/3.3/backward/hash_set.h
	OLD_FILES+=usr/include/c++/3.3/backward/hashtable.h
	OLD_FILES+=usr/include/c++/3.3/backward/heap.h
	OLD_FILES+=usr/include/c++/3.3/backward/iomanip.h
	OLD_FILES+=usr/include/c++/3.3/backward/iostream.h
	OLD_FILES+=usr/include/c++/3.3/backward/istream.h
	OLD_FILES+=usr/include/c++/3.3/backward/iterator.h
	OLD_FILES+=usr/include/c++/3.3/backward/list.h
	OLD_FILES+=usr/include/c++/3.3/backward/map.h
	OLD_FILES+=usr/include/c++/3.3/backward/multimap.h
	OLD_FILES+=usr/include/c++/3.3/backward/multiset.h
	OLD_FILES+=usr/include/c++/3.3/backward/new.h
	OLD_FILES+=usr/include/c++/3.3/backward/ostream.h
	OLD_FILES+=usr/include/c++/3.3/backward/pair.h
	OLD_FILES+=usr/include/c++/3.3/backward/queue.h
	OLD_FILES+=usr/include/c++/3.3/backward/rope.h
	OLD_FILES+=usr/include/c++/3.3/backward/set.h
	OLD_FILES+=usr/include/c++/3.3/backward/slist.h
	OLD_FILES+=usr/include/c++/3.3/backward/stack.h
	OLD_FILES+=usr/include/c++/3.3/backward/stream.h
	OLD_FILES+=usr/include/c++/3.3/backward/streambuf.h
	OLD_FILES+=usr/include/c++/3.3/backward/strstream
	OLD_FILES+=usr/include/c++/3.3/backward/strstream.h
	OLD_FILES+=usr/include/c++/3.3/backward/tempbuf.h
	OLD_FILES+=usr/include/c++/3.3/backward/tree.h
	OLD_FILES+=usr/include/c++/3.3/backward/vector.h
	OLD_DIRS+=usr/include/c++/3.3/backward
	OLD_FILES+=usr/include/c++/3.3/bits/atomicity.h
	OLD_FILES+=usr/include/c++/3.3/bits/basic_file.h
	OLD_FILES+=usr/include/c++/3.3/bits/basic_ios.h
	OLD_FILES+=usr/include/c++/3.3/bits/basic_ios.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/basic_string.h
	OLD_FILES+=usr/include/c++/3.3/bits/basic_string.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/boost_concept_check.h
	OLD_FILES+=usr/include/c++/3.3/bits/c++config.h
	OLD_FILES+=usr/include/c++/3.3/bits/c++io.h
	OLD_FILES+=usr/include/c++/3.3/bits/c++locale.h
	OLD_FILES+=usr/include/c++/3.3/bits/c++locale_internal.h
	OLD_FILES+=usr/include/c++/3.3/bits/char_traits.h
	OLD_FILES+=usr/include/c++/3.3/bits/cmath.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/codecvt.h
	OLD_FILES+=usr/include/c++/3.3/bits/codecvt_specializations.h
	OLD_FILES+=usr/include/c++/3.3/bits/concept_check.h
	OLD_FILES+=usr/include/c++/3.3/bits/cpp_type_traits.h
	OLD_FILES+=usr/include/c++/3.3/bits/ctype_base.h
	OLD_FILES+=usr/include/c++/3.3/bits/ctype_inline.h
	OLD_FILES+=usr/include/c++/3.3/bits/ctype_noninline.h
	OLD_FILES+=usr/include/c++/3.3/bits/deque.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/fpos.h
	OLD_FILES+=usr/include/c++/3.3/bits/fstream.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/functexcept.h
	OLD_FILES+=usr/include/c++/3.3/bits/generic_shadow.h
	OLD_FILES+=usr/include/c++/3.3/bits/gslice.h
	OLD_FILES+=usr/include/c++/3.3/bits/gslice_array.h
	OLD_FILES+=usr/include/c++/3.3/bits/gthr-default.h
	OLD_FILES+=usr/include/c++/3.3/bits/gthr-posix.h
	OLD_FILES+=usr/include/c++/3.3/bits/gthr-single.h
	OLD_FILES+=usr/include/c++/3.3/bits/gthr.h
	OLD_FILES+=usr/include/c++/3.3/bits/indirect_array.h
	OLD_FILES+=usr/include/c++/3.3/bits/ios_base.h
	OLD_FILES+=usr/include/c++/3.3/bits/istream.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/list.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/locale_classes.h
	OLD_FILES+=usr/include/c++/3.3/bits/locale_facets.h
	OLD_FILES+=usr/include/c++/3.3/bits/locale_facets.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/localefwd.h
	OLD_FILES+=usr/include/c++/3.3/bits/mask_array.h
	OLD_FILES+=usr/include/c++/3.3/bits/messages_members.h
	OLD_FILES+=usr/include/c++/3.3/bits/os_defines.h
	OLD_FILES+=usr/include/c++/3.3/bits/ostream.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/pthread_allocimpl.h
	OLD_FILES+=usr/include/c++/3.3/bits/slice.h
	OLD_FILES+=usr/include/c++/3.3/bits/slice_array.h
	OLD_FILES+=usr/include/c++/3.3/bits/sstream.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/stl_algo.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_algobase.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_alloc.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_bvector.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_construct.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_deque.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_function.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_heap.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_iterator.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_iterator_base_funcs.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_iterator_base_types.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_list.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_map.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_multimap.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_multiset.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_numeric.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_pair.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_pthread_alloc.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_queue.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_raw_storage_iter.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_relops.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_set.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_stack.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_tempbuf.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_threads.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_tree.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_uninitialized.h
	OLD_FILES+=usr/include/c++/3.3/bits/stl_vector.h
	OLD_FILES+=usr/include/c++/3.3/bits/stream_iterator.h
	OLD_FILES+=usr/include/c++/3.3/bits/streambuf.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/streambuf_iterator.h
	OLD_FILES+=usr/include/c++/3.3/bits/stringfwd.h
	OLD_FILES+=usr/include/c++/3.3/bits/time_members.h
	OLD_FILES+=usr/include/c++/3.3/bits/type_traits.h
	OLD_FILES+=usr/include/c++/3.3/bits/valarray_array.h
	OLD_FILES+=usr/include/c++/3.3/bits/valarray_array.tcc
	OLD_FILES+=usr/include/c++/3.3/bits/valarray_meta.h
	OLD_FILES+=usr/include/c++/3.3/bits/vector.tcc
	OLD_DIRS+=usr/include/c++/3.3/bits
	OLD_FILES+=usr/include/c++/3.3/bitset
	OLD_FILES+=usr/include/c++/3.3/cassert
	OLD_FILES+=usr/include/c++/3.3/cctype
	OLD_FILES+=usr/include/c++/3.3/cerrno
	OLD_FILES+=usr/include/c++/3.3/cfloat
	OLD_FILES+=usr/include/c++/3.3/ciso646
	OLD_FILES+=usr/include/c++/3.3/climits
	OLD_FILES+=usr/include/c++/3.3/clocale
	OLD_FILES+=usr/include/c++/3.3/cmath
	OLD_FILES+=usr/include/c++/3.3/complex
	OLD_FILES+=usr/include/c++/3.3/csetjmp
	OLD_FILES+=usr/include/c++/3.3/csignal
	OLD_FILES+=usr/include/c++/3.3/cstdarg
	OLD_FILES+=usr/include/c++/3.3/cstddef
	OLD_FILES+=usr/include/c++/3.3/cstdio
	OLD_FILES+=usr/include/c++/3.3/cstdlib
	OLD_FILES+=usr/include/c++/3.3/cstring
	OLD_FILES+=usr/include/c++/3.3/ctime
	OLD_FILES+=usr/include/c++/3.3/cwchar
	OLD_FILES+=usr/include/c++/3.3/cwctype
	OLD_FILES+=usr/include/c++/3.3/cxxabi.h
	OLD_FILES+=usr/include/c++/3.3/deque
	OLD_FILES+=usr/include/c++/3.3/exception
	OLD_FILES+=usr/include/c++/3.3/exception_defines.h
	OLD_FILES+=usr/include/c++/3.3/ext/algorithm
	OLD_FILES+=usr/include/c++/3.3/ext/enc_filebuf.h
	OLD_FILES+=usr/include/c++/3.3/ext/functional
	OLD_FILES+=usr/include/c++/3.3/ext/hash_map
	OLD_FILES+=usr/include/c++/3.3/ext/hash_set
	OLD_FILES+=usr/include/c++/3.3/ext/iterator
	OLD_FILES+=usr/include/c++/3.3/ext/memory
	OLD_FILES+=usr/include/c++/3.3/ext/numeric
	OLD_FILES+=usr/include/c++/3.3/ext/rb_tree
	OLD_FILES+=usr/include/c++/3.3/ext/rope
	OLD_FILES+=usr/include/c++/3.3/ext/ropeimpl.h
	OLD_FILES+=usr/include/c++/3.3/ext/slist
	OLD_FILES+=usr/include/c++/3.3/ext/stdio_filebuf.h
	OLD_FILES+=usr/include/c++/3.3/ext/stl_hash_fun.h
	OLD_FILES+=usr/include/c++/3.3/ext/stl_hashtable.h
	OLD_FILES+=usr/include/c++/3.3/ext/stl_rope.h
	OLD_DIRS+=usr/include/c++/3.3/ext
	OLD_FILES+=usr/include/c++/3.3/fstream
	OLD_FILES+=usr/include/c++/3.3/functional
	OLD_FILES+=usr/include/c++/3.3/iomanip
	OLD_FILES+=usr/include/c++/3.3/ios
	OLD_FILES+=usr/include/c++/3.3/iosfwd
	OLD_FILES+=usr/include/c++/3.3/iostream
	OLD_FILES+=usr/include/c++/3.3/istream
	OLD_FILES+=usr/include/c++/3.3/iterator
	OLD_FILES+=usr/include/c++/3.3/limits
	OLD_FILES+=usr/include/c++/3.3/list
	OLD_FILES+=usr/include/c++/3.3/locale
	OLD_FILES+=usr/include/c++/3.3/map
	OLD_FILES+=usr/include/c++/3.3/memory
	OLD_FILES+=usr/include/c++/3.3/new
	OLD_FILES+=usr/include/c++/3.3/numeric
	OLD_FILES+=usr/include/c++/3.3/ostream
	OLD_FILES+=usr/include/c++/3.3/queue
	OLD_FILES+=usr/include/c++/3.3/set
	OLD_FILES+=usr/include/c++/3.3/sstream
	OLD_FILES+=usr/include/c++/3.3/stack
	OLD_FILES+=usr/include/c++/3.3/stdexcept
	OLD_FILES+=usr/include/c++/3.3/streambuf
	OLD_FILES+=usr/include/c++/3.3/string
	OLD_FILES+=usr/include/c++/3.3/typeinfo
	OLD_FILES+=usr/include/c++/3.3/utility
	OLD_FILES+=usr/include/c++/3.3/valarray
	OLD_FILES+=usr/include/c++/3.3/vector
	# 20040713: fla(4) removed.
	OLD_FILES+=usr/share/man/man4/fla.4.gz
	# 200407XX
	OLD_FILES+=usr/sbin/kernbb
	OLD_FILES+=usr/sbin/ntp-genkeys
	OLD_FILES+=usr/sbin/ntptimeset
	OLD_FILES+=usr/share/man/man8/kernbb.8.gz
	OLD_FILES+=usr/share/man/man8/ntp-genkeys.8.gz
	# 20040627: usbdevs.h and usbdevs_data.h removal
	OLD_FILES+=usr/include/dev/usb/usbdevs.h
	OLD_FILES+=usr/include/dev/usb/usbdevs_data.h
	# 200406XX
	OLD_FILES+=usr/bin/gasp
	OLD_FILES+=usr/bin/gdbreplay
	OLD_FILES+=usr/share/man/man1/gasp.1.gz
	OLD_FILES+=sbin/mountd
	OLD_FILES+=sbin/mount_fdesc
	OLD_FILES+=sbin/mount_umap
	OLD_FILES+=sbin/mount_union
	OLD_FILES+=sbin/mount_msdos
	OLD_FILES+=sbin/mount_null
	OLD_FILES+=sbin/mount_kernfs
	# 200405XX: arl
	OLD_FILES+=usr/sbin/arlconfig
	OLD_FILES+=usr/share/man/man8/arlconfig.8.gz
	# 200403XX
	OLD_FILES+=bin/raidctl
	OLD_FILES+=sbin/raidctl
	OLD_FILES+=usr/bin/sasc
	OLD_FILES+=usr/sbin/sgsc
	OLD_FILES+=usr/sbin/stlload
	OLD_FILES+=usr/sbin/stlstats
	OLD_FILES+=usr/share/man/man1/sasc.1.gz
	OLD_FILES+=usr/share/man/man1/sgsc.1.gz
	OLD_FILES+=usr/share/man/man4/i386/stl.4.gz
	OLD_FILES+=usr/share/man/man8/raidctl.8.gz
	# 20040229: clean_environment() was removed after 3 days
	OLD_FILES+=usr/share/man/man3/clean_environment.3.gz
	# 20040119: installed as `isdntel' in newer systems
	OLD_FILES+=etc/isdn/isdntel.sh
	# 200XYYZZ: /lib transition clitches
	OLD_FILES+=lib/libalias.so
	OLD_FILES+=lib/libatm.so
	OLD_FILES+=lib/libbsdxml.so
	OLD_FILES+=lib/libc.so
	OLD_FILES+=lib/libcam.so
	OLD_FILES+=lib/libcrypt.so
	OLD_FILES+=lib/libcrypto.so
	OLD_FILES+=lib/libdevstat.so
	OLD_FILES+=lib/libedit.so
	OLD_FILES+=lib/libgeom.so
	OLD_FILES+=lib/libipsec.so
	OLD_FILES+=lib/libipx.so
	OLD_FILES+=lib/libkvm.so
	OLD_FILES+=lib/libm.so
	OLD_FILES+=lib/libmd.so
	OLD_FILES+=lib/libncurses.so
	OLD_FILES+=lib/libreadline.so
	OLD_FILES+=lib/libsbuf.so
	OLD_FILES+=lib/libufs.so
	OLD_FILES+=lib/libz.so
	# 200312XX
	OLD_FILES+=bin/cxconfig
	OLD_FILES+=sbin/cxconfig
	OLD_FILES+=usr/share/man/man8/cxconfig.8.gz
	# 20031016: MULTI_DRIVER_MODULE macro removed
	OLD_FILES+=usr/share/man/man9/MULTI_DRIVER_MODULE.9.gz
	# 200309XX
	OLD_FILES+=usr/bin/symorder
	OLD_FILES+=usr/share/man/man1/symorder.1.gz
	# 200308XX
	OLD_FILES+=usr/sbin/amldb
	OLD_FILES+=usr/share/man/man8/amldb.8.gz
	# 200307XX
	OLD_FILES+=sbin/mount_nwfs
	OLD_FILES+=sbin/mount_portalfs
	OLD_FILES+=sbin/mount_smbfs
	# 200306XX
	OLD_FILES+=usr/sbin/dev_mkdb
	OLD_FILES+=usr/share/man/man8/dev_mkdb.8.gz
	# 200304XX
	OLD_FILES+=usr/lib/libcipher.a
	OLD_FILES+=usr/lib/libcipher.so
	OLD_FILES+=usr/lib/libcipher_p.a
	OLD_FILES+=usr/lib/libgmp.a
	OLD_FILES+=usr/lib/libgmp.so
	OLD_FILES+=usr/lib/libgmp_p.a
	OLD_FILES+=usr/lib/libperl.a
	OLD_FILES+=usr/lib/libperl.so
	OLD_FILES+=usr/lib/libperl_p.a
	OLD_FILES+=usr/lib/libposix1e.a
	OLD_FILES+=usr/lib/libposix1e.so
	OLD_FILES+=usr/lib/libposix1e_p.a
	OLD_FILES+=usr/lib/libskey.a
	OLD_FILES+=usr/lib/libskey.so
	OLD_FILES+=usr/lib/libskey_p.a
	OLD_FILES+=usr/libexec/tradcpp0
	OLD_FILES+=usr/libexec/cpp0
	# 200304XX: removal of xten
	OLD_FILES+=usr/libexec/xtend
	OLD_FILES+=usr/sbin/xten
	OLD_FILES+=usr/share/man/man1/xten.1.gz
	OLD_FILES+=usr/share/man/man8/xtend.8.gz
	# 200303XX
	OLD_FILES+=usr/lib/libacl.so
	OLD_FILES+=usr/lib/libdescrypt.so
	OLD_FILES+=usr/lib/libf2c.so
	OLD_FILES+=usr/lib/libg++.so
	OLD_FILES+=usr/lib/libkdb.so
	OLD_FILES+=usr/lib/librsaINTL.so
	OLD_FILES+=usr/lib/libscrypt.so
	OLD_FILES+=usr/lib/libss.so
	# 200302XX
	OLD_FILES+=usr/lib/libacl.a
	OLD_FILES+=usr/lib/libacl_p.a
	OLD_FILES+=usr/lib/libkadm.a
	OLD_FILES+=usr/lib/libkadm.so
	OLD_FILES+=usr/lib/libkadm_p.a
	OLD_FILES+=usr/lib/libkafs.a
	OLD_FILES+=usr/lib/libkafs.so
	OLD_FILES+=usr/lib/libkafs_p.a
	OLD_FILES+=usr/lib/libkdb.a
	OLD_FILES+=usr/lib/libkdb_p.a
	OLD_FILES+=usr/lib/libkrb.a
	OLD_FILES+=usr/lib/libkrb.so
	OLD_FILES+=usr/lib/libkrb_p.a
	OLD_FILES+=usr/share/man/man3/SSL_CIPHER_get_name.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_COMP_add_compression_method.3
	OLD_FILES+=usr/share/man/man3/SSL_CTX_add_extra_chain_cert.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_add_session.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_ctrl.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_flush_sessions.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_free.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_get_verify_mode.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_load_verify_locations.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_new.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_sess_number.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_sess_set_cache_size.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_sess_set_get_cb.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_sessions.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_cert_store.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_cert_verify_callback.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_cipher_list.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_client_CA_list.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_client_cert_cb.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_default_passwd_cb.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_generate_session_id.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_info_callback.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_max_cert_list.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_mode.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_msg_callback.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_options.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_quiet_shutdown.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_session_cache_mode.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_session_id_context.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_ssl_version.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_timeout.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_tmp_dh_callback.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_tmp_rsa_callback.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_set_verify.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_use_certificate.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_SESSION_free.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_SESSION_get_ex_new_index.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_SESSION_get_time.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_accept.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_alert_type_string.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_clear.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_connect.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_do_handshake.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_free.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_SSL_CTX.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_ciphers.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_client_CA_list.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_current_cipher.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_default_timeout.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_error.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_ex_data_X509_STORE_CTX_idx.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_ex_new_index.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_fd.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_peer_cert_chain.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_peer_certificate.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_rbio.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_session.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_verify_result.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_get_version.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_library_init.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_load_client_CA_file.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_new.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_pending.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_read.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_rstate_string.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_session_reused.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_set_bio.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_set_connect_state.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_set_fd.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_set_session.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_set_shutdown.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_set_verify_result.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_shutdown.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_state_string.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_want.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_write.3.gz
	OLD_FILES+=usr/share/man/man3/d2i_SSL_SESSION.3.gz
	# 200301XX
	OLD_FILES+=usr/share/man/man3/des_3cbc_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_3ecb_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_cbc_cksum.3.gz
	OLD_FILES+=usr/share/man/man3/des_cbc_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_cfb_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_ecb_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_enc_read.3.gz
	OLD_FILES+=usr/share/man/man3/des_enc_write.3.gz
	OLD_FILES+=usr/share/man/man3/des_is_weak_key.3.gz
	OLD_FILES+=usr/share/man/man3/des_key_sched.3.gz
	OLD_FILES+=usr/share/man/man3/des_ofb_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_pcbc_encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/des_quad_cksum.3.gz
	OLD_FILES+=usr/share/man/man3/des_random_key.3.gz
	OLD_FILES+=usr/share/man/man3/des_read_2password.3.gz
	OLD_FILES+=usr/share/man/man3/des_read_password.3.gz
	OLD_FILES+=usr/share/man/man3/des_read_pw_string.3.gz
	OLD_FILES+=usr/share/man/man3/des_set_key.3.gz
	OLD_FILES+=usr/share/man/man3/des_set_odd_parity.3.gz
	OLD_FILES+=usr/share/man/man3/des_string_to_2key.3.gz
	OLD_FILES+=usr/share/man/man3/des_string_to_key.3.gz
	# 200212XX
	OLD_FILES+=usr/sbin/kenv
	OLD_FILES+=usr/bin/kenv
	OLD_FILES+=usr/sbin/elf2aout
	# 200210XX
	OLD_FILES+=usr/include/libusbhid.h
	OLD_FILES+=usr/share/man/man3/All_FreeBSD.3.gz
	OLD_FILES+=usr/share/man/man3/CheckRules.3.gz
	OLD_FILES+=usr/share/man/man3/ChunkCanBeRoot.3.gz
	OLD_FILES+=usr/share/man/man3/Clone_Disk.3.gz
	OLD_FILES+=usr/share/man/man3/Collapse_Chunk.3.gz
	OLD_FILES+=usr/share/man/man3/Collapse_Disk.3.gz
	OLD_FILES+=usr/share/man/man3/Create_Chunk.3.gz
	OLD_FILES+=usr/share/man/man3/Create_Chunk_DWIM.3.gz
	OLD_FILES+=usr/share/man/man3/Cyl_Aligned.3.gz
	OLD_FILES+=usr/share/man/man3/Debug_Disk.3.gz
	OLD_FILES+=usr/share/man/man3/Delete_Chunk.3.gz
	OLD_FILES+=usr/share/man/man3/Disk_Names.3.gz
	OLD_FILES+=usr/share/man/man3/Free_Disk.3.gz
	OLD_FILES+=usr/share/man/man3/MakeDev.3.gz
	OLD_FILES+=usr/share/man/man3/MakeDevDisk.3.gz
	OLD_FILES+=usr/share/man/man3/Next_Cyl_Aligned.3.gz
	OLD_FILES+=usr/share/man/man3/Next_Track_Aligned.3.gz
	OLD_FILES+=usr/share/man/man3/Open_Disk.3.gz
	OLD_FILES+=usr/share/man/man3/Prev_Cyl_Aligned.3.gz
	OLD_FILES+=usr/share/man/man3/Prev_Track_Aligned.3.gz
	OLD_FILES+=usr/share/man/man3/Set_Bios_Geom.3.gz
	OLD_FILES+=usr/share/man/man3/Set_Boot_Blocks.3.gz
	OLD_FILES+=usr/share/man/man3/Set_Boot_Mgr.3.gz
	OLD_FILES+=usr/share/man/man3/ShowChunkFlags.3.gz
	OLD_FILES+=usr/share/man/man3/Track_Aligned.3.gz
	OLD_FILES+=usr/share/man/man3/Write_Disk.3.gz
	OLD_FILES+=usr/share/man/man3/slice_type_name.3.gz
	# 200210XX: most games moved to ports
	OLD_FILES+=usr/share/man/man6/adventure.6.gz
	OLD_FILES+=usr/share/man/man6/arithmetic.6.gz
	OLD_FILES+=usr/share/man/man6/atc.6.gz
	OLD_FILES+=usr/share/man/man6/backgammon.6.gz
	OLD_FILES+=usr/share/man/man6/battlestar.6.gz
	OLD_FILES+=usr/share/man/man6/bs.6.gz
	OLD_FILES+=usr/share/man/man6/canfield.6.gz
	OLD_FILES+=usr/share/man/man6/cfscores.6.gz
	OLD_FILES+=usr/share/man/man6/cribbage.6.gz
	OLD_FILES+=usr/share/man/man6/fish.6.gz
	OLD_FILES+=usr/share/man/man6/hack.6.gz
	OLD_FILES+=usr/share/man/man6/hangman.6.gz
	OLD_FILES+=usr/share/man/man6/larn.6.gz
	OLD_FILES+=usr/share/man/man6/mille.6.gz
	OLD_FILES+=usr/share/man/man6/phantasia.6.gz
	OLD_FILES+=usr/share/man/man6/piano.6.gz
	OLD_FILES+=usr/share/man/man6/pig.6.gz
	OLD_FILES+=usr/share/man/man6/quiz.6.gz
	OLD_FILES+=usr/share/man/man6/rain.6.gz
	OLD_FILES+=usr/share/man/man6/robots.6.gz
	OLD_FILES+=usr/share/man/man6/rogue.6.gz
	OLD_FILES+=usr/share/man/man6/sail.6.gz
	OLD_FILES+=usr/share/man/man6/snake.6.gz
	OLD_FILES+=usr/share/man/man6/snscore.6.gz
	OLD_FILES+=usr/share/man/man6/trek.6.gz
	OLD_FILES+=usr/share/man/man6/wargames.6.gz
	OLD_FILES+=usr/share/man/man6/worm.6.gz
	OLD_FILES+=usr/share/man/man6/worms.6.gz
	OLD_FILES+=usr/share/man/man6/wump.6.gz
	# 200207XX
	OLD_FILES+=usr/share/man/man1aout/ar.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/as.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/ld.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/nm.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/ranlib.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/size.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/strings.1aout.gz
	OLD_FILES+=usr/share/man/man1aout/strip.1aout.gz
	OLD_FILES+=bin/mountd
	OLD_FILES+=bin/nfsd
	# 20020707 sbin/nfsd -> usr.sbin/nfsd
	OLD_FILES+=sbin/nfsd
	# 200206XX
	OLD_FILES+=usr/lib/libpam_ssh.a
	OLD_FILES+=usr/lib/libpam_ssh_p.a
	OLD_FILES+=usr/bin/help
	OLD_FILES+=usr/bin/sccs
	.if ${TARGET_ARCH} != "amd64" && ${TARGET} != "arm" && ${TARGET_ARCH} != "i386" && ${TARGET} != "powerpc"
	OLD_FILES+=usr/bin/gdbserver
	.endif
	OLD_FILES+=usr/bin/ssh-keysign
	OLD_FILES+=usr/sbin/gifconfig
	OLD_FILES+=usr/sbin/prefix
	# 200205XX
	OLD_FILES+=usr/bin/doscmd
	# 200204XX
	OLD_FILES+=usr/bin/a2p
	OLD_FILES+=usr/bin/ptx
	OLD_FILES+=usr/bin/pod2text
	OLD_FILES+=usr/bin/pod2man
	OLD_FILES+=usr/bin/pod2latex
	OLD_FILES+=usr/bin/pod2html
	OLD_FILES+=usr/bin/h2ph
	OLD_FILES+=usr/bin/dprofpp
	OLD_FILES+=usr/bin/c2ph
	OLD_FILES+=usr/bin/h2xs
	OLD_FILES+=usr/bin/pl2pm
	OLD_FILES+=usr/bin/splain
	OLD_FILES+=usr/bin/s2p
	OLD_FILES+=usr/bin/find2perl
	OLD_FILES+=usr/sbin/pkg_update
	OLD_FILES+=usr/sbin/scriptdump
	# 20020409 GC kget(1), userconfig is long dead.
	OLD_FILES+=sbin/kget
	OLD_FILES+=usr/share/man/man8/kget.8.gz
	# 200203XX
	OLD_FILES+=usr/lib/libss.a
	OLD_FILES+=usr/lib/libss_p.a
	OLD_FILES+=usr/lib/libtelnet.a
	OLD_FILES+=usr/lib/libtelnet_p.a
	OLD_FILES+=usr/sbin/diskpart
	# 200202XX
	OLD_FILES+=usr/bin/gprof4
	# 200201XX
	OLD_FILES+=usr/sbin/linux
	# 2001XXXX
	OLD_FILES+=usr/bin/joy
	OLD_FILES+=usr/sbin/ibcs2
	OLD_FILES+=usr/sbin/svr4
	OLD_FILES+=usr/bin/chflags
	OLD_FILES+=usr/sbin/uuconv
	OLD_FILES+=usr/sbin/uuchk
	OLD_FILES+=usr/sbin/portmap
	OLD_FILES+=usr/sbin/pmap_set
	OLD_FILES+=usr/sbin/pmap_dump
	OLD_FILES+=usr/sbin/mcon
	OLD_FILES+=usr/sbin/stlstty
	OLD_FILES+=usr/sbin/ispppcontrol
	OLD_FILES+=usr/sbin/rndcontrol
	# 20011001: UUCP migration to ports
	OLD_FILES+=usr/bin/uucp
	OLD_FILES+=usr/bin/uulog
	OLD_FILES+=usr/bin/uuname
	OLD_FILES+=usr/bin/uupick
	OLD_FILES+=usr/bin/uusched
	OLD_FILES+=usr/bin/uustat
	OLD_FILES+=usr/bin/uuto
	OLD_FILES+=usr/bin/uux
	OLD_FILES+=usr/libexec/uucp/uucico
	OLD_FILES+=usr/libexec/uucp/uuxqt
	OLD_FILES+=usr/libexec/uucpd
	OLD_FILES+=usr/share/man/man1/uuconv.1.gz
	OLD_FILES+=usr/share/man/man1/uucp.1.gz
	OLD_FILES+=usr/share/man/man1/uulog.1.gz
	OLD_FILES+=usr/share/man/man1/uuname.1.gz
	OLD_FILES+=usr/share/man/man1/uupick.1.gz
	OLD_FILES+=usr/share/man/man1/uustat.1.gz
	OLD_FILES+=usr/share/man/man1/uuto.1.gz
	OLD_FILES+=usr/share/man/man1/uux.1.gz
	OLD_FILES+=usr/share/man/man8/uuchk.8.gz
	OLD_FILES+=usr/share/man/man8/uucico.8.gz
	OLD_FILES+=usr/share/man/man8/uucpd.8.gz
	OLD_FILES+=usr/share/man/man8/uusched.8.gz
	OLD_FILES+=usr/share/man/man8/uuxqt.8.gz
	# 20010523 mount_portal -> mount_portalfs
	OLD_FILES+=sbin/mount_portal
	OLD_FILES+=usr/share/man/man8/mount_portal.8.gz
	# 200104XX
	OLD_FILES+=usr/lib/libdescrypt.a
	OLD_FILES+=usr/lib/libscrypt.a
	OLD_FILES+=usr/lib/libscrypt_p.a
	OLD_FILES+=usr/sbin/pim6stat
	OLD_FILES+=usr/sbin/pim6sd
	OLD_FILES+=usr/sbin/pim6dd
	# 20010217
	OLD_FILES+=usr/share/doc/bind/misc/dns-setup
	# 20001200
	OLD_FILES+=usr/lib/libgcc_r_pic.a
	# 200009XX
	OLD_FILES+=usr/lib/libRSAglue.a
	OLD_FILES+=usr/lib/libRSAglue.so
	OLD_FILES+=usr/lib/librsaINTL.a
	OLD_FILES+=usr/lib/librsaUSA.a
	OLD_FILES+=usr/lib/librsaUSA.so
	# 200002XX ?
	OLD_FILES+=usr/lib/libf2c.a
	OLD_FILES+=usr/lib/libf2c_p.a
	OLD_FILES+=usr/lib/libg++.a
	OLD_FILES+=usr/lib/libg++_p.a
	# 20001006
	OLD_FILES+=usr/bin/miniperl
	# 20000810
	OLD_FILES+=usr/bin/sperl
	# 200001XX
	OLD_FILES+=usr/sbin/apmconf
	## unsorted
	# do we still support aout builds?
	#OLD_FILES+=usr/lib/aout/c++rt0.o
	#OLD_FILES+=usr/lib/aout/crt0.o
	#OLD_FILES+=usr/lib/aout/gcrt0.o
	#OLD_FILES+=usr/lib/aout/scrt0.o
	#OLD_FILES+=usr/lib/aout/sgcrt0.o
	OLD_FILES+=usr/lib/pam_ftp.so
	OLD_FILES+=usr/share/man/man1/CA.pl.1.gz
	OLD_FILES+=usr/share/man/man1/asn1parse.1.gz
	OLD_FILES+=usr/share/man/man1/ca.1.gz
	OLD_FILES+=usr/share/man/man1/ciphers.1.gz
	OLD_FILES+=usr/share/man/man1/config.1.gz
	OLD_FILES+=usr/share/man/man1/crl.1.gz
	OLD_FILES+=usr/share/man/man1/crl2pkcs7.1.gz
	OLD_FILES+=usr/share/man/man1/dgst.1.gz
	OLD_FILES+=usr/share/man/man1/dhparam.1.gz
	OLD_FILES+=usr/share/man/man1/doscmd.1.gz
	OLD_FILES+=usr/share/man/man1/dsa.1.gz
	OLD_FILES+=usr/share/man/man1/dsaparam.1.gz
	OLD_FILES+=usr/share/man/man1/enc.1.gz
	OLD_FILES+=usr/share/man/man1/gendsa.1.gz
	OLD_FILES+=usr/share/man/man1/genrsa.1.gz
	OLD_FILES+=usr/share/man/man1/getNAME.1.gz
	OLD_FILES+=usr/share/man/man1/nseq.1.gz
	OLD_FILES+=usr/share/man/man1/ocsp.1.gz
	OLD_FILES+=usr/share/man/man1/openssl.1.gz
	OLD_FILES+=usr/share/man/man1/pkcs12.1.gz
	OLD_FILES+=usr/share/man/man1/pkcs7.1.gz
	OLD_FILES+=usr/share/man/man1/pkcs8.1.gz
	OLD_FILES+=usr/share/man/man1/rand.1.gz
	OLD_FILES+=usr/share/man/man1/req.1.gz
	OLD_FILES+=usr/share/man/man1/rsa.1.gz
	OLD_FILES+=usr/share/man/man1/rsautl.1.gz
	OLD_FILES+=usr/share/man/man1/s_client.1.gz
	OLD_FILES+=usr/share/man/man1/s_server.1.gz
	OLD_FILES+=usr/share/man/man1/sess_id.1.gz
	OLD_FILES+=usr/share/man/man1/smime.1.gz
	OLD_FILES+=usr/share/man/man1/speed.1.gz
	OLD_FILES+=usr/share/man/man1/spkac.1.gz
	OLD_FILES+=usr/share/man/man1/verify.1.gz
	OLD_FILES+=usr/share/man/man1/version.1.gz
	OLD_FILES+=usr/share/man/man1/x509.1.gz
	OLD_FILES+=usr/share/man/man3/SSL_COMP_add_compression_method.3.gz
	OLD_FILES+=usr/share/man/man3/SSL_CTX_get_ex_new_index.3.gz
	OLD_FILES+=usr/share/man/man3/archive_entry_dup.3.gz
	OLD_FILES+=usr/share/man/man3/archive_entry_set_tartype.3.gz
	OLD_FILES+=usr/share/man/man3/archive_entry_tartype.3.gz
	OLD_FILES+=usr/share/man/man3/archive_read_data_into_file.3.gz
	OLD_FILES+=usr/share/man/man3/archive_read_open_tar.3.gz
	OLD_FILES+=usr/share/man/man3/archive_read_support_format_gnutar.3.gz
	OLD_FILES+=usr/share/man/man3/cipher.3.gz
	OLD_FILES+=usr/share/man/man3/des_cipher.3.gz
	OLD_FILES+=usr/share/man/man3/des_setkey.3.gz
	OLD_FILES+=usr/share/man/man3/encrypt.3.gz
	OLD_FILES+=usr/share/man/man3/endvfsent.3.gz
	OLD_FILES+=usr/share/man/man3/getvfsbytype.3.gz
	OLD_FILES+=usr/share/man/man3/getvfsent.3.gz
	OLD_FILES+=usr/share/man/man3/isnanf.3.gz
	OLD_FILES+=usr/share/man/man3/libautofs.3.gz
	OLD_FILES+=usr/share/man/man3/pthread_attr_setsstack.3.gz
	OLD_FILES+=usr/share/man/man3/pthread_getcancelstate.3.gz
	OLD_FILES+=usr/share/man/man3/pthread_mutexattr_getpshared.3.gz
	OLD_FILES+=usr/share/man/man3/pthread_mutexattr_setpshared.3.gz
	OLD_FILES+=usr/share/man/man3/set_assertion_failure_callback.3.gz
	OLD_FILES+=usr/share/man/man3/setkey.3.gz
	OLD_FILES+=usr/share/man/man3/setvfsent.3.gz
	OLD_FILES+=usr/share/man/man3/ssl.3.gz
	OLD_FILES+=usr/share/man/man3/vfsisloadable.3.gz
	OLD_FILES+=usr/share/man/man3/vfsload.3.gz
	OLD_FILES+=usr/share/man/man4/als4000.4.gz
	OLD_FILES+=usr/share/man/man4/csa.4.gz
	OLD_FILES+=usr/share/man/man4/emu10k1.4.gz
	OLD_FILES+=usr/share/man/man4/euc.4.gz
	OLD_FILES+=usr/share/man/man4/gusc.4.gz
	OLD_FILES+=usr/share/man/man4/if_fwp.4.gz
	OLD_FILES+=usr/share/man/man4/lomac.4.gz
	OLD_FILES+=usr/share/man/man4/maestro3.4.gz
	OLD_FILES+=usr/share/man/man4/raid.4.gz
	OLD_FILES+=usr/share/man/man4/sbc.4.gz
	OLD_FILES+=usr/share/man/man4/sd.4.gz
	OLD_FILES+=usr/share/man/man4/snc.4.gz
	OLD_FILES+=usr/share/man/man4/st.4.gz
	OLD_FILES+=usr/share/man/man4/uaudio.4.gz
	OLD_FILES+=usr/share/man/man4/utf2.4.gz
	OLD_FILES+=usr/share/man/man4/vinumdebug.4.gz
	OLD_FILES+=usr/share/man/man5/disklabel.5.gz
	OLD_FILES+=usr/share/man/man5/dm.conf.5.gz
	OLD_FILES+=usr/share/man/man5/ranlib.5.gz
	OLD_FILES+=usr/share/man/man5/utf2.5.gz
	OLD_FILES+=usr/share/man/man7/groff_mwww.7.gz
	OLD_FILES+=usr/share/man/man7/mmroff.7.gz
	OLD_FILES+=usr/share/man/man7/mwww.7.gz
	OLD_FILES+=usr/share/man/man8/apm.8.gz
	OLD_FILES+=usr/share/man/man8/apmconf.8.gz
	OLD_FILES+=usr/share/man/man8/apmd.8.gz
	OLD_FILES+=usr/share/man/man8/dm.8.gz
	OLD_FILES+=usr/share/man/man8/pam_ftp.8.gz
	OLD_FILES+=usr/share/man/man8/pam_wheel.8.gz
	OLD_FILES+=usr/share/man/man8/sconfig.8.gz
	OLD_FILES+=usr/share/man/man8/ssl.8.gz
	OLD_FILES+=usr/share/man/man8/wlconfig.8.gz
	OLD_FILES+=usr/share/man/man9/CURSIG.9.gz
	OLD_FILES+=usr/share/man/man9/VFS_INIT.9.gz
	OLD_FILES+=usr/share/man/man9/at_exit.9.gz
	OLD_FILES+=usr/share/man/man9/at_fork.9.gz
	OLD_FILES+=usr/share/man/man9/cdevsw_add.9.gz
	OLD_FILES+=usr/share/man/man9/cdevsw_remove.9.gz
	OLD_FILES+=usr/share/man/man9/cv_waitq_empty.9.gz
	OLD_FILES+=usr/share/man/man9/cv_waitq_remove.9.gz
	OLD_FILES+=usr/share/man/man9/endtsleep.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo_freem.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo_pg_alloc.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo_pg_free.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo_pg_steal.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo_phys_to_kva.9.gz
	OLD_FILES+=usr/share/man/man9/jumbo_vm_init.9.gz
	OLD_FILES+=usr/share/man/man9/mac_biba.9.gz
	OLD_FILES+=usr/share/man/man9/mac_bsdextended.9.gz
	OLD_FILES+=usr/share/man/man9/mono_time.9.gz
	OLD_FILES+=usr/share/man/man9/p1003_1b.9.gz
	OLD_FILES+=usr/share/man/man9/pmap_prefault.9.gz
	OLD_FILES+=usr/share/man/man9/posix4.9.gz
	OLD_FILES+=usr/share/man/man9/resource_query_name.9.gz
	OLD_FILES+=usr/share/man/man9/resource_query_string.9.gz
	OLD_FILES+=usr/share/man/man9/resource_query_unit.9.gz
	OLD_FILES+=usr/share/man/man9/rm_at_exit.9.gz
	OLD_FILES+=usr/share/man/man9/rm_at_fork.9.gz
	OLD_FILES+=usr/share/man/man9/runtime.9.gz
	OLD_FILES+=usr/share/man/man9/sleepinit.9.gz
	OLD_FILES+=usr/share/man/man9/unsleep.9.gz
	OLD_FILES+=usr/share/games/atc/Game_List
	OLD_FILES+=usr/share/games/atc/Killer
	OLD_FILES+=usr/share/games/atc/crossover
	OLD_FILES+=usr/share/games/atc/default
	OLD_FILES+=usr/share/games/atc/easy
	OLD_FILES+=usr/share/games/atc/game_2
	OLD_FILES+=usr/share/games/larn/larnmaze
	OLD_FILES+=usr/share/games/larn/larnopts
	OLD_FILES+=usr/share/games/larn/larn.help
	OLD_FILES+=usr/share/games/quiz.db/africa
	OLD_FILES+=usr/share/games/quiz.db/america
	OLD_FILES+=usr/share/games/quiz.db/areas
	OLD_FILES+=usr/share/games/quiz.db/arith
	OLD_FILES+=usr/share/games/quiz.db/asia
	OLD_FILES+=usr/share/games/quiz.db/babies
	OLD_FILES+=usr/share/games/quiz.db/bard
	OLD_FILES+=usr/share/games/quiz.db/chinese
	OLD_FILES+=usr/share/games/quiz.db/collectives
	OLD_FILES+=usr/share/games/quiz.db/ed
	OLD_FILES+=usr/share/games/quiz.db/elements
	OLD_FILES+=usr/share/games/quiz.db/europe
	OLD_FILES+=usr/share/games/quiz.db/flowers
	OLD_FILES+=usr/share/games/quiz.db/greek
	OLD_FILES+=usr/share/games/quiz.db/inca
	OLD_FILES+=usr/share/games/quiz.db/index
	OLD_FILES+=usr/share/games/quiz.db/latin
	OLD_FILES+=usr/share/games/quiz.db/locomotive
	OLD_FILES+=usr/share/games/quiz.db/midearth
	OLD_FILES+=usr/share/games/quiz.db/morse
	OLD_FILES+=usr/share/games/quiz.db/murders
	OLD_FILES+=usr/share/games/quiz.db/poetry
	OLD_FILES+=usr/share/games/quiz.db/posneg
	OLD_FILES+=usr/share/games/quiz.db/pres
	OLD_FILES+=usr/share/games/quiz.db/province
	OLD_FILES+=usr/share/games/quiz.db/seq-easy
	OLD_FILES+=usr/share/games/quiz.db/seq-hard
	OLD_FILES+=usr/share/games/quiz.db/sexes
	OLD_FILES+=usr/share/games/quiz.db/sov
	OLD_FILES+=usr/share/games/quiz.db/spell
	OLD_FILES+=usr/share/games/quiz.db/state
	OLD_FILES+=usr/share/games/quiz.db/trek
	OLD_FILES+=usr/share/games/quiz.db/ucc
	OLD_FILES+=usr/share/games/cribbage.instr
	OLD_FILES+=usr/share/games/fish.instr
	OLD_FILES+=usr/share/games/wump.info
	OLD_FILES+=usr/games/hide/adventure
	OLD_FILES+=usr/games/hide/arithmetic
	OLD_FILES+=usr/games/hide/atc
	OLD_FILES+=usr/games/hide/backgammon
	OLD_FILES+=usr/games/hide/teachgammon
	OLD_FILES+=usr/games/hide/battlestar
	OLD_FILES+=usr/games/hide/bs
	OLD_FILES+=usr/games/hide/canfield
	OLD_FILES+=usr/games/hide/cribbage
	OLD_FILES+=usr/games/hide/fish
	OLD_FILES+=usr/games/hide/hack
	OLD_FILES+=usr/games/hide/hangman
	OLD_FILES+=usr/games/hide/larn
	OLD_FILES+=usr/games/hide/mille
	OLD_FILES+=usr/games/hide/phantasia
	OLD_FILES+=usr/games/hide/quiz
	OLD_FILES+=usr/games/hide/robots
	OLD_FILES+=usr/games/hide/rogue
	OLD_FILES+=usr/games/hide/sail
	OLD_FILES+=usr/games/hide/snake
	OLD_FILES+=usr/games/hide/trek
	OLD_FILES+=usr/games/hide/worm
	OLD_FILES+=usr/games/hide/wump
	OLD_FILES+=usr/games/adventure
	OLD_FILES+=usr/games/arithmetic
	OLD_FILES+=usr/games/atc
	OLD_FILES+=usr/games/backgammon
	OLD_FILES+=usr/games/teachgammon
	OLD_FILES+=usr/games/battlestar
	OLD_FILES+=usr/games/bs
	OLD_FILES+=usr/games/canfield
	OLD_FILES+=usr/games/cfscores
	OLD_FILES+=usr/games/cribbage
	OLD_FILES+=usr/games/dm
	OLD_FILES+=usr/games/fish
	OLD_FILES+=usr/games/hack
	OLD_FILES+=usr/games/hangman
	OLD_FILES+=usr/games/larn
	OLD_FILES+=usr/games/mille
	OLD_FILES+=usr/games/phantasia
	OLD_FILES+=usr/games/piano
	OLD_FILES+=usr/games/pig
	OLD_FILES+=usr/games/quiz
	OLD_FILES+=usr/games/rain
	OLD_FILES+=usr/games/robots
	OLD_FILES+=usr/games/rogue
	OLD_FILES+=usr/games/sail
	OLD_FILES+=usr/games/snake
	OLD_FILES+=usr/games/snscore
	OLD_FILES+=usr/games/trek
	OLD_FILES+=usr/games/wargames
	OLD_FILES+=usr/games/worm
	OLD_FILES+=usr/games/worms
	OLD_FILES+=usr/games/wump
	OLD_FILES+=sbin/mount_reiserfs
	OLD_FILES+=usr/include/cam/cam_extend.h
	OLD_FILES+=usr/include/dev/wi/wi_hostap.h
	OLD_FILES+=usr/include/disktab.h
	OLD_FILES+=usr/include/g++/FlexLexer.h
	OLD_FILES+=usr/include/g++/PlotFile.h
	OLD_FILES+=usr/include/g++/SFile.h
	OLD_FILES+=usr/include/g++/_G_config.h
	OLD_FILES+=usr/include/g++/algo.h
	OLD_FILES+=usr/include/g++/algobase.h
	OLD_FILES+=usr/include/g++/algorithm
	OLD_FILES+=usr/include/g++/alloc.h
	OLD_FILES+=usr/include/g++/bitset
	OLD_FILES+=usr/include/g++/builtinbuf.h
	OLD_FILES+=usr/include/g++/bvector.h
	OLD_FILES+=usr/include/g++/cassert
	OLD_FILES+=usr/include/g++/cctype
	OLD_FILES+=usr/include/g++/cerrno
	OLD_FILES+=usr/include/g++/cfloat
	OLD_FILES+=usr/include/g++/ciso646
	OLD_FILES+=usr/include/g++/climits
	OLD_FILES+=usr/include/g++/clocale
	OLD_FILES+=usr/include/g++/cmath
	OLD_FILES+=usr/include/g++/complex
	OLD_FILES+=usr/include/g++/complex.h
	OLD_FILES+=usr/include/g++/csetjmp
	OLD_FILES+=usr/include/g++/csignal
	OLD_FILES+=usr/include/g++/cstdarg
	OLD_FILES+=usr/include/g++/cstddef
	OLD_FILES+=usr/include/g++/cstdio
	OLD_FILES+=usr/include/g++/cstdlib
	OLD_FILES+=usr/include/g++/cstring
	OLD_FILES+=usr/include/g++/ctime
	OLD_FILES+=usr/include/g++/cwchar
	OLD_FILES+=usr/include/g++/cwctype
	OLD_FILES+=usr/include/g++/defalloc.h
	OLD_FILES+=usr/include/g++/deque
	OLD_FILES+=usr/include/g++/deque.h
	OLD_FILES+=usr/include/g++/editbuf.h
	OLD_FILES+=usr/include/g++/exception
	OLD_FILES+=usr/include/g++/floatio.h
	OLD_FILES+=usr/include/g++/fstream
	OLD_FILES+=usr/include/g++/fstream.h
	OLD_FILES+=usr/include/g++/function.h
	OLD_FILES+=usr/include/g++/functional
	OLD_FILES+=usr/include/g++/hash_map
	OLD_FILES+=usr/include/g++/hash_map.h
	OLD_FILES+=usr/include/g++/hash_set
	OLD_FILES+=usr/include/g++/hash_set.h
	OLD_FILES+=usr/include/g++/hashtable.h
	OLD_FILES+=usr/include/g++/heap.h
	OLD_FILES+=usr/include/g++/indstream.h
	OLD_FILES+=usr/include/g++/iolibio.h
	OLD_FILES+=usr/include/g++/iomanip
	OLD_FILES+=usr/include/g++/iomanip.h
	OLD_FILES+=usr/include/g++/iosfwd
	OLD_FILES+=usr/include/g++/iostdio.h
	OLD_FILES+=usr/include/g++/iostream
	OLD_FILES+=usr/include/g++/iostream.h
	OLD_FILES+=usr/include/g++/iostreamP.h
	OLD_FILES+=usr/include/g++/istream.h
	OLD_FILES+=usr/include/g++/iterator
	OLD_FILES+=usr/include/g++/iterator.h
	OLD_FILES+=usr/include/g++/libio.h
	OLD_FILES+=usr/include/g++/libioP.h
	OLD_FILES+=usr/include/g++/list
	OLD_FILES+=usr/include/g++/list.h
	OLD_FILES+=usr/include/g++/map
	OLD_FILES+=usr/include/g++/map.h
	OLD_FILES+=usr/include/g++/memory
	OLD_FILES+=usr/include/g++/multimap.h
	OLD_FILES+=usr/include/g++/multiset.h
	OLD_FILES+=usr/include/g++/new
	OLD_FILES+=usr/include/g++/new.h
	OLD_FILES+=usr/include/g++/numeric
	OLD_FILES+=usr/include/g++/ostream.h
	OLD_FILES+=usr/include/g++/pair.h
	OLD_FILES+=usr/include/g++/parsestream.h
	OLD_FILES+=usr/include/g++/pfstream.h
	OLD_FILES+=usr/include/g++/procbuf.h
	OLD_FILES+=usr/include/g++/pthread_alloc
	OLD_FILES+=usr/include/g++/pthread_alloc.h
	OLD_FILES+=usr/include/g++/queue
	OLD_FILES+=usr/include/g++/rope
	OLD_FILES+=usr/include/g++/rope.h
	OLD_FILES+=usr/include/g++/ropeimpl.h
	OLD_FILES+=usr/include/g++/set
	OLD_FILES+=usr/include/g++/set.h
	OLD_FILES+=usr/include/g++/slist
	OLD_FILES+=usr/include/g++/slist.h
	OLD_FILES+=usr/include/g++/sstream
	OLD_FILES+=usr/include/g++/stack
	OLD_FILES+=usr/include/g++/stack.h
	OLD_FILES+=usr/include/g++/std/bastring.cc
	OLD_FILES+=usr/include/g++/std/bastring.h
	OLD_FILES+=usr/include/g++/std/complext.cc
	OLD_FILES+=usr/include/g++/std/complext.h
	OLD_FILES+=usr/include/g++/std/dcomplex.h
	OLD_FILES+=usr/include/g++/std/fcomplex.h
	OLD_FILES+=usr/include/g++/std/gslice.h
	OLD_FILES+=usr/include/g++/std/gslice_array.h
	OLD_FILES+=usr/include/g++/std/indirect_array.h
	OLD_FILES+=usr/include/g++/std/ldcomplex.h
	OLD_FILES+=usr/include/g++/std/mask_array.h
	OLD_FILES+=usr/include/g++/std/slice.h
	OLD_FILES+=usr/include/g++/std/slice_array.h
	OLD_FILES+=usr/include/g++/std/std_valarray.h
	OLD_FILES+=usr/include/g++/std/straits.h
	OLD_FILES+=usr/include/g++/std/valarray_array.h
	OLD_FILES+=usr/include/g++/std/valarray_array.tcc
	OLD_FILES+=usr/include/g++/std/valarray_meta.h
	OLD_FILES+=usr/include/g++/stdexcept
	OLD_FILES+=usr/include/g++/stdiostream.h
	OLD_FILES+=usr/include/g++/stl.h
	OLD_FILES+=usr/include/g++/stl_algo.h
	OLD_FILES+=usr/include/g++/stl_algobase.h
	OLD_FILES+=usr/include/g++/stl_alloc.h
	OLD_FILES+=usr/include/g++/stl_bvector.h
	OLD_FILES+=usr/include/g++/stl_config.h
	OLD_FILES+=usr/include/g++/stl_construct.h
	OLD_FILES+=usr/include/g++/stl_deque.h
	OLD_FILES+=usr/include/g++/stl_function.h
	OLD_FILES+=usr/include/g++/stl_hash_fun.h
	OLD_FILES+=usr/include/g++/stl_hash_map.h
	OLD_FILES+=usr/include/g++/stl_hash_set.h
	OLD_FILES+=usr/include/g++/stl_hashtable.h
	OLD_FILES+=usr/include/g++/stl_heap.h
	OLD_FILES+=usr/include/g++/stl_iterator.h
	OLD_FILES+=usr/include/g++/stl_list.h
	OLD_FILES+=usr/include/g++/stl_map.h
	OLD_FILES+=usr/include/g++/stl_multimap.h
	OLD_FILES+=usr/include/g++/stl_multiset.h
	OLD_FILES+=usr/include/g++/stl_numeric.h
	OLD_FILES+=usr/include/g++/stl_pair.h
	OLD_FILES+=usr/include/g++/stl_queue.h
	OLD_FILES+=usr/include/g++/stl_raw_storage_iter.h
	OLD_FILES+=usr/include/g++/stl_relops.h
	OLD_FILES+=usr/include/g++/stl_rope.h
	OLD_FILES+=usr/include/g++/stl_set.h
	OLD_FILES+=usr/include/g++/stl_slist.h
	OLD_FILES+=usr/include/g++/stl_stack.h
	OLD_FILES+=usr/include/g++/stl_tempbuf.h
	OLD_FILES+=usr/include/g++/stl_tree.h
	OLD_FILES+=usr/include/g++/stl_uninitialized.h
	OLD_FILES+=usr/include/g++/stl_vector.h
	OLD_FILES+=usr/include/g++/stream.h
	OLD_FILES+=usr/include/g++/streambuf.h
	OLD_FILES+=usr/include/g++/strfile.h
	OLD_FILES+=usr/include/g++/string
	OLD_FILES+=usr/include/g++/strstream
	OLD_FILES+=usr/include/g++/strstream.h
	OLD_FILES+=usr/include/g++/tempbuf.h
	OLD_FILES+=usr/include/g++/tree.h
	OLD_FILES+=usr/include/g++/type_traits.h
	OLD_FILES+=usr/include/g++/typeinfo
	OLD_FILES+=usr/include/g++/utility
	OLD_FILES+=usr/include/g++/valarray
	OLD_FILES+=usr/include/g++/vector
	OLD_FILES+=usr/include/g++/vector.h
	OLD_FILES+=usr/include/gmp.h
	OLD_FILES+=usr/include/isc/assertions.h
	OLD_FILES+=usr/include/isc/ctl.h
	OLD_FILES+=usr/include/isc/dst.h
	OLD_FILES+=usr/include/isc/eventlib.h
	OLD_FILES+=usr/include/isc/heap.h
	OLD_FILES+=usr/include/isc/irpmarshall.h
	OLD_FILES+=usr/include/isc/list.h
	OLD_FILES+=usr/include/isc/logging.h
	OLD_FILES+=usr/include/isc/memcluster.h
	OLD_FILES+=usr/include/isc/misc.h
	OLD_FILES+=usr/include/isc/tree.h
	OLD_FILES+=usr/include/machine/ansi.h
	OLD_FILES+=usr/include/machine/apic.h
	OLD_FILES+=usr/include/machine/asc_ioctl.h
	OLD_FILES+=usr/include/machine/asnames.h
	OLD_FILES+=usr/include/machine/bus_at386.h
	OLD_FILES+=usr/include/machine/bus_memio.h
	OLD_FILES+=usr/include/machine/bus_pc98.h
	OLD_FILES+=usr/include/machine/bus_pio.h
	OLD_FILES+=usr/include/machine/cdk.h
	OLD_FILES+=usr/include/machine/comstats.h
	OLD_FILES+=usr/include/machine/console.h
	OLD_FILES+=usr/include/machine/critical.h
	OLD_FILES+=usr/include/machine/cronyx.h
	OLD_FILES+=usr/include/machine/dvcfg.h
	OLD_FILES+=usr/include/machine/globaldata.h
	OLD_FILES+=usr/include/machine/globals.h
	OLD_FILES+=usr/include/machine/gsc.h
	OLD_FILES+=usr/include/machine/i4b_isppp.h
	OLD_FILES+=usr/include/machine/if_wavelan_ieee.h
	OLD_FILES+=usr/include/machine/iic.h
	OLD_FILES+=usr/include/machine/ioctl_ctx.h
	OLD_FILES+=usr/include/machine/ioctl_fd.h
	OLD_FILES+=usr/include/machine/ipl.h
	OLD_FILES+=usr/include/machine/lock.h
	OLD_FILES+=usr/include/machine/mouse.h
	OLD_FILES+=usr/include/machine/mpapic.h
	OLD_FILES+=usr/include/machine/mtpr.h
	OLD_FILES+=usr/include/machine/pc/msdos.h
	OLD_FILES+=usr/include/machine/physio_proc.h
	OLD_FILES+=usr/include/machine/smb.h
	OLD_FILES+=usr/include/machine/spigot.h
	OLD_FILES+=usr/include/machine/types.h
	OLD_FILES+=usr/include/machine/uc_device.h
	OLD_FILES+=usr/include/machine/ultrasound.h
	OLD_FILES+=usr/include/machine/wtio.h
	OLD_FILES+=usr/include/msdosfs/bootsect.h
	OLD_FILES+=usr/include/msdosfs/bpb.h
	OLD_FILES+=usr/include/msdosfs/denode.h
	OLD_FILES+=usr/include/msdosfs/direntry.h
	OLD_FILES+=usr/include/msdosfs/fat.h
	OLD_FILES+=usr/include/msdosfs/msdosfsmount.h
	OLD_FILES+=usr/include/net/hostcache.h
	OLD_FILES+=usr/include/net/if_faith.h
	OLD_FILES+=usr/include/net/if_ieee80211.h
	OLD_FILES+=usr/include/net/if_tunvar.h
	OLD_FILES+=usr/include/net/intrq.h
	OLD_FILES+=usr/include/netatm/kern_include.h
	OLD_FILES+=usr/include/netinet/if_fddi.h
	OLD_FILES+=usr/include/netinet/in_hostcache.h
	OLD_FILES+=usr/include/netinet/ip_flow.h
	OLD_FILES+=usr/include/netinet/ip_fw2.h
	OLD_FILES+=usr/include/netinet6/in6_prefix.h
	OLD_FILES+=usr/include/netns/idp.h
	OLD_FILES+=usr/include/netns/idp_var.h
	OLD_FILES+=usr/include/netns/ns.h
	OLD_FILES+=usr/include/netns/ns_error.h
	OLD_FILES+=usr/include/netns/ns_if.h
	OLD_FILES+=usr/include/netns/ns_pcb.h
	OLD_FILES+=usr/include/netns/sp.h
	OLD_FILES+=usr/include/netns/spidp.h
	OLD_FILES+=usr/include/netns/spp_debug.h
	OLD_FILES+=usr/include/netns/spp_timer.h
	OLD_FILES+=usr/include/netns/spp_var.h
	OLD_FILES+=usr/include/nfs/nfs.h
	OLD_FILES+=usr/include/nfs/nfsm_subs.h
	OLD_FILES+=usr/include/nfs/nfsmount.h
	OLD_FILES+=usr/include/nfs/nfsnode.h
	OLD_FILES+=usr/include/nfs/nfsrtt.h
	OLD_FILES+=usr/include/nfs/nfsrvcache.h
	OLD_FILES+=usr/include/nfs/nfsv2.h
	OLD_FILES+=usr/include/nfs/nqnfs.h
	OLD_FILES+=usr/include/ntfs/ntfs.h
	OLD_FILES+=usr/include/ntfs/ntfs_compr.h
	OLD_FILES+=usr/include/ntfs/ntfs_ihash.h
	OLD_FILES+=usr/include/ntfs/ntfs_inode.h
	OLD_FILES+=usr/include/ntfs/ntfs_subr.h
	OLD_FILES+=usr/include/ntfs/ntfs_vfsops.h
	OLD_FILES+=usr/include/ntfs/ntfsmount.h
	OLD_FILES+=usr/include/nwfs/nwfs.h
	OLD_FILES+=usr/include/nwfs/nwfs_mount.h
	OLD_FILES+=usr/include/nwfs/nwfs_node.h
	OLD_FILES+=usr/include/nwfs/nwfs_subr.h
	OLD_FILES+=usr/include/posix4/_semaphore.h
	OLD_FILES+=usr/include/posix4/aio.h
	OLD_FILES+=usr/include/posix4/ksem.h
	OLD_FILES+=usr/include/posix4/mqueue.h
	OLD_FILES+=usr/include/posix4/posix4.h
	OLD_FILES+=usr/include/posix4/sched.h
	OLD_FILES+=usr/include/posix4/semaphore.h
	OLD_DIRS+=usr/include/posix4
	OLD_FILES+=usr/include/security/_pam_compat.h
	OLD_FILES+=usr/include/security/_pam_macros.h
	OLD_FILES+=usr/include/security/_pam_types.h
	OLD_FILES+=usr/include/security/pam_malloc.h
	OLD_FILES+=usr/include/security/pam_misc.h
	OLD_FILES+=usr/include/skey.h
	OLD_FILES+=usr/include/strhash.h
	OLD_FILES+=usr/include/struct.h
	OLD_FILES+=usr/include/sys/_label.h
	OLD_FILES+=usr/include/sys/_posix.h
	OLD_FILES+=usr/include/sys/bus_private.h
	OLD_FILES+=usr/include/sys/ccdvar.h
	OLD_FILES+=usr/include/sys/diskslice.h
	OLD_FILES+=usr/include/sys/dmap.h
	OLD_FILES+=usr/include/sys/inttypes.h
	OLD_FILES+=usr/include/sys/jumbo.h
	OLD_FILES+=usr/include/sys/mac_policy.h
	OLD_FILES+=usr/include/sys/pbioio.h
	OLD_FILES+=usr/include/sys/syscall-hide.h
	OLD_FILES+=usr/include/sys/tprintf.h
	OLD_FILES+=usr/include/sys/vnioctl.h
	OLD_FILES+=usr/include/sys/wormio.h
	OLD_FILES+=usr/include/telnet.h
	OLD_FILES+=usr/include/ufs/mfs/mfs_extern.h
	OLD_FILES+=usr/include/ufs/mfs/mfsnode.h
	OLD_FILES+=usr/include/values.h
	OLD_FILES+=usr/include/vm/vm_zone.h
	OLD_FILES+=usr/share/examples/etc/usbd.conf
	OLD_FILES+=usr/share/examples/meteor/README
	OLD_FILES+=usr/share/examples/meteor/rgb16.c
	OLD_FILES+=usr/share/examples/meteor/rgb24.c
	OLD_FILES+=usr/share/examples/meteor/test-n.c
	OLD_FILES+=usr/share/examples/meteor/yuvpk.c
	OLD_FILES+=usr/share/examples/meteor/yuvpl.c
	OLD_FILES+=usr/share/examples/worm/README
	OLD_FILES+=usr/share/examples/worm/makecdfs.sh
	OLD_FILES+=usr/share/groff_font/devlj4/Makefile
	OLD_FILES+=usr/share/groff_font/devlj4/text.map
	OLD_FILES+=usr/share/groff_font/devlj4/special.map
	OLD_FILES+=usr/share/misc/nslookup.help
	OLD_FILES+=usr/share/sendmail/cf/feature/nodns.m4
	OLD_FILES+=usr/share/syscons/keymaps/lat-amer.kbd
	OLD_FILES+=usr/share/vi/catalog/ru_SU.KOI8-R
	OLD_FILES+=usr/share/zoneinfo/SystemV/YST9
	OLD_FILES+=usr/share/zoneinfo/SystemV/PST8
	OLD_FILES+=usr/share/zoneinfo/SystemV/EST5EDT
	OLD_FILES+=usr/share/zoneinfo/SystemV/CST6CDT
	OLD_FILES+=usr/share/zoneinfo/SystemV/MST7MDT
	OLD_FILES+=usr/share/zoneinfo/SystemV/PST8PDT
	OLD_FILES+=usr/share/zoneinfo/SystemV/YST9YDT
	OLD_FILES+=usr/share/zoneinfo/SystemV/HST10
	OLD_FILES+=usr/share/zoneinfo/SystemV/MST7
	OLD_FILES+=usr/share/zoneinfo/SystemV/EST5
	OLD_FILES+=usr/share/zoneinfo/SystemV/AST4ADT
	OLD_FILES+=usr/share/zoneinfo/SystemV/CST6
	OLD_FILES+=usr/share/zoneinfo/SystemV/AST4
	OLD_FILES+=usr/share/doc/ntp/accopt.htm
	OLD_FILES+=usr/share/doc/ntp/assoc.htm
	OLD_FILES+=usr/share/doc/ntp/audio.htm
	OLD_FILES+=usr/share/doc/ntp/authopt.htm
	OLD_FILES+=usr/share/doc/ntp/biblio.htm
	OLD_FILES+=usr/share/doc/ntp/build.htm
	OLD_FILES+=usr/share/doc/ntp/clockopt.htm
	OLD_FILES+=usr/share/doc/ntp/config.htm
	OLD_FILES+=usr/share/doc/ntp/confopt.htm
	OLD_FILES+=usr/share/doc/ntp/copyright.htm
	OLD_FILES+=usr/share/doc/ntp/debug.htm
	OLD_FILES+=usr/share/doc/ntp/driver1.htm
	OLD_FILES+=usr/share/doc/ntp/driver10.htm
	OLD_FILES+=usr/share/doc/ntp/driver11.htm
	OLD_FILES+=usr/share/doc/ntp/driver12.htm
	OLD_FILES+=usr/share/doc/ntp/driver16.htm
	OLD_FILES+=usr/share/doc/ntp/driver18.htm
	OLD_FILES+=usr/share/doc/ntp/driver19.htm
	OLD_FILES+=usr/share/doc/ntp/driver2.htm
	OLD_FILES+=usr/share/doc/ntp/driver20.htm
	OLD_FILES+=usr/share/doc/ntp/driver22.htm
	OLD_FILES+=usr/share/doc/ntp/driver23.htm
	OLD_FILES+=usr/share/doc/ntp/driver24.htm
	OLD_FILES+=usr/share/doc/ntp/driver26.htm
	OLD_FILES+=usr/share/doc/ntp/driver27.htm
	OLD_FILES+=usr/share/doc/ntp/driver28.htm
	OLD_FILES+=usr/share/doc/ntp/driver29.htm
	OLD_FILES+=usr/share/doc/ntp/driver3.htm
	OLD_FILES+=usr/share/doc/ntp/driver30.htm
	OLD_FILES+=usr/share/doc/ntp/driver32.htm
	OLD_FILES+=usr/share/doc/ntp/driver33.htm
	OLD_FILES+=usr/share/doc/ntp/driver34.htm
	OLD_FILES+=usr/share/doc/ntp/driver35.htm
	OLD_FILES+=usr/share/doc/ntp/driver36.htm
	OLD_FILES+=usr/share/doc/ntp/driver37.htm
	OLD_FILES+=usr/share/doc/ntp/driver4.htm
	OLD_FILES+=usr/share/doc/ntp/driver5.htm
	OLD_FILES+=usr/share/doc/ntp/driver6.htm
	OLD_FILES+=usr/share/doc/ntp/driver7.htm
	OLD_FILES+=usr/share/doc/ntp/driver8.htm
	OLD_FILES+=usr/share/doc/ntp/driver9.htm
	OLD_FILES+=usr/share/doc/ntp/exec.htm
	OLD_FILES+=usr/share/doc/ntp/extern.htm
	OLD_FILES+=usr/share/doc/ntp/gadget.htm
	OLD_FILES+=usr/share/doc/ntp/hints.htm
	OLD_FILES+=usr/share/doc/ntp/howto.htm
	OLD_FILES+=usr/share/doc/ntp/htmlprimer.htm
	OLD_FILES+=usr/share/doc/ntp/index.htm
	OLD_FILES+=usr/share/doc/ntp/kern.htm
	OLD_FILES+=usr/share/doc/ntp/kernpps.htm
	OLD_FILES+=usr/share/doc/ntp/ldisc.htm
	OLD_FILES+=usr/share/doc/ntp/measure.htm
	OLD_FILES+=usr/share/doc/ntp/miscopt.htm
	OLD_FILES+=usr/share/doc/ntp/monopt.htm
	OLD_FILES+=usr/share/doc/ntp/mx4200data.htm
	OLD_FILES+=usr/share/doc/ntp/notes.htm
	OLD_FILES+=usr/share/doc/ntp/ntpd.htm
	OLD_FILES+=usr/share/doc/ntp/ntpdate.htm
	OLD_FILES+=usr/share/doc/ntp/ntpdc.htm
	OLD_FILES+=usr/share/doc/ntp/ntpq.htm
	OLD_FILES+=usr/share/doc/ntp/ntptime.htm
	OLD_FILES+=usr/share/doc/ntp/ntptrace.htm
	OLD_FILES+=usr/share/doc/ntp/parsedata.htm
	OLD_FILES+=usr/share/doc/ntp/parsenew.htm
	OLD_FILES+=usr/share/doc/ntp/patches.htm
	OLD_FILES+=usr/share/doc/ntp/porting.htm
	OLD_FILES+=usr/share/doc/ntp/pps.htm
	OLD_FILES+=usr/share/doc/ntp/prefer.htm
	OLD_FILES+=usr/share/doc/ntp/qth.htm
	OLD_FILES+=usr/share/doc/ntp/quick.htm
	OLD_FILES+=usr/share/doc/ntp/rdebug.htm
	OLD_FILES+=usr/share/doc/ntp/refclock.htm
	OLD_FILES+=usr/share/doc/ntp/release.htm
	OLD_FILES+=usr/share/doc/ntp/tickadj.htm
	OLD_FILES+=usr/share/doc/papers/nqnfs.ascii.gz
	OLD_FILES+=usr/share/doc/papers/px.ascii.gz
	OLD_FILES+=usr/share/man/man3/exp10.3.gz
	OLD_FILES+=usr/share/man/man3/exp10f.3.gz
	OLD_FILES+=usr/share/man/man3/fpsetsticky.3.gz
	OLD_FILES+=usr/share/man/man3/gss_krb5_compat_des3_mic.3.gz
	OLD_FILES+=usr/share/man/man3/gss_krb5_copy_ccache.3.gz
	OLD_FILES+=usr/share/man/man3/mac_is_present_np.3.gz
	OLD_FILES+=usr/share/man/man3/mbmb.3.gz
	OLD_FILES+=usr/share/man/man3/setrunelocale.3.gz
	OLD_FILES+=usr/share/man/man5/usbd.conf.5.gz
	.if ${TARGET_ARCH} != "i386" && ${TARGET_ARCH} != "amd64"
	OLD_FILES+=usr/share/man/man8/boot_i386.8.gz
	.endif
	.if ${TARGET_ARCH} != "aarch64" && ${TARGET} != "arm" && \
	${TARGET_ARCH} != "powerpc" && ${TARGET_ARCH} != "powerpc64" && \
	${TARGET_ARCH} != "sparc64" && ${TARGET} != "mips"
	OLD_FILES+=usr/share/man/man8/ofwdump.8.gz
	.endif
	OLD_FILES+=usr/share/man/man8/mount_reiserfs.8.gz
	OLD_FILES+=usr/share/man/man9/VFS_START.9.gz
	OLD_FILES+=usr/share/man/man9/cpu_critical_exit.9.gz
	OLD_FILES+=usr/share/man/man9/cpu_critical_enter.9.gz
	OLD_FILES+=usr/share/info/annotate.info.gz
	OLD_FILES+=usr/share/info/tar.info.gz
	OLD_FILES+=usr/share/bsnmp/defs/tree.def
	OLD_FILES+=usr/share/bsnmp/defs/mibII_tree.def
	OLD_FILES+=usr/share/bsnmp/defs/netgraph_tree.def
	OLD_FILES+=usr/share/bsnmp/mibs/FOKUS-MIB.txt
	OLD_FILES+=usr/share/bsnmp/mibs/BEGEMOT-MIB.txt
	OLD_FILES+=usr/share/bsnmp/mibs/BEGEMOT-SNMPD.txt
	OLD_FILES+=usr/share/bsnmp/mibs/BEGEMOT-NETGRAPH.txt
	OLD_FILES+=usr/libdata/msdosfs/iso22dos
	OLD_FILES+=usr/libdata/msdosfs/iso72dos
	OLD_FILES+=usr/libdata/msdosfs/koi2dos
	OLD_FILES+=usr/libdata/msdosfs/koi8u2dos
	# The following files are not obsolete, they just don't get touched at
	# install, so don't add them:
	# - boot/loader.rc
	# - usr/share/tmac/man.local
	# - usr/share/tmac/mm/locale
	# - usr/share/tmac/mm/se_locale
	# - var/yp/Makefile

	# Early entries split OLD_FILES, OLD_LIBS, and OLD_DIRS into separate sections
	# in this file, but this practice was abandoned in the mid-2000s.
	#
	# 20071120: shared library version bump
	OLD_LIBS+=usr/lib/libasn1.so.8
	OLD_LIBS+=usr/lib/libgssapi.so.8
	OLD_LIBS+=usr/lib/libgssapi_krb5.so.8
	OLD_LIBS+=usr/lib/libhdb.so.8
	OLD_LIBS+=usr/lib/libkadm5clnt.so.8
	OLD_LIBS+=usr/lib/libkadm5srv.so.8
	OLD_LIBS+=usr/lib/libkafs5.so.8
	OLD_LIBS+=usr/lib/libkrb5.so.8
	OLD_LIBS+=usr/lib/libobjc.so.2
	OLD_LIBS+=usr/lib32/libgssapi.so.8
	OLD_LIBS+=usr/lib32/libobjc.so.2
	# 20070519: GCC 4.2
	OLD_LIBS+=usr/lib/libg2c.a
	OLD_LIBS+=usr/lib/libg2c.so
	OLD_LIBS+=usr/lib/libg2c.so.2
	OLD_LIBS+=usr/lib/libg2c_p.a
	OLD_LIBS+=usr/lib/libgcc_pic.a
	OLD_LIBS+=usr/lib32/libg2c.a
	OLD_LIBS+=usr/lib32/libg2c.so
	OLD_LIBS+=usr/lib32/libg2c.so.2
	OLD_LIBS+=usr/lib32/libg2c_p.a
	OLD_LIBS+=usr/lib32/libgcc_pic.a
	# 20060729: OpenSSL 0.9.7e -> 0.9.8b upgrade
	OLD_LIBS+=lib/libcrypto.so.4
	OLD_LIBS+=usr/lib/libssl.so.4
	OLD_LIBS+=usr/lib32/libcrypto.so.4
	OLD_LIBS+=usr/lib32/libssl.so.4
	# 20060521: gethostbyaddr(3) ABI change
	OLD_LIBS+=usr/lib/libroken.so.8
	OLD_LIBS+=lib/libatm.so.3
	OLD_LIBS+=lib/libc.so.6
	OLD_LIBS+=lib/libutil.so.5
	OLD_LIBS+=usr/lib32/libatm.so.3
	OLD_LIBS+=usr/lib32/libc.so.6
	OLD_LIBS+=usr/lib32/libutil.so.5
	# 20060413: shared library moved to /usr/lib
	OLD_LIBS+=lib/libgpib.so.1
	# 20060413: libpcap.so.4 moved to /lib/
	OLD_LIBS+=usr/lib/libpcap.so.4
	# 20060412: libpthread.so.2 moved to /lib/
	OLD_LIBS+=usr/lib/libpthread.so.2
	# 20060127: revert libdisk to static-only
	OLD_LIBS+=usr/lib/libdisk.so.3
	# 20051027: libc_r discontinued (removed 20101113)
	OLD_LIBS+=usr/lib/libc_r.a
	OLD_LIBS+=usr/lib/libc_r.so
	OLD_LIBS+=usr/lib/libc_r.so.7
	OLD_LIBS+=usr/lib/libc_r_p.a
	OLD_LIBS+=usr/lib32/libc_r.a
	OLD_LIBS+=usr/lib32/libc_r.so
	OLD_LIBS+=usr/lib32/libc_r.so.7
	OLD_LIBS+=usr/lib32/libc_r_p.a
	# 20050722: bump for 6.0-RELEASE
	OLD_LIBS+=lib/libalias.so.4
	OLD_LIBS+=lib/libatm.so.2
	OLD_LIBS+=lib/libbegemot.so.1
	OLD_LIBS+=lib/libbsdxml.so.1
	OLD_LIBS+=lib/libbsnmp.so.2
	OLD_LIBS+=lib/libc.so.5
	OLD_LIBS+=lib/libcam.so.2
	OLD_LIBS+=lib/libcrypt.so.2
	OLD_LIBS+=lib/libcrypto.so.3
	OLD_LIBS+=lib/libdevstat.so.4
	OLD_LIBS+=lib/libedit.so.4
	OLD_LIBS+=lib/libgeom.so.2
	OLD_LIBS+=lib/libgpib.so.0
	OLD_LIBS+=lib/libipsec.so.1
	OLD_LIBS+=lib/libipx.so.2
	OLD_LIBS+=lib/libkiconv.so.1
	OLD_LIBS+=lib/libkvm.so.2
	OLD_LIBS+=lib/libm.so.3
	OLD_LIBS+=lib/libmd.so.2
	OLD_LIBS+=lib/libncurses.so.5
	OLD_LIBS+=lib/libreadline.so.5
	OLD_LIBS+=lib/libsbuf.so.2
	OLD_LIBS+=lib/libufs.so.2
	OLD_LIBS+=lib/libutil.so.4
	OLD_LIBS+=lib/libz.so.2
	OLD_LIBS+=usr/lib/libarchive.so.1
	OLD_LIBS+=usr/lib/libasn1.so.7
	OLD_LIBS+=usr/lib/libbluetooth.so.1
	OLD_LIBS+=usr/lib/libbz2.so.1
	OLD_LIBS+=usr/lib/libc_r.so.5
	OLD_LIBS+=usr/lib/libcalendar.so.2
	OLD_LIBS+=usr/lib/libcom_err.so.2
	OLD_LIBS+=usr/lib/libdevinfo.so.2
	OLD_LIBS+=usr/lib/libdialog.so.4
	OLD_LIBS+=usr/lib/libfetch.so.3
	OLD_LIBS+=usr/lib/libform.so.2
	OLD_LIBS+=usr/lib/libftpio.so.5
	OLD_LIBS+=usr/lib/libg2c.so.1
	OLD_LIBS+=usr/lib/libgnuregex.so.2
	OLD_LIBS+=usr/lib/libgssapi.so.7
	OLD_LIBS+=usr/lib/libhdb.so.7
	OLD_LIBS+=usr/lib/libhistory.so.5
	OLD_LIBS+=usr/lib/libkadm5clnt.so.7
	OLD_LIBS+=usr/lib/libkadm5srv.so.7
	OLD_LIBS+=usr/lib/libkafs5.so.7
	OLD_LIBS+=usr/lib/libkrb5.so.7
	OLD_LIBS+=usr/lib/libmagic.so.1
	OLD_LIBS+=usr/lib/libmenu.so.2
	OLD_LIBS+=usr/lib/libmilter.so.2
	OLD_LIBS+=usr/lib/libmp.so.4
	OLD_LIBS+=usr/lib/libncp.so.1
	OLD_LIBS+=usr/lib/libnetgraph.so.1
	OLD_LIBS+=usr/lib/libngatm.so.1
	OLD_LIBS+=usr/lib/libobjc.so.1
	OLD_LIBS+=usr/lib/libopie.so.3
	OLD_LIBS+=usr/lib/libpam.so.2
	OLD_LIBS+=usr/lib/libpanel.so.2
	OLD_LIBS+=usr/lib/libpcap.so.3
	OLD_LIBS+=usr/lib/libpmc.so.2
	OLD_LIBS+=usr/lib/libpthread.so.1
	OLD_LIBS+=usr/lib/libradius.so.1
	OLD_LIBS+=usr/lib/libroken.so.7
	OLD_LIBS+=usr/lib/librpcsvc.so.2
	OLD_LIBS+=usr/lib/libsdp.so.1
	OLD_LIBS+=usr/lib/libsmb.so.1
	OLD_LIBS+=usr/lib/libssh.so.2
	OLD_LIBS+=usr/lib/libssl.so.3
	OLD_LIBS+=usr/lib/libstdc++.so.4
	OLD_LIBS+=usr/lib/libtacplus.so.1
	OLD_LIBS+=usr/lib/libthr.so.1
	OLD_LIBS+=usr/lib/libthread_db.so.1
	OLD_LIBS+=usr/lib/libugidfw.so.1
	OLD_LIBS+=usr/lib/libusbhid.so.1
	OLD_LIBS+=usr/lib/libvgl.so.3
	OLD_LIBS+=usr/lib/libwrap.so.3
	OLD_LIBS+=usr/lib/libypclnt.so.1
	OLD_LIBS+=usr/lib/pam_chroot.so.2
	OLD_LIBS+=usr/lib/pam_deny.so.2
	OLD_LIBS+=usr/lib/pam_echo.so.2
	OLD_LIBS+=usr/lib/pam_exec.so.2
	OLD_LIBS+=usr/lib/pam_ftpusers.so.2
	OLD_LIBS+=usr/lib/pam_group.so.2
	OLD_LIBS+=usr/lib/pam_guest.so.2
	OLD_LIBS+=usr/lib/pam_krb5.so.2
	OLD_LIBS+=usr/lib/pam_ksu.so.2
	OLD_LIBS+=usr/lib/pam_lastlog.so.2
	OLD_LIBS+=usr/lib/pam_login_access.so.2
	OLD_LIBS+=usr/lib/pam_nologin.so.2
	OLD_LIBS+=usr/lib/pam_opie.so.2
	OLD_LIBS+=usr/lib/pam_opieaccess.so.2
	OLD_LIBS+=usr/lib/pam_passwdqc.so.2
	OLD_LIBS+=usr/lib/pam_permit.so.2
	OLD_LIBS+=usr/lib/pam_radius.so.2
	OLD_LIBS+=usr/lib/pam_rhosts.so.2
	OLD_LIBS+=usr/lib/pam_rootok.so.2
	OLD_LIBS+=usr/lib/pam_securetty.so.2
	OLD_LIBS+=usr/lib/pam_self.so.2
	OLD_LIBS+=usr/lib/pam_ssh.so.2
	OLD_LIBS+=usr/lib/pam_tacplus.so.2
	OLD_LIBS+=usr/lib/pam_unix.so.2
	OLD_LIBS+=usr/lib/snmp_atm.so.3
	OLD_LIBS+=usr/lib/snmp_mibII.so.3
	OLD_LIBS+=usr/lib/snmp_netgraph.so.3
	OLD_LIBS+=usr/lib/snmp_pf.so.3
	# 200505XX: ?
	OLD_LIBS+=usr/lib/snmp_atm.so.2
	OLD_LIBS+=usr/lib/snmp_mibII.so.2
	OLD_LIBS+=usr/lib/snmp_netgraph.so.2
	OLD_LIBS+=usr/lib/snmp_pf.so.2
	# 2005XXXX: not ready for primetime yet
	OLD_LIBS+=usr/lib/libautofs.so.1
	# 200411XX: libxpg4 removal
	OLD_LIBS+=usr/lib/libxpg4.so.3
	# 200410XX: libm compatibility fix
	OLD_LIBS+=lib/libm.so.2
	# 20041001: version bump
	OLD_LIBS+=lib/libreadline.so.4
	OLD_LIBS+=usr/lib/libhistory.so.4
	OLD_LIBS+=usr/lib/libopie.so.2
	OLD_LIBS+=usr/lib/libpcap.so.2
	# 20040925: bind9 import
	OLD_LIBS+=usr/lib/libisc.so.1
	# 200408XX
	OLD_LIBS+=usr/lib/snmp_netgraph.so.1
	# 200404XX
	OLD_LIBS+=usr/lib/libsnmp.so.1
	OLD_LIBS+=usr/lib/snmp_mibII.so.1
	# 200309XX
	OLD_LIBS+=usr/lib/libasn1.so.6
	OLD_LIBS+=usr/lib/libhdb.so.6
	OLD_LIBS+=usr/lib/libkadm5clnt.so.6
	OLD_LIBS+=usr/lib/libkadm5srv.so.6
	OLD_LIBS+=usr/lib/libkrb5.so.6
	OLD_LIBS+=usr/lib/libroken.so.6
	# 200304XX
	OLD_LIBS+=usr/lib/libc.so.4
	OLD_LIBS+=usr/lib/libc_r.so.4
	OLD_LIBS+=usr/lib/libdevstat.so.2
	OLD_LIBS+=usr/lib/libedit.so.3
	OLD_LIBS+=usr/lib/libgmp.so.3
	OLD_LIBS+=usr/lib/libmp.so.3
	OLD_LIBS+=usr/lib/libpam.so.1
	OLD_LIBS+=usr/lib/libposix1e.so.2
	OLD_LIBS+=usr/lib/libskey.so.2
	OLD_LIBS+=usr/lib/libusbhid.so.0
	OLD_LIBS+=usr/lib/libvgl.so.2
	# 20030218: OpenSSL 0.9.7 import
	OLD_FILES+=usr/include/des.h
	OLD_FILES+=usr/lib/libdes.a
	OLD_FILES+=usr/lib/libdes.so
	OLD_LIBS+=usr/lib/libdes.so.3
	OLD_FILES+=usr/lib/libdes_p.a
	# 200302XX
	OLD_LIBS+=usr/lib/libacl.so.3
	OLD_LIBS+=usr/lib/libasn1.so.5
	OLD_LIBS+=usr/lib/libcrypto.so.2
	OLD_LIBS+=usr/lib/libgssapi.so.5
	OLD_LIBS+=usr/lib/libhdb.so.5
	OLD_LIBS+=usr/lib/libkadm.so.3
	OLD_LIBS+=usr/lib/libkadm5clnt.so.5
	OLD_LIBS+=usr/lib/libkadm5srv.so.5
	OLD_LIBS+=usr/lib/libkafs.so.3
	OLD_LIBS+=usr/lib/libkafs5.so.5
	OLD_LIBS+=usr/lib/libkdb.so.3
	OLD_LIBS+=usr/lib/libkrb.so.3
	OLD_LIBS+=usr/lib/libroken.so.
	OLD_LIBS+=usr/lib/libssl.so.2
	OLD_LIBS+=usr/lib/pam_kerberosIV.so
	# 200208XX
	OLD_LIBS+=usr/lib/libgssapi.so.4
	# 200203XX
	OLD_LIBS+=usr/lib/libss.so.3
	OLD_LIBS+=usr/lib/libusb.so.0
	# 200112XX
	OLD_LIBS+=usr/lib/libfetch.so.2
	# 200110XX
	OLD_LIBS+=usr/lib/libgssapi.so.3
	# 200104XX
	OLD_LIBS+=usr/lib/libdescrypt.so.2
	OLD_LIBS+=usr/lib/libscrypt.so.2
	# 200102XX
	OLD_LIBS+=usr/lib/libcrypto.so.1
	OLD_LIBS+=usr/lib/libssl.so.1
	# 200009XX
	OLD_LIBS+=usr/lib/libRSAglue.so.1
	OLD_LIBS+=usr/lib/librsaINTL.so.1
	OLD_LIBS+=usr/lib/librsaUSA.so.1
	# 200006XX
	OLD_LIBS+=usr/lib/libalias.so.3
	OLD_LIBS+=usr/lib/libfetch.so.1
	OLD_LIBS+=usr/lib/libipsec.so.0
	# 200005XX
	OLD_LIBS+=usr/lib/libxpg4.so.2
	# 200002XX
	OLD_LIBS+=usr/lib/libc.so.3
	OLD_LIBS+=usr/lib/libcurses.so.2
	OLD_LIBS+=usr/lib/libdialog.so.3
	OLD_LIBS+=usr/lib/libedit.so.2
	OLD_LIBS+=usr/lib/libf2c.so.2
	OLD_LIBS+=usr/lib/libftpio.so.4
	OLD_LIBS+=usr/lib/libg++.so.4
	OLD_LIBS+=usr/lib/libhistory.so.3
	OLD_LIBS+=usr/lib/libmytinfo.so.2
	OLD_LIBS+=usr/lib/libncurses.so.3
	OLD_LIBS+=usr/lib/libreadline.so.3
	OLD_LIBS+=usr/lib/libss.so.2
	OLD_LIBS+=usr/lib/libtermcap.so.2
	OLD_LIBS+=usr/lib/libutil.so.2
	OLD_LIBS+=usr/lib/libvgl.so.1
	OLD_LIBS+=usr/lib/libwrap.so.2
	# ???
	OLD_LIBS+=usr/lib/libarchive.so.2
	OLD_LIBS+=usr/lib/libbsnmp.so.1
	OLD_LIBS+=usr/lib/libc_r.so.6
	OLD_LIBS+=usr/lib32/libarchive.so.2
	OLD_LIBS+=usr/lib32/libc_r.so.6
	OLD_LIBS+=usr/lib/libcipher.so.2
	OLD_LIBS+=usr/lib/libgssapi.so.6
	OLD_LIBS+=usr/lib/libkse.so.1
	OLD_LIBS+=usr/lib/liblwres.so.3
	OLD_LIBS+=usr/lib/pam_ftp.so.2

	# 20040925: bind9 import
	OLD_DIRS+=usr/share/doc/bind/html
	OLD_DIRS+=usr/share/doc/bind/misc
	OLD_DIRS+=usr/share/doc/bind/
	# ???
	OLD_DIRS+=usr/include/g++/std
	OLD_DIRS+=usr/include/msdosfs
	OLD_DIRS+=usr/include/ntfs
	OLD_DIRS+=usr/include/nwfs
	OLD_DIRS+=usr/include/ufs/mfs
	# 20011001: UUCP migration to ports
	OLD_DIRS+=usr/libexec/uucp


	.include "tools/build/mk/OptionalObsoleteFiles.inc"
	diff --git a/RELNOTES b/RELNOTES
	index c78f92c68977..d34c504ca760 100644
	--- a/RELNOTES
	+++ b/RELNOTES
	@@ -1,225 +1,229 @@
	Release notes for FreeBSD 13.0.

	This file describes new user-visible features, changes and updates relevant to
	users of binary FreeBSD releases. Each entry should describe the change in no
	more than several sentences and should reference manual pages where an
	interested user can find more information. Entries should wrap after 80
	columns. Each entry should begin with one or more commit IDs on one line,
	specified as a comma separated list and/or range, followed by a colon and a
	newline. Entries should be separated by a newline.

	Changes to this file should not be MFCed.

	+r363679:
	+ Applications using regex(3), e.g. sed/grep, will no longer accept
	+ redundant escapes for most ordinary characters.
	+
	r363253:
	SCTP support has been removed from GENERIC kernel configurations.
	The SCTP stack is now built as sctp.ko and can be dynamically loaded.

	r363233:
	Merge sendmail 8.16.1: See contrib/sendmail/RELEASE_NOTES for details.

	r363180:
	The safexcel(4) crypto offload driver has been added.

	r363084:
	nc(1) now implements SCTP mode, enabled by specifying the --sctp option.

	r362158, r362163:
	struct export_args has changed so that the "user" specified for
	the -maproot and -mapall exports(5) options may be in more than
	16 groups.

	r361884:
	sed(1) has learned about hex escapes (e.g. \x27) and will now do the
	right thing with them, removing the need for printf magic or obnoxious
	escaping in many scenarios.

	r361238, r361798, r361799:
	ZFS will now unconditionally reject read(2) of a directory with EISDIR.
	Additionally, read(2) of a directory is now rejected with EISDIR by
	default and may be re-enabled for non-ZFS filesystems that allow it with
	the sysctl(8) MIB 'security.bsd.allow_read_dir'.

	Aliases for grep to default to '-d skip' may be desired if commonly
	non-recursively grepping a list that includes directories and the
	possibility of EISDIR errors in stderr is not tolerable. Example
	aliases, commented out, have been installed in /root/.cshrc and
	/root/.shrc.

	r361066:
	Add exec.prepare and exec.release hooks for jail(8) and jail.conf(5).
	exec.prepare runs before mounts, so can be used to populate new jails.
	exec.release runs after unmounts, so can be used to remove ephemeral
	jails.

	r360920,r360923,r360924,r360927,r360928,r360931,r360933,r360936:
	Remove support for ARC4, Blowfish, Cast, DES, Triple DES, MD5,
	MD5-KPDK, MD5-HMAC, SHA1-KPDK, and Skipjack algorithms from
	the kernel open cryptographic framework (OCF).

	r360562:
	Remove support for ARC4, Blowfish, Cast, DES, Triple DES,
	MD5-HMAC, and Skipjack algorithms from /dev/crypto.

	r360557:
	Remove support for DES, Triple DES, Blowfish, Cast, and
	Camellia ciphers from IPsec(4). Remove support for MD5-HMAC,
	Keyed MD5, Keyed SHA1, and RIPEMD160-HMAC from IPsec(4).

	r359945:
	Remove support for Triple DES, Blowfish, and MD5 HMAC from
	geli(4).

	r359786-r359787:
	Remove support for DES, Triple DES, and RC4 from in-kernel GSS
	authentication.

	r357627:
	remove elf2aout.

	r357560-r357565:
	init(8), service(8), and cron(8) will now adopt user/class environment
	variables (excluding PATH, by default, which will be overwritten) by
	default. Notably, environment variables for all cron jobs and rc
	services can now be set via login.conf(5).

	r357455:
	sparc64 has been removed from FreeBSD.

	r355677:
	Adds support for NFSv4.2 (RFC-7862) and Extended Attributes
	(RFC-8276) to the NFS client and server.
	NFSv4.2 is comprised of several optional features that can be supported
	in addition to NFSv4.1. This patch adds the following optional features:
	- posix_fadvise(POSIX_FADV_WILLNEED/POSIX_FADV_DONTNEED)
	- posix_fallocate()
	- intra server file range copying via the copy_file_range(2) syscall
	--> Avoiding data tranfer over the wire to/from the NFS client.
	- lseek(SEEK_DATA/SEEK_HOLE)
	- Extended attribute syscalls for "user" namespace attributes as defined
	by RFC-8276.

	For the client, NFSv4.2 is only used if the mount command line option
	minorversion=2 is specified.
	For the server, two new sysctls called vfs.nfsd.server_min_minorversion4
	and vfs.nfsd.server_max_minorversion4 have been added that allow
	sysadmins to limit the minor versions of NFSv4 supported by the nfsd
	server.
	Setting vfs.nfsd.server_max_minorversion4 to 0 or 1 will disable NFSv4.2
	on the server.

	r356263:
	armv5 support has been removed from FreeBSD.

	r354517:
	iwm(4) now supports most Intel 9260, 9460 and 9560 Wi-Fi devices.

	r354269:
	sqlite3 is updated to sqlite3-3.30.1.

	r352668:
	cron(8) now supports the -n (suppress mail on succesful run) and -q
	(suppress logging of command execution) options in the crontab format.
	See the crontab(5) manpage for details.

	r352304:
	ntpd is no longer by default locked in memory. rlimit memlock 32
	or rlimit memlock 0 can be used to restore this behaviour.

	r351770,r352920,r352922,r352923:
	dd(1) now supports conv=fsync, conv=fdatasync, oflag=fsync, oflag=sync,
	and iflag=fullblock flags, compatible with illumos and GNU.

	r351522:
	Add kernel-side support for in-kernel Transport Layer Security
	(KTLS). KTLS permits using sendfile(2) over sockets using
	TLS.

	r351397:
	WPA is updated from 2.8 to 2.9.

	r351361:
	Add probes for lockmgr(9) to the lockstat DTrace provider, add
	corresponding lockstat(1) events, and document the new probes in
	dtrace_lockstat.4.

	r351356:
	Intel RST is a new 'feature' that remaps NVMe devices from
	their normal location to part of the AHCI bar space. This
	will eliminate the need to set the BIOS SATA setting from RST
	to AHCI causing the nvme drive to be erased before FreeBSD
	will see the nvme drive. FreeBSD will now be able to see the
	nvme drive now in the default config.

	r351201, r351372:
	Add a vop_stdioctl() call, so that file systems that do not support
	holes will have a trivial implementation of lseek(SEEK_DATA/SEEK_HOLE).
	The algorithm appears to be compatible with the POSIX draft and
	the implementation in Linux for the case of a file system that
	does not support holes. Prior to this patch, lseek(2) would reply
	-1 with errno set to ENOTTY for SEEK_DATA/SEEK_HOLE on files in
	file systems that do not support holes.
	r351372 maps ENOTTY to EINVAL for lseek(SEEK_DATA/SEEK_HOLE) for
	any other cases, such as a ENOTTY return from vn_bmap_seekhole().

	r350665:
	The fuse driver has been renamed to fusefs(5) and been substantially
	rewritten. The new driver includes many bug fixes and performance
	enhancements, as well as the following user-visible features:
	* Optional kernel-side permissions checks (-o default_permissions)
	* mknod(2), socket(2), and pipe(2) support
	* server side locking with fcntl(2)
	* FUSE operations are now interruptible when mounted with -o intr
	* server side handling of UTIME_NOW during utimensat(2)
	* mount options may be updated with "mount -u"
	* fusefs file system may now be exported over NFS
	* RLIMIT_FSIZE support
	* support for fuse file systems using protocols as old as 7.4

	FUSE file system developers should also take note of the following new
	features:
	* The protocol level has been raised from 7.8 to 7.23
	* kqueue support on /dev/fuse
	* server-initiated cache invalidation via FUSE_NOTIFY_REPLY

	r350471:
	gnop(8) can now configure a delay to be applied to read and write
	request delays. See the -d, -q and -x parameters.

	r350315, r350316:
	Adds a Linux compatible copy_file_range(2) syscall.

	r350307:
	libcap_random(3) has been removed. Applications can use native
	APIs to get random data in capability mode.

	r349529,r349530:
	Add support for using unmapped mbufs with sendfile(2).

	r349352:
	nand(4) and related components have been removed.

	r349349:
	The UEFI loader now supports HTTP boot.

	r349335:
	bhyve(8) now implements a High Definition Audio (HDA) driver, allowing
	guests to play to and record audio data from the host.

	r349286:
	swapon(8) can now erase a swap device immediately before enabling it,
	similar to newfs(8)'s -E option. This behaviour can be specified by
	adding -E to swapon(8)'s command-line parameters, or by adding the
	"trimonce" option to a swap device's /etc/fstab entry.

	r347908-r347923:
	The following network drivers have been removed: bm(4), cs(4), de(4),
	ed(4), ep(4), ex(4), fe(4), pcn(4), sf(4), sn(4), tl(4), tx(4), txp(4),
	vx(4), wb(4), xe(4).

	r347532:
	Wired page accounting has been split into kernel wirings and user
	wirings (e.g., by mlock(2)). Kernel wirings no long count towards
	the global limit, which is renamed to vm.max_user_wired. bhyve -S
	allocates user-wired memory and is now subject to that limit.

	$FreeBSD$
	diff --git a/UPDATING b/UPDATING
	index ee69605d9f28..9cd27c5f4f37 100644
	--- a/UPDATING
	+++ b/UPDATING
	@@ -1,2299 +1,2311 @@
	Updating Information for FreeBSD current users.

	This file is maintained and copyrighted by M. Warner Losh <imp@freebsd.org>.
	See end of file for further details. For commonly done items, please see the
	COMMON ITEMS: section later in the file. These instructions assume that you
	basically know what you are doing. If not, then please consult the FreeBSD
	handbook:

	https://www.freebsd.org/doc/en_US.ISO8859-1/books/handbook/makeworld.html

	Items affecting the ports and packages system can be found in
	/usr/ports/UPDATING. Please read that file before running portupgrade.

	NOTE TO PEOPLE WHO THINK THAT FreeBSD 13.x IS SLOW:
	FreeBSD 13.x has many debugging features turned on, in both the kernel
	and userland. These features attempt to detect incorrect use of
	system primitives, and encourage loud failure through extra sanity
	checking and fail stop semantics. They also substantially impact
	system performance. If you want to do performance measurement,
	benchmarking, and optimization, you'll want to turn them off. This
	includes various WITNESS- related kernel options, INVARIANTS, malloc
	debugging flags in userland, and various verbose features in the
	kernel. Many developers choose to disable these features on build
	machines to maximize performance. (To completely disable malloc
	debugging, define MALLOC_PRODUCTION in /etc/make.conf, or to merely
	disable the most expensive debugging functionality run
	"ln -s 'abort:false,junk:false' /etc/malloc.conf".)

	+20200729:
	+ r363679 has redefined some undefined behavior in regcomp(3); notably,
	+ extraneous escapes of most ordinary characters will no longer be
	+ accepted. An exp-run has identified all of the problems with this in
	+ ports, but other non-ports software may need extra escapes removed to
	+ continue to function.
	+
	+ Because of this change, installworld may encounter the following error
	+ from rtld: Undefined symbol "regcomp@FBSD_1.6" -- It is imperative that
	+ you do not halt installworld. Instead, let it run to completion (whether
	+ successful or not) and run installworld once more.
	+
	20200627:
	A new implementation of bc and dc has been imorted in r362681. This
	implementation corrects non-conformant behavior of the previous bc
	and adds GNU bc compatible options. It offers a number of extensions,
	is much faster on large values, and has support for message catalogs
	(a number of languages are already supported, contributions of further
	languages welcome). The option WITHOUT_GH_BC can be used to build the
	world with the previous versions of bc and dc.

	20200625:
	r362639 changed the internal API used between the NFS kernel modules.
	As such, they all need to be rebuilt from sources.

	20200613:
	r362158 changed the arguments for VFS_CHECKEXP(). As such, any
	out of tree file systems need to be modified and rebuilt.
	Also, any file systems that are modules must be rebuilt.

	20200604:
	read(2) of a directory fd is now rejected by default. root may
	re-enable it for system root only on non-ZFS filesystems with the
	security.bsd.allow_read_dir sysctl(8) MIB if
	security.bsd.suser_enabled=1.

	It may be advised to setup aliases for grep to default to `-d skip` if
	commonly non-recursively grepping a list that includes directories and
	the potential for the resulting stderr output is not tolerable. Example
	aliases are now installed, commented out, in /root/.cshrc and
	/root/.shrc.

	20200523:
	Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have
	been upgraded to 10.0.1. Please see the 20141231 entry below for
	information about prerequisites and upgrading, if you are not already
	using clang 3.5.0 or higher.

	20200512:
	Support for obsolete compilers has been removed from the build system.
	Clang 6 and GCC 6.4 are the minimum supported versions.

	20200424:
	closefrom(2) has been moved under COMPAT12, and replaced in libc with a
	stub that calls close_range(2). If using a custom kernel configuration,
	you may want to ensure that the COMPAT_FREEBSD12 option is included, as
	a slightly older -CURRENT userland and older FreeBSD userlands may not
	be functional without closefrom(2).

	20200414:
	Upstream DTS from Linux 5.6 was merged and they now have the SID
	and THS (Secure ID controller and THermal Sensor) node present.
	The DTB overlays have now been removed from the tree for the H3/H5 and
	A64 SoCs and the aw_sid and aw_thermal driver have been updated to
	deal with upstream DTS. If you are using those overlays you need to
	remove them from loader.conf and update the DTBs on the FAT partition.

	20200310:
	Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have
	been upgraded to 10.0.0. Please see the 20141231 entry below for
	information about prerequisites and upgrading, if you are not already
	using clang 3.5.0 or higher.

	20200309:
	The amd(8) automount daemon has been removed from the source tree.
	As of FreeBSD 10.1 autofs(5) is the preferred tool for automounting.
	amd is still available in the sysutils/am-utils port.

	20200301:
	Removed brooktree driver (bktr.4) from the tree.

	20200229:
	The WITH_GPL_DTC option has been removed. The BSD-licenced device tree
	compiler in usr.bin/dtc is used on all architectures which use dtc, and
	the GPL dtc is available (if needed) from the sysutils/dtc port.

	20200229:
	The WITHOUT_LLVM_LIBUNWIND option has been removed. LLVM's libunwind
	is used by all supported CPU architectures.

	20200229:
	GCC 4.2.1 has been removed from the tree. The WITH_GCC,
	WITH_GCC_BOOTSTRAP, and WITH_GNUCXX options are no longer available.
	Users who wish to build FreeBSD with GCC must use the external toolchain
	ports or packages.

	20200220:
	ncurses has been updated to a newer version (6.2-20200215). Given the ABI
	has changed, users will have to rebuild all the ports that are linked to
	ncurses.

	20200217:
	The size of struct vnet and the magic cookie have changed.
	Users need to recompile libkvm and all modules using VIMAGE
	together with their new kernel.

	20200212:
	Defining the long deprecated NO_CTF, NO_DEBUG_FILES, NO_INSTALLLIB,
	NO_MAN, NO_PROFILE, and NO_WARNS variables is now an error. Update
	your Makefiles and scripts to define MK_<var>=no instead as required.

	One exception to this is that program or library Makefiles should
	define MAN to empty rather than setting MK_MAN=no.

	20200108:
	Clang/LLVM is now the default compiler and LLD the default
	linker for riscv64.

	20200107:
	make universe no longer uses GCC 4.2.1 on any architectures.
	Architectures not supported by in-tree Clang/LLVM require an
	external toolchain package.

	20200104:
	GCC 4.2.1 is now not built by default, as part of the GCC 4.2.1
	retirement plan. Specifically, the GCC, GCC_BOOTSTRAP, and GNUCXX
	options default to off for all supported CPU architectures. As a
	short-term transition aid they may be enabled via WITH_* options.
	GCC 4.2.1 is expected to be removed from the tree on 2020-03-31.

	20200102:
	Support for armv5 has been disconnected and is being removed. The
	machine combination MACHINE=arm MACHINE_ARCH=arm is no longer valid.
	You must now use a MACHINE_ARCH of armv6 or armv7. The default
	MACHINE_ARCH for MACHINE=arm is now armv7.

	20191226:
	Clang/LLVM is now the default compiler for all powerpc architectures.
	LLD is now the default linker for powerpc64. The change for powerpc64
	also includes a change to the ELFv2 ABI, incompatible with the existing
	ABI.

	20191226:
	Kernel-loadable random(4) modules are no longer unloadable.

	20191222:
	Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have
	been upgraded to 9.0.1. Please see the 20141231 entry below for
	information about prerequisites and upgrading, if you are not already
	using clang 3.5.0 or higher.

	20191212:
	r355677 has modified the internal interface used between the
	NFS modules in the kernel. As such, they must all be upgraded
	simultaneously. I will do a version bump for this.

	20191205:
	The root certificates of the Mozilla CA Certificate Store have been
	imported into the base system and can be managed with the certctl(8)
	utility. If you have installed the security/ca_root_nss port or package
	with the ETCSYMLINK option (the default), be advised that there may be
	differences between those included in the port and those included in
	base due to differences in nss branch used as well as general update
	frequency. Note also that certctl(8) cannot manage certs in the
	format used by the security/ca_root_nss port.

	20191120:
	The amd(8) automount daemon has been disabled by default, and will be
	removed in the future. As of FreeBSD 10.1 the autofs(5) is available
	for automounting.

	20191107:
	The nctgpio and wbwd drivers have been moved to the superio bus.
	If you have one of these drivers in a kernel configuration, then
	you should add device superio to it. If you use one of these drivers
	as a module and you compile a custom set of modules, then you should
	add superio to the set.

	20191021:
	KPIs for network drivers to access interface addresses have changed.
	Users need to recompile NIC driver modules together with kernel.

	20191021:
	The net.link.tap.user_open sysctl no longer prevents user opening of
	already created /dev/tapNN devices. Access is still controlled by
	node permissions, just like tun devices. The net.link.tap.user_open
	sysctl is now used only to allow users to perform devfs cloning of
	tap devices, and the subsequent open may not succeed if the user is not
	in the appropriate group. This sysctl may be deprecated/removed
	completely in the future.

	20191009:
	mips, powerpc, and sparc64 are no longer built as part of
	universe / tinderbox unless MAKE_OBSOLETE_GCC is defined. If
	not defined, mips, powerpc, and sparc64 builds will look for
	the xtoolchain binaries and if installed use them for universe
	builds. As llvm 9.0 becomes vetted for these architectures, they
	will be removed from the list.

	20191009:
	Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have
	been upgraded to 9.0.0. Please see the 20141231 entry below for
	information about prerequisites and upgrading, if you are not already
	using clang 3.5.0 or higher.

	20191003:
	The hpt27xx, hptmv, hptnr, and hptrr drivers have been removed from
	GENERIC. They are available as modules and can be loaded by adding
	to /boot/loader.conf hpt27xx_load="YES", hptmv_load="YES",
	hptnr_load="YES", or hptrr_load="YES", respectively.

	20190913:
	ntpd no longer by default locks its pages in memory, allowing them
	to be paged out by the kernel. Use rlimit memlock to restore
	historic BSD behaviour. For example, add "rlimit memlock 32"
	to ntp.conf to lock up to 32 MB of ntpd address space in memory.

	20190823:
	Several of ping6's options have been renamed for better consistency
	with ping. If you use any of -ARWXaghmrtwx, you must update your
	scripts. See ping6(8) for details.

	20190727:
	The vfs.fusefs.sync_unmount and vfs.fusefs.init_backgrounded sysctls
	and the "-o sync_unmount" and "-o init_backgrounded" mount options have
	been removed from mount_fusefs(8). You can safely remove them from
	your scripts, because they had no effect.

	The vfs.fusefs.fix_broken_io, vfs.fusefs.sync_resize,
	vfs.fusefs.refresh_size, vfs.fusefs.mmap_enable,
	vfs.fusefs.reclaim_revoked, and vfs.fusefs.data_cache_invalidate
	sysctls have been removed. If you felt the need to set any of them to
	a non-default value, please tell asomers@FreeBSD.org why.

	20190713:
	Default permissions on the /var/account/acct file (and copies of it
	rotated by periodic daily scripts) are changed from 0644 to 0640
	because the file contains sensitive information that should not be
	world-readable. If the /var/account directory must be created by
	rc.d/accounting, the mode used is now 0750. Admins who use the
	accounting feature are encouraged to change the mode of an existing
	/var/account directory to 0750 or 0700.

	20190620:
	Entropy collection and the /dev/random device are no longer optional
	components. The "device random" option has been removed.
	Implementations of distilling algorithms can still be made loadable
	with "options RANDOM_LOADABLE" (e.g., random_fortuna.ko).

	20190612:
	Clang, llvm, lld, lldb, compiler-rt, libc++, libunwind and openmp have
	been upgraded to 8.0.1. Please see the 20141231 entry below for
	information about prerequisites and upgrading, if you are not already
	using clang 3.5.0 or higher.

	20190608:
	A fix was applied to i386 kernel modules to avoid panics with
	dpcpu or vnet. Users need to recompile i386 kernel modules
	having pcpu or vnet sections or they will refuse to load.

	20190513:
	User-wired pages now have their own counter,
	vm.stats.vm.v_user_wire_count. The vm.max_wired sysctl was renamed
	to vm.max_user_wired and changed from an unsigned int to an unsigned
	long. bhyve VMs wired with the -S are now subject to the user
	wiring limit; the vm.max_user_wired sysctl may need to be tuned to
	avoid running into the limit.

	20190507:
	The IPSEC option has been removed from GENERIC. Users requiring
	ipsec(4) must now load the ipsec(4) kernel module.

	20190507:
	The tap(4) driver has been folded into tun(4), and the module has been
	renamed to tuntap. You should update any kld_list="if_tap" or
	kld_list="if_tun" entries in /etc/rc.conf, if_tap_load="YES" or
	if_tun_load="YES" entries in /boot/loader.conf to load the if_tuntap
	module instead, and "device tap" or "device tun" entries in kernel
	config files to select the tuntap device instead.

	20190418:
	The following knobs have been added related to tradeoffs between
	safe use of the random device and availability in the absence of
	entropy:

	kern.random.initial_seeding.bypass_before_seeding: tunable; set
	non-zero to bypass the random device prior to seeding, or zero to
	block random requests until the random device is initially seeded.
	For now, set to 1 (unsafe) by default to restore pre-r346250 boot
	availability properties.

	kern.random.initial_seeding.read_random_bypassed_before_seeding:
	read-only diagnostic sysctl that is set when bypass is enabled and
	read_random(9) is bypassed, to enable programmatic handling of this
	initial condition, if desired.

	kern.random.initial_seeding.arc4random_bypassed_before_seeding:
	Similar to the above, but for for arc4random(9) initial seeding.

	kern.random.initial_seeding.disable_bypass_warnings: tunable; set
	non-zero to disable warnings in dmesg when the same conditions are
	met as for the diagnostic sysctls above. Defaults to zero, i.e.,
	produce warnings in dmesg when the conditions are met.

	20190416:
	The loadable random module KPI has changed; the random_infra_init()
	routine now requires a 3rd function pointer for a bool (*)(void)
	method that returns true if the random device is seeded (and
	therefore unblocked).

	20190404:
	r345895 reverts r320698. This implies that an nfsuserd(8) daemon
	built from head sources between r320757 (July 6, 2017) and
	r338192 (Aug. 22, 2018) will not work unless the "-use-udpsock"
	is added to the command line.
	nfsuserd daemons built from head sources that are post-r338192 are
	not affected and should continue to work.

	20190320:
	The fuse(4) module has been renamed to fusefs(4) for consistency with
	other filesystems. You should update any kld_load="fuse" entries in
	/etc/rc.conf, fuse_load="YES" entries in /boot/loader.conf, and
	"options FUSE" entries in kernel config files.

	20190304:
	Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to
	8.0.0. Please see the 20141231 entry below for information about
	prerequisites and upgrading, if you are not already using clang 3.5.0
	or higher.

	20190226:
	geom_uzip(4) depends on the new module xz. If geom_uzip is statically
	compiled into your custom kernel, add 'device xz' statement to the
	kernel config.

	20190219:
	drm and drm2 have been removed from the tree. Please see
	https://wiki.freebsd.org/Graphics for the latest information on
	migrating to the drm ports.

	20190131:
	Iflib is no longer unconditionally compiled into the kernel. Drivers
	using iflib and statically compiled into the kernel, now require
	the 'device iflib' config option. For the same drivers loaded as
	modules on kernels not having 'device iflib', the iflib.ko module
	is loaded automatically.

	20190125:
	The IEEE80211_AMPDU_AGE and AH_SUPPORT_AR5416 kernel configuration
	options no longer exist since r343219 and r343427 respectively;
	nothing uses them, so they should be just removed from custom
	kernel config files.

	20181230:
	r342635 changes the way efibootmgr(8) works by requiring users to add
	the -b (bootnum) parameter for commands where the bootnum was previously
	specified with each option. For example 'efibootmgr -B 0001' is now
	'efibootmgr -B -b 0001'.

	20181220:
	r342286 modifies the NFSv4 server so that it obeys vfs.nfsd.nfs_privport
	in the same as it is applied to NFSv2 and 3. This implies that NFSv4
	servers that have vfs.nfsd.nfs_privport set will only allow mounts
	from clients using a reserved port#. Since both the FreeBSD and Linux
	NFSv4 clients use reserved port#s by default, this should not affect
	most NFSv4 mounts.

	20181219:
	The XLP config has been removed. We can't support 64-bit atomics in this
	kernel because it is running in 32-bit mode. XLP users must transition
	to running a 64-bit kernel (XLP64 or XLPN32).

	The mips GXEMUL support has been removed from FreeBSD. MALTA* + qemu is
	the preferred emulator today and we don't need two different ones.

	The old sibyte / swarm / Broadcom BCM1250 support has been
	removed from the mips port.

	20181211:
	Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to
	7.0.1. Please see the 20141231 entry below for information about
	prerequisites and upgrading, if you are not already using clang 3.5.0
	or higher.

	20181211:
	Remove the timed and netdate programs from the base tree. Setting
	the time with these daemons has been obsolete for over a decade.

	20181126:
	On amd64, arm64 and armv7 (architectures that install LLVM's ld.lld
	linker as /usr/bin/ld) GNU ld is no longer installed as ld.bfd, as
	it produces broken binaries when ifuncs are in use. Users needing
	GNU ld should install the binutils port or package.

	20181123:
	The BSD crtbegin and crtend code has been enabled by default. It has
	had extensive testing on amd64, arm64, and i386. It can be disabled
	by building a world with -DWITHOUT_BSD_CRTBEGIN.

	20181115:
	The set of CTM commands (ctm, ctm_smail, ctm_rmail, ctm_dequeue)
	has been converted to a port (misc/ctm) and will be removed from
	FreeBSD-13. It is available as a package (ctm) for all supported
	FreeBSD versions.

	20181110:
	The default newsyslog.conf(5) file has been changed to only include
	files in /etc/newsyslog.conf.d/ and /usr/local/etc/newsyslog.conf.d/ if
	the filenames end in '.conf' and do not begin with a '.'.

	You should check the configuration files in these two directories match
	this naming convention. You can verify which configuration files are
	being included using the command:
	$ newsyslog -Nrv

	20181015:
	Ports for the DRM modules have been simplified. Now, amd64 users should
	just install the drm-kmod port. All others should install
	drm-legacy-kmod.

	Graphics hardware that's newer than about 2010 usually works with
	drm-kmod. For hardware older than 2013, however, some users will need
	to use drm-legacy-kmod if drm-kmod doesn't work for them. Hardware older
	than 2008 usually only works in drm-legacy-kmod. The graphics team can
	only commit to hardware made since 2013 due to the complexity of the
	market and difficulty to test all the older cards effectively. If you
	have hardware supported by drm-kmod, you are strongly encouraged to use
	that as you will get better support.

	Other than KPI chasing, drm-legacy-kmod will not be updated. As outlined
	elsewhere, the drm and drm2 modules will be eliminated from the src base
	soon (with a limited exception for arm). Please update to the package
	asap and report any issues to x11@freebsd.org.

	Generally, anybody using the drm*-kmod packages should add
	WITHOUT_DRM_MODULE=t and WITHOUT_DRM2_MODULE=t to avoid nasty
	cross-threading surprises, especially with automatic driver
	loading from X11 startup. These will become the defaults in 13-current
	shortly.

	20181012:
	The ixlv(4) driver has been renamed to iavf(4). As a consequence,
	custom kernel and module loading configuration files must be updated
	accordingly. Moreover, interfaces previous presented as ixlvN to the
	system are now exposed as iavfN and network configuration files must
	be adjusted as necessary.

	20181009:
	OpenSSL has been updated to version 1.1.1. This update included
	additional various API changes throughout the base system. It is
	important to rebuild third-party software after upgrading. The value
	of __FreeBSD_version has been bumped accordingly.

	20181006:
	The legacy DRM modules and drivers have now been added to the loader's
	module blacklist, in favor of loading them with kld_list in rc.conf(5).
	The module blacklist may be overridden with the loader.conf(5)
	'module_blacklist' variable, but loading them via rc.conf(5) is strongly
	encouraged.

	20181002:
	The cam(4) based nda(4) driver will be used over nvd(4) by default on
	powerpc64. You may set 'options NVME_USE_NVD=1' in your kernel conf or
	loader tunable 'hw.nvme.use_nvd=1' if you wish to use the existing
	driver. Make sure to edit /boot/etc/kboot.conf and fstab to use the
	nda device name.

	20180913:
	Reproducible build mode is now on by default, in preparation for
	FreeBSD 12.0. This eliminates build metadata such as the user,
	host, and time from the kernel (and uname), unless the working tree
	corresponds to a modified checkout from a version control system.
	The previous behavior can be obtained by setting the /etc/src.conf
	knob WITHOUT_REPRODUCIBLE_BUILD.

	20180826:
	The Yarrow CSPRNG has been removed from the kernel as it has not been
	supported by its designers since at least 2003. Fortuna has been the
	default since FreeBSD-11.

	20180822:
	devctl freeze/thaw have gone into the tree, the rc scripts have been
	updated to use them and devmatch has been changed. You should update
	kernel, userland and rc scripts all at the same time.

	20180818:
	The default interpreter has been switched from 4th to Lua.
	LOADER_DEFAULT_INTERP, documented in build(7), will override the default
	interpreter. If you have custom FORTH code you will need to set
	LOADER_DEFAULT_INTERP=4th (valid values are 4th, lua or simp) in
	src.conf for the build. This will create default hard links between
	loader and loader_4th instead of loader and loader_lua, the new default.
	If you are using UEFI it will create the proper hard link to loader.efi.

	bhyve uses userboot.so. It remains 4th-only until some issues are solved
	regarding coexisting with multiple versions of FreeBSD are resolved.

	20180815:
	ls(1) now respects the COLORTERM environment variable used in other
	systems and software to indicate that a colored terminal is both
	supported and desired. If ls(1) is suddenly emitting colors, they may
	be disabled again by either removing the unwanted COLORTERM from your
	environment, or using `ls --color=never`. The ls(1) specific CLICOLOR
	may not be observed in a future release.

	20180808:
	The default pager for most commands has been changed to "less". To
	restore the old behavior, set PAGER="more" and MANPAGER="more -s" in
	your environment.

	20180731:
	The jedec_ts(4) driver has been removed. A superset of its functionality
	is available in the jedec_dimm(4) driver, and the manpage for that
	driver includes migration instructions. If you have "device jedec_ts"
	in your kernel configuration file, it must be removed.

	20180730:
	amd64/GENERIC now has EFI runtime services, EFIRT, enabled by default.
	This should have no effect if the kernel is booted via BIOS/legacy boot.
	EFIRT may be disabled via a loader tunable, efi.rt.disabled, if a system
	has a buggy firmware that prevents a successful boot due to use of
	runtime services.

	20180727:
	Atmel AT91RM9200 and AT91SAM9, Cavium CNS 11xx and XScale
	support has been removed from the tree. These ports were
	obsolete and/or known to be broken for many years.

	20180723:
	loader.efi has been augmented to participate more fully in the
	UEFI boot manager protocol. loader.efi will now look at the
	BootXXXX environment variable to determine if a specific kernel
	or root partition was specified. XXXX is derived from BootCurrent.
	efibootmgr(8) manages these standard UEFI variables.

	20180720:
	zfsloader's functionality has now been folded into loader.
	zfsloader is no longer necessary once you've updated your
	boot blocks. For a transition period, we will install a
	hardlink for zfsloader to loader to allow a smooth transition
	until the boot blocks can be updated (hard link because old
	zfs boot blocks don't understand symlinks).

	20180719:
	ARM64 now have efifb support, if you want to have serial console
	on your arm64 board when an screen is connected and the bootloader
	setup a frame buffer for us to use, just add :
	boot_serial=YES
	boot_multicons=YES
	in /boot/loader.conf
	For Raspberry Pi 3 (RPI) users, this is needed even if you don't have
	an screen connected as the firmware will setup a frame buffer are that
	u-boot will expose as an EFI frame buffer.

	20180719:
	New uid:gid added, ntpd:ntpd (123:123). Be sure to run mergemaster
	or take steps to update /etc/passwd before doing installworld on
	existing systems. Do not skip the "mergemaster -Fp" step before
	installworld, as described in the update procedures near the bottom
	of this document. Also, rc.d/ntpd now starts ntpd(8) as user ntpd
	if the new mac_ntpd(4) policy is available, unless ntpd_flags or
	the ntp config file contain options that change file/dir locations.
	When such options (e.g., "statsdir" or "crypto") are used, ntpd can
	still be run as non-root by setting ntpd_user=ntpd in rc.conf, after
	taking steps to ensure that all required files/dirs are accessible
	by the ntpd user.

	20180717:
	Big endian arm support has been removed.

	20180711:
	The static environment setup in kernel configs is no longer mutually
	exclusive with the loader(8) environment by default. In order to
	restore the previous default behavior of disabling the loader(8)
	environment if a static environment is present, you must specify
	loader_env.disabled=1 in the static environment.

	20180705:
	The ABI of syscalls used by management tools like sockstat and
	netstat has been broken to allow 32-bit binaries to work on
	64-bit kernels without modification. These programs will need
	to match the kernel in order to function. External programs may
	require minor modifications to accommodate a change of type in
	structures from pointers to 64-bit virtual addresses.

	20180702:
	On i386 and amd64 atomics are now inlined. Out of tree modules using
	atomics will need to be rebuilt.

	20180701:
	The '%I' format in the kern.corefile sysctl limits the number of
	core files that a process can generate to the number stored in the
	debug.ncores sysctl. The '%I' format is replaced by the single digit
	index. Previously, if all indexes were taken the kernel would overwrite
	only a core file with the highest index in a filename.
	Currently the system will create a new core file if there is a free
	index or if all slots are taken it will overwrite the oldest one.

	20180630:
	Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to
	6.0.1. Please see the 20141231 entry below for information about
	prerequisites and upgrading, if you are not already using clang 3.5.0
	or higher.

	20180628:
	r335753 introduced a new quoting method. However, etc/devd/devmatch.conf
	needed to be changed to work with it. This change was made with r335763
	and requires a mergemaster / etcupdate / etc to update the installed
	file.

	20180612:
	r334930 changed the interface between the NFS modules, so they all
	need to be rebuilt. r335018 did a __FreeBSD_version bump for this.

	20180530:
	As of r334391 lld is the default amd64 system linker; it is installed
	as /usr/bin/ld. Kernel build workarounds (see 20180510 entry) are no
	longer necessary.

	20180530:
	The kernel / userland interface for devinfo changed, so you'll
	need a new kernel and userland as a pair for it to work (rebuilding
	lib/libdevinfo is all that's required). devinfo and devmatch will
	not work, but everything else will when there's a mismatch.

	20180523:
	The on-disk format for hwpmc callchain records has changed to include
	threadid corresponding to a given record. This changes the field offsets
	and thus requires that libpmcstat be rebuilt before using a kernel
	later than r334108.

	20180517:
	The vxge(4) driver has been removed. This driver was introduced into
	HEAD one week before the Exar left the Ethernet market and is not
	known to be used. If you have device vxge in your kernel config file
	it must be removed.

	20180510:
	The amd64 kernel now requires a ld that supports ifunc to produce a
	working kernel, either lld or a newer binutils. lld is built by default
	on amd64, and the 'buildkernel' target uses it automatically. However,
	it is not the default linker, so building the kernel the traditional
	way requires LD=ld.lld on the command line (or LD=/usr/local/bin/ld for
	binutils port/package). lld will soon be default, and this requirement
	will go away.

	NOTE: As of r334391 lld is the default system linker on amd64, and no
	workaround is necessary.

	20180508:
	The nxge(4) driver has been removed. This driver was for PCI-X 10g
	cards made by s2io/Neterion. The company was acquired by Exar and
	no longer sells or supports Ethernet products. If you have device
	nxge in your kernel config file it must be removed.

	20180504:
	The tz database (tzdb) has been updated to 2018e. This version more
	correctly models time stamps in time zones with negative DST such as
	Europe/Dublin (from 1971 on), Europe/Prague (1946/7), and
	Africa/Windhoek (1994/2017). This does not affect the UT offsets, only
	time zone abbreviations and the tm_isdst flag.

	20180502:
	The ixgb(4) driver has been removed. This driver was for an early and
	uncommon legacy PCI 10GbE for a single ASIC, Intel 82597EX. Intel
	quickly shifted to the long lived ixgbe family. If you have device
	ixgb in your kernel config file it must be removed.

	20180501:
	The lmc(4) driver has been removed. This was a WAN interface
	card that was already reportedly rare in 2003, and had an ambiguous
	license. If you have device lmc in your kernel config file it must
	be removed.

	20180413:
	Support for Arcnet networks has been removed. If you have device
	arcnet or device cm in your kernel config file they must be
	removed.

	20180411:
	Support for FDDI networks has been removed. If you have device
	fddi or device fpa in your kernel config file they must be
	removed.

	20180406:
	In addition to supporting RFC 3164 formatted messages, the
	syslogd(8) service is now capable of parsing RFC 5424 formatted
	log messages. The main benefit of using RFC 5424 is that clients
	may now send log messages with timestamps containing year numbers,
	microseconds and time zone offsets.

	Similarly, the syslog(3) C library function has been altered to
	send RFC 5424 formatted messages to the local system logging
	daemon. On systems using syslogd(8), this change should have no
	negative impact, as long as syslogd(8) and the C library are
	updated at the same time. On systems using a different system
	logging daemon, it may be necessary to make configuration
	adjustments, depending on the software used.

	When using syslog-ng, add the 'syslog-protocol' flag to local
	input sources to enable parsing of RFC 5424 formatted messages:

	source src {
	unix-dgram("/var/run/log" flags(syslog-protocol));
	}

	When using rsyslog, disable the 'SysSock.UseSpecialParser' option
	of the 'imuxsock' module to let messages be processed by the
	regular RFC 3164/5424 parsing pipeline:

	module(load="imuxsock" SysSock.UseSpecialParser="off")

	Do note that these changes only affect communication between local
	applications and syslogd(8). The format that syslogd(8) uses to
	store messages on disk or forward messages to other systems
	remains unchanged. syslogd(8) still uses RFC 3164 for these
	purposes. Options to customize this behaviour will be added in the
	future. Utilities that process log files stored in /var/log are
	thus expected to continue to function as before.

	__FreeBSD_version has been incremented to 1200061 to denote this
	change.

	20180328:
	Support for token ring networks has been removed. If you
	have "device token" in your kernel config you should remove
	it. No device drivers supported token ring.

	20180323:
	makefs was modified to be able to tag ISO9660 El Torito boot catalog
	entries as EFI instead of overloading the i386 tag as done previously.
	The amd64 mkisoimages.sh script used to build amd64 ISO images for
	release was updated to use this. This may mean that makefs must be
	updated before "make cdrom" can be run in the release directory. This
	should be as simple as:

	$ cd $SRCDIR/usr.sbin/makefs
	$ make depend all install

	20180212:
	FreeBSD boot loader enhanced with Lua scripting. It's purely opt-in for
	now by building WITH_LOADER_LUA and WITHOUT_FORTH in /etc/src.conf.
	Co-existence for the transition period will come shortly. Booting is a
	complex environment and test coverage for Lua-enabled loaders has been
	thin, so it would be prudent to assume it might not work and make
	provisions for backup boot methods.

	20180211:
	devmatch functionality has been turned on in devd. It will automatically
	load drivers for unattached devices. This may cause unexpected drivers
	to be loaded. Please report any problems to current@ and
	imp@freebsd.org.

	20180114:
	Clang, llvm, lld, lldb, compiler-rt and libc++ have been upgraded to
	6.0.0. Please see the 20141231 entry below for information about
	prerequisites and upgrading, if you are not already using clang 3.5.0
	or higher.

	20180110:
	LLVM's lld linker is now used as the FreeBSD/amd64 bootstrap linker.
	This means it is used to link the kernel and userland libraries and
	executables, but is not yet installed as /usr/bin/ld by default.

	To revert to ld.bfd as the bootstrap linker, in /etc/src.conf set
	WITHOUT_LLD_BOOTSTRAP=yes

	20180110:
	On i386, pmtimer has been removed. Its functionality has been folded
	into apm. It was a no-op on ACPI in current for a while now (but was
	still needed on i386 in FreeBSD 11 and earlier). Users may need to
	remove it from kernel config files.

	20180104:
	The use of RSS hash from the network card aka flowid has been
	disabled by default for lagg(4) as it's currently incompatible with
	the lacp and loadbalance protocols.

	This can be re-enabled by setting the following in loader.conf:
	net.link.lagg.default_use_flowid="1"

	20180102:
	The SW_WATCHDOG option is no longer necessary to enable the
	hardclock-based software watchdog if no hardware watchdog is
	configured. As before, SW_WATCHDOG will cause the software
	watchdog to be enabled even if a hardware watchdog is configured.

	20171215:
	r326887 fixes the issue described in the 20171214 UPDATING entry.
	r326888 flips the switch back to building GELI support always.

	20171214:
	r362593 broke ZFS + GELI support for reasons unknown. However,
	it also broke ZFS support generally, so GELI has been turned off
	by default as the lesser evil in r326857. If you boot off ZFS and/or
	GELI, it might not be a good time to update.

	20171125:
	PowerPC users must update loader(8) by rebuilding world before
	installing a new kernel, as the protocol connecting them has
	changed. Without the update, loader metadata will not be passed
	successfully to the kernel and users will have to enter their
	root partition at the kernel mountroot prompt to continue booting.
	Newer versions of loader can boot old kernels without issue.

	20171110:
	The LOADER_FIREWIRE_SUPPORT build variable as been renamed to
	WITH/OUT_LOADER_FIREWIRE. LOADER_{NO_,}GELI_SUPPORT has been renamed
	to WITH/OUT_LOADER_GELI.

	20171106:
	The naive and non-compliant support of posix_fallocate(2) in ZFS
	has been removed as of r325320. The system call now returns EINVAL
	when used on a ZFS file. Although the new behavior complies with the
	standard, some consumers are not prepared to cope with it.
	One known victim is lld prior to r325420.

	20171102:
	Building in a FreeBSD src checkout will automatically create object
	directories now rather than store files in the current directory if
	'make obj' was not ran. Calling 'make obj' is no longer necessary.
	This feature can be disabled by setting WITHOUT_AUTO_OBJ=yes in
	/etc/src-env.conf (not /etc/src.conf), or passing the option in the
	environment.

	20171101:
	The default MAKEOBJDIR has changed from /usr/obj/<srcdir> for native
	builds, and /usr/obj/<arch>/<srcdir> for cross-builds, to a unified
	/usr/obj/<srcdir>/<arch>. This behavior can be changed to the old
	format by setting WITHOUT_UNIFIED_OBJDIR=yes in /etc/src-env.conf,
	the environment, or with -DWITHOUT_UNIFIED_OBJDIR when building.
	The UNIFIED_OBJDIR option is a transitional feature that will be
	removed for 12.0 release; please migrate to the new format for any
	tools by looking up the OBJDIR used by 'make -V .OBJDIR' means rather
	than hardcoding paths.

	20171028:
	The native-xtools target no longer installs the files by default to the
	OBJDIR. Use the native-xtools-install target with a DESTDIR to install
	to ${DESTDIR}/${NXTP} where NXTP defaults to /nxb-bin.

	20171021:
	As part of the boot loader infrastructure cleanup, LOADER_*_SUPPORT
	options are changing from controlling the build if defined / undefined
	to controlling the build with explicit 'yes' or 'no' values. They will
	shift to WITH/WITHOUT options to match other options in the system.

	20171010:
	libstand has turned into a private library for sys/boot use only.
	It is no longer supported as a public interface outside of sys/boot.

	20171005:
	The arm port has split armv6 into armv6 and armv7. armv7 is now
	a valid TARGET_ARCH/MACHINE_ARCH setting. If you have an armv7 system
	and are running a kernel from before r324363, you will need to add
	MACHINE_ARCH=armv7 to 'make buildworld' to do a native build.

	20171003:
	When building multiple kernels using KERNCONF, non-existent KERNCONF
	files will produce an error and buildkernel will fail. Previously
	missing KERNCONF files silently failed giving no indication as to
	why, only to subsequently discover during installkernel that the
	desired kernel was never built in the first place.

	20170912:
	The default serial number format for CTL LUNs has changed. This will
	affect users who use /dev/diskid/* device nodes, or whose FibreChannel
	or iSCSI clients care about their LUNs' serial numbers. Users who
	require serial number stability should hardcode serial numbers in
	/etc/ctl.conf .

	20170912:
	For 32-bit arm compiled for hard-float support, soft-floating point
	binaries now always get their shared libraries from
	LD_SOFT_LIBRARY_PATH (in the past, this was only used if
	/usr/libsoft also existed). Only users with a hard-float ld.so, but
	soft-float everything else should be affected.

	20170826:
	The geli password typed at boot is now hidden. To restore the previous
	behavior, see geli(8) for configuration options.

	20170825:
	Move PMTUD blackhole counters to TCPSTATS and remove them from bare
	sysctl values. Minor nit, but requires a rebuild of both world/kernel
	to complete.

	20170814:
	"make check" behavior (made in ^/head@r295380) has been changed to
	execute from a limited sandbox, as opposed to executing from
	${TESTSDIR}.

	Behavioral changes:
	- The "beforecheck" and "aftercheck" targets are now specified.
	- ${CHECKDIR} (added in commit noted above) has been removed.
	- Legacy behavior can be enabled by setting
	WITHOUT_MAKE_CHECK_USE_SANDBOX in src.conf(5) or the environment.

	If the limited sandbox mode is enabled, "make check" will execute
	"make distribution", then install, execute the tests, and clean up the
	sandbox if successful.

	The "make distribution" and "make install" targets are typically run as
	root to set appropriate permissions and ownership at installation time.
	The end-user should set "WITH_INSTALL_AS_USER" in src.conf(5) or the
	environment if executing "make check" with limited sandbox mode using
	an unprivileged user.

	20170808:
	Since the switch to GPT disk labels, fsck for UFS/FFS has been
	unable to automatically find alternate superblocks. As of r322297,
	the information needed to find alternate superblocks has been
	moved to the end of the area reserved for the boot block.
	Filesystems created with a newfs of this vintage or later
	will create the recovery information. If you have a filesystem
	created prior to this change and wish to have a recovery block
	created for your filesystem, you can do so by running fsck in
	foreground mode (i.e., do not use the -p or -y options). As it
	starts, fsck will ask ``SAVE DATA TO FIND ALTERNATE SUPERBLOCKS''
	to which you should answer yes.

	20170728:
	As of r321665, an NFSv4 server configuration that services
	Kerberos mounts or clients that do not support the uid/gid in
	owner/owner_group string capability, must explicitly enable
	the nfsuserd daemon by adding nfsuserd_enable="YES" to the
	machine's /etc/rc.conf file.

	20170722:
	Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 5.0.0.
	Please see the 20141231 entry below for information about prerequisites
	and upgrading, if you are not already using clang 3.5.0 or higher.

	20170701:
	WITHOUT_RCMDS is now the default. Set WITH_RCMDS if you need the
	r-commands (rlogin, rsh, etc.) to be built with the base system.

	20170625:
	The FreeBSD/powerpc platform now uses a 64-bit type for time_t. This is
	a very major ABI incompatible change, so users of FreeBSD/powerpc must
	be careful when performing source upgrades. It is best to run
	'make installworld' from an alternate root system, either a live
	CD/memory stick, or a temporary root partition. Additionally, all ports
	must be recompiled. powerpc64 is largely unaffected, except in the case
	of 32-bit compatibility. All 32-bit binaries will be affected.

	20170623:
	Forward compatibility for the "ino64" project have been committed. This
	will allow most new binaries to run on older kernels in a limited
	fashion. This prevents many of the common foot-shooting actions in the
	upgrade as well as the limited ability to roll back the kernel across
	the ino64 upgrade. Complicated use cases may not work properly, though
	enough simpler ones work to allow recovery in most situations.

	20170620:
	Switch back to the BSDL dtc (Device Tree Compiler). Set WITH_GPL_DTC
	if you require the GPL compiler.

	20170618:
	The internal ABI used for communication between the NFS kernel modules
	was changed by r320085, so __FreeBSD_version was bumped to
	ensure all the NFS related modules are updated together.

	20170617:
	The ABI of struct event was changed by extending the data
	member to 64bit and adding ext fields. For upgrade, same
	precautions as for the entry 20170523 "ino64" must be
	followed.

	20170531:
	The GNU roff toolchain has been removed from base. To render manpages
	which are not supported by mandoc(1), man(1) can fallback on GNU roff
	from ports (and recommends to install it).
	To render roff(7) documents, consider using GNU roff from ports or the
	heirloom doctools roff toolchain from ports via pkg install groff or
	via pkg install heirloom-doctools.

	20170524:
	The ath(4) and ath_hal(4) modules now build piecemeal to allow for
	smaller runtime footprint builds. This is useful for embedded systems
	which only require one chipset support.

	If you load it as a module, make sure this is in /boot/loader.conf:

	if_ath_load="YES"

	This will load the HAL, all chip/RF backends and if_ath_pci.
	If you have if_ath_pci in /boot/loader.conf, ensure it is after
	if_ath or it will not load any HAL chipset support.

	If you want to selectively load things (eg on ye cheape ARM/MIPS
	platforms where RAM is at a premium) you should:

	* load ath_hal
	* load the chip modules in question
	* load ath_rate, ath_dfs
	* load ath_main
	* load if_ath_pci and/or if_ath_ahb depending upon your particular
	bus bind type - this is where probe/attach is done.

	For further comments/feedback, poke adrian@ .

	20170523:
	The "ino64" 64-bit inode project has been committed, which extends
	a number of types to 64 bits. Upgrading in place requires care and
	adherence to the documented upgrade procedure.

	If using a custom kernel configuration ensure that the
	COMPAT_FREEBSD11 option is included (as during the upgrade the
	system will be running the ino64 kernel with the existing world).

	For the safest in-place upgrade begin by removing previous build
	artifacts via "rm -rf /usr/obj/*". Then, carefully follow the full
	procedure documented below under the heading "To rebuild everything and
	install it on the current system." Specifically, a reboot is required
	after installing the new kernel before installing world. While an
	installworld normally works by accident from multiuser after rebooting
	the proper kernel, there are many cases where this will fail across this
	upgrade and installworld from single user is required.

	20170424:
	The NATM framework including the en(4), fatm(4), hatm(4), and
	patm(4) devices has been removed. Consumers should plan a
	migration before the end-of-life date for FreeBSD 11.

	20170420:
	GNU diff has been replaced by a BSD licensed diff. Some features of GNU
	diff has not been implemented, if those are needed a newer version of
	GNU diff is available via the diffutils package under the gdiff name.

	20170413:
	As of r316810 for ipfilter, keep frags is no longer assumed when
	keep state is specified in a rule. r316810 aligns ipfilter with
	documentation in man pages separating keep frags from keep state.
	This allows keep state to be specified without forcing keep frags
	and allows keep frags to be specified independently of keep state.
	To maintain previous behaviour, also specify keep frags with
	keep state (as documented in ipf.conf.5).

	20170407:
	arm64 builds now use the base system LLD 4.0.0 linker by default,
	instead of requiring that the aarch64-binutils port or package be
	installed. To continue using aarch64-binutils, set
	CROSS_BINUTILS_PREFIX=/usr/local/aarch64-freebsd/bin .

	20170405:
	The UDP optimization in entry 20160818 that added the sysctl
	net.inet.udp.require_l2_bcast has been reverted. L2 broadcast
	packets will no longer be treated as L3 broadcast packets.

	20170331:
	Binds and sends to the loopback addresses, IPv6 and IPv4, will now
	use any explicitly assigned loopback address available in the jail
	instead of using the first assigned address of the jail.

	20170329:
	The ctl.ko module no longer implements the iSCSI target frontend:
	cfiscsi.ko does instead.

	If building cfiscsi.ko as a kernel module, the module can be loaded
	via one of the following methods:
	- `cfiscsi_load="YES"` in loader.conf(5).
	- Add `cfiscsi` to `$kld_list` in rc.conf(5).
	- ctladm(8)/ctld(8), when compiled with iSCSI support
	(`WITH_ISCSI=yes` in src.conf(5))

	Please see cfiscsi(4) for more details.

	20170316:
	The mmcsd.ko module now additionally depends on geom_flashmap.ko.
	Also, mmc.ko and mmcsd.ko need to be a matching pair built from the
	same source (previously, the dependency of mmcsd.ko on mmc.ko was
	missing, but mmcsd.ko now will refuse to load if it is incompatible
	with mmc.ko).

	20170315:
	The syntax of ipfw(8) named states was changed to avoid ambiguity.
	If you have used named states in the firewall rules, you need to modify
	them after installworld and before rebooting. Now named states must
	be prefixed with colon.

	20170311:
	The old drm (sys/dev/drm/) drivers for i915 and radeon have been
	removed as the userland we provide cannot use them. The KMS version
	(sys/dev/drm2) supports the same hardware.

	20170302:
	Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 4.0.0.
	Please see the 20141231 entry below for information about prerequisites
	and upgrading, if you are not already using clang 3.5.0 or higher.

	20170221:
	The code that provides support for ZFS .zfs/ directory functionality
	has been reimplemented. It's not possible now to create a snapshot
	by mkdir under .zfs/snapshot/. That should be the only user visible
	change.

	20170216:
	EISA bus support has been removed. The WITH_EISA option is no longer
	valid.

	20170215:
	MCA bus support has been removed.

	20170127:
	The WITH_LLD_AS_LD / WITHOUT_LLD_AS_LD build knobs have been renamed
	WITH_LLD_IS_LD / WITHOUT_LLD_IS_LD, for consistency with CLANG_IS_CC.

	20170112:
	The EM_MULTIQUEUE kernel configuration option is deprecated now that
	the em(4) driver conforms to iflib specifications.

	20170109:
	The igb(4), em(4) and lem(4) ethernet drivers are now implemented via
	IFLIB. If you have a custom kernel configuration that excludes em(4)
	but you use igb(4), you need to re-add em(4) to your custom
	configuration.

	20161217:
	Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.9.1.
	Please see the 20141231 entry below for information about prerequisites
	and upgrading, if you are not already using clang 3.5.0 or higher.

	20161124:
	Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.9.0.
	Please see the 20141231 entry below for information about prerequisites
	and upgrading, if you are not already using clang 3.5.0 or higher.

	20161119:
	The layout of the pmap structure has changed for powerpc to put the pmap
	statistics at the front for all CPU variations. libkvm(3) and all tools
	that link against it need to be recompiled.

	20161030:
	isl(4) and cyapa(4) drivers now require a new driver,
	chromebook_platform(4), to work properly on Chromebook-class hardware.
	On other types of hardware the drivers may need to be configured using
	device hints. Please see the corresponding manual pages for details.

	20161017:
	The urtwn(4) driver was merged into rtwn(4) and now consists of
	rtwn(4) main module + rtwn_usb(4) and rtwn_pci(4) bus-specific
	parts.
	Also, firmware for RTL8188CE was renamed due to possible name
	conflict (rtwnrtl8192cU(B) -> rtwnrtl8192cE(B))

	20161015:
	GNU rcs has been removed from base. It is available as packages:
	- rcs: Latest GPLv3 GNU rcs version.
	- rcs57: Copy of the latest version of GNU rcs (GPLv2) before it was
	removed from base.

	20161008:
	Use of the cc_cdg, cc_chd, cc_hd, or cc_vegas congestion control
	modules now requires that the kernel configuration contain the
	TCP_HHOOK option. (This option is included in the GENERIC kernel.)

	20161003:
	The WITHOUT_ELFCOPY_AS_OBJCOPY src.conf(5) knob has been retired.
	ELF Tool Chain's elfcopy is always installed as /usr/bin/objcopy.

	20160924:
	Relocatable object files with the extension of .So have been renamed
	to use an extension of .pico instead. The purpose of this change is
	to avoid a name clash with shared libraries on case-insensitive file
	systems. On those file systems, foo.So is the same file as foo.so.

	20160918:
	GNU rcs has been turned off by default. It can (temporarily) be built
	again by adding WITH_RCS knob in src.conf.
	Otherwise, GNU rcs is available from packages:
	- rcs: Latest GPLv3 GNU rcs version.
	- rcs57: Copy of the latest version of GNU rcs (GPLv2) from base.

	20160918:
	The backup_uses_rcs functionality has been removed from rc.subr.

	20160908:
	The queue(3) debugging macro, QUEUE_MACRO_DEBUG, has been split into
	two separate components, QUEUE_MACRO_DEBUG_TRACE and
	QUEUE_MACRO_DEBUG_TRASH. Define both for the original
	QUEUE_MACRO_DEBUG behavior.

	20160824:
	r304787 changed some ioctl interfaces between the iSCSI userspace
	programs and the kernel. ctladm, ctld, iscsictl, and iscsid must be
	rebuilt to work with new kernels. __FreeBSD_version has been bumped
	to 1200005.

	20160818:
	The UDP receive code has been updated to only treat incoming UDP
	packets that were addressed to an L2 broadcast address as L3
	broadcast packets. It is not expected that this will affect any
	standards-conforming UDP application. The new behaviour can be
	disabled by setting the sysctl net.inet.udp.require_l2_bcast to
	0.

	20160818:
	Remove the openbsd_poll system call.
	__FreeBSD_version has been bumped because of this.

	20160708:
	The stable/11 branch has been created from head@r302406.

	20160622:
	The libc stub for the pipe(2) system call has been replaced with
	a wrapper that calls the pipe2(2) system call and the pipe(2)
	system call is now only implemented by the kernels that include
	"options COMPAT_FREEBSD10" in their config file (this is the
	default). Users should ensure that this option is enabled in
	their kernel or upgrade userspace to r302092 before upgrading their
	kernel.

	20160527:
	CAM will now strip leading spaces from SCSI disks' serial numbers.
	This will affect users who create UFS filesystems on SCSI disks using
	those disk's diskid device nodes. For example, if /etc/fstab
	previously contained a line like
	"/dev/diskid/DISK-%20%20%20%20%20%20%20ABCDEFG0123456", you should
	change it to "/dev/diskid/DISK-ABCDEFG0123456". Users of geom
	transforms like gmirror may also be affected. ZFS users should
	generally be fine.

	20160523:
	The bitstring(3) API has been updated with new functionality and
	improved performance. But it is binary-incompatible with the old API.
	Objects built with the new headers may not be linked against objects
	built with the old headers.

	20160520:
	The brk and sbrk functions have been removed from libc on arm64.
	Binutils from ports has been updated to not link to these
	functions and should be updated to the latest version before
	installing a new libc.

	20160517:
	The armv6 port now defaults to hard float ABI. Limited support
	for running both hardfloat and soft float on the same system
	is available using the libraries installed with -DWITH_LIBSOFT.
	This has only been tested as an upgrade path for installworld
	and packages may fail or need manual intervention to run. New
	packages will be needed.

	To update an existing self-hosted armv6hf system, you must add
	TARGET_ARCH=armv6 on the make command line for both the build
	and the install steps.

	20160510:
	Kernel modules compiled outside of a kernel build now default to
	installing to /boot/modules instead of /boot/kernel. Many kernel
	modules built this way (such as those in ports) already overrode
	KMODDIR explicitly to install into /boot/modules. However,
	manually building and installing a module from /sys/modules will
	now install to /boot/modules instead of /boot/kernel.

	20160414:
	The CAM I/O scheduler has been committed to the kernel. There should be
	no user visible impact. This does enable NCQ Trim on ada SSDs. While the
	list of known rogues that claim support for this but actually corrupt
	data is believed to be complete, be on the lookout for data
	corruption. The known rogue list is believed to be complete:

	o Crucial MX100, M550 drives with MU01 firmware.
	o Micron M510 and M550 drives with MU01 firmware.
	o Micron M500 prior to MU07 firmware
	o Samsung 830, 840, and 850 all firmwares
	o FCCT M500 all firmwares

	Crucial has firmware http://www.crucial.com/usa/en/support-ssd-firmware
	with working NCQ TRIM. For Micron branded drives, see your sales rep for
	updated firmware. Black listed drives will work correctly because these
	drives work correctly so long as no NCQ TRIMs are sent to them. Given
	this list is the same as found in Linux, it's believed there are no
	other rogues in the market place. All other models from the above
	vendors work.

	To be safe, if you are at all concerned, you can quirk each of your
	drives to prevent NCQ from being sent by setting:
	kern.cam.ada.X.quirks="0x2"
	in loader.conf. If the drive requires the 4k sector quirk, set the
	quirks entry to 0x3.

	20160330:
	The FAST_DEPEND build option has been removed and its functionality is
	now the one true way. The old mkdep(1) style of 'make depend' has
	been removed. See 20160311 for further details.

	20160317:
	Resource range types have grown from unsigned long to uintmax_t. All
	drivers, and anything using libdevinfo, need to be recompiled.

	20160311:
	WITH_FAST_DEPEND is now enabled by default for in-tree and out-of-tree
	builds. It no longer runs mkdep(1) during 'make depend', and the
	'make depend' stage can safely be skipped now as it is auto ran
	when building 'make all' and will generate all SRCS and DPSRCS before
	building anything else. Dependencies are gathered at compile time with
	-MF flags kept in separate .depend files per object file. Users should
	run 'make cleandepend' once if using -DNO_CLEAN to clean out older
	stale .depend files.

	20160306:
	On amd64, clang 3.8.0 can now insert sections of type AMD64_UNWIND into
	kernel modules. Therefore, if you load any kernel modules at boot time,
	please install the boot loaders after you install the kernel, but before
	rebooting, e.g.:

	make buildworld
	make buildkernel KERNCONF=YOUR_KERNEL_HERE
	make installkernel KERNCONF=YOUR_KERNEL_HERE
	make -C sys/boot install
	<reboot in single user>

	Then follow the usual steps, described in the General Notes section,
	below.

	20160305:
	Clang, llvm, lldb and compiler-rt have been upgraded to 3.8.0. Please
	see the 20141231 entry below for information about prerequisites and
	upgrading, if you are not already using clang 3.5.0 or higher.

	20160301:
	The AIO subsystem is now a standard part of the kernel. The
	VFS_AIO kernel option and aio.ko kernel module have been removed.
	Due to stability concerns, asynchronous I/O requests are only
	permitted on sockets and raw disks by default. To enable
	asynchronous I/O requests on all file types, set the
	vfs.aio.enable_unsafe sysctl to a non-zero value.

	20160226:
	The ELF object manipulation tool objcopy is now provided by the
	ELF Tool Chain project rather than by GNU binutils. It should be a
	drop-in replacement, with the addition of arm64 support. The
	(temporary) src.conf knob WITHOUT_ELFCOPY_AS_OBJCOPY knob may be set
	to obtain the GNU version if necessary.

	20160129:
	Building ZFS pools on top of zvols is prohibited by default. That
	feature has never worked safely; it's always been prone to deadlocks.
	Using a zvol as the backing store for a VM guest's virtual disk will
	still work, even if the guest is using ZFS. Legacy behavior can be
	restored by setting vfs.zfs.vol.recursive=1.

	20160119:
	The NONE and HPN patches has been removed from OpenSSH. They are
	still available in the security/openssh-portable port.

	20160113:
	With the addition of ypldap(8), a new _ypldap user is now required
	during installworld. "mergemaster -p" can be used to add the user
	prior to installworld, as documented in the handbook.

	20151216:
	The tftp loader (pxeboot) now uses the option root-path directive. As a
	consequence it no longer looks for a pxeboot.4th file on the tftp
	server. Instead it uses the regular /boot infrastructure as with the
	other loaders.

	20151211:
	The code to start recording plug and play data into the modules has
	been committed. While the old tools will properly build a new kernel,
	a number of warnings about "unknown metadata record 4" will be produced
	for an older kldxref. To avoid such warnings, make sure to rebuild
	the kernel toolchain (or world). Make sure that you have r292078 or
	later when trying to build 292077 or later before rebuilding.

	20151207:
	Debug data files are now built by default with 'make buildworld' and
	installed with 'make installworld'. This facilitates debugging but
	requires more disk space both during the build and for the installed
	world. Debug files may be disabled by setting WITHOUT_DEBUG_FILES=yes
	in src.conf(5).

	20151130:
	r291527 changed the internal interface between the nfsd.ko and
	nfscommon.ko modules. As such, they must both be upgraded to-gether.
	__FreeBSD_version has been bumped because of this.

	20151108:
	Add support for unicode collation strings leads to a change of
	order of files listed by ls(1) for example. To get back to the old
	behaviour, set LC_COLLATE environment variable to "C".

	Databases administrators will need to reindex their databases given
	collation results will be different.

	Due to a bug in install(1) it is recommended to remove the ancient
	locales before running make installworld.

	rm -rf /usr/share/locale/*

	20151030:
	The OpenSSL has been upgraded to 1.0.2d. Any binaries requiring
	libcrypto.so.7 or libssl.so.7 must be recompiled.

	20151020:
	Qlogic 24xx/25xx firmware images were updated from 5.5.0 to 7.3.0.
	Kernel modules isp_2400_multi and isp_2500_multi were removed and
	should be replaced with isp_2400 and isp_2500 modules respectively.

	20151017:
	The build previously allowed using 'make -n' to not recurse into
	sub-directories while showing what commands would be executed, and
	'make -n -n' to recursively show commands. Now 'make -n' will recurse
	and 'make -N' will not.

	20151012:
	If you specify SENDMAIL_MC or SENDMAIL_CF in make.conf, mergemaster
	and etcupdate will now use this file. A custom sendmail.cf is now
	updated via this mechanism rather than via installworld. If you had
	excluded sendmail.cf in mergemaster.rc or etcupdate.conf, you may
	want to remove the exclusion or change it to "always install".
	/etc/mail/sendmail.cf is now managed the same way regardless of
	whether SENDMAIL_MC/SENDMAIL_CF is used. If you are not using
	SENDMAIL_MC/SENDMAIL_CF there should be no change in behavior.

	20151011:
	Compatibility shims for legacy ATA device names have been removed.
	It includes ATA_STATIC_ID kernel option, kern.cam.ada.legacy_aliases
	and kern.geom.raid.legacy_aliases loader tunables, kern.devalias.*
	environment variables, /dev/ad* and /dev/ar* symbolic links.

	20151006:
	Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.7.0.
	Please see the 20141231 entry below for information about prerequisites
	and upgrading, if you are not already using clang 3.5.0 or higher.

	20150924:
	Kernel debug files have been moved to /usr/lib/debug/boot/kernel/,
	and renamed from .symbols to .debug. This reduces the size requirements
	on the boot partition or file system and provides consistency with
	userland debug files.

	When using the supported kernel installation method the
	/usr/lib/debug/boot/kernel directory will be renamed (to kernel.old)
	as is done with /boot/kernel.

	Developers wishing to maintain the historical behavior of installing
	debug files in /boot/kernel/ can set KERN_DEBUGDIR="" in src.conf(5).

	20150827:
	The wireless drivers had undergone changes that remove the 'parent
	interface' from the ifconfig -l output. The rc.d network scripts
	used to check presence of a parent interface in the list, so old
	scripts would fail to start wireless networking. Thus, etcupdate(3)
	or mergemaster(8) run is required after kernel update, to update your
	rc.d scripts in /etc.

	20150827:
	pf no longer supports 'scrub fragment crop' or 'scrub fragment drop-ovl'
	These configurations are now automatically interpreted as
	'scrub fragment reassemble'.

	20150817:
	Kernel-loadable modules for the random(4) device are back. To use
	them, the kernel must have

	device random
	options RANDOM_LOADABLE

	kldload(8) can then be used to load random_fortuna.ko
	or random_yarrow.ko. Please note that due to the indirect
	function calls that the loadable modules need to provide,
	the build-in variants will be slightly more efficient.

	The random(4) kernel option RANDOM_DUMMY has been retired due to
	unpopularity. It was not all that useful anyway.

	20150813:
	The WITHOUT_ELFTOOLCHAIN_TOOLS src.conf(5) knob has been retired.
	Control over building the ELF Tool Chain tools is now provided by
	the WITHOUT_TOOLCHAIN knob.

	20150810:
	The polarity of Pulse Per Second (PPS) capture events with the
	uart(4) driver has been corrected. Prior to this change the PPS
	"assert" event corresponded to the trailing edge of a positive PPS
	pulse and the "clear" event was the leading edge of the next pulse.

	As the width of a PPS pulse in a typical GPS receiver is on the
	order of 1 millisecond, most users will not notice any significant
	difference with this change.

	Anyone who has compensated for the historical polarity reversal by
	configuring a negative offset equal to the pulse width will need to
	remove that workaround.

	20150809:
	The default group assigned to /dev/dri entries has been changed
	from 'wheel' to 'video' with the id of '44'. If you want to have
	access to the dri devices please add yourself to the video group
	with:

	# pw groupmod video -m $USER

	20150806:
	The menu.rc and loader.rc files will now be replaced during
	upgrades. Please migrate local changes to menu.rc.local and
	loader.rc.local instead.

	20150805:
	GNU Binutils versions of addr2line, c++filt, nm, readelf, size,
	strings and strip have been removed. The src.conf(5) knob
	WITHOUT_ELFTOOLCHAIN_TOOLS no longer provides the binutils tools.

	20150728:
	As ZFS requires more kernel stack pages than is the default on some
	architectures e.g. i386, it now warns if KSTACK_PAGES is less than
	ZFS_MIN_KSTACK_PAGES (which is 4 at the time of writing).

	Please consider using 'options KSTACK_PAGES=X' where X is greater
	than or equal to ZFS_MIN_KSTACK_PAGES i.e. 4 in such configurations.

	20150706:
	sendmail has been updated to 8.15.2. Starting with FreeBSD 11.0
	and sendmail 8.15, sendmail uses uncompressed IPv6 addresses by
	default, i.e., they will not contain "::". For example, instead
	of ::1, it will be 0:0:0:0:0:0:0:1. This permits a zero subnet
	to have a more specific match, such as different map entries for
	IPv6:0:0 vs IPv6:0. This change requires that configuration
	data (including maps, files, classes, custom ruleset, etc.) must
	use the same format, so make certain such configuration data is
	upgrading. As a very simple check search for patterns like
	'IPv6:[0-9a-fA-F:]*::' and 'IPv6::'. To return to the old
	behavior, set the m4 option confUSE_COMPRESSED_IPV6_ADDRESSES or
	the cf option UseCompressedIPv6Addresses.

	20150630:
	The default kernel entropy-processing algorithm is now
	Fortuna, replacing Yarrow.

	Assuming you have 'device random' in your kernel config
	file, the configurations allow a kernel option to override
	this default. You may choose ONE of:

	options RANDOM_YARROW # Legacy /dev/random algorithm.
	options RANDOM_DUMMY # Blocking-only driver.

	If you have neither, you get Fortuna. For most people,
	read no further, Fortuna will give a /dev/random that works
	like it always used to, and the difference will be irrelevant.

	If you remove 'device random', you get NO kernel-processed
	entropy at all. This may be acceptable to folks building
	embedded systems, but has complications. Carry on reading,
	and it is assumed you know what you need.

	PLEASE read random(4) and random(9) if you are in the
	habit of tweaking kernel configs, and/or if you are a member
	of the embedded community, wanting specific and not-usual
	behaviour from your security subsystems.

	NOTE!! If you use RANDOM_DUMMY and/or have no 'device
	random', you will NOT have a functioning /dev/random, and
	many cryptographic features will not work, including SSH.
	You may also find strange behaviour from the random(3) set
	of library functions, in particular sranddev(3), srandomdev(3)
	and arc4random(3). The reason for this is that the KERN_ARND
	sysctl only returns entropy if it thinks it has some to
	share, and with RANDOM_DUMMY or no 'device random' this
	will never happen.

	20150623:
	An additional fix for the issue described in the 20150614 sendmail
	entry below has been committed in revision 284717.

	20150616:
	FreeBSD's old make (fmake) has been removed from the system. It is
	available as the devel/fmake port or via pkg install fmake.

	20150615:
	The fix for the issue described in the 20150614 sendmail entry
	below has been committed in revision 284436. The work
	around described in that entry is no longer needed unless the
	default setting is overridden by a confDH_PARAMETERS configuration
	setting of '5' or pointing to a 512 bit DH parameter file.

	20150614:
	ALLOW_DEPRECATED_ATF_TOOLS/ATFFILE support has been removed from
	atf.test.mk (included from bsd.test.mk). Please upgrade devel/atf
	and devel/kyua to version 0.20+ and adjust any calling code to work
	with Kyuafile and kyua.

	20150614:
	The import of openssl to address the FreeBSD-SA-15:10.openssl
	security advisory includes a change which rejects handshakes
	with DH parameters below 768 bits. sendmail releases prior
	to 8.15.2 (not yet released), defaulted to a 512 bit
	DH parameter setting for client connections. To work around
	this interoperability, sendmail can be configured to use a
	2048 bit DH parameter by:

	1. Edit /etc/mail/`hostname`.mc
	2. If a setting for confDH_PARAMETERS does not exist or
	exists and is set to a string beginning with '5',
	replace it with '2'.
	3. If a setting for confDH_PARAMETERS exists and is set to
	a file path, create a new file with:
	openssl dhparam -out /path/to/file 2048
	4. Rebuild the .cf file:
	cd /etc/mail/; make; make install
	5. Restart sendmail:
	cd /etc/mail/; make restart

	A sendmail patch is coming, at which time this file will be
	updated.

	20150604:
	Generation of legacy formatted entries have been disabled by default
	in pwd_mkdb(8), as all base system consumers of the legacy formatted
	entries were converted to use the new format by default when the new,
	machine independent format have been added and supported since FreeBSD
	5.x.

	Please see the pwd_mkdb(8) manual page for further details.

	20150525:
	Clang and llvm have been upgraded to 3.6.1 release. Please see the
	20141231 entry below for information about prerequisites and upgrading,
	if you are not already using 3.5.0 or higher.

	20150521:
	TI platform code switched to using vendor DTS files and this update
	may break existing systems running on Beaglebone, Beaglebone Black,
	and Pandaboard:

	- dtb files should be regenerated/reinstalled. Filenames are the
	same but content is different now
	- GPIO addressing was changed, now each GPIO bank (32 pins per bank)
	has its own /dev/gpiocX device, e.g. pin 121 on /dev/gpioc0 in old
	addressing scheme is now pin 25 on /dev/gpioc3.
	- Pandaboard: /etc/ttys should be updated, serial console device is
	now /dev/ttyu2, not /dev/ttyu0

	20150501:
	soelim(1) from gnu/usr.bin/groff has been replaced by usr.bin/soelim.
	If you need the GNU extension from groff soelim(1), install groff
	from package: pkg install groff, or via ports: textproc/groff.

	20150423:
	chmod, chflags, chown and chgrp now affect symlinks in -R mode as
	defined in symlink(7); previously symlinks were silently ignored.

	20150415:
	The const qualifier has been removed from iconv(3) to comply with
	POSIX. The ports tree is aware of this from r384038 onwards.

	20150416:
	Libraries specified by LIBADD in Makefiles must have a corresponding
	DPADD_<lib> variable to ensure correct dependencies. This is now
	enforced in src.libnames.mk.

	20150324:
	From legacy ata(4) driver was removed support for SATA controllers
	supported by more functional drivers ahci(4), siis(4) and mvs(4).
	Kernel modules ataahci and ataadaptec were removed completely,
	replaced by ahci and mvs modules respectively.

	20150315:
	Clang, llvm and lldb have been upgraded to 3.6.0 release. Please see
	the 20141231 entry below for information about prerequisites and
	upgrading, if you are not already using 3.5.0 or higher.

	20150307:
	The 32-bit PowerPC kernel has been changed to a position-independent
	executable. This can only be booted with a version of loader(8)
	newer than January 31, 2015, so make sure to update both world and
	kernel before rebooting.

	20150217:
	If you are running a -CURRENT kernel since r273872 (Oct 30th, 2014),
	but before r278950, the RNG was not seeded properly. Immediately
	upgrade the kernel to r278950 or later and regenerate any keys (e.g.
	ssh keys or openssl keys) that were generated w/ a kernel from that
	range. This does not affect programs that directly used /dev/random
	or /dev/urandom. All userland uses of arc4random(3) are affected.

	20150210:
	The autofs(4) ABI was changed in order to restore binary compatibility
	with 10.1-RELEASE. The automountd(8) daemon needs to be rebuilt to work
	with the new kernel.

	20150131:
	The powerpc64 kernel has been changed to a position-independent
	executable. This can only be booted with a new version of loader(8),
	so make sure to update both world and kernel before rebooting.

	20150118:
	Clang and llvm have been upgraded to 3.5.1 release. This is a bugfix
	only release, no new features have been added. Please see the 20141231
	entry below for information about prerequisites and upgrading, if you
	are not already using 3.5.0.

	20150107:
	ELF tools addr2line, elfcopy (strip), nm, size, and strings are now
	taken from the ELF Tool Chain project rather than GNU binutils. They
	should be drop-in replacements, with the addition of arm64 support.
	The WITHOUT_ELFTOOLCHAIN_TOOLS= knob may be used to obtain the
	binutils tools, if necessary. See 20150805 for updated information.

	20150105:
	The default Unbound configuration now enables remote control
	using a local socket. Users who have already enabled the
	local_unbound service should regenerate their configuration
	by running "service local_unbound setup" as root.

	20150102:
	The GNU texinfo and GNU info pages have been removed.
	To be able to view GNU info pages please install texinfo from ports.

	20141231:
	Clang, llvm and lldb have been upgraded to 3.5.0 release.

	As of this release, a prerequisite for building clang, llvm and lldb is
	a C++11 capable compiler and C++11 standard library. This means that to
	be able to successfully build the cross-tools stage of buildworld, with
	clang as the bootstrap compiler, your system compiler or cross compiler
	should either be clang 3.3 or later, or gcc 4.8 or later, and your
	system C++ library should be libc++, or libdstdc++ from gcc 4.8 or
	later.

	On any standard FreeBSD 10.x or 11.x installation, where clang and
	libc++ are on by default (that is, on x86 or arm), this should work out
	of the box.

	On 9.x installations where clang is enabled by default, e.g. on x86 and
	powerpc, libc++ will not be enabled by default, so libc++ should be
	built (with clang) and installed first. If both clang and libc++ are
	missing, build clang first, then use it to build libc++.

	On 8.x and earlier installations, upgrade to 9.x first, and then follow
	the instructions for 9.x above.

	Sparc64 and mips users are unaffected, as they still use gcc 4.2.1 by
	default, and do not build clang.

	Many embedded systems are resource constrained, and will not be able to
	build clang in a reasonable time, or in some cases at all. In those
	cases, cross building bootable systems on amd64 is a workaround.

	This new version of clang introduces a number of new warnings, of which
	the following are most likely to appear:

	-Wabsolute-value

	This warns in two cases, for both C and C++:
	* When the code is trying to take the absolute value of an unsigned
	quantity, which is effectively a no-op, and almost never what was
	intended. The code should be fixed, if at all possible. If you are
	sure that the unsigned quantity can be safely cast to signed, without
	loss of information or undefined behavior, you can add an explicit
	cast, or disable the warning.

	* When the code is trying to take an absolute value, but the called
	abs() variant is for the wrong type, which can lead to truncation.
	If you want to disable the warning instead of fixing the code, please
	make sure that truncation will not occur, or it might lead to unwanted
	side-effects.

	-Wtautological-undefined-compare and
	-Wundefined-bool-conversion

	These warn when C++ code is trying to compare 'this' against NULL, while
	'this' should never be NULL in well-defined C++ code. However, there is
	some legacy (pre C++11) code out there, which actively abuses this
	feature, which was less strictly defined in previous C++ versions.

	Squid and openjdk do this, for example. The warning can be turned off
	for C++98 and earlier, but compiling the code in C++11 mode might result
	in unexpected behavior; for example, the parts of the program that are
	unreachable could be optimized away.

	20141222:
	The old NFS client and server (kernel options NFSCLIENT, NFSSERVER)
	kernel sources have been removed. The .h files remain, since some
	utilities include them. This will need to be fixed later.
	If "mount -t oldnfs ..." is attempted, it will fail.
	If the "-o" option on mountd(8), nfsd(8) or nfsstat(1) is used,
	the utilities will report errors.

	20141121:
	The handling of LOCAL_LIB_DIRS has been altered to skip addition of
	directories to top level SUBDIR variable when their parent
	directory is included in LOCAL_DIRS. Users with build systems with
	such hierarchies and without SUBDIR entries in the parent
	directory Makefiles should add them or add the directories to
	LOCAL_DIRS.

	20141109:
	faith(4) and faithd(8) have been removed from the base system. Faith
	has been obsolete for a very long time.

	20141104:
	vt(4), the new console driver, is enabled by default. It brings
	support for Unicode and double-width characters, as well as
	support for UEFI and integration with the KMS kernel video
	drivers.

	You may need to update your console settings in /etc/rc.conf,
	most probably the keymap. During boot, /etc/rc.d/syscons will
	indicate what you need to do.

	vt(4) still has issues and lacks some features compared to
	syscons(4). See the wiki for up-to-date information:
	https://wiki.freebsd.org/Newcons

	If you want to keep using syscons(4), you can do so by adding
	the following line to /boot/loader.conf:
	kern.vty=sc

	20141102:
	pjdfstest has been integrated into kyua as an opt-in test suite.
	Please see share/doc/pjdfstest/README for more details on how to
	execute it.

	20141009:
	gperf has been removed from the base system for architectures
	that use clang. Ports that require gperf will obtain it from the
	devel/gperf port.

	20140923:
	pjdfstest has been moved from tools/regression/pjdfstest to
	contrib/pjdfstest .

	20140922:
	At svn r271982, The default linux compat kernel ABI has been adjusted
	to 2.6.18 in support of the linux-c6 compat ports infrastructure
	update. If you wish to continue using the linux-f10 compat ports,
	add compat.linux.osrelease=2.6.16 to your local sysctl.conf. Users are
	encouraged to update their linux-compat packages to linux-c6 during
	their next update cycle.

	20140729:
	The ofwfb driver, used to provide a graphics console on PowerPC when
	using vt(4), no longer allows mmap() of all physical memory. This
	will prevent Xorg on PowerPC with some ATI graphics cards from
	initializing properly unless x11-servers/xorg-server is updated to
	1.12.4_8 or newer.

	20140723:
	The xdev targets have been converted to using TARGET and
	TARGET_ARCH instead of XDEV and XDEV_ARCH.

	20140719:
	The default unbound configuration has been modified to address
	issues with reverse lookups on networks that use private
	address ranges. If you use the local_unbound service, run
	"service local_unbound setup" as root to regenerate your
	configuration, then "service local_unbound reload" to load the
	new configuration.

	20140709:
	The GNU texinfo and GNU info pages are not built and installed
	anymore, WITH_INFO knob has been added to allow to built and install
	them again.
	UPDATE: see 20150102 entry on texinfo's removal

	20140708:
	The GNU readline library is now an INTERNALLIB - that is, it is
	statically linked into consumers (GDB and variants) in the base
	system, and the shared library is no longer installed. The
	devel/readline port is available for third party software that
	requires readline.

	20140702:
	The Itanium architecture (ia64) has been removed from the list of
	known architectures. This is the first step in the removal of the
	architecture.

	20140701:
	Commit r268115 has added NFSv4.1 server support, merged from
	projects/nfsv4.1-server. Since this includes changes to the
	internal interfaces between the NFS related modules, a full
	build of the kernel and modules will be necessary.
	__FreeBSD_version has been bumped.

	20140629:
	The WITHOUT_VT_SUPPORT kernel config knob has been renamed
	WITHOUT_VT. (The other _SUPPORT knobs have a consistent meaning
	which differs from the behaviour controlled by this knob.)

	20140619:
	Maximal length of the serial number in CTL was increased from 16 to
	64 chars, that breaks ABI. All CTL-related tools, such as ctladm
	and ctld, need to be rebuilt to work with a new kernel.

	20140606:
	The libatf-c and libatf-c++ major versions were downgraded to 0 and
	1 respectively to match the upstream numbers. They were out of
	sync because, when they were originally added to FreeBSD, the
	upstream versions were not respected. These libraries are private
	and not yet built by default, so renumbering them should be a
	non-issue. However, unclean source trees will yield broken test
	programs once the operator executes "make delete-old-libs" after a
	"make installworld".

	Additionally, the atf-sh binary was made private by moving it into
	/usr/libexec/. Already-built shell test programs will keep the
	path to the old binary so they will break after "make delete-old"
	is run.

	If you are using WITH_TESTS=yes (not the default), wipe the object
	tree and rebuild from scratch to prevent spurious test failures.
	This is only needed once: the misnumbered libraries and misplaced
	binaries have been added to OptionalObsoleteFiles.inc so they will
	be removed during a clean upgrade.

	20140512:
	Clang and llvm have been upgraded to 3.4.1 release.

	20140508:
	We bogusly installed src.opts.mk in /usr/share/mk. This file should
	be removed to avoid issues in the future (and has been added to
	ObsoleteFiles.inc).

	20140505:
	/etc/src.conf now affects only builds of the FreeBSD src tree. In the
	past, it affected all builds that used the bsd.*.mk files. The old
	behavior was a bug, but people may have relied upon it. To get this
	behavior back, you can .include /etc/src.conf from /etc/make.conf
	(which is still global and isn't changed). This also changes the
	behavior of incremental builds inside the tree of individual
	directories. Set MAKESYSPATH to ".../share/mk" to do that.
	Although this has survived make universe and some upgrade scenarios,
	other upgrade scenarios may have broken. At least one form of
	temporary breakage was fixed with MAKESYSPATH settings for buildworld
	as well... In cases where MAKESYSPATH isn't working with this
	setting, you'll need to set it to the full path to your tree.

	One side effect of all this cleaning up is that bsd.compiler.mk
	is no longer implicitly included by bsd.own.mk. If you wish to
	use COMPILER_TYPE, you must now explicitly include bsd.compiler.mk
	as well.

	20140430:
	The lindev device has been removed since /dev/full has been made a
	standard device. __FreeBSD_version has been bumped.

	20140424:
	The knob WITHOUT_VI was added to the base system, which controls
	building ex(1), vi(1), etc. Older releases of FreeBSD required ex(1)
	in order to reorder files share/termcap and didn't build ex(1) as a
	build tool, so building/installing with WITH_VI is highly advised for
	build hosts for older releases.

	This issue has been fixed in stable/9 and stable/10 in r277022 and
	r276991, respectively.

	20140418:
	The YES_HESIOD knob has been removed. It has been obsolete for
	a decade. Please move to using WITH_HESIOD instead or your builds
	will silently lack HESIOD.

	20140405:
	The uart(4) driver has been changed with respect to its handling
	of the low-level console. Previously the uart(4) driver prevented
	any process from changing the baudrate or the CLOCAL and HUPCL
	control flags. By removing the restrictions, operators can make
	changes to the serial console port without having to reboot.
	However, when getty(8) is started on the serial device that is
	associated with the low-level console, a misconfigured terminal
	line in /etc/ttys will now have a real impact.
	Before upgrading the kernel, make sure that /etc/ttys has the
	serial console device configured as 3wire without baudrate to
	preserve the previous behaviour. E.g:
	ttyu0 "/usr/libexec/getty 3wire" vt100 on secure

	20140306:
	Support for libwrap (TCP wrappers) in rpcbind was disabled by default
	to improve performance. To re-enable it, if needed, run rpcbind
	with command line option -W.

	20140226:
	Switched back to the GPL dtc compiler due to updates in the upstream
	dts files not being supported by the BSDL dtc compiler. You will need
	to rebuild your kernel toolchain to pick up the new compiler. Core dumps
	may result while building dtb files during a kernel build if you fail
	to do so. Set WITHOUT_GPL_DTC if you require the BSDL compiler.

	20140216:
	Clang and llvm have been upgraded to 3.4 release.

	20140216:
	The nve(4) driver has been removed. Please use the nfe(4) driver
	for NVIDIA nForce MCP Ethernet adapters instead.

	20140212:
	An ABI incompatibility crept into the libc++ 3.4 import in r261283.
	This could cause certain C++ applications using shared libraries built
	against the previous version of libc++ to crash. The incompatibility
	has now been fixed, but any C++ applications or shared libraries built
	between r261283 and r261801 should be recompiled.

	20140204:
	OpenSSH will now ignore errors caused by kernel lacking of Capsicum
	capability mode support. Please note that enabling the feature in
	kernel is still highly recommended.

	20140131:
	OpenSSH is now built with sandbox support, and will use sandbox as
	the default privilege separation method. This requires Capsicum
	capability mode support in kernel.

	20140128:
	The libelf and libdwarf libraries have been updated to newer
	versions from upstream. Shared library version numbers for
	these two libraries were bumped. Any ports or binaries
	requiring these two libraries should be recompiled.
	__FreeBSD_version is bumped to 1100006.

	20140110:
	If a Makefile in a tests/ directory was auto-generating a Kyuafile
	instead of providing an explicit one, this would prevent such
	Makefile from providing its own Kyuafile in the future during
	NO_CLEAN builds. This has been fixed in the Makefiles but manual
	intervention is needed to clean an objdir if you use NO_CLEAN:
	# find /usr/obj -name Kyuafile \| xargs rm -f

	20131213:
	The behavior of gss_pseudo_random() for the krb5 mechanism
	has changed, for applications requesting a longer random string
	than produced by the underlying enctype's pseudo-random() function.
	In particular, the random string produced from a session key of
	enctype aes256-cts-hmac-sha1-96 or aes256-cts-hmac-sha1-96 will
	be different at the 17th octet and later, after this change.
	The counter used in the PRF+ construction is now encoded as a
	big-endian integer in accordance with RFC 4402.
	__FreeBSD_version is bumped to 1100004.

	20131108:
	The WITHOUT_ATF build knob has been removed and its functionality
	has been subsumed into the more generic WITHOUT_TESTS. If you were
	using the former to disable the build of the ATF libraries, you
	should change your settings to use the latter.

	20131025:
	The default version of mtree is nmtree which is obtained from
	NetBSD. The output is generally the same, but may vary
	slightly. If you found you need identical output adding
	"-F freebsd9" to the command line should do the trick. For the
	time being, the old mtree is available as fmtree.

	20131014:
	libbsdyml has been renamed to libyaml and moved to /usr/lib/private.
	This will break ports-mgmt/pkg. Rebuild the port, or upgrade to pkg
	1.1.4_8 and verify bsdyml not linked in, before running "make
	delete-old-libs":
	# make -C /usr/ports/ports-mgmt/pkg build deinstall install clean
	or
	# pkg install pkg; ldd /usr/local/sbin/pkg \| grep bsdyml

	20131010:
	The stable/10 branch has been created in subversion from head
	revision r256279.

	COMMON ITEMS:

	General Notes
	-------------
	Sometimes, obscure build problems are the result of environment
	poisoning. This can happen because the make utility reads its
	environment when searching for values for global variables. To run
	your build attempts in an "environmental clean room", prefix all make
	commands with 'env -i '. See the env(1) manual page for more details.
	Occasionally a build failure will occur with "make -j" due to a race
	condition. If this happens try building again without -j, and please
	report a bug if it happens consistently.

	When upgrading from one major version to another it is generally best to
	upgrade to the latest code in the currently installed branch first, then
	do an upgrade to the new branch. This is the best-tested upgrade path,
	and has the highest probability of being successful. Please try this
	approach if you encounter problems with a major version upgrade. Since
	the stable 4.x branch point, one has generally been able to upgrade from
	anywhere in the most recent stable branch to head / current (or even the
	last couple of stable branches). See the top of this file when there's
	an exception.

	The update process will emit an error on an attempt to perform a build
	or install from a FreeBSD version below the earliest supported version.
	When updating from an older version the update should be performed one
	major release at a time, including running `make delete-old` at each
	step.

	When upgrading a live system, having a root shell around before
	installing anything can help undo problems. Not having a root shell
	around can lead to problems if pam has changed too much from your
	starting point to allow continued authentication after the upgrade.

	This file should be read as a log of events. When a later event changes
	information of a prior event, the prior event should not be deleted.
	Instead, a pointer to the entry with the new information should be
	placed in the old entry. Readers of this file should also sanity check
	older entries before relying on them blindly. Authors of new entries
	should write them with this in mind.

	ZFS notes
	---------
	When upgrading the boot ZFS pool to a new version, always follow
	these two steps:

	1.) recompile and reinstall the ZFS boot loader and boot block
	(this is part of "make buildworld" and "make installworld")

	2.) update the ZFS boot block on your boot drive

	The following example updates the ZFS boot block on the first
	partition (freebsd-boot) of a GPT partitioned drive ada0:
	"gpart bootcode -p /boot/gptzfsboot -i 1 ada0"

	Non-boot pools do not need these updates.

	To build a kernel
	-----------------
	If you are updating from a prior version of FreeBSD (even one just
	a few days old), you should follow this procedure. It is the most
	failsafe as it uses a /usr/obj tree with a fresh mini-buildworld,

	make kernel-toolchain
	make -DALWAYS_CHECK_MAKE buildkernel KERNCONF=YOUR_KERNEL_HERE
	make -DALWAYS_CHECK_MAKE installkernel KERNCONF=YOUR_KERNEL_HERE

	To test a kernel once
	---------------------
	If you just want to boot a kernel once (because you are not sure
	if it works, or if you want to boot a known bad kernel to provide
	debugging information) run
	make installkernel KERNCONF=YOUR_KERNEL_HERE KODIR=/boot/testkernel
	nextboot -k testkernel

	To rebuild everything and install it on the current system.
	-----------------------------------------------------------
	# Note: sometimes if you are running current you gotta do more than
	# is listed here if you are upgrading from a really old current.

	<make sure you have good level 0 dumps>
	make buildworld
	make buildkernel KERNCONF=YOUR_KERNEL_HERE
	make installkernel KERNCONF=YOUR_KERNEL_HERE
	[1]
	<reboot in single user> [3]
	mergemaster -Fp [5]
	make installworld
	mergemaster -Fi [4]
	make delete-old [6]
	<reboot>

	To cross-install current onto a separate partition
	--------------------------------------------------
	# In this approach we use a separate partition to hold
	# current's root, 'usr', and 'var' directories. A partition
	# holding "/", "/usr" and "/var" should be about 2GB in
	# size.

	<make sure you have good level 0 dumps>
	<boot into -stable>
	make buildworld
	make buildkernel KERNCONF=YOUR_KERNEL_HERE
	<maybe newfs current's root partition>
	<mount current's root partition on directory ${CURRENT_ROOT}>
	make installworld DESTDIR=${CURRENT_ROOT} -DDB_FROM_SRC
	make distribution DESTDIR=${CURRENT_ROOT} # if newfs'd
	make installkernel KERNCONF=YOUR_KERNEL_HERE DESTDIR=${CURRENT_ROOT}
	cp /etc/fstab ${CURRENT_ROOT}/etc/fstab # if newfs'd
	<edit ${CURRENT_ROOT}/etc/fstab to mount "/" from the correct partition>
	<reboot into current>
	<do a "native" rebuild/install as described in the previous section>
	<maybe install compatibility libraries from ports/misc/compat*>
	<reboot>


	To upgrade in-place from stable to current
	----------------------------------------------
	<make sure you have good level 0 dumps>
	make buildworld [9]
	make buildkernel KERNCONF=YOUR_KERNEL_HERE [8]
	make installkernel KERNCONF=YOUR_KERNEL_HERE
	[1]
	<reboot in single user> [3]
	mergemaster -Fp [5]
	make installworld
	mergemaster -Fi [4]
	make delete-old [6]
	<reboot>

	Make sure that you've read the UPDATING file to understand the
	tweaks to various things you need. At this point in the life
	cycle of current, things change often and you are on your own
	to cope. The defaults can also change, so please read ALL of
	the UPDATING entries.

	Also, if you are tracking -current, you must be subscribed to
	freebsd-current@freebsd.org. Make sure that before you update
	your sources that you have read and understood all the recent
	messages there. If in doubt, please track -stable which has
	much fewer pitfalls.

	[1] If you have third party modules, such as vmware, you
	should disable them at this point so they don't crash your
	system on reboot.

	[3] From the bootblocks, boot -s, and then do
	fsck -p
	mount -u /
	mount -a
	sh /etc/rc.d/zfs start # mount zfs filesystem, if needed
	cd src # full path to source
	adjkerntz -i # if CMOS is wall time
	Also, when doing a major release upgrade, it is required that
	you boot into single user mode to do the installworld.

	[4] Note: This step is non-optional. Failure to do this step
	can result in a significant reduction in the functionality of the
	system. Attempting to do it by hand is not recommended and those
	that pursue this avenue should read this file carefully, as well
	as the archives of freebsd-current and freebsd-hackers mailing lists
	for potential gotchas. The -U option is also useful to consider.
	See mergemaster(8) for more information.

	[5] Usually this step is a no-op. However, from time to time
	you may need to do this if you get unknown user in the following
	step. It never hurts to do it all the time. You may need to
	install a new mergemaster (cd src/usr.sbin/mergemaster && make
	install) after the buildworld before this step if you last updated
	from current before 20130425 or from -stable before 20130430.

	[6] This only deletes old files and directories. Old libraries
	can be deleted by "make delete-old-libs", but you have to make
	sure that no program is using those libraries anymore.

	[8] The new kernel must be able to run existing binaries used by an
	installworld. When upgrading across major versions, the new kernel's
	configuration must include the correct COMPAT_FREEBSD<n> option for
	existing binaries (e.g. COMPAT_FREEBSD11 to run 11.x binaries). Failure
	to do so may leave you with a system that is hard to boot to recover. A
	GENERIC kernel will include suitable compatibility options to run
	binaries from older branches. Note that the ability to run binaries
	from unsupported branches is not guaranteed.

	Make sure that you merge any new devices from GENERIC since the
	last time you updated your kernel config file. Options also
	change over time, so you may need to adjust your custom kernels
	for these as well.

	[9] If CPUTYPE is defined in your /etc/make.conf, make sure to use the
	"?=" instead of the "=" assignment operator, so that buildworld can
	override the CPUTYPE if it needs to.

	MAKEOBJDIRPREFIX must be defined in an environment variable, and
	not on the command line, or in /etc/make.conf. buildworld will
	warn if it is improperly defined.
	FORMAT:

	This file contains a list, in reverse chronological order, of major
	breakages in tracking -current. It is not guaranteed to be a complete
	list of such breakages, and only contains entries since September 23, 2011.
	If you need to see UPDATING entries from before that date, you will need
	to fetch an UPDATING file from an older FreeBSD release.

	Copyright information:

	Copyright 1998-2009 M. Warner Losh <imp@FreeBSD.org>

	Redistribution, publication, translation and use, with or without
	modification, in full or in part, in any form or format of this
	document are permitted without further permission from the author.

	THIS DOCUMENT IS PROVIDED BY WARNER LOSH ``AS IS'' AND ANY EXPRESS OR
	IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	DISCLAIMED. IN NO EVENT SHALL WARNER LOSH BE LIABLE FOR ANY DIRECT,
	INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
	IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	POSSIBILITY OF SUCH DAMAGE.

	Contact Warner Losh if you have any questions about your use of
	this document.

	$FreeBSD$
	diff --git a/bin/ps/ps.c b/bin/ps/ps.c
	index f6d32e5411fe..d3cfc669d581 100644
	--- a/bin/ps/ps.c
	+++ b/bin/ps/ps.c
	@@ -1,1470 +1,1473 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1990, 1993, 1994
	* The Regents of the University of California. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	* ------+---------+---------+-------- + --------+---------+---------+---------*
	* Copyright (c) 2004 - Garance Alistair Drosehn <gad@FreeBSD.org>.
	* All rights reserved.
	*
	* Significant modifications made to bring `ps' options somewhat closer
	* to the standard for `ps' as described in SingleUnixSpec-v3.
	* ------+---------+---------+-------- + --------+---------+---------+---------*
	*/

	#ifndef lint
	static const char copyright[] =
	"@(#) Copyright (c) 1990, 1993, 1994\n\
	The Regents of the University of California. All rights reserved.\n";
	#endif /* not lint */

	#if 0
	#ifndef lint
	static char sccsid[] = "@(#)ps.c 8.4 (Berkeley) 4/2/94";
	#endif /* not lint */
	#endif

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/jail.h>
	#include <sys/proc.h>
	#include <sys/user.h>
	#include <sys/stat.h>
	#include <sys/ioctl.h>
	#include <sys/sysctl.h>
	#include <sys/mount.h>

	#include <ctype.h>
	#include <err.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <grp.h>
	#include <jail.h>
	#include <kvm.h>
	#include <limits.h>
	#include <locale.h>
	#include <paths.h>
	#include <pwd.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <unistd.h>
	#include <libxo/xo.h>

	#include "ps.h"

	#define _PATH_PTS "/dev/pts/"

	#define W_SEP " \t" /* "Whitespace" list separators */
	#define T_SEP "," /* "Terminate-element" list separators */

	#ifdef LAZY_PS
	#define DEF_UREAD 0
	#define OPT_LAZY_f "f"
	#else
	#define DEF_UREAD 1 /* Always do the more-expensive read. */
	#define OPT_LAZY_f /* I.e., the `-f' option is not added. */
	#endif

	/*
	* isdigit takes an `int', but expects values in the range of unsigned char.
	* This wrapper ensures that values from a 'char' end up in the correct range.
	*/
	#define isdigitch(Anychar) isdigit((u_char)(Anychar))

	int cflag; /* -c */
	int eval; /* Exit value */
	time_t now; /* Current time(3) value */
	int rawcpu; /* -C */
	int sumrusage; /* -S */
	int termwidth; /* Width of the screen (0 == infinity). */
	int showthreads; /* will threads be shown? */

	struct velisthead varlist = STAILQ_HEAD_INITIALIZER(varlist);

	static int forceuread = DEF_UREAD; /* Do extra work to get u-area. */
	static kvm_t *kd;
	static int needcomm; /* -o "command" */
	static int needenv; /* -e */
	static int needuser; /* -o "user" */
	static int optfatal; /* Fatal error parsing some list-option. */
	static int pid_max; /* kern.max_pid */

	static enum sort { DEFAULT, SORTMEM, SORTCPU } sortby = DEFAULT;

	struct listinfo;
	typedef int addelem_rtn(struct listinfo _inf, const char _elem);

	struct listinfo {
	int count;
	int maxcount;
	int elemsize;
	addelem_rtn *addelem;
	const char *lname;
	union {
	gid_t *gids;
	int *jids;
	pid_t *pids;
	dev_t *ttys;
	uid_t *uids;
	void *ptr;
	} l;
	};

	static int addelem_gid(struct listinfo , const char );
	static int addelem_jid(struct listinfo , const char );
	static int addelem_pid(struct listinfo , const char );
	static int addelem_tty(struct listinfo , const char );
	static int addelem_uid(struct listinfo , const char );
	static void add_list(struct listinfo , const char );
	static void descendant_sort(KINFO *, int);
	static void format_output(KINFO *);
	static void expand_list(struct listinfo );
	static const char *
	fmt(char *()(kvm_t , const struct kinfo_proc , int),
	KINFO , char , char *, int);
	static void free_list(struct listinfo *);
	static void init_list(struct listinfo , addelem_rtn, int, const char );
	static char kludge_oldps_options(const char , char , const char );
	static int pscomp(const void , const void );
	static void saveuser(KINFO *);
	static void scanvars(void);
	static void sizevars(void);
	static void pidmax_init(void);
	static void usage(void);

	static char dfmt[] = "pid,tt,state,time,command";
	static char jfmt[] = "user,pid,ppid,pgid,sid,jobc,state,tt,time,command";
	static char lfmt[] = "uid,pid,ppid,cpu,pri,nice,vsz,rss,mwchan,state,"
	"tt,time,command";
	static char o1[] = "pid";
	static char o2[] = "tt,state,time,command";
	static char ufmt[] = "user,pid,%cpu,%mem,vsz,rss,tt,state,start,time,command";
	static char vfmt[] = "pid,state,time,sl,re,pagein,vsz,rss,lim,tsiz,"
	"%cpu,%mem,command";
	static char Zfmt[] = "label";

	#define PS_ARGS "AaCcde" OPT_LAZY_f "G:gHhjJ:LlM:mN:O:o:p:rSTt:U:uvwXxZ"

	int
	main(int argc, char *argv[])
	{
	struct listinfo gidlist, jidlist, pgrplist, pidlist;
	struct listinfo ruidlist, sesslist, ttylist, uidlist;
	struct kinfo_proc *kp;
	KINFO kinfo = NULL, next_KINFO;
	KINFO_STR *ks;
	struct varent *vent;
	struct winsize ws = { .ws_row = 0 };
	const char nlistf, memf, *str;
	char *cols;
	int all, ch, elem, flag, _fmt, i, lineno, linelen, left;
	int descendancy, nentries, nkept, nselectors;
	int prtheader, wflag, what, xkeep, xkeep_implied;
	int fwidthmin, fwidthmax;
	char errbuf[_POSIX2_LINE_MAX];
	char fmtbuf[_POSIX2_LINE_MAX];

	(void) setlocale(LC_ALL, "");
	time(&now); /* Used by routines in print.c. */

	/*
	* Compute default output line length before processing options.
	* If COLUMNS is set, use it. Otherwise, if this is part of an
	* interactive job (i.e. one associated with a terminal), use
	* the terminal width. "Interactive" is determined by whether
	* any of stdout, stderr, or stdin is a terminal. The intent
	* is that "ps", "ps \| more", and "ps \| grep" all use the same
	* default line length unless -w is specified.
	*
	* If not interactive, the default length was traditionally 79.
	* It has been changed to unlimited. This is mostly for the
	* benefit of non-interactive scripts, which arguably should
	* use -ww, but is compatible with Linux.
	*/
	if ((cols = getenv("COLUMNS")) != NULL && *cols != '\0')
	termwidth = atoi(cols);
	else if ((ioctl(STDOUT_FILENO, TIOCGWINSZ, (char *)&ws) == -1 &&
	ioctl(STDERR_FILENO, TIOCGWINSZ, (char *)&ws) == -1 &&
	ioctl(STDIN_FILENO, TIOCGWINSZ, (char *)&ws) == -1) \|\|
	ws.ws_col == 0)
	termwidth = UNLIMITED;
	else
	termwidth = ws.ws_col - 1;

	/*
	* Hide a number of option-processing kludges in a separate routine,
	* to support some historical BSD behaviors, such as `ps axu'.
	*/
	if (argc > 1)
	argv[1] = kludge_oldps_options(PS_ARGS, argv[1], argv[2]);

	pidmax_init();

	all = descendancy = _fmt = nselectors = optfatal = 0;
	prtheader = showthreads = wflag = xkeep_implied = 0;
	xkeep = -1; /* Neither -x nor -X. */
	init_list(&gidlist, addelem_gid, sizeof(gid_t), "group");
	init_list(&jidlist, addelem_jid, sizeof(int), "jail id");
	init_list(&pgrplist, addelem_pid, sizeof(pid_t), "process group");
	init_list(&pidlist, addelem_pid, sizeof(pid_t), "process id");
	init_list(&ruidlist, addelem_uid, sizeof(uid_t), "ruser");
	init_list(&sesslist, addelem_pid, sizeof(pid_t), "session id");
	init_list(&ttylist, addelem_tty, sizeof(dev_t), "tty");
	init_list(&uidlist, addelem_uid, sizeof(uid_t), "user");
	memf = _PATH_DEVNULL;
	nlistf = NULL;

	argc = xo_parse_args(argc, argv);
	if (argc < 0)
	exit(1);

	while ((ch = getopt(argc, argv, PS_ARGS)) != -1)
	switch (ch) {
	case 'A':
	/*
	* Exactly the same as `-ax'. This has been
	* added for compatibility with SUSv3, but for
	* now it will not be described in the man page.
	*/
	nselectors++;
	all = xkeep = 1;
	break;
	case 'a':
	nselectors++;
	all = 1;
	break;
	case 'C':
	rawcpu = 1;
	break;
	case 'c':
	cflag = 1;
	break;
	case 'd':
	descendancy = 1;
	break;
	case 'e': /* XXX set ufmt */
	needenv = 1;
	break;
	#ifdef LAZY_PS
	case 'f':
	if (getuid() == 0 \|\| getgid() == 0)
	forceuread = 1;
	break;
	#endif
	case 'G':
	add_list(&gidlist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	case 'g':
	#if 0
	/*-
	* XXX - This SUSv3 behavior is still under debate
	* since it conflicts with the (undocumented)
	* `-g' option. So we skip it for now.
	*/
	add_list(&pgrplist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	#else
	/* The historical BSD-ish (from SunOS) behavior. */
	break; /* no-op */
	#endif
	case 'H':
	showthreads = KERN_PROC_INC_THREAD;
	break;
	case 'h':
	prtheader = ws.ws_row > 5 ? ws.ws_row : 22;
	break;
	case 'J':
	add_list(&jidlist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	case 'j':
	parsefmt(jfmt, 0);
	_fmt = 1;
	jfmt[0] = '\0';
	break;
	case 'L':
	showkey();
	exit(0);
	case 'l':
	parsefmt(lfmt, 0);
	_fmt = 1;
	lfmt[0] = '\0';
	break;
	case 'M':
	memf = optarg;
	break;
	case 'm':
	sortby = SORTMEM;
	break;
	case 'N':
	nlistf = optarg;
	break;
	case 'O':
	parsefmt(o1, 1);
	parsefmt(optarg, 1);
	parsefmt(o2, 1);
	o1[0] = o2[0] = '\0';
	_fmt = 1;
	break;
	case 'o':
	parsefmt(optarg, 1);
	_fmt = 1;
	break;
	case 'p':
	add_list(&pidlist, optarg);
	/*
	* Note: `-p' does not set xkeep, but any values
	* from pidlist are checked before xkeep is. That
	* way they are always matched, even if the user
	* specifies `-X'.
	*/
	nselectors++;
	break;
	#if 0
	case 'R':
	/*-
	* XXX - This un-standard option is still under
	* debate. This is what SUSv3 defines as
	* the `-U' option, and while it would be
	* nice to have, it could cause even more
	* confusion to implement it as `-R'.
	*/
	add_list(&ruidlist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	#endif
	case 'r':
	sortby = SORTCPU;
	break;
	case 'S':
	sumrusage = 1;
	break;
	#if 0
	case 's':
	/*-
	* XXX - This non-standard option is still under
	* debate. This is supported on Solaris,
	* Linux, and IRIX, but conflicts with `-s'
	* on NetBSD and maybe some older BSD's.
	*/
	add_list(&sesslist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	#endif
	case 'T':
	if ((optarg = ttyname(STDIN_FILENO)) == NULL)
	xo_errx(1, "stdin: not a terminal");
	/* FALLTHROUGH */
	case 't':
	add_list(&ttylist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	case 'U':
	/* This is what SUSv3 defines as the `-u' option. */
	add_list(&uidlist, optarg);
	xkeep_implied = 1;
	nselectors++;
	break;
	case 'u':
	parsefmt(ufmt, 0);
	sortby = SORTCPU;
	_fmt = 1;
	ufmt[0] = '\0';
	break;
	case 'v':
	parsefmt(vfmt, 0);
	sortby = SORTMEM;
	_fmt = 1;
	vfmt[0] = '\0';
	break;
	case 'w':
	if (wflag)
	termwidth = UNLIMITED;
	else if (termwidth < 131 && termwidth != UNLIMITED)
	termwidth = 131;
	wflag++;
	break;
	case 'X':
	/*
	* Note that `-X' and `-x' are not standard "selector"
	* options. For most selector-options, we check all
	* processes to see if any are matched by the given
	* value(s). After we have a set of all the matched
	* processes, then `-X' and `-x' govern whether we
	* modify that matched set for processes which do
	* not have a controlling terminal. `-X' causes
	* those processes to be deleted from the matched
	* set, while `-x' causes them to be kept.
	*/
	xkeep = 0;
	break;
	case 'x':
	xkeep = 1;
	break;
	case 'Z':
	parsefmt(Zfmt, 0);
	Zfmt[0] = '\0';
	break;
	case '?':
	default:
	usage();
	}
	argc -= optind;
	argv += optind;

	/*
	* If there arguments after processing all the options, attempt
	* to treat them as a list of process ids.
	*/
	while (*argv) {
	if (!isdigitch(**argv))
	break;
	add_list(&pidlist, *argv);
	argv++;
	}
	if (*argv) {
	xo_warnx("illegal argument: %s\n", *argv);
	usage();
	}
	if (optfatal)
	exit(1); /* Error messages already printed. */
	if (xkeep < 0) /* Neither -X nor -x was specified. */
	xkeep = xkeep_implied;

	kd = kvm_openfiles(nlistf, memf, NULL, O_RDONLY, errbuf);
	if (kd == NULL)
	xo_errx(1, "%s", errbuf);

	if (!_fmt)
	parsefmt(dfmt, 0);

	if (nselectors == 0) {
	uidlist.l.ptr = malloc(sizeof(uid_t));
	if (uidlist.l.ptr == NULL)
	xo_errx(1, "malloc failed");
	nselectors = 1;
	uidlist.count = uidlist.maxcount = 1;
	*uidlist.l.uids = getuid();
	}

	/*
	* scan requested variables, noting what structures are needed,
	* and adjusting header widths as appropriate.
	*/
	scanvars();

	/*
	* Get process list. If the user requested just one selector-
	* option, then kvm_getprocs can be asked to return just those
	* processes. Otherwise, have it return all processes, and
	* then this routine will search that full list and select the
	* processes which match any of the user's selector-options.
	*/
	what = showthreads != 0 ? KERN_PROC_ALL : KERN_PROC_PROC;
	flag = 0;
	if (nselectors == 1) {
	if (gidlist.count == 1) {
	what = KERN_PROC_RGID \| showthreads;
	flag = *gidlist.l.gids;
	nselectors = 0;
	} else if (pgrplist.count == 1) {
	what = KERN_PROC_PGRP \| showthreads;
	flag = *pgrplist.l.pids;
	nselectors = 0;
	} else if (pidlist.count == 1 && !descendancy) {
	what = KERN_PROC_PID \| showthreads;
	flag = *pidlist.l.pids;
	nselectors = 0;
	} else if (ruidlist.count == 1) {
	what = KERN_PROC_RUID \| showthreads;
	flag = *ruidlist.l.uids;
	nselectors = 0;
	} else if (sesslist.count == 1) {
	what = KERN_PROC_SESSION \| showthreads;
	flag = *sesslist.l.pids;
	nselectors = 0;
	} else if (ttylist.count == 1) {
	what = KERN_PROC_TTY \| showthreads;
	flag = *ttylist.l.ttys;
	nselectors = 0;
	} else if (uidlist.count == 1) {
	what = KERN_PROC_UID \| showthreads;
	flag = *uidlist.l.uids;
	nselectors = 0;
	} else if (all) {
	/* No need for this routine to select processes. */
	nselectors = 0;
	}
	}

	/*
	* select procs
	*/
	nentries = -1;
	kp = kvm_getprocs(kd, what, flag, &nentries);
	/*
	* Ignore ESRCH to preserve behaviour of "ps -p nonexistent-pid"
	* not reporting an error.
	*/
	if ((kp == NULL && errno != ESRCH) \|\| (kp != NULL && nentries < 0))
	xo_errx(1, "%s", kvm_geterr(kd));
	nkept = 0;
	if (descendancy)
	for (elem = 0; elem < pidlist.count; elem++)
	for (i = 0; i < nentries; i++)
	if (kp[i].ki_ppid == pidlist.l.pids[elem]) {
	if (pidlist.count >= pidlist.maxcount)
	expand_list(&pidlist);
	pidlist.l.pids[pidlist.count++] = kp[i].ki_pid;
	}
	if (nentries > 0) {
	if ((kinfo = malloc(nentries * sizeof(*kinfo))) == NULL)
	xo_errx(1, "malloc failed");
	for (i = nentries; --i >= 0; ++kp) {
	/*
	* If the user specified multiple selection-criteria,
	* then keep any process matched by the inclusive OR
	* of all the selection-criteria given.
	*/
	if (pidlist.count > 0) {
	for (elem = 0; elem < pidlist.count; elem++)
	if (kp->ki_pid == pidlist.l.pids[elem])
	goto keepit;
	}
	/*
	* Note that we had to process pidlist before
	* filtering out processes which do not have
	* a controlling terminal.
	*/
	if (xkeep == 0) {
	if ((kp->ki_tdev == NODEV \|\|
	(kp->ki_flag & P_CONTROLT) == 0))
	continue;
	}
	if (nselectors == 0)
	goto keepit;
	if (gidlist.count > 0) {
	for (elem = 0; elem < gidlist.count; elem++)
	if (kp->ki_rgid == gidlist.l.gids[elem])
	goto keepit;
	}
	if (jidlist.count > 0) {
	for (elem = 0; elem < jidlist.count; elem++)
	if (kp->ki_jid == jidlist.l.jids[elem])
	goto keepit;
	}
	if (pgrplist.count > 0) {
	for (elem = 0; elem < pgrplist.count; elem++)
	if (kp->ki_pgid ==
	pgrplist.l.pids[elem])
	goto keepit;
	}
	if (ruidlist.count > 0) {
	for (elem = 0; elem < ruidlist.count; elem++)
	if (kp->ki_ruid ==
	ruidlist.l.uids[elem])
	goto keepit;
	}
	if (sesslist.count > 0) {
	for (elem = 0; elem < sesslist.count; elem++)
	if (kp->ki_sid == sesslist.l.pids[elem])
	goto keepit;
	}
	if (ttylist.count > 0) {
	for (elem = 0; elem < ttylist.count; elem++)
	if (kp->ki_tdev == ttylist.l.ttys[elem])
	goto keepit;
	}
	if (uidlist.count > 0) {
	for (elem = 0; elem < uidlist.count; elem++)
	if (kp->ki_uid == uidlist.l.uids[elem])
	goto keepit;
	}
	/*
	* This process did not match any of the user's
	* selector-options, so skip the process.
	*/
	continue;

	keepit:
	next_KINFO = &kinfo[nkept];
	next_KINFO->ki_p = kp;
	next_KINFO->ki_d.level = 0;
	next_KINFO->ki_d.prefix = NULL;
	next_KINFO->ki_pcpu = getpcpu(next_KINFO);
	if (sortby == SORTMEM)
	next_KINFO->ki_memsize = kp->ki_tsize +
	kp->ki_dsize + kp->ki_ssize;
	if (needuser)
	saveuser(next_KINFO);
	nkept++;
	}
	}

	sizevars();

	if (nkept == 0) {
	printheader();
	xo_finish();
	exit(1);
	}

	/*
	* sort proc list
	*/
	qsort(kinfo, nkept, sizeof(KINFO), pscomp);

	/*
	* We want things in descendant order
	*/
	if (descendancy)
	descendant_sort(kinfo, nkept);


	/*
	* Prepare formatted output.
	*/
	for (i = 0; i < nkept; i++)
	format_output(&kinfo[i]);

	/*
	* Print header.
	*/
	xo_open_container("process-information");
	printheader();
	if (xo_get_style(NULL) != XO_STYLE_TEXT)
	termwidth = UNLIMITED;

	/*
	* Output formatted lines.
	*/
	xo_open_list("process");
	for (i = lineno = 0; i < nkept; i++) {
	linelen = 0;
	xo_open_instance("process");
	STAILQ_FOREACH(vent, &varlist, next_ve) {
	ks = STAILQ_FIRST(&kinfo[i].ki_ks);
	STAILQ_REMOVE_HEAD(&kinfo[i].ki_ks, ks_next);
	/* Truncate rightmost column if necessary. */
	fwidthmax = _POSIX2_LINE_MAX;
	if (STAILQ_NEXT(vent, next_ve) == NULL &&
	termwidth != UNLIMITED && ks->ks_str != NULL) {
	left = termwidth - linelen;
	if (left > 0 && left < (int)strlen(ks->ks_str))
	fwidthmax = left;
	}

	str = ks->ks_str;
	if (str == NULL)
	str = "-";
	/* No padding for the last column, if it's LJUST. */
	fwidthmin = (xo_get_style(NULL) != XO_STYLE_TEXT \|\|
	(STAILQ_NEXT(vent, next_ve) == NULL &&
	(vent->var->flag & LJUST))) ? 0 : vent->var->width;
	snprintf(fmtbuf, sizeof(fmtbuf), "{:%s/%%%s%d..%dhs}",
	vent->var->field ? vent->var->field : vent->var->name,
	(vent->var->flag & LJUST) ? "-" : "",
	fwidthmin, fwidthmax);
	xo_emit(fmtbuf, str);
	linelen += fwidthmin;

	if (ks->ks_str != NULL) {
	free(ks->ks_str);
	ks->ks_str = NULL;
	}
	free(ks);
	ks = NULL;

	if (STAILQ_NEXT(vent, next_ve) != NULL) {
	xo_emit("{P: }");
	linelen++;
	}
	}
	xo_emit("\n");
	xo_close_instance("process");
	if (prtheader && lineno++ == prtheader - 4) {
	xo_emit("\n");
	printheader();
	lineno = 0;
	}
	}
	xo_close_list("process");
	xo_close_container("process-information");
	xo_finish();

	free_list(&gidlist);
	free_list(&jidlist);
	free_list(&pidlist);
	free_list(&pgrplist);
	free_list(&ruidlist);
	free_list(&sesslist);
	free_list(&ttylist);
	free_list(&uidlist);
	for (i = 0; i < nkept; i++)
	free(kinfo[i].ki_d.prefix);
	free(kinfo);

	exit(eval);
	}

	static int
	addelem_gid(struct listinfo inf, const char elem)
	{
	struct group *grp;
	const char *nameorID;
	char *endp;
	u_long bigtemp;

	if (*elem == '\0' \|\| strlen(elem) >= MAXLOGNAME) {
	if (*elem == '\0')
	xo_warnx("Invalid (zero-length) %s name", inf->lname);
	else
	xo_warnx("%s name too long: %s", inf->lname, elem);
	optfatal = 1;
	return (0); /* Do not add this value. */
	}

	/*
	* SUSv3 states that `ps -G grouplist' should match "real-group
	* ID numbers", and does not mention group-names. I do want to
	* also support group-names, so this tries for a group-id first,
	* and only tries for a name if that doesn't work. This is the
	* opposite order of what is done in addelem_uid(), but in
	* practice the order would only matter for group-names which
	* are all-numeric.
	*/
	grp = NULL;
	nameorID = "named";
	errno = 0;
	bigtemp = strtoul(elem, &endp, 10);
	if (errno == 0 && *endp == '\0' && bigtemp <= GID_MAX) {
	nameorID = "name or ID matches";
	grp = getgrgid((gid_t)bigtemp);
	}
	if (grp == NULL)
	grp = getgrnam(elem);
	if (grp == NULL) {
	xo_warnx("No %s %s '%s'", inf->lname, nameorID, elem);
	optfatal = 1;
	return (0);
	}
	if (inf->count >= inf->maxcount)
	expand_list(inf);
	inf->l.gids[(inf->count)++] = grp->gr_gid;
	return (1);
	}

	static int
	addelem_jid(struct listinfo inf, const char elem)
	{
	int tempid;

	if (*elem == '\0') {
	warnx("Invalid (zero-length) jail id");
	optfatal = 1;
	return (0); /* Do not add this value. */
	}

	tempid = jail_getid(elem);
	if (tempid < 0) {
	warnx("Invalid %s: %s", inf->lname, elem);
	optfatal = 1;
	return (0);
	}

	if (inf->count >= inf->maxcount)
	expand_list(inf);
	inf->l.jids[(inf->count)++] = tempid;
	return (1);
	}

	static int
	addelem_pid(struct listinfo inf, const char elem)
	{
	char *endp;
	long tempid;

	if (*elem == '\0') {
	xo_warnx("Invalid (zero-length) process id");
	optfatal = 1;
	return (0); /* Do not add this value. */
	}

	errno = 0;
	tempid = strtol(elem, &endp, 10);
	if (*endp != '\0' \|\| tempid < 0 \|\| elem == endp) {
	xo_warnx("Invalid %s: %s", inf->lname, elem);
	errno = ERANGE;
	} else if (errno != 0 \|\| tempid > pid_max) {
	xo_warnx("%s too large: %s", inf->lname, elem);
	errno = ERANGE;
	}
	if (errno == ERANGE) {
	optfatal = 1;
	return (0);
	}
	if (inf->count >= inf->maxcount)
	expand_list(inf);
	inf->l.pids[(inf->count)++] = tempid;
	return (1);
	}

	/*-
	* The user can specify a device via one of three formats:
	* 1) fully qualified, e.g.: /dev/ttyp0 /dev/console /dev/pts/0
	* 2) missing "/dev", e.g.: ttyp0 console pts/0
	* 3) two-letters, e.g.: p0 co 0
	* (matching letters that would be seen in the "TT" column)
	*/
	static int
	addelem_tty(struct listinfo inf, const char elem)
	{
	const char *ttypath;
	struct stat sb;
	char pathbuf[PATH_MAX], pathbuf2[PATH_MAX], pathbuf3[PATH_MAX];

	ttypath = NULL;
	pathbuf2[0] = '\0';
	pathbuf3[0] = '\0';
	switch (*elem) {
	case '/':
	ttypath = elem;
	break;
	case 'c':
	if (strcmp(elem, "co") == 0) {
	ttypath = _PATH_CONSOLE;
	break;
	}
	/* FALLTHROUGH */
	default:
	strlcpy(pathbuf, _PATH_DEV, sizeof(pathbuf));
	strlcat(pathbuf, elem, sizeof(pathbuf));
	ttypath = pathbuf;
	if (strncmp(pathbuf, _PATH_TTY, strlen(_PATH_TTY)) == 0)
	break;
	if (strncmp(pathbuf, _PATH_PTS, strlen(_PATH_PTS)) == 0)
	break;
	if (strcmp(pathbuf, _PATH_CONSOLE) == 0)
	break;
	/* Check to see if /dev/tty${elem} exists */
	strlcpy(pathbuf2, _PATH_TTY, sizeof(pathbuf2));
	strlcat(pathbuf2, elem, sizeof(pathbuf2));
	if (stat(pathbuf2, &sb) == 0 && S_ISCHR(sb.st_mode)) {
	/* No need to repeat stat() && S_ISCHR() checks */
	ttypath = NULL;
	break;
	}
	/* Check to see if /dev/pts/${elem} exists */
	strlcpy(pathbuf3, _PATH_PTS, sizeof(pathbuf3));
	strlcat(pathbuf3, elem, sizeof(pathbuf3));
	if (stat(pathbuf3, &sb) == 0 && S_ISCHR(sb.st_mode)) {
	/* No need to repeat stat() && S_ISCHR() checks */
	ttypath = NULL;
	break;
	}
	break;
	}
	if (ttypath) {
	if (stat(ttypath, &sb) == -1) {
	if (pathbuf3[0] != '\0')
	xo_warn("%s, %s, and %s", pathbuf3, pathbuf2,
	ttypath);
	else
	xo_warn("%s", ttypath);
	optfatal = 1;
	return (0);
	}
	if (!S_ISCHR(sb.st_mode)) {
	if (pathbuf3[0] != '\0')
	xo_warnx("%s, %s, and %s: Not a terminal",
	pathbuf3, pathbuf2, ttypath);
	else
	xo_warnx("%s: Not a terminal", ttypath);
	optfatal = 1;
	return (0);
	}
	}
	if (inf->count >= inf->maxcount)
	expand_list(inf);
	inf->l.ttys[(inf->count)++] = sb.st_rdev;
	return (1);
	}

	static int
	addelem_uid(struct listinfo inf, const char elem)
	{
	struct passwd *pwd;
	char *endp;
	u_long bigtemp;

	if (*elem == '\0' \|\| strlen(elem) >= MAXLOGNAME) {
	if (*elem == '\0')
	xo_warnx("Invalid (zero-length) %s name", inf->lname);
	else
	xo_warnx("%s name too long: %s", inf->lname, elem);
	optfatal = 1;
	return (0); /* Do not add this value. */
	}

	pwd = getpwnam(elem);
	if (pwd == NULL) {
	errno = 0;
	bigtemp = strtoul(elem, &endp, 10);
	if (errno != 0 \|\| *endp != '\0' \|\| bigtemp > UID_MAX)
	xo_warnx("No %s named '%s'", inf->lname, elem);
	else {
	/* The string is all digits, so it might be a userID. */
	pwd = getpwuid((uid_t)bigtemp);
	if (pwd == NULL)
	xo_warnx("No %s name or ID matches '%s'",
	inf->lname, elem);
	}
	}
	if (pwd == NULL) {
	/*
	* These used to be treated as minor warnings (and the
	* option was simply ignored), but now they are fatal
	* errors (and the command will be aborted).
	*/
	optfatal = 1;
	return (0);
	}
	if (inf->count >= inf->maxcount)
	expand_list(inf);
	inf->l.uids[(inf->count)++] = pwd->pw_uid;
	return (1);
	}

	static void
	add_list(struct listinfo inf, const char argp)
	{
	const char *savep;
	char cp, endp;
	int toolong;
	char elemcopy[PATH_MAX];

	if (*argp == '\0')
	inf->addelem(inf, argp);
	while (*argp != '\0') {
	while (argp != '\0' && strchr(W_SEP, argp) != NULL)
	argp++;
	savep = argp;
	toolong = 0;
	cp = elemcopy;
	if (strchr(T_SEP, *argp) == NULL) {
	endp = elemcopy + sizeof(elemcopy) - 1;
	while (*argp != '\0' && cp <= endp &&
	strchr(W_SEP T_SEP, *argp) == NULL)
	cp++ = argp++;
	if (cp > endp)
	toolong = 1;
	}
	if (!toolong) {
	*cp = '\0';
	/*
	* Add this single element to the given list.
	*/
	inf->addelem(inf, elemcopy);
	} else {
	/*
	* The string is too long to copy. Find the end
	* of the string to print out the warning message.
	*/
	while (*argp != '\0' && strchr(W_SEP T_SEP,
	*argp) == NULL)
	argp++;
	xo_warnx("Value too long: %.*s", (int)(argp - savep),
	savep);
	optfatal = 1;
	}
	/*
	* Skip over any number of trailing whitespace characters,
	* but only one (at most) trailing element-terminating
	* character.
	*/
	while (argp != '\0' && strchr(W_SEP, argp) != NULL)
	argp++;
	if (argp != '\0' && strchr(T_SEP, argp) != NULL) {
	argp++;
	/* Catch case where string ended with a comma. */
	if (*argp == '\0')
	inf->addelem(inf, argp);
	}
	}
	}

	static void
	descendant_sort(KINFO *ki, int items)
	{
	int dst, lvl, maxlvl, n, ndst, nsrc, siblings, src;
	unsigned char *path;
	KINFO kn;

	/*
	* First, sort the entries by descendancy, tracking the descendancy
	* depth in the ki_d.level field.
	*/
	src = 0;
	maxlvl = 0;
	while (src < items) {
	if (ki[src].ki_d.level) {
	src++;
	continue;
	}
	for (nsrc = 1; src + nsrc < items; nsrc++)
	if (!ki[src + nsrc].ki_d.level)
	break;

	for (dst = 0; dst < items; dst++) {
	if (ki[dst].ki_p->ki_pid == ki[src].ki_p->ki_pid)
	continue;
	if (ki[dst].ki_p->ki_pid == ki[src].ki_p->ki_ppid)
	break;
	}

	if (dst == items) {
	src += nsrc;
	continue;
	}

	for (ndst = 1; dst + ndst < items; ndst++)
	if (ki[dst + ndst].ki_d.level <= ki[dst].ki_d.level)
	break;

	for (n = src; n < src + nsrc; n++) {
	ki[n].ki_d.level += ki[dst].ki_d.level + 1;
	if (maxlvl < ki[n].ki_d.level)
	maxlvl = ki[n].ki_d.level;
	}

	while (nsrc) {
	if (src < dst) {
	kn = ki[src];
	memmove(ki + src, ki + src + 1,
	(dst - src + ndst - 1) * sizeof *ki);
	ki[dst + ndst - 1] = kn;
	nsrc--;
	dst--;
	ndst++;
	} else if (src != dst + ndst) {
	kn = ki[src];
	memmove(ki + dst + ndst + 1, ki + dst + ndst,
	(src - dst - ndst) * sizeof *ki);
	ki[dst + ndst] = kn;
	ndst++;
	nsrc--;
	src++;
	} else {
	ndst += nsrc;
	src += nsrc;
	nsrc = 0;
	}
	}
	}

	/*
	* Now populate ki_d.prefix (instead of ki_d.level) with the command
	* prefix used to show descendancies.
	*/
	path = malloc((maxlvl + 7) / 8);
	memset(path, '\0', (maxlvl + 7) / 8);
	for (src = 0; src < items; src++) {
	if ((lvl = ki[src].ki_d.level) == 0) {
	ki[src].ki_d.prefix = NULL;
	continue;
	}
	if ((ki[src].ki_d.prefix = malloc(lvl * 2 + 1)) == NULL)
	xo_errx(1, "malloc failed");
	for (n = 0; n < lvl - 2; n++) {
	ki[src].ki_d.prefix[n * 2] =
	path[n / 8] & 1 << (n % 8) ? '\|' : ' ';
	ki[src].ki_d.prefix[n * 2 + 1] = ' ';
	}
	if (n == lvl - 2) {
	/* Have I any more siblings? */
	for (siblings = 0, dst = src + 1; dst < items; dst++) {
	if (ki[dst].ki_d.level > lvl)
	continue;
	if (ki[dst].ki_d.level == lvl)
	siblings = 1;
	break;
	}
	if (siblings)
	path[n / 8] \|= 1 << (n % 8);
	else
	path[n / 8] &= ~(1 << (n % 8));
	ki[src].ki_d.prefix[n * 2] = siblings ? '\|' : '`';
	ki[src].ki_d.prefix[n * 2 + 1] = '-';
	n++;
	}
	strcpy(ki[src].ki_d.prefix + n * 2, "- ");
	}
	free(path);
	}

	static void *
	expand_list(struct listinfo *inf)
	{
	void *newlist;
	int newmax;

	newmax = (inf->maxcount + 1) << 1;
	newlist = realloc(inf->l.ptr, newmax * inf->elemsize);
	if (newlist == NULL) {
	free(inf->l.ptr);
	xo_errx(1, "realloc to %d %ss failed", newmax, inf->lname);
	}
	inf->maxcount = newmax;
	inf->l.ptr = newlist;

	return (newlist);
	}

	static void
	free_list(struct listinfo *inf)
	{

	inf->count = inf->elemsize = inf->maxcount = 0;
	if (inf->l.ptr != NULL)
	free(inf->l.ptr);
	inf->addelem = NULL;
	inf->lname = NULL;
	inf->l.ptr = NULL;
	}

	static void
	init_list(struct listinfo *inf, addelem_rtn artn, int elemsize,
	const char *lname)
	{

	inf->count = inf->maxcount = 0;
	inf->elemsize = elemsize;
	inf->addelem = artn;
	inf->lname = lname;
	inf->l.ptr = NULL;
	}

	VARENT *
	find_varentry(VAR *v)
	{
	struct varent *vent;

	STAILQ_FOREACH(vent, &varlist, next_ve) {
	if (strcmp(vent->var->name, v->name) == 0)
	return vent;
	}
	return NULL;
	}

	static void
	scanvars(void)
	{
	struct varent *vent;
	VAR *v;

	STAILQ_FOREACH(vent, &varlist, next_ve) {
	v = vent->var;
	if (v->flag & USER)
	needuser = 1;
	if (v->flag & COMM)
	needcomm = 1;
	}
	}

	static void
	format_output(KINFO *ki)
	{
	struct varent *vent;
	VAR *v;
	KINFO_STR *ks;
	char *str;
	int len;

	STAILQ_INIT(&ki->ki_ks);
	STAILQ_FOREACH(vent, &varlist, next_ve) {
	v = vent->var;
	str = (v->oproc)(ki, vent);
	ks = malloc(sizeof(*ks));
	if (ks == NULL)
	xo_errx(1, "malloc failed");
	ks->ks_str = str;
	STAILQ_INSERT_TAIL(&ki->ki_ks, ks, ks_next);
	if (str != NULL) {
	len = strlen(str);
	} else
	len = 1; /* "-" */
	if (v->width < len)
	v->width = len;
	}
	}

	static void
	sizevars(void)
	{
	struct varent *vent;
	VAR *v;
	int i;

	STAILQ_FOREACH(vent, &varlist, next_ve) {
	v = vent->var;
	i = strlen(vent->header);
	if (v->width < i)
	v->width = i;
	}
	}

	static const char *
	fmt(char *(fn)(kvm_t , const struct kinfo_proc , int), KINFO *ki,
	char comm, char thread, int maxlen)
	{
	const char *s;

	s = fmt_argv((*fn)(kd, ki->ki_p, termwidth), comm,
	showthreads && ki->ki_p->ki_numthreads > 1 ? thread : NULL, maxlen);
	return (s);
	}

	#define UREADOK(ki) (forceuread \|\| (ki->ki_p->ki_flag & P_INMEM))

	static void
	saveuser(KINFO *ki)
	{
	+ char tdname[COMMLEN + 1];
	char *argsp;

	if (ki->ki_p->ki_flag & P_INMEM) {
	/*
	* The u-area might be swapped out, and we can't get
	* at it because we have a crashdump and no swap.
	* If it's here fill in these fields, otherwise, just
	* leave them 0.
	*/
	ki->ki_valid = 1;
	} else
	ki->ki_valid = 0;
	/*
	* save arguments if needed
	*/
	if (needcomm) {
	- if (ki->ki_p->ki_stat == SZOMB)
	+ if (ki->ki_p->ki_stat == SZOMB) {
	ki->ki_args = strdup("<defunct>");
	- else if (UREADOK(ki) \|\| (ki->ki_p->ki_args != NULL))
	+ } else if (UREADOK(ki) \|\| (ki->ki_p->ki_args != NULL)) {
	+ (void)snprintf(tdname, sizeof(tdname), "%s%s",
	+ ki->ki_p->ki_tdname, ki->ki_p->ki_moretdname);
	ki->ki_args = fmt(kvm_getargv, ki,
	- ki->ki_p->ki_comm, ki->ki_p->ki_tdname, MAXCOMLEN);
	- else {
	+ ki->ki_p->ki_comm, tdname, COMMLEN * 2 + 1);
	+ } else {
	asprintf(&argsp, "(%s)", ki->ki_p->ki_comm);
	ki->ki_args = argsp;
	}
	if (ki->ki_args == NULL)
	xo_errx(1, "malloc failed");
	} else {
	ki->ki_args = NULL;
	}
	if (needenv) {
	if (UREADOK(ki))
	ki->ki_env = fmt(kvm_getenvv, ki,
	(char )NULL, (char )NULL, 0);
	else
	ki->ki_env = strdup("()");
	if (ki->ki_env == NULL)
	xo_errx(1, "malloc failed");
	} else {
	ki->ki_env = NULL;
	}
	}

	/* A macro used to improve the readability of pscomp(). */
	#define DIFF_RETURN(a, b, field) do { \
	if ((a)->field != (b)->field) \
	return (((a)->field < (b)->field) ? -1 : 1); \
	} while (0)

	static int
	pscomp(const void a, const void b)
	{
	const KINFO ka, kb;

	ka = a;
	kb = b;
	/* SORTCPU and SORTMEM are sorted in descending order. */
	if (sortby == SORTCPU)
	DIFF_RETURN(kb, ka, ki_pcpu);
	if (sortby == SORTMEM)
	DIFF_RETURN(kb, ka, ki_memsize);
	/*
	* TTY's are sorted in ascending order, except that all NODEV
	* processes come before all processes with a device.
	*/
	if (ka->ki_p->ki_tdev != kb->ki_p->ki_tdev) {
	if (ka->ki_p->ki_tdev == NODEV)
	return (-1);
	if (kb->ki_p->ki_tdev == NODEV)
	return (1);
	DIFF_RETURN(ka, kb, ki_p->ki_tdev);
	}

	/* PID's and TID's (threads) are sorted in ascending order. */
	DIFF_RETURN(ka, kb, ki_p->ki_pid);
	DIFF_RETURN(ka, kb, ki_p->ki_tid);
	return (0);
	}
	#undef DIFF_RETURN

	/*
	* ICK (all for getopt), would rather hide the ugliness
	* here than taint the main code.
	*
	* ps foo -> ps -foo
	* ps 34 -> ps -p34
	*
	* The old convention that 't' with no trailing tty arg means the users
	* tty, is only supported if argv[1] doesn't begin with a '-'. This same
	* feature is available with the option 'T', which takes no argument.
	*/
	static char *
	kludge_oldps_options(const char optlist, char origval, const char *nextarg)
	{
	size_t len;
	char argp, cp, newopts, ns, optp, pidp;

	/*
	* See if the original value includes any option which takes an
	* argument (and will thus use up the remainder of the string).
	*/
	argp = NULL;
	if (optlist != NULL) {
	for (cp = origval; *cp != '\0'; cp++) {
	optp = strchr(optlist, *cp);
	if ((optp != NULL) && *(optp + 1) == ':') {
	argp = cp;
	break;
	}
	}
	}
	if (argp != NULL && *origval == '-')
	return (origval);

	/*
	* if last letter is a 't' flag with no argument (in the context
	* of the oldps options -- option string NOT starting with a '-' --
	* then convert to 'T' (meaning this terminal, i.e. ttyname(0)).
	*
	* However, if a flag accepting a string argument is found earlier
	* in the option string (including a possible `t' flag), then the
	* remainder of the string must be the argument to that flag; so
	* do not modify that argument. Note that a trailing `t' would
	* cause argp to be set, if argp was not already set by some
	* earlier option.
	*/
	len = strlen(origval);
	cp = origval + len - 1;
	pidp = NULL;
	if (cp == 't' && origval != '-' && cp == argp) {
	if (nextarg == NULL \|\| nextarg == '-' \|\| isdigitch(nextarg))
	*cp = 'T';
	} else if (argp == NULL) {
	/*
	* The original value did not include any option which takes
	* an argument (and that would include `p' and `t'), so check
	* the value for trailing number, or comma-separated list of
	* numbers, which we will treat as a pid request.
	*/
	if (isdigitch(*cp)) {
	while (cp >= origval && (cp == ',' \|\| isdigitch(cp)))
	--cp;
	pidp = cp + 1;
	}
	}

	/*
	* If nothing needs to be added to the string, then return
	* the "original" (although possibly modified) value.
	*/
	if (*origval == '-' && pidp == NULL)
	return (origval);

	/*
	* Create a copy of the string to add '-' and/or 'p' to the
	* original value.
	*/
	if ((newopts = ns = malloc(len + 3)) == NULL)
	xo_errx(1, "malloc failed");

	if (*origval != '-')
	ns++ = '-'; / add option flag */

	if (pidp == NULL)
	strcpy(ns, origval);
	else {
	/*
	* Copy everything before the pid string, add the `p',
	* and then copy the pid string.
	*/
	len = pidp - origval;
	memcpy(ns, origval, len);
	ns += len;
	*ns++ = 'p';
	strcpy(ns, pidp);
	}

	return (newopts);
	}

	static void
	pidmax_init(void)
	{
	size_t intsize;

	intsize = sizeof(pid_max);
	if (sysctlbyname("kern.pid_max", &pid_max, &intsize, NULL, 0) < 0) {
	xo_warn("unable to read kern.pid_max");
	pid_max = 99999;
	}
	}

	static void __dead2
	usage(void)
	{
	#define SINGLE_OPTS "[-aCcde" OPT_LAZY_f "HhjlmrSTuvwXxZ]"

	(void)xo_error("%s\n%s\n%s\n%s\n",
	"usage: ps " SINGLE_OPTS " [-O fmt \| -o fmt] [-G gid[,gid...]]",
	" [-J jid[,jid...]] [-M core] [-N system]",
	" [-p pid[,pid...]] [-t tty[,tty...]] [-U user[,user...]]",
	" ps [-L]");
	exit(1);
	}
	diff --git a/bin/sh/main.c b/bin/sh/main.c
	index 4d0aae96215f..8df24dba312e 100644
	--- a/bin/sh/main.c
	+++ b/bin/sh/main.c
	@@ -1,339 +1,343 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1991, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* Kenneth Almquist.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#ifndef lint
	static char const copyright[] =
	"@(#) Copyright (c) 1991, 1993\n\
	The Regents of the University of California. All rights reserved.\n";
	#endif /* not lint */

	#ifndef lint
	#if 0
	static char sccsid[] = "@(#)main.c 8.6 (Berkeley) 5/28/95";
	#endif
	#endif /* not lint */
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <stdio.h>
	#include <signal.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <fcntl.h>
	#include <locale.h>
	#include <errno.h>

	#include "shell.h"
	#include "main.h"
	#include "mail.h"
	#include "options.h"
	#include "output.h"
	#include "parser.h"
	#include "nodes.h"
	#include "expand.h"
	#include "eval.h"
	#include "jobs.h"
	#include "input.h"
	#include "trap.h"
	#include "var.h"
	#include "show.h"
	#include "memalloc.h"
	#include "error.h"
	#include "mystring.h"
	#include "exec.h"
	#include "cd.h"
	#include "redir.h"
	#include "builtins.h"

	int rootpid;
	int rootshell;
	struct jmploc main_handler;
	int localeisutf8, initial_localeisutf8;

	static void reset(void);
	static void cmdloop(int);
	static void read_profile(const char *);
	static char find_dot_file(char );

	/*
	* Main routine. We initialize things, parse the arguments, execute
	* profiles if we're a login shell, and then call cmdloop to execute
	* commands. The setjmp call sets up the location to jump to when an
	* exception occurs. When an exception occurs the variable "state"
	* is used to figure out how far we had gotten.
	*/

	int
	main(int argc, char *argv[])
	{
	struct stackmark smark, smark2;
	volatile int state;
	char *shinit;

	(void) setlocale(LC_ALL, "");
	initcharset();
	state = 0;
	if (setjmp(main_handler.loc)) {
	if (state == 0 \|\| iflag == 0 \|\| ! rootshell \|\|
	exception == EXEXIT)
	exitshell(exitstatus);
	reset();
	if (exception == EXINT)
	out2fmt_flush("\n");
	popstackmark(&smark);
	FORCEINTON; /* enable interrupts */
	if (state == 1)
	goto state1;
	else if (state == 2)
	goto state2;
	else if (state == 3)
	goto state3;
	else
	goto state4;
	}
	handler = &main_handler;
	#ifdef DEBUG
	opentrace();
	trputs("Shell args: "); trargs(argv);
	#endif
	rootpid = getpid();
	rootshell = 1;
	INTOFF;
	initvar();
	setstackmark(&smark);
	setstackmark(&smark2);
	procargs(argc, argv);
	pwd_init(iflag);
	INTON;
	if (iflag)
	chkmail(1);
	if (argv[0] && argv[0][0] == '-') {
	state = 1;
	read_profile("/etc/profile");
	state1:
	state = 2;
	if (privileged == 0)
	read_profile("${HOME-}/.profile");
	else
	read_profile("/etc/suid_profile");
	}
	state2:
	state = 3;
	if (!privileged && iflag) {
	if ((shinit = lookupvar("ENV")) != NULL && *shinit != '\0') {
	state = 3;
	read_profile(shinit);
	}
	}
	state3:
	state = 4;
	popstackmark(&smark2);
	if (minusc) {
	evalstring(minusc, sflag ? 0 : EV_EXIT);
	}
	state4:
	if (sflag \|\| minusc == NULL) {
	cmdloop(1);
	}
	exitshell(exitstatus);
	/NOTREACHED/
	return 0;
	}

	static void
	reset(void)
	{
	reseteval();
	resetinput();
	}

	/*
	* Read and execute commands. "Top" is nonzero for the top level command
	* loop; it turns on prompting if the shell is interactive.
	*/

	static void
	cmdloop(int top)
	{
	union node *n;
	struct stackmark smark;
	int inter;
	int numeof = 0;

	TRACE(("cmdloop(%d) called\n", top));
	setstackmark(&smark);
	for (;;) {
	if (pendingsig)
	dotrap();
	inter = 0;
	if (iflag && top) {
	inter++;
	showjobs(1, SHOWJOBS_DEFAULT);
	chkmail(0);
	flushout(&output);
	}
	n = parsecmd(inter);
	/* showtree(n); DEBUG */
	if (n == NEOF) {
	if (!top \|\| numeof >= 50)
	break;
	if (!stoppedjobs()) {
	if (!Iflag)
	break;
	out2fmt_flush("\nUse \"exit\" to leave shell.\n");
	}
	numeof++;
	} else if (n != NULL && nflag == 0) {
	job_warning = (job_warning == 2) ? 1 : 0;
	numeof = 0;
	evaltree(n, 0);
	}
	popstackmark(&smark);
	setstackmark(&smark);
	if (evalskip != 0) {
	if (evalskip == SKIPRETURN)
	evalskip = 0;
	break;
	}
	}
	popstackmark(&smark);
	+ if (top && iflag) {
	+ out2c('\n');
	+ flushout(out2);
	+ }
	}



	/*
	* Read /etc/profile or .profile. Return on error.
	*/

	static void
	read_profile(const char *name)
	{
	int fd;
	const char *expandedname;

	expandedname = expandstr(name);
	if (expandedname == NULL)
	return;
	INTOFF;
	if ((fd = open(expandedname, O_RDONLY \| O_CLOEXEC)) >= 0)
	setinputfd(fd, 1);
	INTON;
	if (fd < 0)
	return;
	cmdloop(0);
	popfile();
	}



	/*
	* Read a file containing shell functions.
	*/

	void
	readcmdfile(const char *name)
	{
	setinputfile(name, 1);
	cmdloop(0);
	popfile();
	}



	/*
	* Take commands from a file. To be compatible we should do a path
	* search for the file, which is necessary to find sub-commands.
	*/


	static char *
	find_dot_file(char *basename)
	{
	char *fullname;
	const char *opt;
	const char *path = pathval();
	struct stat statb;

	/* don't try this for absolute or relative paths */
	if( strchr(basename, '/'))
	return basename;

	while ((fullname = padvance(&path, &opt, basename)) != NULL) {
	if ((stat(fullname, &statb) == 0) && S_ISREG(statb.st_mode)) {
	/*
	* Don't bother freeing here, since it will
	* be freed by the caller.
	*/
	return fullname;
	}
	stunalloc(fullname);
	}
	return basename;
	}

	int
	dotcmd(int argc, char **argv)
	{
	char filename, fullname;

	if (argc < 2)
	error("missing filename");

	exitstatus = 0;

	/*
	* Because we have historically not supported any options,
	* only treat "--" specially.
	*/
	filename = argc > 2 && strcmp(argv[1], "--") == 0 ? argv[2] : argv[1];

	fullname = find_dot_file(filename);
	setinputfile(fullname, 1);
	commandname = fullname;
	cmdloop(0);
	popfile();
	return exitstatus;
	}


	int
	exitcmd(int argc, char **argv)
	{
	if (stoppedjobs())
	return 0;
	if (argc > 1)
	exitshell(number(argv[1]));
	else
	exitshell_savedstatus();
	}
	diff --git a/cddl/lib/Makefile b/cddl/lib/Makefile
	index 69a0e381dfec..b65983dd64a7 100644
	--- a/cddl/lib/Makefile
	+++ b/cddl/lib/Makefile
	@@ -1,38 +1,33 @@
	# $FreeBSD$

	.include <src.opts.mk>

	-SUBDIR= ${_drti} \
	+SUBDIR= drti \
	libavl \
	libctf \
	- ${_libdtrace} \
	+ libdtrace \
	libnvpair \
	libumem \
	libuutil \
	${_libzfs_core} \
	${_libzfs} \
	${_libzpool} \

	SUBDIR.${MK_TESTS}+= tests

	.if ${MK_ZFS} != "no"
	_libzfs_core= libzfs_core
	_libzfs= libzfs
	.if ${MK_LIBTHR} != "no"
	_libzpool= libzpool
	.endif
	.endif

	-.if ${MACHINE_CPUARCH} != "sparc64"
	-_drti= drti
	-_libdtrace= libdtrace
	-.endif
	-
	SUBDIR_DEPEND_libdtrace= libctf
	SUBDIR_DEPEND_libzfs_core= libnvpair
	SUBDIR_DEPEND_libzfs= libavl libnvpair libumem libuutil libzfs_core
	SUBDIR_DEPEND_libzpool= libavl libnvpair libumem

	SUBDIR_PARALLEL=

	.include <bsd.subdir.mk>
	diff --git a/cddl/lib/libdtrace/Makefile b/cddl/lib/libdtrace/Makefile
	index bc58855ffc5d..11565b03b0ac 100644
	--- a/cddl/lib/libdtrace/Makefile
	+++ b/cddl/lib/libdtrace/Makefile
	@@ -1,133 +1,130 @@
	# $FreeBSD$

	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/common
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libgen/common

	LIB= dtrace
	SRCS= dt_aggregate.c \
	dt_as.c \
	dt_buf.c \
	dt_cc.c \
	dt_cg.c \
	dt_consume.c \
	dt_decl.c \
	dt_dis.c \
	dt_dof.c \
	dt_error.c \
	dt_errtags.c \
	dt_grammar.y \
	dt_handle.c \
	dt_ident.c \
	dt_isadep.c \
	dt_inttab.c \
	dt_lex.l \
	dt_link.c \
	dt_list.c \
	dt_map.c \
	dt_module.c \
	dt_names.c \
	dt_open.c \
	dt_options.c \
	dt_parser.c \
	dt_pcb.c \
	dt_pid.c \
	dt_pq.c \
	dt_pragma.c \
	dt_print.c \
	dt_printf.c \
	dt_proc.c \
	dt_program.c \
	dt_provider.c \
	dt_regset.c \
	dt_string.c \
	dt_strtab.c \
	dt_subr.c \
	dt_sugar.c \
	dt_work.c \
	dt_xlator.c \
	gmatch.c

	DSRCS= errno.d \
	io.d \
	ip.d \
	psinfo.d \
	sctp.d \
	siftr.d \
	signal.d \
	tcp.d \
	socket.d \
	udp.d \
	udplite.d \
	unistd.d

	FILES= ${DSRCS}
	FILESDIR= /usr/lib/dtrace
	FILESMODE= ${NOBINMODE}

	WARNS?= 1

	CFLAGS+= -I${.OBJDIR} -I${.CURDIR} \
	-I${SRCTOP}/sys/cddl/dev/dtrace/${MACHINE_ARCH} \
	-I${SRCTOP}/sys/cddl/compat/opensolaris \
	-I${SRCTOP}/cddl/compat/opensolaris/include \
	-I${OPENSOLARIS_USR_DISTDIR}/head \
	-I${OPENSOLARIS_USR_DISTDIR}/lib/libctf/common \
	-I${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common \
	-I${OPENSOLARIS_SYS_DISTDIR}/uts/common

	#CFLAGS+= -DYYDEBUG

	.if ${MACHINE_CPUARCH} == "aarch64"
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/aarch64
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/aarch64
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/aarch64
	.elif ${MACHINE_CPUARCH} == "amd64" \|\| ${MACHINE_CPUARCH} == "i386"
	CFLAGS+= -I${SRCTOP}/sys/cddl/dev/dtrace/x86
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/intel -DDIS_MEM
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/i386
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/${MACHINE_ARCH}
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/x86
	.elif ${MACHINE_CPUARCH} == "arm"
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/arm
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/arm
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/arm
	.elif ${MACHINE_CPUARCH} == "mips"
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/mips
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/mips
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/mips
	.elif ${MACHINE_CPUARCH} == "powerpc"
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/powerpc
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/powerpc
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/powerpc
	.elif ${MACHINE_CPUARCH} == "riscv"
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/riscv
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/riscv
	.PATH: ${SRCTOP}/sys/cddl/dev/dtrace/riscv
	-.elif ${MACHINE_CPUARCH} == "sparc64"
	-CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/sparc
	-.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libdtrace/sparc
	.else
	# temporary hack
	CFLAGS+= -I${OPENSOLARIS_SYS_DISTDIR}/uts/intel
	.endif

	.if ${MACHINE_ARCH} == "i386" \|\| ${MACHINE_ARCH} == "amd64"
	SRCS+= dis_tables.c
	DSRCS+= regs_x86.d
	.endif

	LFLAGS+=-l

	YFLAGS+=-d

	LIBADD= ctf elf proc pthread rtld_db

	CLEANFILES= dt_errtags.c dt_names.c

	.include <bsd.lib.mk>

	dt_errtags.c: ${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common/dt_errtags.h
	sh ${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common/mkerrtags.sh < ${.ALLSRC} > ${.TARGET}

	dt_names.c: ${OPENSOLARIS_SYS_DISTDIR}/uts/common/sys/dtrace.h
	sh ${OPENSOLARIS_USR_DISTDIR}/lib/libdtrace/common/mknames.sh < ${.ALLSRC} > ${.TARGET}
	diff --git a/cddl/lib/libzpool/Makefile b/cddl/lib/libzpool/Makefile
	index 1c545e0260d1..576b89b7725a 100644
	--- a/cddl/lib/libzpool/Makefile
	+++ b/cddl/lib/libzpool/Makefile
	@@ -1,84 +1,82 @@
	# $FreeBSD$

	.include "${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/Makefile.files"

	# ZFS_COMMON_SRCS
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs
	# LUA_SRCS
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lua
	# ZFS_SHARED_SRCS
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/zfs
	# LZ4_COMMON_SRCS
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
	# KERNEL_SRCS
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libzpool/common
	# LIST_SRCS
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/os
	# ATOMIC_SRCS
	.if exists(${SRCTOP}/sys/cddl/contrib/opensolaris/common/atomic/${MACHINE_ARCH}/opensolaris_atomic.S)
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/atomic/${MACHINE_ARCH}
	ATOMIC_SRCS= opensolaris_atomic.S
	-.if ${MACHINE_ARCH} != "sparc64"
	ACFLAGS+= -Wa,--noexecstack
	-.endif
	.else
	.PATH: ${SRCTOP}/sys/cddl/compat/opensolaris/kern
	ATOMIC_SRCS= opensolaris_atomic.c
	.endif
	# UNICODE_SRCS
	.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/unicode
	# LIBCMDUTILS_SRCS
	.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libcmdutils/common

	LIB= zpool

	ZFS_COMMON_SRCS= ${ZFS_COMMON_OBJS:C/.o$/.c/} trim_map.c
	ZFS_SHARED_SRCS= ${ZFS_SHARED_OBJS:C/.o$/.c/}
	LZ4_COMMON_SRCS= lz4.c
	LUA_SRCS= ${LUA_OBJS:C/.o$/.c/}
	KERNEL_SRCS= kernel.c taskq.c util.c
	LIST_SRCS= list.c
	UNICODE_SRCS= u8_textprep.c
	LIBCMDUTILS_SRCS=nicenum.c

	SRCS= ${ZFS_COMMON_SRCS} ${ZFS_SHARED_SRCS} ${LUA_SRCS} \
	${LZ4_COMMON_SRCS} ${KERNEL_SRCS} ${LIST_SRCS} ${ATOMIC_SRCS} \
	${UNICODE_SRCS} ${LIBCMDUTILS_SRCS}

	WARNS?= 0
	CFLAGS+= -I${SRCTOP}/sys/cddl/compat/opensolaris
	CFLAGS+= -I${SRCTOP}/cddl/compat/opensolaris/include
	CFLAGS+= -I${SRCTOP}/cddl/compat/opensolaris/lib/libumem
	CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libzfs/common
	CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libzpool/common
	CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs
	CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lua
	CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/zfs
	CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
	CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common
	CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/head
	CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libnvpair
	CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libcmdutils
	# XXX: pthread doesn't have mutex_owned() equivalent, so we need to look
	# into libthr private structures. That's sooo evil, but it's only for
	# ZFS debugging tools needs.
	CFLAGS+= -DWANTS_MUTEX_OWNED
	CFLAGS+= -I${SRCTOP}/lib/libpthread/thread
	CFLAGS+= -I${SRCTOP}/lib/libpthread/sys
	CFLAGS+= -I${SRCTOP}/lib/libthr/arch/${MACHINE_CPUARCH}/include
	CFLAGS.lz4.c+= -D_FAKE_KERNEL
	CFLAGS.gcc+= -fms-extensions

	LIBADD= md pthread z nvpair avl umem

	# atomic.S doesn't like profiling.
	MK_PROFILE= no

	CSTD= c99

	# Since there are many asserts in this library, it makes no sense to compile
	# it without debugging.

	CFLAGS+= -g -DDEBUG=1

	.include <bsd.lib.mk>
	diff --git a/contrib/ipfilter/iplang/iplang_y.y b/contrib/ipfilter/iplang/iplang_y.y
	index 98c8f1a983ea..f223b1eb8b32 100644
	--- a/contrib/ipfilter/iplang/iplang_y.y
	+++ b/contrib/ipfilter/iplang/iplang_y.y
	@@ -1,1858 +1,1849 @@
	/* $FreeBSD$ */

	%{
	/*
	* Copyright (C) 2012 by Darren Reed.
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*
	* Id: iplang_y.y,v 2.9.2.4 2006/03/17 12:11:29 darrenr Exp $
	* $FreeBSD$
	*/

	#include <stdio.h>
	#include <string.h>
	#include <fcntl.h>
	#if !defined(__SVR4) && !defined(__svr4__)
	# include <strings.h>
	#else
	# include <sys/byteorder.h>
	#endif
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/time.h>
	#include <stdlib.h>
	#include <unistd.h>
	#include <stddef.h>
	#include <sys/socket.h>
	#include <net/if.h>
	#include <netinet/in.h>
	#include <netinet/in_systm.h>
	#include <netinet/ip.h>
	#ifndef linux
	# include <netinet/ip_var.h>
	# include <net/route.h>
	# include <netinet/if_ether.h>
	#endif
	#include <netdb.h>
	#include <arpa/nameser.h>
	#include <arpa/inet.h>
	#include <resolv.h>
	#include <ctype.h>
	#include "ipsend.h"
	#include "ip_compat.h"
	#include "ipf.h"
	#include "iplang.h"

	-#if !defined(__NetBSD__) && (!defined(__FreeBSD_version) && \
	- __FreeBSD_version < 400020) && (!SOLARIS \|\| SOLARIS2 < 10)
	-extern struct ether_addr ether_aton __P((char ));
	-#endif
	-
	extern int opts;
	extern struct ipopt_names ionames[];
	extern int state, state, lineNum, token;
	extern int yylineno;
	extern char yytext[];
	extern FILE *yyin;
	int yylex __P((void));
	#define YYDEBUG 1
	-#if !defined(ultrix) && !defined(hpux)
	int yydebug = 1;
	-#else
	-extern int yydebug;
	-#endif

	iface_t iflist = NULL, *iftail = &iflist;
	iface_t *cifp = NULL;
	arp_t arplist = NULL, arptail = &arplist, carp = NULL;
	struct in_addr defrouter;
	send_t sending;
	char *sclass = NULL;
	u_short c_chksum __P((u_short *, u_int, u_long));
	u_long p_chksum __P((u_short *, u_int));

	u_long ipbuffer[67584/sizeof(u_long)]; /* 66K */
	aniphdr_t aniphead = NULL, canip = NULL, **aniptail = &aniphead;
	ip_t *ip = NULL;
	udphdr_t *udp = NULL;
	tcphdr_t *tcp = NULL;
	icmphdr_t *icmp = NULL;

	struct statetoopt {
	int sto_st;
	int sto_op;
	};

	struct in_addr getipv4addr __P((char *arg));
	u_short getportnum __P((char , char ));
	struct ether_addr geteaddr __P((char , struct ether_addr *));
	void *new_header __P((int));
	void free_aniplist __P((void));
	void inc_anipheaders __P((int));
	void new_data __P((void));
	void set_datalen __P((char **));
	void set_datafile __P((char **));
	void set_data __P((char **));
	void new_packet __P((void));
	void set_ipv4proto __P((char **));
	void set_ipv4src __P((char **));
	void set_ipv4dst __P((char **));
	void set_ipv4off __P((char **));
	void set_ipv4v __P((char **));
	void set_ipv4hl __P((char **));
	void set_ipv4ttl __P((char **));
	void set_ipv4tos __P((char **));
	void set_ipv4id __P((char **));
	void set_ipv4sum __P((char **));
	void set_ipv4len __P((char **));
	void new_tcpheader __P((void));
	void set_tcpsport __P((char **));
	void set_tcpdport __P((char **));
	void set_tcpseq __P((char **));
	void set_tcpack __P((char **));
	void set_tcpoff __P((char **));
	void set_tcpurp __P((char **));
	void set_tcpwin __P((char **));
	void set_tcpsum __P((char **));
	void set_tcpflags __P((char **));
	void set_tcpopt __P((int, char **));
	void end_tcpopt __P((void));
	void new_udpheader __P((void));
	void set_udplen __P((char **));
	void set_udpsum __P((char **));
	void prep_packet __P((void));
	void packet_done __P((void));
	void new_interface __P((void));
	void check_interface __P((void));
	void set_ifname __P((char **));
	void set_ifmtu __P((int));
	void set_ifv4addr __P((char **));
	void set_ifeaddr __P((char **));
	void new_arp __P((void));
	void set_arpeaddr __P((char **));
	void set_arpv4addr __P((char **));
	void reset_send __P((void));
	void set_sendif __P((char **));
	void set_sendvia __P((char **));
	void set_defaultrouter __P((char **));
	void new_icmpheader __P((void));
	void set_icmpcode __P((int));
	void set_icmptype __P((int));
	void set_icmpcodetok __P((char **));
	void set_icmptypetok __P((char **));
	void set_icmpid __P((int));
	void set_icmpseq __P((int));
	void set_icmpotime __P((int));
	void set_icmprtime __P((int));
	void set_icmpttime __P((int));
	void set_icmpmtu __P((int));
	void set_redir __P((int, char **));
	void new_ipv4opt __P((void));
	void set_icmppprob __P((int));
	void add_ipopt __P((int, void *));
	void end_ipopt __P((void));
	void set_secclass __P((char **));
	void free_anipheader __P((void));
	void end_ipv4 __P((void));
	void end_icmp __P((void));
	void end_udp __P((void));
	void end_tcp __P((void));
	void end_data __P((void));
	void yyerror __P((char *));
	void iplang __P((FILE *));
	int arp_getipv4 __P((char , char ));
	int yyparse __P((void));
	%}
	%union {
	char *str;
	int num;
	}
	%token <num> IL_NUMBER
	%type <num> number digits optnumber
	%token <str> IL_TOKEN
	%type <str> token optoken
	%token IL_HEXDIGIT IL_COLON IL_DOT IL_EOF IL_COMMENT
	%token IL_INTERFACE IL_IFNAME IL_MTU IL_EADDR
	%token IL_IPV4 IL_V4PROTO IL_V4SRC IL_V4DST IL_V4OFF IL_V4V IL_V4HL IL_V4TTL
	%token IL_V4TOS IL_V4SUM IL_V4LEN IL_V4OPT IL_V4ID
	%token IL_TCP IL_SPORT IL_DPORT IL_TCPFL IL_TCPSEQ IL_TCPACK IL_TCPOFF
	%token IL_TCPWIN IL_TCPSUM IL_TCPURP IL_TCPOPT IL_TCPO_NOP IL_TCPO_EOL
	%token IL_TCPO_MSS IL_TCPO_WSCALE IL_TCPO_TS
	%token IL_UDP IL_UDPLEN IL_UDPSUM
	%token IL_ICMP IL_ICMPTYPE IL_ICMPCODE
	%token IL_SEND IL_VIA
	%token IL_ARP
	%token IL_DEFROUTER
	%token IL_SUM IL_OFF IL_LEN IL_V4ADDR IL_OPT
	%token IL_DATA IL_DLEN IL_DVALUE IL_DFILE
	%token IL_IPO_NOP IL_IPO_RR IL_IPO_ZSU IL_IPO_MTUP IL_IPO_MTUR IL_IPO_EOL
	%token IL_IPO_TS IL_IPO_TR IL_IPO_SEC IL_IPO_LSRR IL_IPO_ESEC
	%token IL_IPO_SATID IL_IPO_SSRR IL_IPO_ADDEXT IL_IPO_VISA IL_IPO_IMITD
	%token IL_IPO_EIP IL_IPO_FINN IL_IPO_SECCLASS IL_IPO_CIPSO IL_IPO_ENCODE
	%token <str> IL_IPS_RESERV4 IL_IPS_TOPSECRET IL_IPS_SECRET IL_IPS_RESERV3
	%token <str> IL_IPS_CONFID IL_IPS_UNCLASS IL_IPS_RESERV2 IL_IPS_RESERV1
	%token IL_ICMP_ECHOREPLY IL_ICMP_UNREACH IL_ICMP_UNREACH_NET
	%token IL_ICMP_UNREACH_HOST IL_ICMP_UNREACH_PROTOCOL IL_ICMP_UNREACH_PORT
	%token IL_ICMP_UNREACH_NEEDFRAG IL_ICMP_UNREACH_SRCFAIL
	%token IL_ICMP_UNREACH_NET_UNKNOWN IL_ICMP_UNREACH_HOST_UNKNOWN
	%token IL_ICMP_UNREACH_ISOLATED IL_ICMP_UNREACH_NET_PROHIB
	%token IL_ICMP_UNREACH_HOST_PROHIB IL_ICMP_UNREACH_TOSNET
	%token IL_ICMP_UNREACH_TOSHOST IL_ICMP_UNREACH_FILTER_PROHIB
	%token IL_ICMP_UNREACH_HOST_PRECEDENCE IL_ICMP_UNREACH_PRECEDENCE_CUTOFF
	%token IL_ICMP_SOURCEQUENCH IL_ICMP_REDIRECT IL_ICMP_REDIRECT_NET
	%token IL_ICMP_REDIRECT_HOST IL_ICMP_REDIRECT_TOSNET
	%token IL_ICMP_REDIRECT_TOSHOST IL_ICMP_ECHO IL_ICMP_ROUTERADVERT
	%token IL_ICMP_ROUTERSOLICIT IL_ICMP_TIMXCEED IL_ICMP_TIMXCEED_INTRANS
	%token IL_ICMP_TIMXCEED_REASS IL_ICMP_PARAMPROB IL_ICMP_PARAMPROB_OPTABSENT
	%token IL_ICMP_TSTAMP IL_ICMP_TSTAMPREPLY IL_ICMP_IREQ IL_ICMP_IREQREPLY
	%token IL_ICMP_MASKREQ IL_ICMP_MASKREPLY IL_ICMP_SEQ IL_ICMP_ID
	%token IL_ICMP_OTIME IL_ICMP_RTIME IL_ICMP_TTIME

	%%
	file: line
	\| line file
	\| IL_COMMENT
	\| IL_COMMENT file
	;

	line: iface
	\| arp
	\| send
	\| defrouter
	\| ipline
	;

	iface: ifhdr '{' ifaceopts '}' ';' { check_interface(); }
	;

	ifhdr: IL_INTERFACE { new_interface(); }
	;

	ifaceopts:
	ifaceopt
	\| ifaceopt ifaceopts
	;

	ifaceopt:
	IL_IFNAME token { set_ifname(&$2); }
	\| IL_MTU number { set_ifmtu($2); }
	\| IL_V4ADDR token { set_ifv4addr(&$2); }
	\| IL_EADDR token { set_ifeaddr(&$2); }
	;

	send: sendhdr '{' sendbody '}' ';' { packet_done(); }
	\| sendhdr ';' { packet_done(); }
	;

	sendhdr:
	IL_SEND { reset_send(); }
	;

	sendbody:
	sendopt
	\| sendbody sendopt
	;

	sendopt:
	IL_IFNAME token { set_sendif(&$2); }
	\| IL_VIA token { set_sendvia(&$2); }
	;

	arp: arphdr '{' arpbody '}' ';'
	;

	arphdr: IL_ARP { new_arp(); }
	;

	arpbody:
	arpopt
	\| arpbody arpopt
	;

	arpopt: IL_V4ADDR token { set_arpv4addr(&$2); }
	\| IL_EADDR token { set_arpeaddr(&$2); }
	;

	defrouter:
	IL_DEFROUTER token { set_defaultrouter(&$2); }
	;

	bodyline:
	ipline
	\| tcp tcpline
	\| udp udpline
	\| icmp icmpline
	\| data dataline
	;

	ipline: ipv4 '{' ipv4body '}' ';' { end_ipv4(); }
	;

	ipv4: IL_IPV4 { new_packet(); }

	ipv4body:
	ipv4type
	\| ipv4type ipv4body
	\| bodyline
	;

	ipv4type:
	IL_V4PROTO token { set_ipv4proto(&$2); }
	\| IL_V4SRC token { set_ipv4src(&$2); }
	\| IL_V4DST token { set_ipv4dst(&$2); }
	\| IL_V4OFF token { set_ipv4off(&$2); }
	\| IL_V4V token { set_ipv4v(&$2); }
	\| IL_V4HL token { set_ipv4hl(&$2); }
	\| IL_V4ID token { set_ipv4id(&$2); }
	\| IL_V4TTL token { set_ipv4ttl(&$2); }
	\| IL_V4TOS token { set_ipv4tos(&$2); }
	\| IL_V4SUM token { set_ipv4sum(&$2); }
	\| IL_V4LEN token { set_ipv4len(&$2); }
	\| ipv4opt '{' ipv4optlist '}' ';' { end_ipopt(); }
	;

	tcp: IL_TCP { new_tcpheader(); }
	;

	tcpline:
	'{' tcpheader '}' ';' { end_tcp(); }
	;

	tcpheader:
	tcpbody
	\| tcpbody tcpheader
	\| bodyline
	;

	tcpbody:
	IL_SPORT token { set_tcpsport(&$2); }
	\| IL_DPORT token { set_tcpdport(&$2); }
	\| IL_TCPSEQ token { set_tcpseq(&$2); }
	\| IL_TCPACK token { set_tcpack(&$2); }
	\| IL_TCPOFF token { set_tcpoff(&$2); }
	\| IL_TCPURP token { set_tcpurp(&$2); }
	\| IL_TCPWIN token { set_tcpwin(&$2); }
	\| IL_TCPSUM token { set_tcpsum(&$2); }
	\| IL_TCPFL token { set_tcpflags(&$2); }
	\| IL_TCPOPT '{' tcpopts '}' ';' { end_tcpopt(); }
	;

	tcpopts:
	\| tcpopt tcpopts
	;

	tcpopt: IL_TCPO_NOP ';' { set_tcpopt(IL_TCPO_NOP, NULL); }
	\| IL_TCPO_EOL ';' { set_tcpopt(IL_TCPO_EOL, NULL); }
	\| IL_TCPO_MSS optoken { set_tcpopt(IL_TCPO_MSS,&$2);}
	\| IL_TCPO_WSCALE optoken { set_tcpopt(IL_TCPO_WSCALE,&$2);}
	\| IL_TCPO_TS optoken { set_tcpopt(IL_TCPO_TS, &$2);}
	;

	udp: IL_UDP { new_udpheader(); }
	;

	udpline:
	'{' udpheader '}' ';' { end_udp(); }
	;


	udpheader:
	udpbody
	\| udpbody udpheader
	\| bodyline
	;

	udpbody:
	IL_SPORT token { set_tcpsport(&$2); }
	\| IL_DPORT token { set_tcpdport(&$2); }
	\| IL_UDPLEN token { set_udplen(&$2); }
	\| IL_UDPSUM token { set_udpsum(&$2); }
	;

	icmp: IL_ICMP { new_icmpheader(); }
	;

	icmpline:
	'{' icmpbody '}' ';' { end_icmp(); }
	;

	icmpbody:
	icmpheader
	\| icmpheader bodyline
	;

	icmpheader:
	IL_ICMPTYPE icmptype
	\| IL_ICMPTYPE icmptype icmpcode
	;

	icmpcode:
	IL_ICMPCODE token { set_icmpcodetok(&$2); }
	;

	icmptype:
	IL_ICMP_ECHOREPLY ';' { set_icmptype(ICMP_ECHOREPLY); }
	\| IL_ICMP_ECHOREPLY '{' icmpechoopts '}' ';'
	\| unreach
	\| IL_ICMP_SOURCEQUENCH ';' { set_icmptype(ICMP_SOURCEQUENCH); }
	\| redirect
	\| IL_ICMP_ROUTERADVERT ';' { set_icmptype(ICMP_ROUTERADVERT); }
	\| IL_ICMP_ROUTERSOLICIT ';' { set_icmptype(ICMP_ROUTERSOLICIT); }
	\| IL_ICMP_ECHO ';' { set_icmptype(ICMP_ECHO); }
	\| IL_ICMP_ECHO '{' icmpechoopts '}' ';'
	\| IL_ICMP_TIMXCEED ';' { set_icmptype(ICMP_TIMXCEED); }
	\| IL_ICMP_TIMXCEED '{' exceed '}' ';'
	\| IL_ICMP_TSTAMP ';' { set_icmptype(ICMP_TSTAMP); }
	\| IL_ICMP_TSTAMPREPLY ';' { set_icmptype(ICMP_TSTAMPREPLY); }
	\| IL_ICMP_TSTAMPREPLY '{' icmptsopts '}' ';'
	\| IL_ICMP_IREQ ';' { set_icmptype(ICMP_IREQ); }
	\| IL_ICMP_IREQREPLY ';' { set_icmptype(ICMP_IREQREPLY); }
	\| IL_ICMP_IREQREPLY '{' data dataline '}' ';'
	\| IL_ICMP_MASKREQ ';' { set_icmptype(ICMP_MASKREQ); }
	\| IL_ICMP_MASKREPLY ';' { set_icmptype(ICMP_MASKREPLY); }
	\| IL_ICMP_MASKREPLY '{' token '}' ';'
	\| IL_ICMP_PARAMPROB ';' { set_icmptype(ICMP_PARAMPROB); }
	\| IL_ICMP_PARAMPROB '{' paramprob '}' ';'
	\| IL_TOKEN ';' { set_icmptypetok(&$1); }
	;

	icmpechoopts:
	\| icmpechoopts icmpecho
	;

	icmpecho:
	IL_ICMP_SEQ number { set_icmpseq($2); }
	\| IL_ICMP_ID number { set_icmpid($2); }
	;

	icmptsopts:
	\| icmptsopts icmpts ';'
	;

	icmpts: IL_ICMP_OTIME number { set_icmpotime($2); }
	\| IL_ICMP_RTIME number { set_icmprtime($2); }
	\| IL_ICMP_TTIME number { set_icmpttime($2); }
	;

	unreach:
	IL_ICMP_UNREACH
	\| IL_ICMP_UNREACH '{' unreachopts '}' ';'
	;

	unreachopts:
	IL_ICMP_UNREACH_NET line
	\| IL_ICMP_UNREACH_HOST line
	\| IL_ICMP_UNREACH_PROTOCOL line
	\| IL_ICMP_UNREACH_PORT line
	\| IL_ICMP_UNREACH_NEEDFRAG number ';' { set_icmpmtu($2); }
	\| IL_ICMP_UNREACH_SRCFAIL line
	\| IL_ICMP_UNREACH_NET_UNKNOWN line
	\| IL_ICMP_UNREACH_HOST_UNKNOWN line
	\| IL_ICMP_UNREACH_ISOLATED line
	\| IL_ICMP_UNREACH_NET_PROHIB line
	\| IL_ICMP_UNREACH_HOST_PROHIB line
	\| IL_ICMP_UNREACH_TOSNET line
	\| IL_ICMP_UNREACH_TOSHOST line
	\| IL_ICMP_UNREACH_FILTER_PROHIB line
	\| IL_ICMP_UNREACH_HOST_PRECEDENCE line
	\| IL_ICMP_UNREACH_PRECEDENCE_CUTOFF line
	;

	redirect:
	IL_ICMP_REDIRECT
	\| IL_ICMP_REDIRECT '{' redirectopts '}' ';'
	;

	redirectopts:
	\| IL_ICMP_REDIRECT_NET token { set_redir(0, &$2); }
	\| IL_ICMP_REDIRECT_HOST token { set_redir(1, &$2); }
	\| IL_ICMP_REDIRECT_TOSNET token { set_redir(2, &$2); }
	\| IL_ICMP_REDIRECT_TOSHOST token { set_redir(3, &$2); }
	;

	exceed:
	IL_ICMP_TIMXCEED_INTRANS line
	\| IL_ICMP_TIMXCEED_REASS line
	;

	paramprob:
	IL_ICMP_PARAMPROB_OPTABSENT
	\| IL_ICMP_PARAMPROB_OPTABSENT paraprobarg

	paraprobarg:
	'{' number '}' ';' { set_icmppprob($2); }
	;

	ipv4opt: IL_V4OPT { new_ipv4opt(); }
	;

	ipv4optlist:
	\| ipv4opts ipv4optlist
	;

	ipv4opts:
	IL_IPO_NOP ';' { add_ipopt(IL_IPO_NOP, NULL); }
	\| IL_IPO_RR optnumber { add_ipopt(IL_IPO_RR, &$2); }
	\| IL_IPO_ZSU ';' { add_ipopt(IL_IPO_ZSU, NULL); }
	\| IL_IPO_MTUP ';' { add_ipopt(IL_IPO_MTUP, NULL); }
	\| IL_IPO_MTUR ';' { add_ipopt(IL_IPO_MTUR, NULL); }
	\| IL_IPO_ENCODE ';' { add_ipopt(IL_IPO_ENCODE, NULL); }
	\| IL_IPO_TS ';' { add_ipopt(IL_IPO_TS, NULL); }
	\| IL_IPO_TR ';' { add_ipopt(IL_IPO_TR, NULL); }
	\| IL_IPO_SEC ';' { add_ipopt(IL_IPO_SEC, NULL); }
	\| IL_IPO_SECCLASS secclass { add_ipopt(IL_IPO_SECCLASS, sclass); }
	\| IL_IPO_LSRR token { add_ipopt(IL_IPO_LSRR,&$2); }
	\| IL_IPO_ESEC ';' { add_ipopt(IL_IPO_ESEC, NULL); }
	\| IL_IPO_CIPSO ';' { add_ipopt(IL_IPO_CIPSO, NULL); }
	\| IL_IPO_SATID optnumber { add_ipopt(IL_IPO_SATID,&$2);}
	\| IL_IPO_SSRR token { add_ipopt(IL_IPO_SSRR,&$2); }
	\| IL_IPO_ADDEXT ';' { add_ipopt(IL_IPO_ADDEXT, NULL); }
	\| IL_IPO_VISA ';' { add_ipopt(IL_IPO_VISA, NULL); }
	\| IL_IPO_IMITD ';' { add_ipopt(IL_IPO_IMITD, NULL); }
	\| IL_IPO_EIP ';' { add_ipopt(IL_IPO_EIP, NULL); }
	\| IL_IPO_FINN ';' { add_ipopt(IL_IPO_FINN, NULL); }
	;

	secclass:
	IL_IPS_RESERV4 ';' { set_secclass(&$1); }
	\| IL_IPS_TOPSECRET ';' { set_secclass(&$1); }
	\| IL_IPS_SECRET ';' { set_secclass(&$1); }
	\| IL_IPS_RESERV3 ';' { set_secclass(&$1); }
	\| IL_IPS_CONFID ';' { set_secclass(&$1); }
	\| IL_IPS_UNCLASS ';' { set_secclass(&$1); }
	\| IL_IPS_RESERV2 ';' { set_secclass(&$1); }
	\| IL_IPS_RESERV1 ';' { set_secclass(&$1); }
	;

	data: IL_DATA { new_data(); }
	;

	dataline:
	'{' databody '}' ';' { end_data(); }
	;

	databody: dataopts
	\| dataopts databody
	;

	dataopts:
	IL_DLEN token { set_datalen(&$2); }
	\| IL_DVALUE token { set_data(&$2); }
	\| IL_DFILE token { set_datafile(&$2); }
	;

	token: IL_TOKEN ';'
	;

	optoken: ';' { $$ = ""; }
	\| token
	;

	number: digits ';'
	;

	optnumber: ';' { $$ = 0; }
	\| number
	;

	digits: IL_NUMBER
	\| digits IL_NUMBER
	;
	%%

	struct statetoopt toipopts[] = {
	{ IL_IPO_NOP, IPOPT_NOP },
	{ IL_IPO_RR, IPOPT_RR },
	{ IL_IPO_ZSU, IPOPT_ZSU },
	{ IL_IPO_MTUP, IPOPT_MTUP },
	{ IL_IPO_MTUR, IPOPT_MTUR },
	{ IL_IPO_ENCODE, IPOPT_ENCODE },
	{ IL_IPO_TS, IPOPT_TS },
	{ IL_IPO_TR, IPOPT_TR },
	{ IL_IPO_SEC, IPOPT_SECURITY },
	{ IL_IPO_SECCLASS, IPOPT_SECURITY },
	{ IL_IPO_LSRR, IPOPT_LSRR },
	{ IL_IPO_ESEC, IPOPT_E_SEC },
	{ IL_IPO_CIPSO, IPOPT_CIPSO },
	{ IL_IPO_SATID, IPOPT_SATID },
	{ IL_IPO_SSRR, IPOPT_SSRR },
	{ IL_IPO_ADDEXT, IPOPT_ADDEXT },
	{ IL_IPO_VISA, IPOPT_VISA },
	{ IL_IPO_IMITD, IPOPT_IMITD },
	{ IL_IPO_EIP, IPOPT_EIP },
	{ IL_IPO_FINN, IPOPT_FINN },
	{ 0, 0 }
	};

	struct statetoopt tosecopts[] = {
	{ IL_IPS_RESERV4, IPSO_CLASS_RES4 },
	{ IL_IPS_TOPSECRET, IPSO_CLASS_TOPS },
	{ IL_IPS_SECRET, IPSO_CLASS_SECR },
	{ IL_IPS_RESERV3, IPSO_CLASS_RES3 },
	{ IL_IPS_CONFID, IPSO_CLASS_CONF },
	{ IL_IPS_UNCLASS, IPSO_CLASS_UNCL },
	{ IL_IPS_RESERV2, IPSO_CLASS_RES2 },
	{ IL_IPS_RESERV1, IPSO_CLASS_RES1 },
	{ 0, 0 }
	};

	#ifdef bsdi
	struct ether_addr *
	ether_aton(s)
	char *s;
	{
	static struct ether_addr n;
	u_int i[6];

	if (sscanf(s, " %x:%x:%x:%x:%x:%x ", &i[0], &i[1],
	&i[2], &i[3], &i[4], &i[5]) == 6) {
	n.ether_addr_octet[0] = (u_char)i[0];
	n.ether_addr_octet[1] = (u_char)i[1];
	n.ether_addr_octet[2] = (u_char)i[2];
	n.ether_addr_octet[3] = (u_char)i[3];
	n.ether_addr_octet[4] = (u_char)i[4];
	n.ether_addr_octet[5] = (u_char)i[5];
	return &n;
	}
	return NULL;
	}
	#endif


	struct in_addr getipv4addr(arg)
	char *arg;
	{
	struct hostent *hp;
	struct in_addr in;

	in.s_addr = 0xffffffff;

	if ((hp = gethostbyname(arg)))
	bcopy(hp->h_addr, &in.s_addr, sizeof(struct in_addr));
	else
	in.s_addr = inet_addr(arg);
	return in;
	}


	u_short getportnum(pr, name)
	char pr, name;
	{
	struct servent *sp;

	if (!(sp = getservbyname(name, pr)))
	return htons(atoi(name));
	return sp->s_port;
	}


	struct ether_addr *geteaddr(arg, buf)
	char *arg;
	struct ether_addr *buf;
	{
	struct ether_addr *e;

	#if !defined(hpux) && !defined(linux)
	e = ether_aton(arg);
	if (!e)
	fprintf(stderr, "Invalid ethernet address: %s\n", arg);
	else
	# ifdef __FreeBSD__
	bcopy(e->octet, buf->octet, sizeof(e->octet));
	# else
	bcopy(e->ether_addr_octet, buf->ether_addr_octet,
	sizeof(e->ether_addr_octet));
	# endif
	return e;
	#else
	return NULL;
	#endif
	}


	void *new_header(type)
	int type;
	{
	aniphdr_t aip, oip = canip;
	int sz = 0;

	aip = (aniphdr_t )calloc(1, sizeof(aip));
	*aniptail = aip;
	aniptail = &aip->ah_next;
	aip->ah_p = type;
	aip->ah_prev = oip;
	canip = aip;

	if (type == IPPROTO_UDP)
	sz = sizeof(udphdr_t);
	else if (type == IPPROTO_TCP)
	sz = sizeof(tcphdr_t);
	else if (type == IPPROTO_ICMP)
	sz = sizeof(icmphdr_t);
	else if (type == IPPROTO_IP)
	sz = sizeof(ip_t);

	if (oip)
	canip->ah_data = oip->ah_data + oip->ah_len;
	else
	canip->ah_data = (char *)ipbuffer;

	/*
	* Increase the size fields in all wrapping headers.
	*/
	for (aip = aniphead; aip; aip = aip->ah_next) {
	aip->ah_len += sz;
	if (aip->ah_p == IPPROTO_IP)
	aip->ah_ip->ip_len += sz;
	else if (aip->ah_p == IPPROTO_UDP)
	aip->ah_udp->uh_ulen += sz;
	}
	return (void *)canip->ah_data;
	}


	void free_aniplist()
	{
	aniphdr_t aip, *aipp = &aniphead;

	while ((aip = *aipp)) {
	*aipp = aip->ah_next;
	free(aip);
	}
	aniptail = &aniphead;
	}


	void inc_anipheaders(inc)
	int inc;
	{
	aniphdr_t *aip;

	for (aip = aniphead; aip; aip = aip->ah_next) {
	aip->ah_len += inc;
	if (aip->ah_p == IPPROTO_IP)
	aip->ah_ip->ip_len += inc;
	else if (aip->ah_p == IPPROTO_UDP)
	aip->ah_udp->uh_ulen += inc;
	}
	}


	void new_data()
	{
	(void) new_header(-1);
	canip->ah_len = 0;
	}


	void set_datalen(arg)
	char **arg;
	{
	int len;

	len = strtol(*arg, NULL, 0);
	inc_anipheaders(len);
	free(*arg);
	*arg = NULL;
	}


	void set_data(arg)
	char **arg;
	{
	u_char s = (u_char )arg, t = (u_char *)canip->ah_data, c;
	int len = 0, todo = 0, quote = 0, val = 0;

	while ((c = *s++)) {
	if (todo) {
	if (ISDIGIT(c)) {
	todo--;
	if (c > '7') {
	fprintf(stderr, "octal with %c!\n", c);
	break;
	}
	val <<= 3;
	val \|= (c - '0');
	}
	if (!ISDIGIT(c) \|\| !todo) {
	*t++ = (u_char)(val & 0xff);
	todo = 0;
	}
	if (todo)
	continue;
	}
	if (quote) {
	if (ISDIGIT(c)) {
	todo = 2;
	if (c > '7') {
	fprintf(stderr, "octal with %c!\n", c);
	break;
	}
	val = (c - '0');
	} else {
	switch (c)
	{
	case '\"' :
	*t++ = '\"';
	break;
	case '\\' :
	*t++ = '\\';
	break;
	case 'n' :
	*t++ = '\n';
	break;
	case 'r' :
	*t++ = '\r';
	break;
	case 't' :
	*t++ = '\t';
	break;
	}
	}
	quote = 0;
	continue;
	}

	if (c == '\\')
	quote = 1;
	else
	*t++ = c;
	}
	if (todo)
	*t++ = (u_char)(val & 0xff);
	if (quote)
	*t++ = '\\';
	len = t - (u_char *)canip->ah_data;
	inc_anipheaders(len - canip->ah_len);
	canip->ah_len = len;
	}


	void set_datafile(arg)
	char **arg;
	{
	struct stat sb;
	char file = arg;
	int fd, len;

	if ((fd = open(file, O_RDONLY)) == -1) {
	perror("open");
	exit(-1);
	}

	if (fstat(fd, &sb) == -1) {
	perror("fstat");
	exit(-1);
	}

	if ((sb.st_size + aniphead->ah_len ) > 65535) {
	fprintf(stderr, "data file %s too big to include.\n", file);
	close(fd);
	return;
	}
	if ((len = read(fd, canip->ah_data, sb.st_size)) == -1) {
	perror("read");
	close(fd);
	return;
	}
	inc_anipheaders(len);
	canip->ah_len += len;
	close(fd);
	}


	void new_packet()
	{
	static u_short id = 0;

	if (!aniphead)
	bzero((char *)ipbuffer, sizeof(ipbuffer));

	ip = (ip_t *)new_header(IPPROTO_IP);
	ip->ip_v = IPVERSION;
	ip->ip_hl = sizeof(ip_t) >> 2;
	ip->ip_len = sizeof(ip_t);
	ip->ip_ttl = 63;
	ip->ip_id = htons(id++);
	}


	void set_ipv4proto(arg)
	char **arg;
	{
	struct protoent *pr;

	if ((pr = getprotobyname(*arg)))
	ip->ip_p = pr->p_proto;
	else
	if (!(ip->ip_p = atoi(*arg)))
	fprintf(stderr, "unknown protocol %s\n", *arg);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4src(arg)
	char **arg;
	{
	ip->ip_src = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4dst(arg)
	char **arg;
	{
	ip->ip_dst = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4off(arg)
	char **arg;
	{
	ip->ip_off = htons(strtol(*arg, NULL, 0));
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4v(arg)
	char **arg;
	{
	ip->ip_v = strtol(*arg, NULL, 0);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4hl(arg)
	char **arg;
	{
	int newhl, inc;

	newhl = strtol(*arg, NULL, 0);
	inc = (newhl - ip->ip_hl) << 2;
	ip->ip_len += inc;
	ip->ip_hl = newhl;
	canip->ah_len += inc;
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4ttl(arg)
	char **arg;
	{
	ip->ip_ttl = strtol(*arg, NULL, 0);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4tos(arg)
	char **arg;
	{
	ip->ip_tos = strtol(*arg, NULL, 0);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4id(arg)
	char **arg;
	{
	ip->ip_id = htons(strtol(*arg, NULL, 0));
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4sum(arg)
	char **arg;
	{
	ip->ip_sum = strtol(*arg, NULL, 0);
	free(*arg);
	*arg = NULL;
	}


	void set_ipv4len(arg)
	char **arg;
	{
	int len;

	len = strtol(*arg, NULL, 0);
	inc_anipheaders(len - ip->ip_len);
	ip->ip_len = len;
	free(*arg);
	*arg = NULL;
	}


	void new_tcpheader()
	{

	if ((ip->ip_p) && (ip->ip_p != IPPROTO_TCP)) {
	fprintf(stderr, "protocol %d specified with TCP!\n", ip->ip_p);
	return;
	}
	ip->ip_p = IPPROTO_TCP;

	tcp = (tcphdr_t *)new_header(IPPROTO_TCP);
	tcp->th_win = htons(4096);
	tcp->th_off = sizeof(*tcp) >> 2;
	}


	void set_tcpsport(arg)
	char **arg;
	{
	u_short *port;
	char *pr;

	if (ip->ip_p == IPPROTO_UDP) {
	port = &udp->uh_sport;
	pr = "udp";
	} else {
	port = &tcp->th_sport;
	pr = "udp";
	}

	port = getportnum(pr, arg);
	free(*arg);
	*arg = NULL;
	}


	void set_tcpdport(arg)
	char **arg;
	{
	u_short *port;
	char *pr;

	if (ip->ip_p == IPPROTO_UDP) {
	port = &udp->uh_dport;
	pr = "udp";
	} else {
	port = &tcp->th_dport;
	pr = "udp";
	}

	port = getportnum(pr, arg);
	free(*arg);
	*arg = NULL;
	}


	void set_tcpseq(arg)
	char **arg;
	{
	tcp->th_seq = htonl(strtol(*arg, NULL, 0));
	free(*arg);
	*arg = NULL;
	}


	void set_tcpack(arg)
	char **arg;
	{
	tcp->th_ack = htonl(strtol(*arg, NULL, 0));
	free(*arg);
	*arg = NULL;
	}


	void set_tcpoff(arg)
	char **arg;
	{
	int off;

	off = strtol(*arg, NULL, 0);
	inc_anipheaders((off - tcp->th_off) << 2);
	tcp->th_off = off;
	free(*arg);
	*arg = NULL;
	}


	void set_tcpurp(arg)
	char **arg;
	{
	tcp->th_urp = htons(strtol(*arg, NULL, 0));
	free(*arg);
	*arg = NULL;
	}


	void set_tcpwin(arg)
	char **arg;
	{
	tcp->th_win = htons(strtol(*arg, NULL, 0));
	free(*arg);
	*arg = NULL;
	}


	void set_tcpsum(arg)
	char **arg;
	{
	tcp->th_sum = strtol(*arg, NULL, 0);
	free(*arg);
	*arg = NULL;
	}


	void set_tcpflags(arg)
	char **arg;
	{
	static char flags[] = "ASURPF";
	static int flagv[] = { TH_ACK, TH_SYN, TH_URG, TH_RST, TH_PUSH,
	TH_FIN } ;
	char s, t;

	for (s = arg; s; s++)
	if (!(t = strchr(flags, *s))) {
	if (s - *arg) {
	fprintf(stderr, "unknown TCP flag %c\n", *s);
	break;
	}
	tcp->th_flags = strtol(*arg, NULL, 0);
	break;
	} else
	tcp->th_flags \|= flagv[t - flags];
	free(*arg);
	*arg = NULL;
	}


	void set_tcpopt(state, arg)
	int state;
	char **arg;
	{
	u_char *s;
	int val, len, val2, pad, optval;

	if (arg && *arg)
	val = atoi(*arg);
	else
	val = 0;

	s = (u_char )tcp + sizeof(tcp) + canip->ah_optlen;
	switch (state)
	{
	case IL_TCPO_EOL :
	optval = 0;
	len = 1;
	break;
	case IL_TCPO_NOP :
	optval = 1;
	len = 1;
	break;
	case IL_TCPO_MSS :
	optval = 2;
	len = 4;
	break;
	case IL_TCPO_WSCALE :
	optval = 3;
	len = 3;
	break;
	case IL_TCPO_TS :
	optval = 8;
	len = 10;
	break;
	default :
	optval = 0;
	len = 0;
	break;
	}

	if (len > 1) {
	/*
	* prepend padding - if required.
	*/
	if (len & 3)
	for (pad = 4 - (len & 3); pad; pad--) {
	*s++ = 1;
	canip->ah_optlen++;
	}
	/*
	* build tcp option
	*/
	*s++ = (u_char)optval;
	*s++ = (u_char)len;
	if (len > 2) {
	if (len == 3) { /* 1 byte - char */
	*s++ = (u_char)val;
	} else if (len == 4) { /* 2 bytes - short */
	*s++ = (u_char)((val >> 8) & 0xff);
	*s++ = (u_char)(val & 0xff);
	} else if (len >= 6) { /* 4 bytes - long */
	val2 = htonl(val);
	bcopy((char *)&val2, s, 4);
	}
	s += (len - 2);
	}
	} else
	*s++ = (u_char)optval;

	canip->ah_lastopt = optval;
	canip->ah_optlen += len;

	if (arg && *arg) {
	free(*arg);
	*arg = NULL;
	}
	}


	void end_tcpopt()
	{
	int pad;
	char s = (char )tcp;

	s += sizeof(*tcp) + canip->ah_optlen;
	/*
	* pad out so that we have a multiple of 4 bytes in size fo the
	* options. make sure last byte is EOL.
	*/
	if (canip->ah_optlen & 3) {
	if (canip->ah_lastopt != 1) {
	for (pad = 3 - (canip->ah_optlen & 3); pad; pad--) {
	*s++ = 1;
	canip->ah_optlen++;
	}
	canip->ah_optlen++;
	} else {
	s -= 1;

	for (pad = 3 - (canip->ah_optlen & 3); pad; pad--) {
	*s++ = 1;
	canip->ah_optlen++;
	}
	}
	*s++ = 0;
	}
	tcp->th_off = (sizeof(*tcp) + canip->ah_optlen) >> 2;
	inc_anipheaders(canip->ah_optlen);
	}


	void new_udpheader()
	{
	if ((ip->ip_p) && (ip->ip_p != IPPROTO_UDP)) {
	fprintf(stderr, "protocol %d specified with UDP!\n", ip->ip_p);
	return;
	}
	ip->ip_p = IPPROTO_UDP;

	udp = (udphdr_t *)new_header(IPPROTO_UDP);
	udp->uh_ulen = sizeof(*udp);
	}


	void set_udplen(arg)
	char **arg;
	{
	int len;

	len = strtol(*arg, NULL, 0);
	inc_anipheaders(len - udp->uh_ulen);
	udp->uh_ulen = len;
	free(*arg);
	*arg = NULL;
	}


	void set_udpsum(arg)
	char **arg;
	{
	udp->uh_sum = strtol(*arg, NULL, 0);
	free(*arg);
	*arg = NULL;
	}


	void prep_packet()
	{
	iface_t *ifp;
	struct in_addr gwip;

	ifp = sending.snd_if;
	if (!ifp) {
	fprintf(stderr, "no interface defined for sending!\n");
	return;
	}
	if (ifp->if_fd == -1)
	ifp->if_fd = initdevice(ifp->if_name, 5);
	gwip = sending.snd_gw;
	if (!gwip.s_addr) {
	if (aniphead == NULL) {
	fprintf(stderr,
	"no destination address defined for sending\n");
	return;
	}
	gwip = aniphead->ah_ip->ip_dst;
	}
	(void) send_ip(ifp->if_fd, ifp->if_MTU, (ip_t *)ipbuffer, gwip, 2);
	}


	void packet_done()
	{
	char outline[80];
	int i, j, k;
	u_char s = (u_char )ipbuffer, t = (u_char )outline;

	if (opts & OPT_VERBOSE) {
	ip->ip_len = htons(ip->ip_len);
	for (i = ntohs(ip->ip_len), j = 0; i; i--, j++, s++) {
	if (j && !(j & 0xf)) {
	*t++ = '\n';
	*t = '\0';
	fputs(outline, stdout);
	fflush(stdout);
	t = (u_char *)outline;
	*t = '\0';
	}
	sprintf((char )t, "%02x", s & 0xff);
	t += 2;
	if (!((j + 1) & 0xf)) {
	s -= 15;
	sprintf((char *)t, " ");
	t += 8;
	for (k = 16; k; k--, s++)
	t++ = (isprint(s) ? *s : '.');
	s--;
	}

	if ((j + 1) & 0xf)
	*t++ = ' ';;
	}

	if (j & 0xf) {
	for (k = 16 - (j & 0xf); k; k--) {
	*t++ = ' ';
	*t++ = ' ';
	*t++ = ' ';
	}
	sprintf((char *)t, " ");
	t += 7;
	s -= j & 0xf;
	for (k = j & 0xf; k; k--, s++)
	t++ = (isprint(s) ? *s : '.');
	*t++ = '\n';
	*t = '\0';
	}
	fputs(outline, stdout);
	fflush(stdout);
	ip->ip_len = ntohs(ip->ip_len);
	}

	prep_packet();
	free_aniplist();
	}


	void new_interface()
	{
	cifp = (iface_t *)calloc(1, sizeof(iface_t));
	*iftail = cifp;
	iftail = &cifp->if_next;
	cifp->if_fd = -1;
	}


	void check_interface()
	{
	if (!cifp->if_name \|\| !*cifp->if_name)
	fprintf(stderr, "No interface name given!\n");
	if (!cifp->if_MTU \|\| !*cifp->if_name)
	fprintf(stderr, "Interface %s has an MTU of 0!\n",
	cifp->if_name);
	}


	void set_ifname(arg)
	char **arg;
	{
	cifp->if_name = *arg;
	*arg = NULL;
	}


	void set_ifmtu(arg)
	int arg;
	{
	cifp->if_MTU = arg;
	}


	void set_ifv4addr(arg)
	char **arg;
	{
	cifp->if_addr = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	void set_ifeaddr(arg)
	char **arg;
	{
	(void) geteaddr(*arg, &cifp->if_eaddr);
	free(*arg);
	*arg = NULL;
	}


	void new_arp()
	{
	carp = (arp_t *)calloc(1, sizeof(arp_t));
	*arptail = carp;
	arptail = &carp->arp_next;
	}


	void set_arpeaddr(arg)
	char **arg;
	{
	(void) geteaddr(*arg, &carp->arp_eaddr);
	free(*arg);
	*arg = NULL;
	}


	void set_arpv4addr(arg)
	char **arg;
	{
	carp->arp_addr = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	int arp_getipv4(ip, addr)
	char *ip;
	char *addr;
	{
	arp_t *a;

	for (a = arplist; a; a = a->arp_next)
	if (!bcmp(ip, (char *)&a->arp_addr, 4)) {
	bcopy((char *)&a->arp_eaddr, addr, 6);
	return 0;
	}
	return -1;
	}


	void reset_send()
	{
	sending.snd_if = iflist;
	sending.snd_gw = defrouter;
	}


	void set_sendif(arg)
	char **arg;
	{
	iface_t *ifp;

	for (ifp = iflist; ifp; ifp = ifp->if_next)
	if (ifp->if_name && !strcmp(ifp->if_name, *arg))
	break;
	sending.snd_if = ifp;
	if (!ifp)
	fprintf(stderr, "couldn't find interface %s\n", *arg);
	free(*arg);
	*arg = NULL;
	}


	void set_sendvia(arg)
	char **arg;
	{
	sending.snd_gw = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	void set_defaultrouter(arg)
	char **arg;
	{
	defrouter = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	void new_icmpheader()
	{
	if ((ip->ip_p) && (ip->ip_p != IPPROTO_ICMP)) {
	fprintf(stderr, "protocol %d specified with ICMP!\n",
	ip->ip_p);
	return;
	}
	ip->ip_p = IPPROTO_ICMP;
	icmp = (icmphdr_t *)new_header(IPPROTO_ICMP);
	}


	void set_icmpcode(code)
	int code;
	{
	icmp->icmp_code = code;
	}


	void set_icmptype(type)
	int type;
	{
	icmp->icmp_type = type;
	}


	void set_icmpcodetok(code)
	char **code;
	{
	char *s;
	int i;

	for (i = 0; (s = icmpcodes[i]); i++)
	if (!strcmp(s, *code)) {
	icmp->icmp_code = i;
	break;
	}
	if (!s)
	fprintf(stderr, "unknown ICMP code %s\n", *code);
	free(*code);
	*code = NULL;
	}


	void set_icmptypetok(type)
	char **type;
	{
	char *s;
	int i, done = 0;

	for (i = 0; !(s = icmptypes[i]) \|\| strcmp(s, "END"); i++)
	if (s && !strcmp(s, *type)) {
	icmp->icmp_type = i;
	done = 1;
	break;
	}
	if (!done)
	fprintf(stderr, "unknown ICMP type %s\n", *type);
	free(*type);
	*type = NULL;
	}


	void set_icmpid(arg)
	int arg;
	{
	icmp->icmp_id = htons(arg);
	}


	void set_icmpseq(arg)
	int arg;
	{
	icmp->icmp_seq = htons(arg);
	}


	void set_icmpotime(arg)
	int arg;
	{
	icmp->icmp_otime = htonl(arg);
	}


	void set_icmprtime(arg)
	int arg;
	{
	icmp->icmp_rtime = htonl(arg);
	}


	void set_icmpttime(arg)
	int arg;
	{
	icmp->icmp_ttime = htonl(arg);
	}


	void set_icmpmtu(arg)
	int arg;
	{
	#if BSD >= 199306
	icmp->icmp_nextmtu = htons(arg);
	#endif
	}


	void set_redir(redir, arg)
	int redir;
	char **arg;
	{
	icmp->icmp_code = redir;
	icmp->icmp_gwaddr = getipv4addr(*arg);
	free(*arg);
	*arg = NULL;
	}


	void set_icmppprob(num)
	int num;
	{
	icmp->icmp_pptr = num;
	}


	void new_ipv4opt()
	{
	new_header(-2);
	}


	void add_ipopt(state, ptr)
	int state;
	void *ptr;
	{
	struct ipopt_names *io;
	struct statetoopt *sto;
	char numbuf[16], arg, *param = ptr;
	int inc, hlen;

	if (state == IL_IPO_RR \|\| state == IL_IPO_SATID) {
	if (param)
	sprintf(numbuf, "%d", (int )param);
	else
	strcpy(numbuf, "0");
	arg = numbuf;
	} else
	arg = param ? *param : NULL;

	if (canip->ah_next) {
	fprintf(stderr, "cannot specify options after data body\n");
	return;
	}
	for (sto = toipopts; sto->sto_st; sto++)
	if (sto->sto_st == state)
	break;
	if (!sto->sto_st) {
	fprintf(stderr, "No mapping for state %d to IP option\n",
	state);
	return;
	}

	hlen = sizeof(ip_t) + canip->ah_optlen;
	for (io = ionames; io->on_name; io++)
	if (io->on_value == sto->sto_op)
	break;
	canip->ah_lastopt = io->on_value;

	if (io->on_name) {
	inc = addipopt((char *)ip + hlen, io, hlen - sizeof(ip_t),arg);
	if (inc > 0) {
	while (inc & 3) {
	((char )ip)[sizeof(ip) + inc] = IPOPT_NOP;
	canip->ah_lastopt = IPOPT_NOP;
	inc++;
	}
	hlen += inc;
	}
	}

	canip->ah_optlen = hlen - sizeof(ip_t);

	if (state != IL_IPO_RR && state != IL_IPO_SATID)
	if (param && *param) {
	free(*param);
	*param = NULL;
	}
	sclass = NULL;
	}


	void end_ipopt()
	{
	int pad;
	char s, buf = (char *)ip;

	/*
	* pad out so that we have a multiple of 4 bytes in size fo the
	* options. make sure last byte is EOL.
	*/
	if (canip->ah_lastopt == IPOPT_NOP) {
	buf[sizeof(*ip) + canip->ah_optlen - 1] = IPOPT_EOL;
	} else if (canip->ah_lastopt != IPOPT_EOL) {
	s = buf + sizeof(*ip) + canip->ah_optlen;

	for (pad = 3 - (canip->ah_optlen & 3); pad; pad--) {
	*s++ = IPOPT_NOP;
	*s = IPOPT_EOL;
	canip->ah_optlen++;
	}
	canip->ah_optlen++;
	} else {
	s = buf + sizeof(*ip) + canip->ah_optlen - 1;

	for (pad = 3 - (canip->ah_optlen & 3); pad; pad--) {
	*s++ = IPOPT_NOP;
	*s = IPOPT_EOL;
	canip->ah_optlen++;
	}
	}
	ip->ip_hl = (sizeof(*ip) + canip->ah_optlen) >> 2;
	inc_anipheaders(canip->ah_optlen);
	free_anipheader();
	}


	void set_secclass(arg)
	char **arg;
	{
	sclass = *arg;
	*arg = NULL;
	}


	void free_anipheader()
	{
	aniphdr_t *aip;

	aip = canip;
	if ((canip = aip->ah_prev)) {
	canip->ah_next = NULL;
	aniptail = &canip->ah_next;
	}

	if (canip)
	free(aip);
	}


	void end_ipv4()
	{
	aniphdr_t *aip;

	ip->ip_sum = 0;
	ip->ip_len = htons(ip->ip_len);
	ip->ip_sum = chksum((u_short *)ip, ip->ip_hl << 2);
	ip->ip_len = ntohs(ip->ip_len);
	free_anipheader();
	for (aip = aniphead, ip = NULL; aip; aip = aip->ah_next)
	if (aip->ah_p == IPPROTO_IP)
	ip = aip->ah_ip;
	}


	void end_icmp()
	{
	aniphdr_t *aip;

	icmp->icmp_cksum = 0;
	icmp->icmp_cksum = chksum((u_short *)icmp, canip->ah_len);
	free_anipheader();
	for (aip = aniphead, icmp = NULL; aip; aip = aip->ah_next)
	if (aip->ah_p == IPPROTO_ICMP)
	icmp = aip->ah_icmp;
	}


	void end_udp()
	{
	u_long sum;
	aniphdr_t *aip;
	ip_t iptmp;

	bzero((char *)&iptmp, sizeof(iptmp));
	iptmp.ip_p = ip->ip_p;
	iptmp.ip_src = ip->ip_src;
	iptmp.ip_dst = ip->ip_dst;
	iptmp.ip_len = htons(ip->ip_len - (ip->ip_hl << 2));
	sum = p_chksum((u_short *)&iptmp, (u_int)sizeof(iptmp));
	udp->uh_ulen = htons(udp->uh_ulen);
	udp->uh_sum = c_chksum((u_short *)udp, (u_int)ntohs(iptmp.ip_len), sum);
	free_anipheader();
	for (aip = aniphead, udp = NULL; aip; aip = aip->ah_next)
	if (aip->ah_p == IPPROTO_UDP)
	udp = aip->ah_udp;
	}


	void end_tcp()
	{
	u_long sum;
	aniphdr_t *aip;
	ip_t iptmp;

	bzero((char *)&iptmp, sizeof(iptmp));
	iptmp.ip_p = ip->ip_p;
	iptmp.ip_src = ip->ip_src;
	iptmp.ip_dst = ip->ip_dst;
	iptmp.ip_len = htons(ip->ip_len - (ip->ip_hl << 2));
	sum = p_chksum((u_short *)&iptmp, (u_int)sizeof(iptmp));
	tcp->th_sum = 0;
	tcp->th_sum = c_chksum((u_short *)tcp, (u_int)ntohs(iptmp.ip_len), sum);
	free_anipheader();
	for (aip = aniphead, tcp = NULL; aip; aip = aip->ah_next)
	if (aip->ah_p == IPPROTO_TCP)
	tcp = aip->ah_tcp;
	}


	void end_data()
	{
	free_anipheader();
	}


	void iplang(fp)
	FILE *fp;
	{
	yyin = fp;

	yydebug = (opts & OPT_DEBUG) ? 1 : 0;

	while (!feof(fp))
	yyparse();
	}


	u_short c_chksum(buf, len, init)
	u_short *buf;
	u_int len;
	u_long init;
	{
	u_long sum = init;
	int nwords = len >> 1;

	for(; nwords > 0; nwords--)
	sum += *buf++;
	sum = (sum>>16) + (sum & 0xffff);
	sum += (sum >>16);
	return (~sum);
	}


	u_long p_chksum(buf,len)
	u_short *buf;
	u_int len;
	{
	u_long sum = 0;
	int nwords = len >> 1;

	for(; nwords > 0; nwords--)
	sum += *buf++;
	return sum;
	}
	diff --git a/contrib/ipfilter/ipsend/arp.c b/contrib/ipfilter/ipsend/arp.c
	index 05f255ea47d2..31b70d3e8987 100644
	--- a/contrib/ipfilter/ipsend/arp.c
	+++ b/contrib/ipfilter/ipsend/arp.c
	@@ -1,139 +1,135 @@
	/* $FreeBSD$ */

	/*
	* arp.c (C) 1995-1998 Darren Reed
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*/
	#if !defined(lint)
	static const char sccsid[] = "@(#)arp.c 1.4 1/11/96 (C)1995 Darren Reed";
	static const char rcsid[] = "@(#)$Id$";
	#endif
	#include <sys/types.h>
	#include <sys/socket.h>
	-#if !defined(ultrix) && !defined(hpux) && !defined(__hpux) && !defined(__osf__) && !defined(_AIX51)
	# include <sys/sockio.h>
	-#endif
	#include <sys/ioctl.h>
	#include <netinet/in_systm.h>
	#include <netinet/in.h>
	#include <net/if.h>
	#include <netinet/if_ether.h>
	-#ifndef ultrix
	# include <net/if_arp.h>
	-#endif
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <netinet/ip_var.h>
	#include <netinet/tcp.h>
	#include <stdio.h>
	#include <errno.h>
	#include <netdb.h>
	#include "ipsend.h"
	#include "iplang/iplang.h"


	/*
	* lookup host and return
	* its IP address in address
	* (4 bytes)
	*/
	int resolve(host, address)
	char host, address;
	{
	struct hostent *hp;
	u_long add;

	add = inet_addr(host);
	if (add == -1)
	{
	if (!(hp = gethostbyname(host)))
	{
	fprintf(stderr, "unknown host: %s\n", host);
	return -1;
	}
	bcopy((char )hp->h_addr, (char )address, 4);
	return 0;
	}
	bcopy((char*)&add, address, 4);
	return 0;
	}

	/*
	* ARP for the MAC address corresponding
	* to the IP address. This taken from
	* some BSD program, I cant remember which.
	*/
	int arp(ip, ether)
	char *ip;
	char *ether;
	{
	static int sfd = -1;
	static char ethersave[6], ipsave[4];
	struct arpreq ar;
	struct sockaddr_in *sin, san;
	struct hostent *hp;
	int fd;

	#ifdef IPSEND
	if (arp_getipv4(ip, ether) == 0)
	return 0;
	#endif
	if (!bcmp(ipsave, ip, 4)) {
	bcopy(ethersave, ether, 6);
	return 0;
	}
	fd = -1;
	bzero((char *)&ar, sizeof(ar));
	sin = (struct sockaddr_in *)&ar.arp_pa;
	sin->sin_family = AF_INET;
	bcopy(ip, (char *)&sin->sin_addr.s_addr, 4);
	if ((hp = gethostbyaddr(ip, 4, AF_INET)))
	# if SOLARIS && (SOLARIS2 >= 10)
	if (!(ether_hostton(hp->h_name, (struct ether_addr *)ether)))
	# else
	if (!(ether_hostton(hp->h_name, ether)))
	# endif
	goto savearp;

	if (sfd == -1)
	if ((sfd = socket(AF_INET, SOCK_DGRAM, 0)) == -1)
	{
	perror("arp: socket");
	return -1;
	}
	tryagain:
	if (ioctl(sfd, SIOCGARP, (caddr_t)&ar) == -1)
	{
	if (fd == -1)
	{
	bzero((char *)&san, sizeof(san));
	san.sin_family = AF_INET;
	san.sin_port = htons(1);
	bcopy(ip, &san.sin_addr.s_addr, 4);
	fd = socket(AF_INET, SOCK_DGRAM, 0);
	(void) sendto(fd, ip, 4, 0,
	(struct sockaddr *)&san, sizeof(san));
	sleep(1);
	(void) close(fd);
	goto tryagain;
	}
	fprintf(stderr, "(%s):", inet_ntoa(sin->sin_addr));
	if (errno != ENXIO)
	perror("SIOCGARP");
	return -1;
	}

	if ((ar.arp_ha.sa_data[0] == 0) && (ar.arp_ha.sa_data[1] == 0) &&
	(ar.arp_ha.sa_data[2] == 0) && (ar.arp_ha.sa_data[3] == 0) &&
	(ar.arp_ha.sa_data[4] == 0) && (ar.arp_ha.sa_data[5] == 0)) {
	fprintf(stderr, "(%s):", inet_ntoa(sin->sin_addr));
	return -1;
	}

	bcopy(ar.arp_ha.sa_data, ether, 6);
	savearp:
	bcopy(ether, ethersave, 6);
	bcopy(ip, ipsave, 4);
	return 0;
	}
	diff --git a/contrib/ipfilter/ipsend/ipresend.c b/contrib/ipfilter/ipsend/ipresend.c
	index ea0b4211c101..1e7b9049169a 100644
	--- a/contrib/ipfilter/ipsend/ipresend.c
	+++ b/contrib/ipfilter/ipsend/ipresend.c
	@@ -1,141 +1,133 @@
	/* $FreeBSD$ */

	/*
	* ipresend.c (C) 1995-1998 Darren Reed
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*
	*/
	#if !defined(lint)
	static const char sccsid[] = "%W% %G% (C)1995 Darren Reed";
	static const char rcsid[] = "@(#)$Id$";
	#endif
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/time.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netinet/in_systm.h>
	#include <netinet/ip.h>
	#include <netinet/ip_var.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <unistd.h>
	#include <netdb.h>
	#include <string.h>
	#include "ipsend.h"


	extern char *optarg;
	extern int optind;
	#ifndef NO_IPF
	extern struct ipread pcap, iphex, iptext;
	#endif

	int opts = 0;
	#ifndef DEFAULT_DEVICE
	# ifdef sun
	char default_device[] = "le0";
	# else
	-# ifdef ultrix
	-char default_device[] = "ln0";
	-# else
	-# ifdef __bsdi__
	-char default_device[] = "ef0";
	-# else
	char default_device[] = "lan0";
	-# endif
	-# endif
	# endif
	#else
	char default_device[] = DEFAULT_DEVICE;
	#endif


	static void usage __P((char *));
	int main __P((int, char **));


	static void usage(prog)
	char *prog;
	{
	fprintf(stderr, "Usage: %s [options] <-r filename\|-R filename>\n\
	\t\t-r filename\tsnoop data file to resend\n\
	\t\t-R filename\tlibpcap data file to resend\n\
	\toptions:\n\
	\t\t-d device\tSend out on this device\n\
	\t\t-g gateway\tIP gateway to use if non-local dest.\n\
	\t\t-m mtu\t\tfake MTU to use when sending out\n\
	", prog);
	exit(1);
	}


	int main(argc, argv)
	int argc;
	char **argv;
	{
	struct in_addr gwip;
	struct ipread *ipr = NULL;
	char name = argv[0], gateway = NULL, *dev = NULL;
	char *resend = NULL;
	int mtu = 1500, c;

	while ((c = getopt(argc, argv, "EHPRSTXd:g:m:r:")) != -1)
	switch (c)
	{
	case 'd' :
	dev = optarg;
	break;
	case 'g' :
	gateway = optarg;
	break;
	case 'm' :
	mtu = atoi(optarg);
	if (mtu < 28)
	{
	fprintf(stderr, "mtu must be > 28\n");
	exit(1);
	}
	case 'r' :
	resend = optarg;
	break;
	case 'R' :
	opts \|= OPT_RAW;
	break;
	#ifndef NO_IPF
	case 'H' :
	ipr = &iphex;
	break;
	case 'P' :
	ipr = &pcap;
	break;
	case 'X' :
	ipr = &iptext;
	break;
	#endif
	default :
	fprintf(stderr, "Unknown option \"%c\"\n", c);
	usage(name);
	}

	if (!ipr \|\| !resend)
	usage(name);

	gwip.s_addr = 0;
	if (gateway && resolve(gateway, (char *)&gwip) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", gateway);
	exit(2);
	}

	if (!dev)
	dev = default_device;

	printf("Device: %s\n", dev);
	printf("Gateway: %s\n", inet_ntoa(gwip));
	printf("mtu: %d\n", mtu);

	return ip_resend(dev, mtu, ipr, gwip, resend);
	}
	diff --git a/contrib/ipfilter/ipsend/ipsend.c b/contrib/ipfilter/ipsend/ipsend.c
	index 95a1bb1e5c78..b7617734f66e 100644
	--- a/contrib/ipfilter/ipsend/ipsend.c
	+++ b/contrib/ipfilter/ipsend/ipsend.c
	@@ -1,424 +1,416 @@
	/* $FreeBSD$ */
	/*
	* ipsend.c (C) 1995-1998 Darren Reed
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*/
	#if !defined(lint)
	static const char sccsid[] = "@(#)ipsend.c 1.5 12/10/95 (C)1995 Darren Reed";
	static const char rcsid[] = "@(#)$Id$";
	#endif
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/time.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netinet/in_systm.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <unistd.h>
	#include <netdb.h>
	#include <string.h>
	#include <netinet/ip.h>
	# include <netinet/ip_var.h>
	#include "ipsend.h"
	#include "ipf.h"
	# include <netinet/udp_var.h>


	extern char *optarg;
	extern int optind;
	extern void iplang __P((FILE *));

	char options[68];
	int opts;
	-# ifdef ultrix
	-char default_device[] = "ln0";
	-# else
	-# ifdef __bsdi__
	-char default_device[] = "ef0";
	-# else
	char default_device[] = "le0";
	-# endif /* __bsdi__ */
	-# endif /* ultrix */


	static void usage __P((char *));
	static void do_icmp __P((ip_t , char ));
	void udpcksum(ip_t , struct udphdr , int);
	int main __P((int, char **));


	static void usage(prog)
	char *prog;
	{
	fprintf(stderr, "Usage: %s [options] dest [flags]\n\
	\toptions:\n\
	\t\t-d\tdebug mode\n\
	\t\t-i device\tSend out on this device\n\
	\t\t-f fragflags\tcan set IP_MF or IP_DF\n\
	\t\t-g gateway\tIP gateway to use if non-local dest.\n\
	\t\t-I code,type[,gw[,dst[,src]]]\tSet ICMP protocol\n\
	\t\t-m mtu\t\tfake MTU to use when sending out\n\
	\t\t-P protocol\tSet protocol by name\n\
	\t\t-s src\t\tsource address for IP packet\n\
	\t\t-T\t\tSet TCP protocol\n\
	\t\t-t port\t\tdestination port\n\
	\t\t-U\t\tSet UDP protocol\n\
	\t\t-v\tverbose mode\n\
	\t\t-w <window>\tSet the TCP window size\n\
	", prog);
	fprintf(stderr, "Usage: %s [-dv] -L <filename>\n\
	\toptions:\n\
	\t\t-d\tdebug mode\n\
	\t\t-L filename\tUse IP language for sending packets\n\
	\t\t-v\tverbose mode\n\
	", prog);
	exit(1);
	}


	static void do_icmp(ip, args)
	ip_t *ip;
	char *args;
	{
	struct icmp *ic;
	char *s;

	ip->ip_p = IPPROTO_ICMP;
	ip->ip_len += sizeof(*ic);
	ic = (struct icmp *)(ip + 1);
	bzero((char )ic, sizeof(ic));
	if (!(s = strchr(args, ',')))
	{
	fprintf(stderr, "ICMP args missing: ,\n");
	return;
	}
	*s++ = '\0';
	ic->icmp_type = atoi(args);
	ic->icmp_code = atoi(s);
	if (ic->icmp_type == ICMP_REDIRECT && strchr(s, ','))
	{
	char *t;

	t = strtok(s, ",");
	t = strtok(NULL, ",");
	if (resolve(t, (char *)&ic->icmp_gwaddr) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", t);
	exit(2);
	}
	if ((t = strtok(NULL, ",")))
	{
	if (resolve(t, (char *)&ic->icmp_ip.ip_dst) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", t);
	exit(2);
	}
	if ((t = strtok(NULL, ",")))
	{
	if (resolve(t,
	(char *)&ic->icmp_ip.ip_src) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", t);
	exit(2);
	}
	}
	}
	}
	}


	int send_packets(dev, mtu, ip, gwip)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	{
	int wfd;

	wfd = initdevice(dev, 5);
	if (wfd == -1)
	return -1;
	return send_packet(wfd, mtu, ip, gwip);
	}

	void
	udpcksum(ip_t ip, struct udphdr udp, int len)
	{
	union pseudoh {
	struct hdr {
	u_short len;
	u_char ttl;
	u_char proto;
	u_32_t src;
	u_32_t dst;
	} h;
	u_short w[6];
	} ph;
	u_32_t temp32;
	u_short *opts;

	ph.h.len = htons(len);
	ph.h.ttl = 0;
	ph.h.proto = IPPROTO_UDP;
	ph.h.src = ip->ip_src.s_addr;
	ph.h.dst = ip->ip_dst.s_addr;
	temp32 = 0;
	opts = &ph.w[0];
	temp32 += opts[0] + opts[1] + opts[2] + opts[3] + opts[4] + opts[5];
	temp32 = (temp32 >> 16) + (temp32 & 65535);
	temp32 += (temp32 >> 16);
	udp->uh_sum = temp32 & 65535;
	udp->uh_sum = chksum((u_short *)udp, len);
	if (udp->uh_sum == 0)
	udp->uh_sum = 0xffff;
	}

	int main(argc, argv)
	int argc;
	char **argv;
	{
	FILE *langfile = NULL;
	struct in_addr gwip;
	tcphdr_t *tcp;
	udphdr_t *udp;
	ip_t *ip;
	char *name = argv[0], host[MAXHOSTNAMELEN + 1];
	char gateway = NULL, dev = NULL;
	char src = NULL, dst, *s;
	int mtu = 1500, olen = 0, c, nonl = 0;

	/*
	* 65535 is maximum packet size...you never know...
	*/
	ip = (ip_t *)calloc(1, 65536);
	tcp = (tcphdr_t *)(ip + 1);
	udp = (udphdr_t *)tcp;
	ip->ip_len = sizeof(*ip);
	IP_HL_A(ip, sizeof(*ip) >> 2);

	while ((c = getopt(argc, argv, "I:L:P:TUdf:i:g:m:o:s:t:vw:")) != -1) {
	switch (c)
	{
	case 'I' :
	nonl++;
	if (ip->ip_p)
	{
	fprintf(stderr, "Protocol already set: %d\n",
	ip->ip_p);
	break;
	}
	do_icmp(ip, optarg);
	break;
	case 'L' :
	if (nonl) {
	fprintf(stderr,
	"Incorrect usage of -L option.\n");
	usage(name);
	}
	if (!strcmp(optarg, "-"))
	langfile = stdin;
	else if (!(langfile = fopen(optarg, "r"))) {
	fprintf(stderr, "can't open file %s\n",
	optarg);
	exit(1);
	}
	iplang(langfile);
	return 0;
	case 'P' :
	{
	struct protoent *p;

	nonl++;
	if (ip->ip_p)
	{
	fprintf(stderr, "Protocol already set: %d\n",
	ip->ip_p);
	break;
	}
	if ((p = getprotobyname(optarg)))
	ip->ip_p = p->p_proto;
	else
	fprintf(stderr, "Unknown protocol: %s\n",
	optarg);
	break;
	}
	case 'T' :
	nonl++;
	if (ip->ip_p)
	{
	fprintf(stderr, "Protocol already set: %d\n",
	ip->ip_p);
	break;
	}
	ip->ip_p = IPPROTO_TCP;
	ip->ip_len += sizeof(tcphdr_t);
	break;
	case 'U' :
	nonl++;
	if (ip->ip_p)
	{
	fprintf(stderr, "Protocol already set: %d\n",
	ip->ip_p);
	break;
	}
	ip->ip_p = IPPROTO_UDP;
	ip->ip_len += sizeof(udphdr_t);
	break;
	case 'd' :
	opts \|= OPT_DEBUG;
	break;
	case 'f' :
	nonl++;
	ip->ip_off = strtol(optarg, NULL, 0);
	break;
	case 'g' :
	nonl++;
	gateway = optarg;
	break;
	case 'i' :
	nonl++;
	dev = optarg;
	break;
	case 'm' :
	nonl++;
	mtu = atoi(optarg);
	if (mtu < 28)
	{
	fprintf(stderr, "mtu must be > 28\n");
	exit(1);
	}
	break;
	case 'o' :
	nonl++;
	olen = buildopts(optarg, options, (IP_HL(ip) - 5) << 2);
	break;
	case 's' :
	nonl++;
	src = optarg;
	break;
	case 't' :
	nonl++;
	if (ip->ip_p == IPPROTO_TCP \|\| ip->ip_p == IPPROTO_UDP)
	tcp->th_dport = htons(atoi(optarg));
	break;
	case 'v' :
	opts \|= OPT_VERBOSE;
	break;
	case 'w' :
	nonl++;
	if (ip->ip_p == IPPROTO_TCP)
	tcp->th_win = atoi(optarg);
	else
	fprintf(stderr, "set protocol to TCP first\n");
	break;
	default :
	fprintf(stderr, "Unknown option \"%c\"\n", c);
	usage(name);
	}
	}

	if (argc - optind < 1)
	usage(name);
	dst = argv[optind++];

	if (!src)
	{
	gethostname(host, sizeof(host));
	src = host;
	}

	if (resolve(src, (char *)&ip->ip_src) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", src);
	exit(2);
	}

	if (resolve(dst, (char *)&ip->ip_dst) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", dst);
	exit(2);
	}

	if (!gateway)
	gwip = ip->ip_dst;
	else if (resolve(gateway, (char *)&gwip) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", gateway);
	exit(2);
	}

	if (olen)
	{
	int hlen;
	char *p;

	printf("Options: %d\n", olen);
	hlen = sizeof(*ip) + olen;
	IP_HL_A(ip, hlen >> 2);
	ip->ip_len += olen;
	p = (char *)malloc(65536);
	if (p == NULL)
	{
	fprintf(stderr, "malloc failed\n");
	exit(2);
	}

	bcopy(ip, p, sizeof(*ip));
	bcopy(options, p + sizeof(*ip), olen);
	bcopy(ip + 1, p + hlen, ip->ip_len - hlen);
	ip = (ip_t *)p;

	if (ip->ip_p == IPPROTO_TCP) {
	tcp = (tcphdr_t *)(p + hlen);
	} else if (ip->ip_p == IPPROTO_UDP) {
	udp = (udphdr_t *)(p + hlen);
	}
	}

	if (ip->ip_p == IPPROTO_TCP)
	for (s = argv[optind]; s && (c = *s); s++)
	switch(c)
	{
	case 'S' : case 's' :
	tcp->th_flags \|= TH_SYN;
	break;
	case 'A' : case 'a' :
	tcp->th_flags \|= TH_ACK;
	break;
	case 'F' : case 'f' :
	tcp->th_flags \|= TH_FIN;
	break;
	case 'R' : case 'r' :
	tcp->th_flags \|= TH_RST;
	break;
	case 'P' : case 'p' :
	tcp->th_flags \|= TH_PUSH;
	break;
	case 'U' : case 'u' :
	tcp->th_flags \|= TH_URG;
	break;
	}

	if (!dev)
	dev = default_device;
	printf("Device: %s\n", dev);
	printf("Source: %s\n", inet_ntoa(ip->ip_src));
	printf("Dest: %s\n", inet_ntoa(ip->ip_dst));
	printf("Gateway: %s\n", inet_ntoa(gwip));
	if (ip->ip_p == IPPROTO_TCP && tcp->th_flags)
	printf("Flags: %#x\n", tcp->th_flags);
	printf("mtu: %d\n", mtu);

	if (ip->ip_p == IPPROTO_UDP) {
	udp->uh_sum = 0;
	udpcksum(ip, udp, ip->ip_len - (IP_HL(ip) << 2));
	}
	#ifdef DOSOCKET
	if (ip->ip_p == IPPROTO_TCP && tcp->th_dport)
	return do_socket(dev, mtu, ip, gwip);
	#endif
	return send_packets(dev, mtu, ip, gwip);
	}
	diff --git a/contrib/ipfilter/ipsend/iptest.c b/contrib/ipfilter/ipsend/iptest.c
	index bc93106c8b89..ffabaf37b5bc 100644
	--- a/contrib/ipfilter/ipsend/iptest.c
	+++ b/contrib/ipfilter/ipsend/iptest.c
	@@ -1,205 +1,197 @@
	/* $FreeBSD$ */

	/*
	* ipsend.c (C) 1995-1998 Darren Reed
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*
	*/
	#if !defined(lint)
	static const char sccsid[] = "%W% %G% (C)1995 Darren Reed";
	static const char rcsid[] = "@(#)$Id$";
	#endif
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/time.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netinet/in_systm.h>
	#include <netinet/ip.h>
	#include <netinet/ip_var.h>
	#include <stdio.h>
	#include <netdb.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <string.h>
	#include "ipsend.h"


	extern char *optarg;
	extern int optind;

	char options[68];
	# ifdef sun
	char default_device[] = "le0";
	# else
	-# ifdef ultrix
	-char default_device[] = "ln0";
	-# else
	-# ifdef __bsdi__
	-char default_device[] = "ef0";
	-# else
	char default_device[] = "lan0";
	-# endif
	-# endif
	# endif

	static void usage __P((char *));
	int main __P((int, char **));


	static void usage(prog)
	char *prog;
	{
	fprintf(stderr, "Usage: %s [options] dest\n\
	\toptions:\n\
	\t\t-d device\tSend out on this device\n\
	\t\t-g gateway\tIP gateway to use if non-local dest.\n\
	\t\t-m mtu\t\tfake MTU to use when sending out\n\
	\t\t-p pointtest\t\n\
	\t\t-s src\t\tsource address for IP packet\n\
	\t\t-1 \t\tPerform test 1 (IP header)\n\
	\t\t-2 \t\tPerform test 2 (IP options)\n\
	\t\t-3 \t\tPerform test 3 (ICMP)\n\
	\t\t-4 \t\tPerform test 4 (UDP)\n\
	\t\t-5 \t\tPerform test 5 (TCP)\n\
	\t\t-6 \t\tPerform test 6 (overlapping fragments)\n\
	\t\t-7 \t\tPerform test 7 (random packets)\n\
	", prog);
	exit(1);
	}


	int main(argc, argv)
	int argc;
	char **argv;
	{
	struct tcpiphdr *ti;
	struct in_addr gwip;
	ip_t *ip;
	char *name = argv[0], host[MAXHOSTNAMELEN + 1];
	char gateway = NULL, dev = NULL;
	char src = NULL, dst;
	int mtu = 1500, tests = 0, pointtest = 0, c;

	/*
	* 65535 is maximum packet size...you never know...
	*/
	ip = (ip_t *)calloc(1, 65536);
	ti = (struct tcpiphdr *)ip;
	ip->ip_len = sizeof(*ip);
	IP_HL_A(ip, sizeof(*ip) >> 2);

	while ((c = getopt(argc, argv, "1234567d:g:m:p:s:")) != -1)
	switch (c)
	{
	case '1' :
	case '2' :
	case '3' :
	case '4' :
	case '5' :
	case '6' :
	case '7' :
	tests = c - '0';
	break;
	case 'd' :
	dev = optarg;
	break;
	case 'g' :
	gateway = optarg;
	break;
	case 'm' :
	mtu = atoi(optarg);
	if (mtu < 28)
	{
	fprintf(stderr, "mtu must be > 28\n");
	exit(1);
	}
	break;
	case 'p' :
	pointtest = atoi(optarg);
	break;
	case 's' :
	src = optarg;
	break;
	default :
	fprintf(stderr, "Unknown option \"%c\"\n", c);
	usage(name);
	}

	if ((argc <= optind) \|\| !argv[optind])
	usage(name);
	dst = argv[optind++];

	if (!src)
	{
	gethostname(host, sizeof(host));
	host[sizeof(host) - 1] = '\0';
	src = host;
	}

	if (resolve(dst, (char *)&ip->ip_dst) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", dst);
	exit(2);
	}

	if (resolve(src, (char *)&ip->ip_src) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", src);
	exit(2);
	}

	if (!gateway)
	gwip = ip->ip_dst;
	else if (resolve(gateway, (char *)&gwip) == -1)
	{
	fprintf(stderr,"Cant resolve %s\n", gateway);
	exit(2);
	}


	if (!dev)
	dev = default_device;
	printf("Device: %s\n", dev);
	printf("Source: %s\n", inet_ntoa(ip->ip_src));
	printf("Dest: %s\n", inet_ntoa(ip->ip_dst));
	printf("Gateway: %s\n", inet_ntoa(gwip));
	printf("mtu: %d\n", mtu);

	switch (tests)
	{
	case 1 :
	ip_test1(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	case 2 :
	ip_test2(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	case 3 :
	ip_test3(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	case 4 :
	ip_test4(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	case 5 :
	ip_test5(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	case 6 :
	ip_test6(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	case 7 :
	ip_test7(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	default :
	ip_test1(dev, mtu, (ip_t *)ti, gwip, pointtest);
	ip_test2(dev, mtu, (ip_t *)ti, gwip, pointtest);
	ip_test3(dev, mtu, (ip_t *)ti, gwip, pointtest);
	ip_test4(dev, mtu, (ip_t *)ti, gwip, pointtest);
	ip_test5(dev, mtu, (ip_t *)ti, gwip, pointtest);
	ip_test6(dev, mtu, (ip_t *)ti, gwip, pointtest);
	ip_test7(dev, mtu, (ip_t *)ti, gwip, pointtest);
	break;
	}
	return 0;
	}
	diff --git a/contrib/ipfilter/ipsend/iptests.c b/contrib/ipfilter/ipsend/iptests.c
	index af8772cc2097..86b850d319bb 100644
	--- a/contrib/ipfilter/ipsend/iptests.c
	+++ b/contrib/ipfilter/ipsend/iptests.c
	@@ -1,1404 +1,1397 @@
	/* $FreeBSD$ */

	/*
	* Copyright (C) 2012 by Darren Reed.
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*
	*/
	#if !defined(lint)
	static const char sccsid[] = "%W% %G% (C)1995 Darren Reed";
	static const char rcsid[] = "@(#)$Id$";
	#endif
	#include <sys/param.h>
	#include <sys/types.h>
	#if defined(__NetBSD__) && defined(__vax__)
	/*
	* XXX need to declare boolean_t for _KERNEL <sys/files.h>
	* which ends up including <sys/device.h> for vax. See PR#32907
	* for further details.
	*/
	typedef int boolean_t;
	#endif
	#include <sys/time.h>
	# ifdef __NetBSD__
	# include <machine/lock.h>
	# include <machine/mutex.h>
	# endif
	# define _KERNEL
	# define KERNEL
	# if !defined(solaris) && !defined(linux) && !defined(__sgi) && !defined(hpux)
	# include <sys/file.h>
	# else
	# ifdef solaris
	# include <sys/dditypes.h>
	# endif
	# endif
	# undef _KERNEL
	# undef KERNEL
	-#if !defined(solaris) && !defined(linux) && !defined(__sgi)
	+#if !defined(solaris)
	# include <nlist.h>
	# include <sys/user.h>
	# include <sys/proc.h>
	#endif
	-#if !defined(ultrix) && !defined(hpux) && !defined(linux) && \
	- !defined(__sgi) && !defined(__osf__) && !defined(_AIX51)
	# include <kvm.h>
	-#endif
	-#ifndef ultrix
	# include <sys/socket.h>
	-#endif
	#if defined(solaris)
	# include <sys/stream.h>
	#else
	# include <sys/socketvar.h>
	#endif
	#ifdef sun
	#include <sys/systm.h>
	#include <sys/session.h>
	#endif
	-#if BSD >= 199103
	# include <sys/sysctl.h>
	# include <sys/filedesc.h>
	# include <paths.h>
	-#endif
	#include <netinet/in_systm.h>
	#include <sys/socket.h>
	#include <net/if.h>
	# if defined(__FreeBSD__)
	# include "radix_ipf.h"
	# endif
	# if !defined(solaris)
	# include <net/route.h>
	# endif
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netinet/ip.h>
	-#if defined(__SVR4) \|\| defined(__svr4__) \|\| defined(__sgi)
	+#if defined(__SVR4) \|\| defined(__svr4__)
	# include <sys/sysmacros.h>
	#endif
	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <string.h>
	# include <netinet/ip_var.h>
	-# if !defined(__hpux) && !defined(solaris)
	+# if !defined(solaris)
	# include <netinet/in_pcb.h>
	# endif
	#include "ipsend.h"
	# include <netinet/tcp_timer.h>
	# include <netinet/tcp_var.h>
	#if defined(__NetBSD_Version__) && (__NetBSD_Version__ >= 106000000)
	# define USE_NANOSLEEP
	#endif


	#ifdef USE_NANOSLEEP
	# define PAUSE() ts.tv_sec = 0; ts.tv_nsec = 10000000; \
	(void) nanosleep(&ts, NULL)
	#else
	# define PAUSE() tv.tv_sec = 0; tv.tv_usec = 10000; \
	(void) select(0, NULL, NULL, NULL, &tv)
	#endif


	void ip_test1(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	udphdr_t *u;
	int nfd, i = 0, len, id = getpid();

	IP_HL_A(ip, sizeof(*ip) >> 2);
	IP_V_A(ip, IPVERSION);
	ip->ip_tos = 0;
	ip->ip_off = 0;
	ip->ip_ttl = 60;
	ip->ip_p = IPPROTO_UDP;
	ip->ip_sum = 0;
	u = (udphdr_t *)(ip + 1);
	u->uh_sport = htons(1);
	u->uh_dport = htons(9);
	u->uh_sum = 0;
	u->uh_ulen = htons(sizeof(*u) + 4);
	ip->ip_len = sizeof(*ip) + ntohs(u->uh_ulen);
	len = ip->ip_len;

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	if (!ptest \|\| (ptest == 1)) {
	/*
	* Part1: hl < len
	*/
	ip->ip_id = 0;
	printf("1.1. sending packets with ip_hl < ip_len\n");
	for (i = 0; i < ((sizeof(*ip) + ntohs(u->uh_ulen)) >> 2); i++) {
	IP_HL_A(ip, i >> 2);
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 2)) {
	/*
	* Part2: hl > len
	*/
	ip->ip_id = 0;
	printf("1.2. sending packets with ip_hl > ip_len\n");
	for (; i < ((sizeof(ip) 2 + ntohs(u->uh_ulen)) >> 2); i++) {
	IP_HL_A(ip, i >> 2);
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 3)) {
	/*
	* Part3: v < 4
	*/
	ip->ip_id = 0;
	printf("1.3. ip_v < 4\n");
	IP_HL_A(ip, sizeof(*ip) >> 2);
	for (i = 0; i < 4; i++) {
	IP_V_A(ip, i);
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 4)) {
	/*
	* Part4: v > 4
	*/
	ip->ip_id = 0;
	printf("1.4. ip_v > 4\n");
	for (i = 5; i < 16; i++) {
	IP_V_A(ip, i);
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 5)) {
	/*
	* Part5: len < packet
	*/
	ip->ip_id = 0;
	IP_V_A(ip, IPVERSION);
	i = ip->ip_len + 1;
	printf("1.5.0 ip_len < packet size (size++, long packets)\n");
	for (; i < (ip->ip_len * 2); i++) {
	ip->ip_id = htons(id++);
	ip->ip_sum = 0;
	ip->ip_sum = chksum((u_short *)ip, IP_HL(ip) << 2);
	(void) send_ether(nfd, (char *)ip, i, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	printf("1.5.1 ip_len < packet size (ip_len-, short packets)\n");
	for (i = len; i > 0; i--) {
	ip->ip_id = htons(id++);
	ip->ip_len = i;
	ip->ip_sum = 0;
	ip->ip_sum = chksum((u_short *)ip, IP_HL(ip) << 2);
	(void) send_ether(nfd, (char *)ip, len, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 6)) {
	/*
	* Part6: len > packet
	*/
	ip->ip_id = 0;
	printf("1.6.0 ip_len > packet size (increase ip_len)\n");
	for (i = len + 1; i < (len * 2); i++) {
	ip->ip_id = htons(id++);
	ip->ip_len = i;
	ip->ip_sum = 0;
	ip->ip_sum = chksum((u_short *)ip, IP_HL(ip) << 2);
	(void) send_ether(nfd, (char *)ip, len, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	ip->ip_len = len;
	printf("1.6.1 ip_len > packet size (size--, short packets)\n");
	for (i = len; i > 0; i--) {
	ip->ip_id = htons(id++);
	ip->ip_sum = 0;
	ip->ip_sum = chksum((u_short *)ip, IP_HL(ip) << 2);
	(void) send_ether(nfd, (char *)ip, i, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 7)) {
	/*
	* Part7: 0 length fragment
	*/
	printf("1.7.0 Zero length fragments (ip_off = 0x2000)\n");
	ip->ip_id = 0;
	ip->ip_len = sizeof(*ip);
	ip->ip_off = htons(IP_MF);
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("1.7.1 Zero length fragments (ip_off = 0x3000)\n");
	ip->ip_id = 0;
	ip->ip_len = sizeof(*ip);
	ip->ip_off = htons(IP_MF);
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("1.7.2 Zero length fragments (ip_off = 0xa000)\n");
	ip->ip_id = 0;
	ip->ip_len = sizeof(*ip);
	ip->ip_off = htons(0xa000);
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("1.7.3 Zero length fragments (ip_off = 0x0100)\n");
	ip->ip_id = 0;
	ip->ip_len = sizeof(*ip);
	ip->ip_off = htons(0x0100);
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();
	}

	if (!ptest \|\| (ptest == 8)) {
	struct timeval tv;

	gettimeofday(&tv, NULL);
	srand(tv.tv_sec ^ getpid() ^ tv.tv_usec);
	/*
	* Part8.1: 63k packet + 1k fragment at offset 0x1ffe
	* Mark it as being ICMP (so it doesn't get junked), but
	* don't bother about the ICMP header, we're not worrying
	* about that here.
	*/
	ip->ip_p = IPPROTO_ICMP;
	ip->ip_off = htons(IP_MF);
	u->uh_dport = htons(9);
	ip->ip_id = htons(id++);
	printf("1.8.1 63k packet + 1k fragment at offset 0x1ffe\n");
	ip->ip_len = 768 + 20 + 8;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);

	ip->ip_len = MIN(768 + 20, mtu - 68);
	i = 512;
	for (; i < (63 * 1024 + 768); i += 768) {
	ip->ip_off = htons(IP_MF \| (i >> 3));
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	ip->ip_len = 896 + 20;
	ip->ip_off = htons(i >> 3);
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	putchar('\n');
	fflush(stdout);

	/*
	* Part8.2: 63k packet + 1k fragment at offset 0x1ffe
	* Mark it as being ICMP (so it doesn't get junked), but
	* don't bother about the ICMP header, we're not worrying
	* about that here. (Lossage here)
	*/
	ip->ip_p = IPPROTO_ICMP;
	ip->ip_off = htons(IP_MF);
	u->uh_dport = htons(9);
	ip->ip_id = htons(id++);
	printf("1.8.2 63k packet + 1k fragment at offset 0x1ffe\n");
	ip->ip_len = 768 + 20 + 8;
	if ((rand() & 0x1f) != 0) {
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	} else
	printf("skip 0\n");

	ip->ip_len = MIN(768 + 20, mtu - 68);
	i = 512;
	for (; i < (63 * 1024 + 768); i += 768) {
	ip->ip_off = htons(IP_MF \| (i >> 3));
	if ((rand() & 0x1f) != 0) {
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	} else
	printf("skip %d\n", i);
	fflush(stdout);
	PAUSE();
	}
	ip->ip_len = 896 + 20;
	ip->ip_off = htons(i >> 3);
	if ((rand() & 0x1f) != 0) {
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	} else
	printf("skip\n");
	putchar('\n');
	fflush(stdout);

	/*
	* Part8.3: 33k packet - test for not dealing with -ve length
	* Mark it as being ICMP (so it doesn't get junked), but
	* don't bother about the ICMP header, we're not worrying
	* about that here.
	*/
	ip->ip_p = IPPROTO_ICMP;
	ip->ip_off = htons(IP_MF);
	u->uh_dport = htons(9);
	ip->ip_id = htons(id++);
	printf("1.8.3 33k packet\n");
	ip->ip_len = 768 + 20 + 8;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);

	ip->ip_len = MIN(768 + 20, mtu - 68);
	i = 512;
	for (; i < (32 * 1024 + 768); i += 768) {
	ip->ip_off = htons(IP_MF \| (i >> 3));
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	ip->ip_len = 896 + 20;
	ip->ip_off = htons(i >> 3);
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	printf("%d\r", i);
	putchar('\n');
	fflush(stdout);
	}

	ip->ip_len = len;
	ip->ip_off = 0;
	if (!ptest \|\| (ptest == 9)) {
	/*
	* Part9: off & 0x8000 == 0x8000
	*/
	ip->ip_id = 0;
	ip->ip_off = htons(0x8000);
	printf("1.9. ip_off & 0x8000 == 0x8000\n");
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();
	}

	ip->ip_off = 0;

	if (!ptest \|\| (ptest == 10)) {
	/*
	* Part10: ttl = 255
	*/
	ip->ip_id = 0;
	ip->ip_ttl = 255;
	printf("1.10.0 ip_ttl = 255\n");
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	ip->ip_ttl = 128;
	printf("1.10.1 ip_ttl = 128\n");
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	ip->ip_ttl = 0;
	printf("1.10.2 ip_ttl = 0\n");
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();
	}

	(void) close(nfd);
	}


	void ip_test2(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	int nfd;
	u_char *s;


	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	IP_HL_A(ip, 6);
	ip->ip_len = IP_HL(ip) << 2;
	s = (u_char *)(ip + 1);
	s[IPOPT_OPTVAL] = IPOPT_NOP;
	s++;
	if (!ptest \|\| (ptest == 1)) {
	/*
	* Test 1: option length > packet length,
	* header length == packet length
	*/
	s[IPOPT_OPTVAL] = IPOPT_TS;
	s[IPOPT_OLEN] = 4;
	s[IPOPT_OFFSET] = IPOPT_MINOFF;
	ip->ip_p = IPPROTO_IP;
	printf("2.1 option length > packet length\n");
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();
	}

	IP_HL_A(ip, 7);
	ip->ip_len = IP_HL(ip) << 2;
	if (!ptest \|\| (ptest == 1)) {
	/*
	* Test 2: options have length = 0
	*/
	printf("2.2.1 option length = 0, RR\n");
	s[IPOPT_OPTVAL] = IPOPT_RR;
	s[IPOPT_OLEN] = 0;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("2.2.2 option length = 0, TS\n");
	s[IPOPT_OPTVAL] = IPOPT_TS;
	s[IPOPT_OLEN] = 0;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("2.2.3 option length = 0, SECURITY\n");
	s[IPOPT_OPTVAL] = IPOPT_SECURITY;
	s[IPOPT_OLEN] = 0;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("2.2.4 option length = 0, LSRR\n");
	s[IPOPT_OPTVAL] = IPOPT_LSRR;
	s[IPOPT_OLEN] = 0;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("2.2.5 option length = 0, SATID\n");
	s[IPOPT_OPTVAL] = IPOPT_SATID;
	s[IPOPT_OLEN] = 0;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();

	printf("2.2.6 option length = 0, SSRR\n");
	s[IPOPT_OPTVAL] = IPOPT_SSRR;
	s[IPOPT_OLEN] = 0;
	(void) send_ip(nfd, mtu, ip, gwip, 1);
	fflush(stdout);
	PAUSE();
	}

	(void) close(nfd);
	}


	/*
	* test 3 (ICMP)
	*/
	void ip_test3(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	static int ict1[10] = { 8, 9, 10, 13, 14, 15, 16, 17, 18, 0 };
	static int ict2[8] = { 3, 9, 10, 13, 14, 17, 18, 0 };
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	struct icmp *icp;
	int nfd, i;

	IP_HL_A(ip, sizeof(*ip) >> 2);
	IP_V_A(ip, IPVERSION);
	ip->ip_tos = 0;
	ip->ip_off = 0;
	ip->ip_ttl = 60;
	ip->ip_p = IPPROTO_ICMP;
	ip->ip_sum = 0;
	ip->ip_len = sizeof(ip) + sizeof(icp);
	icp = (struct icmp )((char )ip + (IP_HL(ip) << 2));

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	if (!ptest \|\| (ptest == 1)) {
	/*
	* Type 0 - 31, 255, code = 0
	*/
	bzero((char )icp, sizeof(icp));
	for (i = 0; i < 32; i++) {
	icp->icmp_type = i;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.1.%d ICMP type %d code 0 (all 0's)\r", i, i);
	}
	icp->icmp_type = 255;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.1.%d ICMP type %d code 0 (all 0's)\r", i, 255);
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 2)) {
	/*
	* Type 3, code = 0 - 31
	*/
	icp->icmp_type = 3;
	for (i = 0; i < 32; i++) {
	icp->icmp_code = i;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.2.%d ICMP type 3 code %d (all 0's)\r", i, i);
	}
	}

	if (!ptest \|\| (ptest == 3)) {
	/*
	* Type 4, code = 0,127,128,255
	*/
	icp->icmp_type = 4;
	icp->icmp_code = 0;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.3.1 ICMP type 4 code 0 (all 0's)\r");
	icp->icmp_code = 127;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.3.2 ICMP type 4 code 127 (all 0's)\r");
	icp->icmp_code = 128;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.3.3 ICMP type 4 code 128 (all 0's)\r");
	icp->icmp_code = 255;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.3.4 ICMP type 4 code 255 (all 0's)\r");
	}

	if (!ptest \|\| (ptest == 4)) {
	/*
	* Type 5, code = 0,127,128,255
	*/
	icp->icmp_type = 5;
	icp->icmp_code = 0;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.4.1 ICMP type 5 code 0 (all 0's)\r");
	icp->icmp_code = 127;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.4.2 ICMP type 5 code 127 (all 0's)\r");
	icp->icmp_code = 128;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.4.3 ICMP type 5 code 128 (all 0's)\r");
	icp->icmp_code = 255;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.4.4 ICMP type 5 code 255 (all 0's)\r");
	}

	if (!ptest \|\| (ptest == 5)) {
	/*
	* Type 8-10;13-18, code - 0,127,128,255
	*/
	for (i = 0; ict1[i]; i++) {
	icp->icmp_type = ict1[i];
	icp->icmp_code = 0;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type 5 code 0 (all 0's)\r",
	i * 4);
	icp->icmp_code = 127;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type 5 code 127 (all 0's)\r",
	i * 4 + 1);
	icp->icmp_code = 128;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type 5 code 128 (all 0's)\r",
	i * 4 + 2);
	icp->icmp_code = 255;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type 5 code 255 (all 0's)\r",
	i * 4 + 3);
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 6)) {
	/*
	* Type 12, code - 0,127,128,129,255
	*/
	icp->icmp_type = 12;
	icp->icmp_code = 0;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.6.1 ICMP type 12 code 0 (all 0's)\r");
	icp->icmp_code = 127;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.6.2 ICMP type 12 code 127 (all 0's)\r");
	icp->icmp_code = 128;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.6.3 ICMP type 12 code 128 (all 0's)\r");
	icp->icmp_code = 129;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.6.4 ICMP type 12 code 129 (all 0's)\r");
	icp->icmp_code = 255;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.6.5 ICMP type 12 code 255 (all 0's)\r");
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 7)) {
	/*
	* Type 3;9-10;13-14;17-18 - shorter packets
	*/
	ip->ip_len = sizeof(ip) + sizeof(icp) / 2;
	for (i = 0; ict2[i]; i++) {
	icp->icmp_type = ict1[i];
	icp->icmp_code = 0;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type %d code 0 (all 0's)\r",
	i * 4, icp->icmp_type);
	icp->icmp_code = 127;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type %d code 127 (all 0's)\r",
	i * 4 + 1, icp->icmp_type);
	icp->icmp_code = 128;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type %d code 128 (all 0's)\r",
	i * 4 + 2, icp->icmp_type);
	icp->icmp_code = 255;
	(void) send_icmp(nfd, mtu, ip, gwip);
	PAUSE();
	printf("3.5.%d ICMP type %d code 127 (all 0's)\r",
	i * 4 + 3, icp->icmp_type);
	}
	putchar('\n');
	}
	}


	/* Perform test 4 (UDP) */

	void ip_test4(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	udphdr_t *u;
	int nfd, i;


	IP_HL_A(ip, sizeof(*ip) >> 2);
	IP_V_A(ip, IPVERSION);
	ip->ip_tos = 0;
	ip->ip_off = 0;
	ip->ip_ttl = 60;
	ip->ip_p = IPPROTO_UDP;
	ip->ip_sum = 0;
	u = (udphdr_t )((char )ip + (IP_HL(ip) << 2));
	u->uh_sport = htons(1);
	u->uh_dport = htons(1);
	u->uh_ulen = htons(sizeof(*u) + 4);

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	if (!ptest \|\| (ptest == 1)) {
	/*
	* Test 1. ulen > packet
	*/
	u->uh_ulen = htons(sizeof(*u) + 4);
	ip->ip_len = (IP_HL(ip) << 2) + ntohs(u->uh_ulen);
	printf("4.1 UDP uh_ulen > packet size - short packets\n");
	for (i = ntohs(u->uh_ulen) * 2; i > sizeof(*u) + 4; i--) {
	u->uh_ulen = htons(i);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 2)) {
	/*
	* Test 2. ulen < packet
	*/
	u->uh_ulen = htons(sizeof(*u) + 4);
	ip->ip_len = (IP_HL(ip) << 2) + ntohs(u->uh_ulen);
	printf("4.2 UDP uh_ulen < packet size - short packets\n");
	for (i = ntohs(u->uh_ulen) * 2; i > sizeof(*u) + 4; i--) {
	ip->ip_len = i;
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 3)) {
	/*
	* Test 3: sport = 0, sport = 1, sport = 32767
	* sport = 32768, sport = 65535
	*/
	u->uh_ulen = sizeof(*u) + 4;
	ip->ip_len = (IP_HL(ip) << 2) + ntohs(u->uh_ulen);
	printf("4.3.1 UDP sport = 0\n");
	u->uh_sport = 0;
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("0\n");
	fflush(stdout);
	PAUSE();
	printf("4.3.2 UDP sport = 1\n");
	u->uh_sport = htons(1);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("1\n");
	fflush(stdout);
	PAUSE();
	printf("4.3.3 UDP sport = 32767\n");
	u->uh_sport = htons(32767);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("32767\n");
	fflush(stdout);
	PAUSE();
	printf("4.3.4 UDP sport = 32768\n");
	u->uh_sport = htons(32768);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("32768\n");
	putchar('\n');
	fflush(stdout);
	PAUSE();
	printf("4.3.5 UDP sport = 65535\n");
	u->uh_sport = htons(65535);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("65535\n");
	fflush(stdout);
	PAUSE();
	}

	if (!ptest \|\| (ptest == 4)) {
	/*
	* Test 4: dport = 0, dport = 1, dport = 32767
	* dport = 32768, dport = 65535
	*/
	u->uh_ulen = ntohs(sizeof(*u) + 4);
	u->uh_sport = htons(1);
	ip->ip_len = (IP_HL(ip) << 2) + ntohs(u->uh_ulen);
	printf("4.4.1 UDP dport = 0\n");
	u->uh_dport = 0;
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("0\n");
	fflush(stdout);
	PAUSE();
	printf("4.4.2 UDP dport = 1\n");
	u->uh_dport = htons(1);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("1\n");
	fflush(stdout);
	PAUSE();
	printf("4.4.3 UDP dport = 32767\n");
	u->uh_dport = htons(32767);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("32767\n");
	fflush(stdout);
	PAUSE();
	printf("4.4.4 UDP dport = 32768\n");
	u->uh_dport = htons(32768);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("32768\n");
	fflush(stdout);
	PAUSE();
	printf("4.4.5 UDP dport = 65535\n");
	u->uh_dport = htons(65535);
	(void) send_udp(nfd, 1500, ip, gwip);
	printf("65535\n");
	fflush(stdout);
	PAUSE();
	}

	if (!ptest \|\| (ptest == 5)) {
	/*
	* Test 5: sizeof(ip_t) <= MTU <= sizeof(udphdr_t) +
	* sizeof(ip_t)
	*/
	printf("4.5 UDP 20 <= MTU <= 32\n");
	for (i = sizeof(*ip); i <= ntohs(u->uh_ulen); i++) {
	(void) send_udp(nfd, i, ip, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}
	}


	/* Perform test 5 (TCP) */

	void ip_test5(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	tcphdr_t *t;
	int nfd, i;

	t = (tcphdr_t )((char )ip + (IP_HL(ip) << 2));
	t->th_x2 = 0;
	TCP_OFF_A(t, 0);
	t->th_sport = htons(1);
	t->th_dport = htons(1);
	t->th_win = htons(4096);
	t->th_urp = 0;
	t->th_sum = 0;
	t->th_seq = htonl(1);
	t->th_ack = 0;
	ip->ip_len = sizeof(ip_t) + sizeof(tcphdr_t);

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	if (!ptest \|\| (ptest == 1)) {
	/*
	* Test 1: flags variations, 0 - 3f
	*/
	TCP_OFF_A(t, sizeof(*t) >> 2);
	printf("5.1 Test TCP flag combinations\n");
	for (i = 0; i <= (TH_URG\|TH_ACK\|TH_PUSH\|TH_RST\|TH_SYN\|TH_FIN);
	i++) {
	t->th_flags = i;
	(void) send_tcp(nfd, mtu, ip, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}

	if (!ptest \|\| (ptest == 2)) {
	t->th_flags = TH_SYN;
	/*
	* Test 2: seq = 0, seq = 1, seq = 0x7fffffff, seq=0x80000000,
	* seq = 0xa000000, seq = 0xffffffff
	*/
	printf("5.2.1 TCP seq = 0\n");
	t->th_seq = htonl(0);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.2.2 TCP seq = 1\n");
	t->th_seq = htonl(1);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.2.3 TCP seq = 0x7fffffff\n");
	t->th_seq = htonl(0x7fffffff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.2.4 TCP seq = 0x80000000\n");
	t->th_seq = htonl(0x80000000);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.2.5 TCP seq = 0xc0000000\n");
	t->th_seq = htonl(0xc0000000);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.2.6 TCP seq = 0xffffffff\n");
	t->th_seq = htonl(0xffffffff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();
	}

	if (!ptest \|\| (ptest == 3)) {
	t->th_flags = TH_ACK;
	/*
	* Test 3: ack = 0, ack = 1, ack = 0x7fffffff, ack = 0x8000000
	* ack = 0xa000000, ack = 0xffffffff
	*/
	printf("5.3.1 TCP ack = 0\n");
	t->th_ack = 0;
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.3.2 TCP ack = 1\n");
	t->th_ack = htonl(1);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.3.3 TCP ack = 0x7fffffff\n");
	t->th_ack = htonl(0x7fffffff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.3.4 TCP ack = 0x80000000\n");
	t->th_ack = htonl(0x80000000);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.3.5 TCP ack = 0xc0000000\n");
	t->th_ack = htonl(0xc0000000);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.3.6 TCP ack = 0xffffffff\n");
	t->th_ack = htonl(0xffffffff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();
	}

	if (!ptest \|\| (ptest == 4)) {
	t->th_flags = TH_SYN;
	/*
	* Test 4: win = 0, win = 32768, win = 65535
	*/
	printf("5.4.1 TCP win = 0\n");
	t->th_seq = htonl(0);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.4.2 TCP win = 32768\n");
	t->th_seq = htonl(0x7fff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.4.3 TCP win = 65535\n");
	t->th_win = htons(0xffff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();
	}

	#if !defined(linux) && !defined(__SVR4) && !defined(__svr4__) && \
	!defined(__sgi) && !defined(__hpux) && !defined(__osf__)
	{
	struct tcpcb *tcbp, tcb;
	struct tcpiphdr ti;
	struct sockaddr_in sin;
	int fd;
	socklen_t slen;

	bzero((char *)&sin, sizeof(sin));

	for (i = 1; i < 63; i++) {
	fd = socket(AF_INET, SOCK_STREAM, 0);
	bzero((char *)&sin, sizeof(sin));
	sin.sin_addr.s_addr = ip->ip_dst.s_addr;
	sin.sin_port = htons(i);
	sin.sin_family = AF_INET;
	if (!connect(fd, (struct sockaddr *)&sin, sizeof(sin)))
	break;
	close(fd);
	}

	if (i == 63) {
	printf("Couldn't open a TCP socket between ports 1 and 63\n");
	printf("to host %s for test 5 and 6 - skipping.\n",
	inet_ntoa(ip->ip_dst));
	goto skip_five_and_six;
	}

	bcopy((char )ip, (char )&ti, sizeof(*ip));
	t->th_dport = htons(i);
	slen = sizeof(sin);
	if (!getsockname(fd, (struct sockaddr *)&sin, &slen))
	t->th_sport = sin.sin_port;
	if (!(tcbp = find_tcp(fd, &ti))) {
	printf("Can't find PCB\n");
	goto skip_five_and_six;
	}
	KMCPY(&tcb, tcbp, sizeof(tcb));
	ti.ti_win = tcb.rcv_adv;
	ti.ti_seq = htonl(tcb.snd_nxt - 1);
	ti.ti_ack = tcb.rcv_nxt;

	if (!ptest \|\| (ptest == 5)) {
	/*
	* Test 5: urp
	*/
	t->th_flags = TH_ACK\|TH_URG;
	printf("5.5.1 TCP Urgent pointer, sport %hu dport %hu\n",
	ntohs(t->th_sport), ntohs(t->th_dport));
	t->th_urp = htons(1);
	(void) send_tcp(nfd, mtu, ip, gwip);
	PAUSE();

	t->th_seq = htonl(tcb.snd_nxt);
	ip->ip_len = sizeof(ip_t) + sizeof(tcphdr_t) + 1;
	t->th_urp = htons(0x7fff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	PAUSE();
	t->th_urp = htons(0x8000);
	(void) send_tcp(nfd, mtu, ip, gwip);
	PAUSE();
	t->th_urp = htons(0xffff);
	(void) send_tcp(nfd, mtu, ip, gwip);
	PAUSE();
	t->th_urp = 0;
	t->th_flags &= ~TH_URG;
	ip->ip_len = sizeof(ip_t) + sizeof(tcphdr_t);
	}

	if (!ptest \|\| (ptest == 6)) {
	/*
	* Test 6: data offset, off = 0, off is inside, off is outside
	*/
	t->th_flags = TH_ACK;
	printf("5.6.1 TCP off = 1-15, len = 40\n");
	for (i = 1; i < 16; i++) {
	TCP_OFF_A(t, ntohs(i));
	(void) send_tcp(nfd, mtu, ip, gwip);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	ip->ip_len = sizeof(ip_t) + sizeof(tcphdr_t);
	}

	(void) close(fd);
	}
	skip_five_and_six:
	#endif
	t->th_seq = htonl(1);
	t->th_ack = htonl(1);
	TCP_OFF_A(t, 0);

	if (!ptest \|\| (ptest == 7)) {
	t->th_flags = TH_SYN;
	/*
	* Test 7: sport = 0, sport = 1, sport = 32767
	* sport = 32768, sport = 65535
	*/
	printf("5.7.1 TCP sport = 0\n");
	t->th_sport = 0;
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.7.2 TCP sport = 1\n");
	t->th_sport = htons(1);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.7.3 TCP sport = 32767\n");
	t->th_sport = htons(32767);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.7.4 TCP sport = 32768\n");
	t->th_sport = htons(32768);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.7.5 TCP sport = 65535\n");
	t->th_sport = htons(65535);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();
	}

	if (!ptest \|\| (ptest == 8)) {
	t->th_sport = htons(1);
	t->th_flags = TH_SYN;
	/*
	* Test 8: dport = 0, dport = 1, dport = 32767
	* dport = 32768, dport = 65535
	*/
	printf("5.8.1 TCP dport = 0\n");
	t->th_dport = 0;
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.8.2 TCP dport = 1\n");
	t->th_dport = htons(1);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.8.3 TCP dport = 32767\n");
	t->th_dport = htons(32767);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.8.4 TCP dport = 32768\n");
	t->th_dport = htons(32768);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();

	printf("5.8.5 TCP dport = 65535\n");
	t->th_dport = htons(65535);
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();
	}

	/* LAND attack - self connect, so make src & dst ip/port the same */
	if (!ptest \|\| (ptest == 9)) {
	printf("5.9 TCP LAND attack. sport = 25, dport = 25\n");
	/* chose SMTP port 25 */
	t->th_sport = htons(25);
	t->th_dport = htons(25);
	t->th_flags = TH_SYN;
	ip->ip_src = ip->ip_dst;
	(void) send_tcp(nfd, mtu, ip, gwip);
	fflush(stdout);
	PAUSE();
	}

	/* TCP options header checking */
	/* 0 length options, etc */
	}


	/* Perform test 6 (exhaust mbuf test) */

	void ip_test6(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	udphdr_t *u;
	int nfd, i, j, k;

	IP_V_A(ip, IPVERSION);
	ip->ip_tos = 0;
	ip->ip_off = 0;
	ip->ip_ttl = 60;
	ip->ip_p = IPPROTO_UDP;
	ip->ip_sum = 0;
	u = (udphdr_t *)(ip + 1);
	u->uh_sport = htons(1);
	u->uh_dport = htons(9);
	u->uh_sum = 0;

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	u->uh_ulen = htons(7168);

	printf("6. Exhaustive mbuf test.\n");
	printf(" Send 7k packet in 768 & 128 byte fragments, 128 times.\n");
	printf(" Total of around 8,900 packets\n");
	for (i = 0; i < 128; i++) {
	/*
	* First send the entire packet in 768 byte chunks.
	*/
	ip->ip_len = sizeof(ip) + 768 + sizeof(u);
	IP_HL_A(ip, sizeof(*ip) >> 2);
	ip->ip_off = htons(IP_MF);
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d %d\r", i, 0);
	fflush(stdout);
	PAUSE();
	/*
	* And again using 128 byte chunks.
	*/
	ip->ip_len = sizeof(ip) + 128 + sizeof(u);
	ip->ip_off = htons(IP_MF);
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d %d\r", i, 0);
	fflush(stdout);
	PAUSE();

	for (j = 768; j < 3584; j += 768) {
	ip->ip_len = sizeof(*ip) + 768;
	ip->ip_off = htons(IP_MF\|(j>>3));
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d %d\r", i, j);
	fflush(stdout);
	PAUSE();

	ip->ip_len = sizeof(*ip) + 128;
	for (k = j - 768; k < j; k += 128) {
	ip->ip_off = htons(IP_MF\|(k>>3));
	(void) send_ip(nfd, 1500, ip, gwip, 1);
	printf("%d %d\r", i, k);
	fflush(stdout);
	PAUSE();
	}
	}
	}
	putchar('\n');
	}


	/* Perform test 7 (random packets) */

	static u_long tbuf[64];

	void ip_test7(dev, mtu, ip, gwip, ptest)
	char *dev;
	int mtu;
	ip_t *ip;
	struct in_addr gwip;
	int ptest;
	{
	ip_t *pip;
	#ifdef USE_NANOSLEEP
	struct timespec ts;
	#else
	struct timeval tv;
	#endif
	int nfd, i, j;
	u_char *s;

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return;

	pip = (ip_t *)tbuf;

	srand(time(NULL) ^ (getpid() * getppid()));

	printf("7. send 1024 random IP packets.\n");

	for (i = 0; i < 512; i++) {
	for (s = (u_char *)pip, j = 0; j < sizeof(tbuf); j++, s++)
	*s = (rand() >> 13) & 0xff;
	IP_V_A(pip, IPVERSION);
	bcopy((char )&ip->ip_dst, (char )&pip->ip_dst,
	sizeof(struct in_addr));
	pip->ip_sum = 0;
	pip->ip_len &= 0xff;
	(void) send_ip(nfd, mtu, pip, gwip, 0);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');

	for (i = 0; i < 512; i++) {
	for (s = (u_char *)pip, j = 0; j < sizeof(tbuf); j++, s++)
	*s = (rand() >> 13) & 0xff;
	IP_V_A(pip, IPVERSION);
	pip->ip_off &= htons(0xc000);
	bcopy((char )&ip->ip_dst, (char )&pip->ip_dst,
	sizeof(struct in_addr));
	pip->ip_sum = 0;
	pip->ip_len &= 0xff;
	(void) send_ip(nfd, mtu, pip, gwip, 0);
	printf("%d\r", i);
	fflush(stdout);
	PAUSE();
	}
	putchar('\n');
	}
	diff --git a/contrib/ipfilter/ipsend/sock.c b/contrib/ipfilter/ipsend/sock.c
	index d7eae8a13196..66e1a0aa897e 100644
	--- a/contrib/ipfilter/ipsend/sock.c
	+++ b/contrib/ipfilter/ipsend/sock.c
	@@ -1,455 +1,322 @@
	/* $FreeBSD$ */
	/*
	* sock.c (C) 1995-1998 Darren Reed
	*
	* See the IPFILTER.LICENCE file for details on licencing.
	*
	*/
	#if !defined(lint)
	static const char sccsid[] = "@(#)sock.c 1.2 1/11/96 (C)1995 Darren Reed";
	static const char rcsid[] = "@(#)$Id$";
	#endif
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/time.h>
	#include <sys/stat.h>
	#if defined(__NetBSD__) && defined(__vax__)
	/*
	* XXX need to declare boolean_t for _KERNEL <sys/files.h>
	* which ends up including <sys/device.h> for vax. See PR#32907
	* for further details.
	*/
	typedef int boolean_t;
	#endif
	-#ifndef ultrix
	#include <fcntl.h>
	-#endif
	-#if (__FreeBSD_version >= 300000)
	# include <sys/dirent.h>
	-#else
	-# include <sys/dir.h>
	-#endif
	# ifdef __NetBSD__
	# include <machine/lock.h>
	# endif
	# ifdef __FreeBSD__
	# define _WANT_FILE
	# else
	# define _KERNEL
	# define KERNEL
	# endif
	-# ifdef ultrix
	-# undef LOCORE
	-# include <sys/smp_lock.h>
	-# endif
	# include <sys/file.h>
	# ifdef __FreeBSD__
	# undef _WANT_FILE
	# else
	# undef _KERNEL
	# undef KERNEL
	# endif
	#include <nlist.h>
	#include <sys/user.h>
	#include <sys/socket.h>
	#include <sys/socketvar.h>
	#include <sys/proc.h>
	-#if !defined(ultrix) && !defined(hpux) && !defined(__osf__)
	# include <kvm.h>
	-#endif
	#ifdef sun
	#include <sys/systm.h>
	#include <sys/session.h>
	#endif
	-#if BSD >= 199103
	#include <sys/sysctl.h>
	#include <sys/filedesc.h>
	#include <paths.h>
	-#endif
	#include <math.h>
	#include <netinet/in.h>
	#include <netinet/in_systm.h>
	#include <netinet/ip.h>
	#include <netinet/tcp.h>
	#include <net/if.h>
	# include <net/route.h>
	#include <netinet/ip_var.h>
	#define _WANT_INPCB
	#include <netinet/in_pcb.h>
	#include <netinet/tcp_timer.h>
	#define _WANT_TCPCB
	#include <netinet/tcp_var.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <stddef.h>
	#include <pwd.h>
	#include "ipsend.h"


	int nproc;
	struct proc *proc;

	#ifndef KMEM
	# ifdef _PATH_KMEM
	# define KMEM _PATH_KMEM
	# endif
	#endif
	#ifndef KERNEL
	# ifdef _PATH_UNIX
	# define KERNEL _PATH_UNIX
	# endif
	#endif
	#ifndef KMEM
	# define KMEM "/dev/kmem"
	#endif
	#ifndef KERNEL
	# define KERNEL "/vmunix"
	#endif


	#if BSD < 199103
	static struct proc *getproc __P((void));
	#else
	static struct kinfo_proc *getproc __P((void));
	#endif


	int kmemcpy(buf, pos, n)
	char *buf;
	void *pos;
	int n;
	{
	static int kfd = -1;
	off_t offset = (u_long)pos;

	if (kfd == -1)
	kfd = open(KMEM, O_RDONLY);

	if (lseek(kfd, offset, SEEK_SET) == -1)
	{
	perror("lseek");
	return -1;
	}
	if (read(kfd, buf, n) == -1)
	{
	perror("read");
	return -1;
	}
	return n;
	}

	struct nlist names[4] = {
	{ "_proc" },
	{ "_nproc" },
	-#ifdef ultrix
	- { "_u" },
	-#else
	{ NULL },
	-#endif
	{ NULL }
	};

	-#if BSD < 199103
	-static struct proc *getproc()
	-{
	- struct proc *p;
	- pid_t pid = getpid();
	- int siz, n;
	-
	- n = nlist(KERNEL, names);
	- if (n != 0)
	- {
	- fprintf(stderr, "nlist(%#x) == %d\n", names, n);
	- return NULL;
	- }
	- if (KMCPY(&nproc, names[1].n_value, sizeof(nproc)) == -1)
	- {
	- fprintf(stderr, "read nproc (%#x)\n", names[1].n_value);
	- return NULL;
	- }
	- siz = nproc * sizeof(struct proc);
	- if (KMCPY(&p, names[0].n_value, sizeof(p)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x,%d) proc\n",
	- names[0].n_value, &p, sizeof(p));
	- return NULL;
	- }
	- proc = (struct proc *)malloc(siz);
	- if (KMCPY(proc, p, siz) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x,%d) proc\n",
	- p, proc, siz);
	- return NULL;
	- }
	-
	- p = proc;
	-
	- for (n = nproc; n; n--, p++)
	- if (p->p_pid == pid)
	- break;
	- if (!n)
	- return NULL;
	-
	- return p;
	-}
	-
	-
	-struct tcpcb *find_tcp(fd, ti)
	- int fd;
	- struct tcpiphdr *ti;
	-{
	- struct tcpcb *t;
	- struct inpcb *i;
	- struct socket *s;
	- struct user *up;
	- struct proc *p;
	- struct file f, *o;
	-
	- if (!(p = getproc()))
	- return NULL;
	- up = (struct user )malloc(sizeof(up));
	-#ifndef ultrix
	- if (KMCPY(up, p->p_uarea, sizeof(*up)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x) failed\n", p, p->p_uarea);
	- return NULL;
	- }
	-#else
	- if (KMCPY(up, names[2].n_value, sizeof(*up)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x) failed\n", p, names[2].n_value);
	- return NULL;
	- }
	-#endif
	-
	- o = (struct file *)calloc(up->u_lastfile + 1, sizeof(o));
	- if (KMCPY(o, up->u_ofile, (up->u_lastfile + 1) * sizeof(*o)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x,%d) - u_ofile - failed\n",
	- up->u_ofile, o, sizeof(*o));
	- return NULL;
	- }
	- f = (struct file )calloc(1, sizeof(f));
	- if (KMCPY(f, o[fd], sizeof(*f)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x,%d) - o[fd] - failed\n",
	- up->u_ofile[fd], f, sizeof(*f));
	- return NULL;
	- }
	-
	- s = (struct socket )calloc(1, sizeof(s));
	- if (KMCPY(s, f->f_data, sizeof(*s)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x,%d) - f_data - failed\n",
	- o[fd], s, sizeof(*s));
	- return NULL;
	- }
	-
	- i = (struct inpcb )calloc(1, sizeof(i));
	- if (KMCPY(i, s->so_pcb, sizeof(*i)) == -1)
	- {
	- fprintf(stderr, "kvm_read(%#x,%#x,%d) - so_pcb - failed\n",
	- s->so_pcb, i, sizeof(*i));
	- return NULL;
	- }
	-
	- t = (struct tcpcb )calloc(1, sizeof(t));
	- if (KMCPY(t, i->inp_ppcb, sizeof(*t)) == -1)
	- {
	- fprintf(stderr, "read(%#x,%#x,%d) - inp_ppcb - failed\n",
	- i->inp_ppcb, t, sizeof(*t));
	- return NULL;
	- }
	- return (struct tcpcb *)i->inp_ppcb;
	-}
	-#else
	static struct kinfo_proc *getproc()
	{
	static struct kinfo_proc kp;
	pid_t pid = getpid();
	int mib[4];
	size_t n;

	mib[0] = CTL_KERN;
	mib[1] = KERN_PROC;
	mib[2] = KERN_PROC_PID;
	mib[3] = pid;

	n = sizeof(kp);
	if (sysctl(mib, 4, &kp, &n, NULL, 0) == -1)
	{
	perror("sysctl");
	return NULL;
	}
	return &kp;
	}


	struct tcpcb *find_tcp(tfd, ti)
	int tfd;
	struct tcpiphdr *ti;
	{
	struct tcpcb *t;
	struct inpcb *i;
	struct socket *s;
	struct filedesc *fd;
	struct kinfo_proc *p;
	struct file f, *o;

	if (!(p = getproc()))
	return NULL;

	fd = (struct filedesc )malloc(sizeof(fd));
	if (fd == NULL)
	return NULL;
	-#if defined( __FreeBSD_version) && __FreeBSD_version >= 500013
	+#if defined( __FreeBSD_version)
	if (KMCPY(fd, p->ki_fd, sizeof(*fd)) == -1)
	{
	fprintf(stderr, "read(%#lx,%#lx) failed\n",
	(u_long)p, (u_long)p->ki_fd);
	free(fd);
	return NULL;
	}
	#else
	if (KMCPY(fd, p->kp_proc.p_fd, sizeof(*fd)) == -1)
	{
	fprintf(stderr, "read(%#lx,%#lx) failed\n",
	(u_long)p, (u_long)p->kp_proc.p_fd);
	free(fd);
	return NULL;
	}
	#endif

	o = NULL;
	f = NULL;
	s = NULL;
	i = NULL;
	t = NULL;

	o = (struct file *)calloc(fd->fd_lastfile + 1, sizeof(o));
	if (KMCPY(o, fd->fd_ofiles, (fd->fd_lastfile + 1) * sizeof(*o)) == -1)
	{
	fprintf(stderr, "read(%#lx,%#lx,%lu) - u_ofile - failed\n",
	(u_long)fd->fd_ofiles, (u_long)o, (u_long)sizeof(*o));
	goto finderror;
	}
	f = (struct file )calloc(1, sizeof(f));
	if (KMCPY(f, o[tfd], sizeof(*f)) == -1)
	{
	fprintf(stderr, "read(%#lx,%#lx,%lu) - o[tfd] - failed\n",
	(u_long)o[tfd], (u_long)f, (u_long)sizeof(*f));
	goto finderror;
	}

	s = (struct socket )calloc(1, sizeof(s));
	if (KMCPY(s, f->f_data, sizeof(*s)) == -1)
	{
	fprintf(stderr, "read(%#lx,%#lx,%lu) - f_data - failed\n",
	(u_long)f->f_data, (u_long)s, (u_long)sizeof(*s));
	goto finderror;
	}

	i = (struct inpcb )calloc(1, sizeof(i));
	if (KMCPY(i, s->so_pcb, sizeof(*i)) == -1)
	{
	fprintf(stderr, "kvm_read(%#lx,%#lx,%lu) - so_pcb - failed\n",
	(u_long)s->so_pcb, (u_long)i, (u_long)sizeof(*i));
	goto finderror;
	}

	t = (struct tcpcb )calloc(1, sizeof(t));
	if (KMCPY(t, i->inp_ppcb, sizeof(*t)) == -1)
	{
	fprintf(stderr, "read(%#lx,%#lx,%lu) - inp_ppcb - failed\n",
	(u_long)i->inp_ppcb, (u_long)t, (u_long)sizeof(*t));
	goto finderror;
	}
	return (struct tcpcb *)i->inp_ppcb;

	finderror:
	if (o != NULL)
	free(o);
	if (f != NULL)
	free(f);
	if (s != NULL)
	free(s);
	if (i != NULL)
	free(i);
	if (t != NULL)
	free(t);
	return NULL;
	}
	-#endif /* BSD < 199301 */

	int do_socket(dev, mtu, ti, gwip)
	char *dev;
	int mtu;
	struct tcpiphdr *ti;
	struct in_addr gwip;
	{
	struct sockaddr_in rsin, lsin;
	struct tcpcb *t, tcb;
	int fd, nfd;
	socklen_t len;

	printf("Dest. Port: %d\n", ti->ti_dport);

	fd = socket(AF_INET, SOCK_STREAM, 0);
	if (fd == -1)
	{
	perror("socket");
	return -1;
	}

	if (fcntl(fd, F_SETFL, FNDELAY) == -1)
	{
	perror("fcntl");
	return -1;
	}

	bzero((char *)&lsin, sizeof(lsin));
	lsin.sin_family = AF_INET;
	bcopy((char )&ti->ti_src, (char )&lsin.sin_addr,
	sizeof(struct in_addr));
	if (bind(fd, (struct sockaddr *)&lsin, sizeof(lsin)) == -1)
	{
	perror("bind");
	return -1;
	}
	len = sizeof(lsin);
	(void) getsockname(fd, (struct sockaddr *)&lsin, &len);
	ti->ti_sport = lsin.sin_port;
	printf("sport %d\n", ntohs(lsin.sin_port));

	nfd = initdevice(dev, 1);
	if (nfd == -1)
	return -1;

	if (!(t = find_tcp(fd, ti)))
	return -1;

	bzero((char *)&rsin, sizeof(rsin));
	rsin.sin_family = AF_INET;
	bcopy((char )&ti->ti_dst, (char )&rsin.sin_addr,
	sizeof(struct in_addr));
	rsin.sin_port = ti->ti_dport;
	if (connect(fd, (struct sockaddr *)&rsin, sizeof(rsin)) == -1 &&
	errno != EINPROGRESS)
	{
	perror("connect");
	return -1;
	}
	KMCPY(&tcb, t, sizeof(tcb));
	ti->ti_win = tcb.rcv_adv;
	ti->ti_seq = tcb.snd_nxt - 1;
	ti->ti_ack = tcb.rcv_nxt;

	if (send_tcp(nfd, mtu, (ip_t *)ti, gwip) == -1)
	return -1;
	(void)write(fd, "Hello World\n", 12);
	sleep(2);
	close(fd);
	return 0;
	}
	diff --git a/contrib/netbsd-tests/lib/libc/regex/data/meta.in b/contrib/netbsd-tests/lib/libc/regex/data/meta.in
	index 4533d3591bc6..eb24075aea62 100644
	--- a/contrib/netbsd-tests/lib/libc/regex/data/meta.in
	+++ b/contrib/netbsd-tests/lib/libc/regex/data/meta.in
	@@ -1,21 +1,23 @@
	# metacharacters, backslashes
	a.c & abc abc
	a[bc]d & abd abd
	a\c & ac a*c
	a\\b & a\b a\b
	a\\\b & a\b a\*b
	-a\bc & abc abc
	+# Begin FreeBSD
	+a\bc &C EESCAPE
	+# End FreeBSD
	a\ &C EESCAPE
	a\\bc & a\bc a\bc
	\{ bC BADRPT
	a\[b & a[b a[b
	a[b &C EBRACK
	# trailing $ is a peculiar special case for the BRE code
	a$ & a a
	a$ & a$
	a\$ & a
	a\$ & a$ a$
	a\\$ & a
	a\\$ & a$
	a\\$ & a\$
	a\\$ & a\ a\
	diff --git a/contrib/netbsd-tests/lib/libc/regex/data/subexp.in b/contrib/netbsd-tests/lib/libc/regex/data/subexp.in
	index d3efe2eab270..e3d376bb7cb3 100644
	--- a/contrib/netbsd-tests/lib/libc/regex/data/subexp.in
	+++ b/contrib/netbsd-tests/lib/libc/regex/data/subexp.in
	@@ -1,60 +1,60 @@
	# subexpressions
	a(b)(c)d - abcd abcd b,c
	a(((b)))c - abc abc b,b,b
	a(b\|(c))d - abd abd b,-
	a(b*\|c\|e)d - abbd abbd bb
	a(b*\|c\|e)d - acd acd c
	a(b*\|c\|e)d - ad ad @d
	a(b?)c - abc abc b
	a(b?)c - ac ac @c
	a(b+)c - abc abc b
	a(b+)c - abbbc abbbc bbb
	a(b*)c - ac ac @c
	(a\|ab)(bc([de]+)f\|cde) - abcdef abcdef a,bcdef,de
	# Begin FreeBSD
	-a$b\\|c$d b ab\|cd ab\|cd b\|c
	+a$b\|c$d b ab\|cd ab\|cd b\|c
	# End FreeBSD
	# the regression tester only asks for 9 subexpressions
	a(b)(c)(d)(e)(f)(g)(h)(i)(j)k - abcdefghijk abcdefghijk b,c,d,e,f,g,h,i,j
	a(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)l - abcdefghijkl abcdefghijkl b,c,d,e,f,g,h,i,j,k
	a([bc]?)c - abc abc b
	a([bc]?)c - ac ac @c
	a([bc]+)c - abc abc b
	a([bc]+)c - abcc abcc bc
	a([bc]+)bc - abcbc abcbc bc
	a(bb+\|b)b - abb abb b
	a(bbb+\|bb+\|b)b - abb abb b
	a(bbb+\|bb+\|b)b - abbb abbb bb
	a(bbb+\|bb+\|b)bb - abbb abbb b
	(.). - abcdef abcdef abcdef
	(a) - bc @b @b

	# do we get the right subexpression when it is used more than once?
	a(b\|c)*d - ad ad -
	a(b\|c)*d - abcd abcd c
	a(b\|c)+d - abd abd b
	a(b\|c)+d - abcd abcd c
	a(b\|c?)+d - ad ad @d
	a(b\|c?)+d - abcd abcd @d
	a(b\|c){0,0}d - ad ad -
	a(b\|c){0,1}d - ad ad -
	a(b\|c){0,1}d - abd abd b
	a(b\|c){0,2}d - ad ad -
	a(b\|c){0,2}d - abcd abcd c
	a(b\|c){0,}d - ad ad -
	a(b\|c){0,}d - abcd abcd c
	a(b\|c){1,1}d - abd abd b
	a(b\|c){1,1}d - acd acd c
	a(b\|c){1,2}d - abd abd b
	a(b\|c){1,2}d - abcd abcd c
	a(b\|c){1,}d - abd abd b
	a(b\|c){1,}d - abcd abcd c
	a(b\|c){2,2}d - acbd acbd b
	a(b\|c){2,2}d - abcd abcd c
	a(b\|c){2,4}d - abcd abcd c
	a(b\|c){2,4}d - abcbd abcbd b
	a(b\|c){2,4}d - abcbcd abcbcd c
	a(b\|c){2,}d - abcd abcd c
	a(b\|c){2,}d - abcbd abcbd b
	a(b+\|((c)*))+d - abd abd @d,@d,-
	a(b+\|((c)*))+d - abcd abcd @d,@d,-
	diff --git a/contrib/openbsm/CREDITS b/contrib/openbsm/CREDITS
	index 18b3ad7dc03f..2721de1e487d 100644
	--- a/contrib/openbsm/CREDITS
	+++ b/contrib/openbsm/CREDITS
	@@ -1,42 +1,45 @@
	OpenBSM Credits

	The following organizations and individuals have contributed substantially to
	the development of OpenBSM:

	Apple Inc.
	McAfee Research, McAfee, Inc.
	SPARTA, Inc.
	Robert Watson
	Wayne Salamon
	Suresh Krishnaswamy
	Kevin Van Vechten
	Tom Rhodes
	Wojciech Koszek
	Chunyang Yuan
	Poul-Henning Kamp
	Christian Brueffer
	Olivier Houchard
	Christian Peron
	Martin Fong
	Pawel Worach
	Martin Englund
	Ruslan Ermilov
	Martin Voros
	Diego Giagio
	Alex Samorukov
	Eric Hall
	Xin LI
	Stacey Son
	Todd Heberlein
	Gary Hoo
	Dave Bertouille
	Jonathan Anderson
	Pawel Jakub Dawidek
	Joel Dahl
	Ryan Steinmetz
	The FreeBSD Foundation
	Brooks Davis
	+ Mateusz Piotrowski
	+ Alan Somers
	+ Aniket Pandey

	In addition, Coverity, Inc.'s Prevent(tm) static analysis tool and Gimpel
	Software's FlexeLint tool were used to identify a number of bugs in the
	OpenBSM implementation.
	diff --git a/contrib/openbsm/bin/auditd/auditd.c b/contrib/openbsm/bin/auditd/auditd.c
	index a165cf314e08..bd00a6b16191 100644
	--- a/contrib/openbsm/bin/auditd/auditd.c
	+++ b/contrib/openbsm/bin/auditd/auditd.c
	@@ -1,869 +1,869 @@
	/*-
	* Copyright (c) 2004-2009 Apple Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of Apple Inc. ("Apple") nor the names of
	* its contributors may be used to endorse or promote products derived
	* from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	* DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/types.h>

	#include <config/config.h>

	#include <sys/dirent.h>
	#ifdef HAVE_FULL_QUEUE_H
	#include <sys/queue.h>
	#else /* !HAVE_FULL_QUEUE_H */
	#include <compat/queue.h>
	#endif /* !HAVE_FULL_QUEUE_H */
	#include <sys/mman.h>
	#include <sys/param.h>
	#include <sys/stat.h>
	#include <sys/wait.h>

	#include <bsm/audit.h>
	#include <bsm/audit_uevents.h>
	#include <bsm/auditd_lib.h>
	#include <bsm/libbsm.h>

	#include <err.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <grp.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <time.h>
	#include <unistd.h>
	#include <signal.h>
	#include <string.h>

	#include "auditd.h"

	#ifndef HAVE_STRLCPY
	#include <compat/strlcpy.h>
	#endif

	/*
	* XXX The following are temporary until these can be added to the kernel
	* audit.h header.
	*/
	#ifndef AUDIT_TRIGGER_INITIALIZE
	#define AUDIT_TRIGGER_INITIALIZE 7
	#endif
	#ifndef AUDIT_TRIGGER_EXPIRE_TRAILS
	#define AUDIT_TRIGGER_EXPIRE_TRAILS 8
	#endif


	/*
	* LaunchD flag (Mac OS X and, maybe, FreeBSD only.) See launchd(8) and
	* http://wiki.freebsd.org/launchd for more information.
	*
	* In order for auditd to work "on demand" with launchd(8) it can't:
	* call daemon(3)
	* call fork and having the parent process exit
	* change uids or gids.
	* set up the current working directory or chroot.
	* set the session id
	* change stdio to /dev/null.
	* call setrusage(2)
	* call setpriority(2)
	* Ignore SIGTERM.
	* auditd (in 'launchd mode') is launched on demand so it must catch
	* SIGTERM to exit cleanly.
	*/
	static int launchd_flag = 0;

	/*
	* The GID of the audit review group (if used). The audit trail files and
	* system logs (Mac OS X only) can only be reviewed by members of this group
	* or the audit administrator (aka. "root").
	*/
	static gid_t audit_review_gid = -1;

	/*
	* The path and file name of the last audit trail file.
	*/
	static char *lastfile = NULL;

	/*
	* Error starting auditd. Run warn script and exit.
	*/
	static void
	fail_exit(void)
	{

	audit_warn_nostart();
	exit(1);
	}

	/*
	* Follow the 'current' symlink to get the active trail file name.
	*/
	static char *
	get_curfile(void)
	{
	char *cf;
	int len;

	cf = malloc(MAXPATHLEN);
	if (cf == NULL) {
	auditd_log_err("malloc failed: %m");
	return (NULL);
	}

	len = readlink(AUDIT_CURRENT_LINK, cf, MAXPATHLEN - 1);
	if (len < 0) {
	free(cf);
	return (NULL);
	}

	/* readlink() doesn't terminate string. */
	cf[len] = '\0';

	return (cf);
	}

	/*
	* Close the previous audit trail file.
	*/
	static int
	close_lastfile(char *TS)
	{
	char *ptr;
	char *oldname;

	/* If lastfile is NULL try to get it from the 'current' link. */
	if (lastfile == NULL)
	lastfile = get_curfile();

	if (lastfile != NULL) {
	oldname = strdup(lastfile);
	if (oldname == NULL)
	return (-1);

	/* Rename the last file -- append timestamp. */
	if ((ptr = strstr(lastfile, NOT_TERMINATED)) != NULL) {
	memcpy(ptr, TS, POSTFIX_LEN);
	if (auditd_rename(oldname, lastfile) != 0)
	auditd_log_err(
	"Could not rename %s to %s: %m", oldname,
	lastfile);
	else {
	/*
	* Remove the 'current' symlink since the link
	* is now invalid.
	*/
	(void) unlink(AUDIT_CURRENT_LINK);
	auditd_log_notice("renamed %s to %s",
	oldname, lastfile);
	audit_warn_closefile(lastfile);
	}
	} else
	auditd_log_err("Could not rename %s to %s", oldname,
	lastfile);
	free(lastfile);
	free(oldname);
	lastfile = NULL;
	}
	return (0);
	}

	/*
	* Create the new file name, swap with existing audit file.
	*/
	static int
	swap_audit_file(void)
	{
	int err;
	char newfile, name;
	char TS[TIMESTAMP_LEN + 1];
	time_t tt;

	if (getTSstr(tt, TS, sizeof(TS)) != 0)
	return (-1);
	/*
	* If prefix and suffix are the same, it means that records are
	* being produced too fast. We don't want to rename now, because
	* next trail file can get the same name and once that one is
	* terminated also within one second it will overwrite the current
	* one. Just keep writing to the same trail and wait for the next
	* trigger from the kernel.
	* FREEBSD KERNEL WAS UPDATED TO KEEP SENDING TRIGGERS, WHICH MIGHT
	* NOT BE THE CASE FOR OTHER OSES.
	* If the kernel will not keep sending triggers, trail file will not
	* be terminated.
	*/
	if (lastfile == NULL) {
	name = NULL;
	} else {
	name = strrchr(lastfile, '/');
	if (name != NULL)
	name++;
	}
	if (name != NULL && strncmp(name, TS, TIMESTAMP_LEN) == 0) {
	auditd_log_debug("Not ready to terminate trail file yet.");
	return (0);
	}
	err = auditd_swap_trail(TS, &newfile, audit_review_gid,
	audit_warn_getacdir);
	if (err != ADE_NOERR) {
	auditd_log_err("%s: %m", auditd_strerror(err));
	if (err != ADE_ACTL)
	return (-1);
	}

	/*
	* Only close the last file if were in an auditing state before
	* calling swap_audit_file(). We may need to recover from a crash.
	*/
	if (auditd_get_state() == AUD_STATE_ENABLED)
	close_lastfile(TS);


	/*
	* auditd_swap_trail() potentially enables auditing (if not already
	* enabled) so updated the cached state as well.
	*/
	auditd_set_state(AUD_STATE_ENABLED);

	/*
	* Create 'current' symlink. Recover from crash, if needed.
	*/
	if (auditd_new_curlink(newfile) != 0)
	auditd_log_err("auditd_new_curlink(\"%s\") failed: %s: %m",
	newfile, auditd_strerror(err));

	lastfile = newfile;
	auditd_log_notice("New audit file is %s", newfile);

	return (0);
	}

	/*
	* Create a new audit log trail file and swap with the current one, if any.
	*/
	static int
	do_trail_file(void)
	{
	int err;

	/*
	* First, refresh the list of audit log directories.
	*/
	err = auditd_read_dirs(audit_warn_soft, audit_warn_hard);
	if (err) {
	auditd_log_err("auditd_read_dirs(): %s",
	auditd_strerror(err));
	if (err == ADE_HARDLIM)
	audit_warn_allhard();
	if (err != ADE_SOFTLIM)
	return (-1);
	else
	audit_warn_allsoft();
	/* continue on with soft limit error */
	}

	/*
	* Create a new file and swap with the one being used in kernel.
	*/
	if (swap_audit_file() == -1) {
	/*
	* XXX Faulty directory listing? - user should be given
	* XXX an opportunity to change the audit_control file
	* XXX switch to a reduced mode of auditing?
	*/
	return (-1);
	}

	/*
	* Finally, see if there are any trail files to expire.
	*/
	err = auditd_expire_trails(audit_warn_expired);
	if (err)
	auditd_log_err("auditd_expire_trails(): %s",
	auditd_strerror(err));

	return (0);
	}

	/*
	* Start up auditing.
	*/
	static void
	audit_setup(void)
	{
	int err;

	/* Configure trail files distribution. */
	err = auditd_set_dist();
	if (err) {
	auditd_log_err("auditd_set_dist() %s: %m",
	auditd_strerror(err));
	} else
	auditd_log_debug("Configured trail files distribution.");

	if (do_trail_file() == -1) {
	auditd_log_err("Error creating audit trail file");
	fail_exit();
	}

	/* Generate an audit record. */
	err = auditd_gen_record(AUE_audit_startup, NULL);
	if (err)
	auditd_log_err("auditd_gen_record(AUE_audit_startup) %s: %m",
	auditd_strerror(err));

	if (auditd_config_controls() == 0)
	auditd_log_info("Audit controls init successful");
	else
	auditd_log_err("Audit controls init failed");
	}


	/*
	* Close auditd pid file and trigger mechanism.
	*/
	static int
	close_misc(void)
	{

	auditd_close_dirs();
	if (unlink(AUDITD_PIDFILE) == -1 && errno != ENOENT) {
	auditd_log_err("Couldn't remove %s: %m", AUDITD_PIDFILE);
	return (1);
	}
	endac();

	if (auditd_close_trigger() != 0) {
	auditd_log_err("Error closing trigger messaging mechanism");
	return (1);
	}
	return (0);
	}

	/*
	* Close all log files, control files, and tell the audit system.
	*/
	static int
	close_all(void)
	{
	int err_ret = 0;
	char TS[TIMESTAMP_LEN + 1];
	int err;
	int cond;
	time_t tt;

	err = auditd_gen_record(AUE_audit_shutdown, NULL);
	if (err)
	auditd_log_err("auditd_gen_record(AUE_audit_shutdown) %s: %m",
	auditd_strerror(err));

	/* Flush contents. */
	cond = AUC_DISABLED;
	err_ret = audit_set_cond(&cond);
	if (err_ret != 0) {
	auditd_log_err("Disabling audit failed! : %s", strerror(errno));
	err_ret = 1;
	}

	/*
	* Updated the cached state that auditing has been disabled.
	*/
	auditd_set_state(AUD_STATE_DISABLED);

	if (getTSstr(tt, TS, sizeof(TS)) == 0)
	close_lastfile(TS);
	if (lastfile != NULL)
	free(lastfile);

	err_ret += close_misc();

	if (err_ret) {
	auditd_log_err("Could not unregister");
	audit_warn_postsigterm();
	}

	auditd_log_info("Finished");
	return (err_ret);
	}

	/*
	* Register the daemon with the signal handler and the auditd pid file.
	*/
	static int
	register_daemon(void)
	{
	struct sigaction action;
	FILE * pidfile;
	int fd;
	pid_t pid;

	/* Set up the signal hander. */
	action.sa_handler = auditd_relay_signal;
	/*
	* sa_flags must not include SA_RESTART, so that read(2) will be
	* interruptible in auditd_wait_for_events
	*/
	action.sa_flags = 0;
	sigemptyset(&action.sa_mask);
	if (sigaction(SIGTERM, &action, NULL) != 0) {
	auditd_log_err(
	"Could not set signal handler for SIGTERM");
	fail_exit();
	}
	if (sigaction(SIGCHLD, &action, NULL) != 0) {
	auditd_log_err(
	"Could not set signal handler for SIGCHLD");
	fail_exit();
	}
	if (sigaction(SIGHUP, &action, NULL) != 0) {
	auditd_log_err(
	"Could not set signal handler for SIGHUP");
	fail_exit();
	}
	if (sigaction(SIGALRM, &action, NULL) != 0) {
	auditd_log_err(
	"Could not set signal handler for SIGALRM");
	fail_exit();
	}

	if ((pidfile = fopen(AUDITD_PIDFILE, "a")) == NULL) {
	auditd_log_err("Could not open PID file");
	audit_warn_tmpfile();
	return (-1);
	}

	/* Attempt to lock the pid file; if a lock is present, exit. */
	fd = fileno(pidfile);
	if (flock(fd, LOCK_EX \| LOCK_NB) < 0) {
	auditd_log_err(
	"PID file is locked (is another auditd running?).");
	audit_warn_ebusy();
	return (-1);
	}

	pid = getpid();
	ftruncate(fd, 0);
	if (fprintf(pidfile, "%u\n", pid) < 0) {
	/* Should not start the daemon. */
	fail_exit();
	}

	fflush(pidfile);
	return (0);
	}

	/*
	* Handle the audit trigger event.
	*
	* We suppress (ignore) duplicated triggers in close succession in order to
	* try to avoid thrashing-like behavior. However, not all triggers can be
	* ignored, as triggers generally represent edge triggers, not level
	* triggers, and won't be retransmitted if the condition persists. Of
	* specific concern is the rotate trigger -- if one is dropped, then it will
	* not be retransmitted, and the log file will grow in an unbounded fashion.
	*/
	#define DUPLICATE_INTERVAL 30
	void
	auditd_handle_trigger(int trigger)
	{
	static int last_trigger, last_warning;
	static time_t last_time;
	struct timeval ts;
	struct timezone tzp;
	time_t tt;
	int au_state;
	int err = 0;

	/*
	* Suppress duplicate messages from the kernel within the specified
	* interval.
	*/
	if (gettimeofday(&ts, &tzp) == 0) {
	tt = (time_t)ts.tv_sec;
	switch (trigger) {
	case AUDIT_TRIGGER_LOW_SPACE:
	case AUDIT_TRIGGER_NO_SPACE:
	/*
	* Triggers we can suppress. Of course, we also need
	* to rate limit the warnings, so apply the same
	* interval limit on syslog messages.
	*/
	if ((trigger == last_trigger) &&
	(tt < (last_time + DUPLICATE_INTERVAL))) {
	if (tt >= (last_warning + DUPLICATE_INTERVAL))
	auditd_log_info(
	"Suppressing duplicate trigger %d",
	trigger);
	return;
	}
	last_warning = tt;
	break;

	case AUDIT_TRIGGER_ROTATE_KERNEL:
	case AUDIT_TRIGGER_ROTATE_USER:
	case AUDIT_TRIGGER_READ_FILE:
	case AUDIT_TRIGGER_CLOSE_AND_DIE:
	case AUDIT_TRIGGER_INITIALIZE:
	/*
	* Triggers that we cannot suppress.
	*/
	break;
	}

	/*
	* Only update last_trigger after aborting due to a duplicate
	* trigger, not before, or we will never allow that trigger
	* again.
	*/
	last_trigger = trigger;
	last_time = tt;
	}

	au_state = auditd_get_state();

	/*
	* Message processing is done here.
	*/
	switch(trigger) {
	case AUDIT_TRIGGER_LOW_SPACE:
	auditd_log_notice("Got low space trigger");
	if (do_trail_file() == -1)
	auditd_log_err("Error swapping audit file");
	break;

	case AUDIT_TRIGGER_NO_SPACE:
	auditd_log_notice("Got no space trigger");
	if (do_trail_file() == -1)
	auditd_log_err("Error swapping audit file");
	break;

	case AUDIT_TRIGGER_ROTATE_KERNEL:
	case AUDIT_TRIGGER_ROTATE_USER:
	auditd_log_info("Got open new trigger from %s", trigger ==
	AUDIT_TRIGGER_ROTATE_KERNEL ? "kernel" : "user");
	if (au_state == AUD_STATE_ENABLED && do_trail_file() == -1)
	auditd_log_err("Error swapping audit file");
	break;

	case AUDIT_TRIGGER_READ_FILE:
	auditd_log_info("Got read file trigger");
	if (au_state == AUD_STATE_ENABLED) {
	if (auditd_config_controls() == -1)
	auditd_log_err("Error setting audit controls");
	else if (do_trail_file() == -1)
	auditd_log_err("Error swapping audit file");
	}
	break;

	case AUDIT_TRIGGER_CLOSE_AND_DIE:
	auditd_log_info("Got close and die trigger");
	if (au_state == AUD_STATE_ENABLED)
	err = close_all();
	/*
	* Running under launchd don't exit. Wait for launchd to
	* send SIGTERM.
	*/
	if (!launchd_flag) {
	auditd_log_info("auditd exiting.");
	exit (err);
	}
	break;

	case AUDIT_TRIGGER_INITIALIZE:
	auditd_log_info("Got audit initialize trigger");
	if (au_state == AUD_STATE_DISABLED)
	audit_setup();
	break;

	case AUDIT_TRIGGER_EXPIRE_TRAILS:
	auditd_log_info("Got audit expire trails trigger");
	err = auditd_expire_trails(audit_warn_expired);
	if (err)
	auditd_log_err("auditd_expire_trails(): %s",
	auditd_strerror(err));
	break;

	default:
	auditd_log_err("Got unknown trigger %d", trigger);
	break;
	}
	}

	/*
	* Reap our children.
	*/
	void
	auditd_reap_children(void)
	{
	pid_t child;
	int wstatus;

	while ((child = waitpid(-1, &wstatus, WNOHANG)) > 0) {
	if (!wstatus)
	continue;
	auditd_log_info("warn process [pid=%d] %s %d.", child,
	((WIFEXITED(wstatus)) ? "exited with non-zero status" :
	"exited as a result of signal"),
	((WIFEXITED(wstatus)) ? WEXITSTATUS(wstatus) :
	WTERMSIG(wstatus)));
	}
	}

	/*
	* Reap any children and terminate. If under launchd don't shutdown auditing
	* but just the other stuff.
	*/
	void
	auditd_terminate(void)
	{
	int ret;

	auditd_reap_children();

	if (launchd_flag)
	ret = close_misc();
	else
	ret = close_all();

	exit(ret);
	}

	/*
	* Configure the audit controls in the kernel: the event to class mapping,
	* kernel preselection mask, etc.
	*/
	int
	auditd_config_controls(void)
	{
	int cnt, err;
	int ret = 0;

	/*
	* Configure event to class mappings in kernel.
	*/
	cnt = auditd_set_evcmap();
	if (cnt < 0) {
	auditd_log_err("auditd_set_evcmap() failed: %m");
	ret = -1;
	} else if (cnt == 0) {
	auditd_log_err("No events to class mappings registered.");
	ret = -1;
	} else
	auditd_log_debug("Registered %d event to class mappings.", cnt);

	/*
	* Configure non-attributable event mask in kernel.
	*/
	err = auditd_set_namask();
	if (err) {
	auditd_log_err("auditd_set_namask() %s: %m",
	auditd_strerror(err));
	ret = -1;
	} else
	auditd_log_debug("Registered non-attributable event mask.");

	/*
	* Configure audit policy in kernel.
	*/
	err = auditd_set_policy();
	if (err) {
	auditd_log_err("auditd_set_policy() %s: %m",
	auditd_strerror(err));
	ret = -1;
	} else
	auditd_log_debug("Set audit policy in kernel.");

	/*
	* Configure audit trail log size in kernel.
	*/
	err = auditd_set_fsize();
	if (err) {
	auditd_log_err("audit_set_fsize() %s: %m",
	auditd_strerror(err));
	ret = -1;
	} else
	auditd_log_debug("Set audit trail size in kernel.");

	/*
	* Configure audit trail queue size in kernel.
	*/
	err = auditd_set_qsize();
	if (err) {
	- auditd_log_err("audit_set_qsize() %s: %m",
	+ auditd_log_err("auditd_set_qsize() %s: %m",
	auditd_strerror(err));
	ret = -1;
	} else
	auditd_log_debug("Set audit trail queue in kernel.");

	/*
	* Configure audit trail volume minimum free percentage of blocks in
	* kernel.
	*/
	err = auditd_set_minfree();
	if (err) {
	auditd_log_err("auditd_set_minfree() %s: %m",
	auditd_strerror(err));
	ret = -1;
	} else
	auditd_log_debug(
	"Set audit trail min free percent in kernel.");

	/*
	* Configure host address in the audit kernel information.
	*/
	err = auditd_set_host();
	if (err) {
	if (err == ADE_PARSE) {
	auditd_log_notice(
	"audit_control(5) may be missing 'host:' field");
	} else {
	auditd_log_err("auditd_set_host() %s: %m",
	auditd_strerror(err));
	ret = -1;
	}
	} else
	auditd_log_debug(
	"Set audit host address information in kernel.");

	return (ret);
	}

	/*
	* Setup and initialize auditd.
	*/
	static void
	setup(void)
	{
	int err;

	if (auditd_open_trigger(launchd_flag) < 0) {
	auditd_log_err("Error opening trigger messaging mechanism");
	fail_exit();
	}

	/*
	* To prevent event feedback cycles and avoid auditd becoming
	* stalled if auditing is suspended, auditd and its children run
	* without their events being audited. We allow the uid, tid, and
	* mask fields to be implicitly set to zero, but do set the pid. We
	* run this after opening the trigger device to avoid configuring
	* audit state without audit present in the system.
	*/
	err = auditd_prevent_audit();
	if (err) {
	auditd_log_err("auditd_prevent_audit() %s: %m",
	auditd_strerror(err));
	fail_exit();
	}

	/*
	* Make sure auditd auditing state is correct.
	*/
	auditd_set_state(AUD_STATE_INIT);

	/*
	* If under launchd, don't start auditing. Wait for a trigger to
	* do so.
	*/
	if (!launchd_flag)
	audit_setup();
	}

	int
	main(int argc, char **argv)
	{
	int ch;
	int debug = 0;
	#ifdef AUDIT_REVIEW_GROUP
	struct group *grp;
	#endif

	while ((ch = getopt(argc, argv, "dl")) != -1) {
	switch(ch) {
	case 'd':
	/* Debug option. */
	debug = 1;
	break;

	case 'l':
	/* Be launchd friendly. */
	launchd_flag = 1;
	break;

	case '?':
	default:
	(void)fprintf(stderr,
	"usage: auditd [-d] [-l]\n");
	exit(1);
	}
	}

	audit_review_gid = getgid();

	#ifdef AUDIT_REVIEW_GROUP
	/*
	* XXXRW: Currently, this code falls back to the daemon gid, which is
	* likely the wheel group. Is there a better way to deal with this?
	*/
	grp = getgrnam(AUDIT_REVIEW_GROUP);
	if (grp != NULL)
	audit_review_gid = grp->gr_gid;
	#endif

	auditd_openlog(debug, audit_review_gid);

	if (launchd_flag)
	auditd_log_info("started by launchd...");
	else
	auditd_log_info("starting...");

	#ifdef AUDIT_REVIEW_GROUP
	if (grp == NULL)
	auditd_log_info(
	"Audit review group '%s' not available, using daemon gid (%d)",
	AUDIT_REVIEW_GROUP, audit_review_gid);
	#endif
	if (debug == 0 && launchd_flag == 0 && daemon(0, 0) == -1) {
	auditd_log_err("Failed to daemonize");
	exit(1);
	}

	if (register_daemon() == -1) {
	auditd_log_err("Could not register as daemon");
	exit(1);
	}

	setup();

	/*
	* auditd_wait_for_events() shouldn't return unless something is wrong.
	*/
	auditd_wait_for_events();

	auditd_log_err("abnormal exit.");
	close_all();
	exit(-1);
	}
	diff --git a/contrib/openbsm/bin/auditdistd/auditdistd.c b/contrib/openbsm/bin/auditdistd/auditdistd.c
	index 696f0488b27a..8468353f4cbc 100644
	--- a/contrib/openbsm/bin/auditdistd/auditdistd.c
	+++ b/contrib/openbsm/bin/auditdistd/auditdistd.c
	@@ -1,798 +1,798 @@
	/*-
	* Copyright (c) 2012 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Pawel Jakub Dawidek under sponsorship from
	* the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <config/config.h>

	#include <sys/param.h>
	#if defined(HAVE_SYS_ENDIAN_H) && defined(HAVE_BSWAP)
	#include <sys/endian.h>
	#else /* !HAVE_SYS_ENDIAN_H \|\| !HAVE_BSWAP */
	#ifdef HAVE_MACHINE_ENDIAN_H
	#include <machine/endian.h>
	#else /* !HAVE_MACHINE_ENDIAN_H */
	#ifdef HAVE_ENDIAN_H
	#include <endian.h>
	#else /* !HAVE_ENDIAN_H */
	#error "No supported endian.h"
	#endif /* !HAVE_ENDIAN_H */
	#endif /* !HAVE_MACHINE_ENDIAN_H */
	#include <compat/endian.h>
	#endif /* !HAVE_SYS_ENDIAN_H \|\| !HAVE_BSWAP */
	#include <sys/queue.h>
	#include <sys/wait.h>

	#include <ctype.h>
	#include <err.h>
	#include <errno.h>
	#include <fcntl.h>
	#ifdef HAVE_LIBUTIL_H
	#include <libutil.h>
	#endif
	#include <signal.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <strings.h>
	#include <unistd.h>

	#include <openssl/hmac.h>

	#ifndef HAVE_PIDFILE_OPEN
	#include <compat/pidfile.h>
	#endif
	#ifndef HAVE_STRLCPY
	#include <compat/strlcpy.h>
	#endif
	#ifndef HAVE_SIGTIMEDWAIT
	#include "sigtimedwait.h"
	#endif

	#include "auditdistd.h"
	#include "pjdlog.h"
	#include "proto.h"
	#include "subr.h"
	#include "synch.h"

	/* Path to configuration file. */
	const char *cfgpath = ADIST_CONFIG;
	/* Auditdistd configuration. */
	static struct adist_config *adcfg;
	/* Was SIGINT or SIGTERM signal received? */
	bool sigexit_received = false;
	/* PID file handle. */
	struct pidfh *pfh;

	/* How often check for hooks running for too long. */
	#define SIGNALS_CHECK_INTERVAL 5

	static void
	usage(void)
	{

	errx(EX_USAGE, "[-dFhl] [-c config] [-P pidfile]");
	}

	void
	descriptors_cleanup(struct adist_host *adhost)
	{
	struct adist_host *adh;
	struct adist_listen *lst;

	TAILQ_FOREACH(adh, &adcfg->adc_hosts, adh_next) {
	if (adh == adhost)
	continue;
	if (adh->adh_remote != NULL) {
	proto_close(adh->adh_remote);
	adh->adh_remote = NULL;
	}
	}
	TAILQ_FOREACH(lst, &adcfg->adc_listen, adl_next) {
	if (lst->adl_conn != NULL)
	proto_close(lst->adl_conn);
	}
	(void)pidfile_close(pfh);
	pjdlog_fini();
	}

	static void
	child_cleanup(struct adist_host *adhost)
	{

	if (adhost->adh_conn != NULL) {
	PJDLOG_ASSERT(adhost->adh_role == ADIST_ROLE_SENDER);
	proto_close(adhost->adh_conn);
	adhost->adh_conn = NULL;
	}
	adhost->adh_worker_pid = 0;
	}

	static void
	child_exit_log(const char *type, unsigned int pid, int status)
	{

	if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
	pjdlog_debug(1, "%s process exited gracefully (pid=%u).",
	type, pid);
	} else if (WIFSIGNALED(status)) {
	pjdlog_error("%s process killed (pid=%u, signal=%d).",
	type, pid, WTERMSIG(status));
	} else {
	pjdlog_error("%s process exited ungracefully (pid=%u, exitcode=%d).",
	type, pid, WIFEXITED(status) ? WEXITSTATUS(status) : -1);
	}
	}

	static void
	child_exit(void)
	{
	struct adist_host *adhost;
	bool restart;
	int status;
	pid_t pid;

	restart = false;
	while ((pid = wait3(&status, WNOHANG, NULL)) > 0) {
	/* Find host related to the process that just exited. */
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (pid == adhost->adh_worker_pid)
	break;
	}
	if (adhost == NULL) {
	child_exit_log("Sandbox", pid, status);
	} else {
	if (adhost->adh_role == ADIST_ROLE_SENDER)
	restart = true;
	pjdlog_prefix_set("[%s] (%s) ", adhost->adh_name,
	role2str(adhost->adh_role));
	child_exit_log("Worker", pid, status);
	child_cleanup(adhost);
	pjdlog_prefix_set("%s", "");
	}
	}
	if (!restart)
	return;
	/* We have some sender processes to restart. */
	sleep(1);
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_role != ADIST_ROLE_SENDER)
	continue;
	if (adhost->adh_worker_pid != 0)
	continue;
	pjdlog_prefix_set("[%s] (%s) ", adhost->adh_name,
	role2str(adhost->adh_role));
	pjdlog_info("Restarting sender process.");
	adist_sender(adcfg, adhost);
	pjdlog_prefix_set("%s", "");
	}
	}

	/* TODO */
	static void
	adist_reload(void)
	{

	pjdlog_info("Reloading configuration is not yet implemented.");
	}

	static void
	terminate_workers(void)
	{
	struct adist_host *adhost;

	pjdlog_info("Termination signal received, exiting.");
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_worker_pid == 0)
	continue;
	pjdlog_info("Terminating worker process (adhost=%s, role=%s, pid=%u).",
	adhost->adh_name, role2str(adhost->adh_role),
	adhost->adh_worker_pid);
	if (kill(adhost->adh_worker_pid, SIGTERM) == 0)
	continue;
	pjdlog_errno(LOG_WARNING,
	"Unable to send signal to worker process (adhost=%s, role=%s, pid=%u).",
	adhost->adh_name, role2str(adhost->adh_role),
	adhost->adh_worker_pid);
	}
	}

	static void
	listen_accept(struct adist_listen *lst)
	{
	unsigned char rnd[32], hash[32], resp[32];
	struct adist_host *adhost;
	struct proto_conn *conn;
	char adname[ADIST_HOSTSIZE];
	char laddr[256], raddr[256];
	char welcome[8];
	int status, version;
	pid_t pid;

	proto_local_address(lst->adl_conn, laddr, sizeof(laddr));
	pjdlog_debug(1, "Accepting connection to %s.", laddr);

	if (proto_accept(lst->adl_conn, &conn) == -1) {
	pjdlog_errno(LOG_ERR, "Unable to accept connection to %s",
	laddr);
	return;
	}

	proto_local_address(conn, laddr, sizeof(laddr));
	proto_remote_address(conn, raddr, sizeof(raddr));
	pjdlog_info("Connection from %s to %s.", raddr, laddr);

	/* Error in setting timeout is not critical, but why should it fail? */
	if (proto_timeout(conn, ADIST_TIMEOUT) < 0)
	pjdlog_errno(LOG_WARNING, "Unable to set connection timeout");

	/*
	* Before receiving any data see if remote host is known.
	*/
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_role != ADIST_ROLE_RECEIVER)
	continue;
	if (!proto_address_match(conn, adhost->adh_remoteaddr))
	continue;
	break;
	}
	if (adhost == NULL) {
	pjdlog_error("Client %s is not known.", raddr);
	goto close;
	}
	/* Ok, remote host is known. */

	/* Exchange welcome message, which include version number. */
	bzero(welcome, sizeof(welcome));
	if (proto_recv(conn, welcome, sizeof(welcome)) == -1) {
	pjdlog_errno(LOG_WARNING,
	"Unable to receive welcome message from %s",
	adhost->adh_remoteaddr);
	goto close;
	}
	if (strncmp(welcome, "ADIST", 5) != 0 \|\| !isdigit(welcome[5]) \|\|
	!isdigit(welcome[6]) \|\| welcome[7] != '\0') {
	pjdlog_warning("Invalid welcome message from %s.",
	adhost->adh_remoteaddr);
	goto close;
	}

	version = MIN(ADIST_VERSION, atoi(welcome + 5));

	(void)snprintf(welcome, sizeof(welcome), "ADIST%02d", version);
	if (proto_send(conn, welcome, sizeof(welcome)) == -1) {
	pjdlog_errno(LOG_WARNING,
	"Unable to send welcome message to %s",
	adhost->adh_remoteaddr);
	goto close;
	}

	if (proto_recv(conn, adname, sizeof(adhost->adh_name)) < 0) {
	pjdlog_errno(LOG_ERR, "Unable to receive hostname from %s",
	raddr);
	goto close;
	}

	/* Find host now that we have hostname. */
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_role != ADIST_ROLE_RECEIVER)
	continue;
	if (!proto_address_match(conn, adhost->adh_remoteaddr))
	continue;
	if (strcmp(adhost->adh_name, adname) != 0)
	continue;
	break;
	}
	if (adhost == NULL) {
	pjdlog_error("No configuration for host %s from address %s.",
	adname, raddr);
	goto close;
	}

	adhost->adh_version = version;
	pjdlog_debug(1, "Version %d negotiated with %s.", adhost->adh_version,
	adhost->adh_remoteaddr);

	/* Now that we know host name setup log prefix. */
	pjdlog_prefix_set("[%s] (%s) ", adhost->adh_name,
	role2str(adhost->adh_role));

	if (adist_random(rnd, sizeof(rnd)) == -1) {
	pjdlog_error("Unable to generate challenge.");
	goto close;
	}
	pjdlog_debug(1, "Challenge generated.");

	if (proto_send(conn, rnd, sizeof(rnd)) == -1) {
	pjdlog_errno(LOG_ERR, "Unable to send challenge to %s",
	adhost->adh_remoteaddr);
	goto close;
	}
	pjdlog_debug(1, "Challenge sent.");

	if (proto_recv(conn, resp, sizeof(resp)) == -1) {
	pjdlog_errno(LOG_ERR, "Unable to receive response from %s",
	adhost->adh_remoteaddr);
	goto close;
	}
	pjdlog_debug(1, "Response received.");

	if (HMAC(EVP_sha256(), adhost->adh_password,
	(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
	NULL) == NULL) {
	pjdlog_error("Unable to generate hash.");
	goto close;
	}
	pjdlog_debug(1, "Hash generated.");

	if (memcmp(resp, hash, sizeof(hash)) != 0) {
	pjdlog_error("Invalid response from %s (wrong password?).",
	adhost->adh_remoteaddr);
	goto close;
	}
	pjdlog_info("Sender authenticated.");

	if (proto_recv(conn, rnd, sizeof(rnd)) == -1) {
	pjdlog_errno(LOG_ERR, "Unable to receive challenge from %s",
	adhost->adh_remoteaddr);
	goto close;
	}
	pjdlog_debug(1, "Challenge received.");

	if (HMAC(EVP_sha256(), adhost->adh_password,
	(int)strlen(adhost->adh_password), rnd, (int)sizeof(rnd), hash,
	NULL) == NULL) {
	pjdlog_error("Unable to generate response.");
	goto close;
	}
	pjdlog_debug(1, "Response generated.");

	if (proto_send(conn, hash, sizeof(hash)) == -1) {
	pjdlog_errno(LOG_ERR, "Unable to send response to %s",
	adhost->adh_remoteaddr);
	goto close;
	}
	pjdlog_debug(1, "Response sent.");

	if (adhost->adh_worker_pid != 0) {
	pjdlog_debug(1,
	"Receiver process exists (pid=%u), stopping it.",
	(unsigned int)adhost->adh_worker_pid);
	/* Stop child process. */
	if (kill(adhost->adh_worker_pid, SIGINT) == -1) {
	pjdlog_errno(LOG_ERR,
	"Unable to stop worker process (pid=%u)",
	(unsigned int)adhost->adh_worker_pid);
	/*
	* Other than logging the problem we
	* ignore it - nothing smart to do.
	*/
	}
	/* Wait for it to exit. */
	else if ((pid = waitpid(adhost->adh_worker_pid,
	&status, 0)) != adhost->adh_worker_pid) {
	/* We can only log the problem. */
	pjdlog_errno(LOG_ERR,
	"Waiting for worker process (pid=%u) failed",
	(unsigned int)adhost->adh_worker_pid);
	} else {
	child_exit_log("Worker", adhost->adh_worker_pid,
	status);
	}
	child_cleanup(adhost);
	}

	adhost->adh_remote = conn;
	adist_receiver(adcfg, adhost);

	pjdlog_prefix_set("%s", "");
	return;
	close:
	proto_close(conn);
	pjdlog_prefix_set("%s", "");
	}

	static void
	connection_migrate(struct adist_host *adhost)
	{
	struct proto_conn *conn;
	int16_t val = 0;

	pjdlog_prefix_set("[%s] (%s) ", adhost->adh_name,
	role2str(adhost->adh_role));

	PJDLOG_ASSERT(adhost->adh_role == ADIST_ROLE_SENDER);

	if (proto_recv(adhost->adh_conn, &val, sizeof(val)) < 0) {
	pjdlog_errno(LOG_WARNING,
	"Unable to receive connection command");
	return;
	}
	if (proto_set("tls:fingerprint", adhost->adh_fingerprint) == -1) {
	val = errno;
	pjdlog_errno(LOG_WARNING, "Unable to set fingerprint");
	goto out;
	}
	if (proto_connect(adhost->adh_localaddr[0] != '\0' ?
	adhost->adh_localaddr : NULL,
	adhost->adh_remoteaddr, -1, &conn) < 0) {
	val = errno;
	pjdlog_errno(LOG_WARNING, "Unable to connect to %s",
	adhost->adh_remoteaddr);
	goto out;
	}
	val = 0;
	out:
	if (proto_send(adhost->adh_conn, &val, sizeof(val)) < 0) {
	pjdlog_errno(LOG_WARNING,
	"Unable to send reply to connection request");
	}
	if (val == 0 && proto_connection_send(adhost->adh_conn, conn) < 0)
	pjdlog_errno(LOG_WARNING, "Unable to send connection");

	pjdlog_prefix_set("%s", "");
	}

	static void
	check_signals(void)
	{
	struct timespec sigtimeout;
	sigset_t mask;
	int signo;

	sigtimeout.tv_sec = 0;
	sigtimeout.tv_nsec = 0;

	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0);

	while ((signo = sigtimedwait(&mask, NULL, &sigtimeout)) != -1) {
	switch (signo) {
	case SIGINT:
	case SIGTERM:
	sigexit_received = true;
	terminate_workers();
	exit(EX_OK);
	break;
	case SIGCHLD:
	child_exit();
	break;
	case SIGHUP:
	adist_reload();
	break;
	default:
	PJDLOG_ABORT("Unexpected signal (%d).", signo);
	}
	}
	}

	static void
	main_loop(void)
	{
	struct adist_host *adhost;
	struct adist_listen *lst;
	struct timeval seltimeout;
	int fd, maxfd, ret;
	fd_set rfds;

	seltimeout.tv_sec = SIGNALS_CHECK_INTERVAL;
	seltimeout.tv_usec = 0;

	pjdlog_info("Started successfully.");

	for (;;) {
	check_signals();

	/* Setup descriptors for select(2). */
	FD_ZERO(&rfds);
	maxfd = -1;
	TAILQ_FOREACH(lst, &adcfg->adc_listen, adl_next) {
	if (lst->adl_conn == NULL)
	continue;
	fd = proto_descriptor(lst->adl_conn);
	PJDLOG_ASSERT(fd >= 0);
	FD_SET(fd, &rfds);
	maxfd = fd > maxfd ? fd : maxfd;
	}
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_role == ADIST_ROLE_SENDER) {
	- /* Only sender workers asks for connections. */
	+ /* Only sender workers ask for connections. */
	PJDLOG_ASSERT(adhost->adh_conn != NULL);
	fd = proto_descriptor(adhost->adh_conn);
	PJDLOG_ASSERT(fd >= 0);
	FD_SET(fd, &rfds);
	maxfd = fd > maxfd ? fd : maxfd;
	} else {
	PJDLOG_ASSERT(adhost->adh_conn == NULL);
	}
	}

	PJDLOG_ASSERT(maxfd + 1 <= (int)FD_SETSIZE);
	ret = select(maxfd + 1, &rfds, NULL, NULL, &seltimeout);
	if (ret == 0) {
	/*
	* select(2) timed out, so there should be no
	* descriptors to check.
	*/
	continue;
	} else if (ret == -1) {
	if (errno == EINTR)
	continue;
	KEEP_ERRNO((void)pidfile_remove(pfh));
	pjdlog_exit(EX_OSERR, "select() failed");
	}
	PJDLOG_ASSERT(ret > 0);

	/*
	* Check for signals before we do anything to update our
	* info about terminated workers in the meantime.
	*/
	check_signals();

	TAILQ_FOREACH(lst, &adcfg->adc_listen, adl_next) {
	if (lst->adl_conn == NULL)
	continue;
	if (FD_ISSET(proto_descriptor(lst->adl_conn), &rfds))
	listen_accept(lst);
	}
	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_role == ADIST_ROLE_SENDER) {
	PJDLOG_ASSERT(adhost->adh_conn != NULL);
	if (FD_ISSET(proto_descriptor(adhost->adh_conn),
	&rfds)) {
	connection_migrate(adhost);
	}
	} else {
	PJDLOG_ASSERT(adhost->adh_conn == NULL);
	}
	}
	}
	}

	static void
	adist_config_dump(struct adist_config *cfg)
	{
	struct adist_host *adhost;
	struct adist_listen *lst;

	pjdlog_debug(2, "Configuration:");
	pjdlog_debug(2, " Global:");
	pjdlog_debug(2, " pidfile: %s", cfg->adc_pidfile);
	pjdlog_debug(2, " timeout: %d", cfg->adc_timeout);
	if (TAILQ_EMPTY(&cfg->adc_listen)) {
	pjdlog_debug(2, " Sender only, not listening.");
	} else {
	pjdlog_debug(2, " Listening on:");
	TAILQ_FOREACH(lst, &cfg->adc_listen, adl_next) {
	pjdlog_debug(2, " listen: %s", lst->adl_addr);
	pjdlog_debug(2, " conn: %p", lst->adl_conn);
	}
	}
	pjdlog_debug(2, " Hosts:");
	TAILQ_FOREACH(adhost, &cfg->adc_hosts, adh_next) {
	pjdlog_debug(2, " name: %s", adhost->adh_name);
	pjdlog_debug(2, " role: %s", role2str(adhost->adh_role));
	pjdlog_debug(2, " version: %d", adhost->adh_version);
	pjdlog_debug(2, " localaddr: %s", adhost->adh_localaddr);
	pjdlog_debug(2, " remoteaddr: %s", adhost->adh_remoteaddr);
	pjdlog_debug(2, " remote: %p", adhost->adh_remote);
	pjdlog_debug(2, " directory: %s", adhost->adh_directory);
	pjdlog_debug(2, " compression: %d", adhost->adh_compression);
	pjdlog_debug(2, " checksum: %d", adhost->adh_checksum);
	pjdlog_debug(2, " pid: %ld", (long)adhost->adh_worker_pid);
	pjdlog_debug(2, " conn: %p", adhost->adh_conn);
	}
	}

	static void
	dummy_sighandler(int sig __unused)
	{
	/* Nothing to do. */
	}

	int
	main(int argc, char *argv[])
	{
	struct adist_host *adhost;
	struct adist_listen *lst;
	const char execpath, pidfile;
	bool foreground, launchd;
	pid_t otherpid;
	int debuglevel;
	sigset_t mask;

	execpath = argv[0];
	if (execpath[0] != '/') {
	errx(EX_USAGE,
	"auditdistd requires execution with an absolute path.");
	}

	/*
	* We are executed from proto to create sandbox.
	*/
	if (argc > 1 && strcmp(argv[1], "proto") == 0) {
	argc -= 2;
	argv += 2;
	if (proto_exec(argc, argv) == -1)
	err(EX_USAGE, "Unable to execute proto");
	}

	foreground = false;
	debuglevel = 0;
	launchd = false;
	pidfile = NULL;

	for (;;) {
	int ch;

	ch = getopt(argc, argv, "c:dFhlP:");
	if (ch == -1)
	break;
	switch (ch) {
	case 'c':
	cfgpath = optarg;
	break;
	case 'd':
	debuglevel++;
	break;
	case 'F':
	foreground = true;
	break;
	case 'l':
	launchd = true;
	break;
	case 'P':
	pidfile = optarg;
	break;
	case 'h':
	default:
	usage();
	}
	}
	argc -= optind;
	argv += optind;

	pjdlog_init(PJDLOG_MODE_STD);
	pjdlog_debug_set(debuglevel);

	if (proto_set("execpath", execpath) == -1)
	pjdlog_exit(EX_TEMPFAIL, "Unable to set executable name");
	if (proto_set("user", ADIST_USER) == -1)
	pjdlog_exit(EX_TEMPFAIL, "Unable to set proto user");
	if (proto_set("tcp:port", ADIST_TCP_PORT) == -1)
	pjdlog_exit(EX_TEMPFAIL, "Unable to set default TCP port");

	/*
	* When path to the configuration file is relative, obtain full path,
	* so we can always find the file, even after daemonizing and changing
	* working directory to /.
	*/
	if (cfgpath[0] != '/') {
	const char *newcfgpath;

	newcfgpath = realpath(cfgpath, NULL);
	if (newcfgpath == NULL) {
	pjdlog_exit(EX_CONFIG,
	"Unable to obtain full path of %s", cfgpath);
	}
	cfgpath = newcfgpath;
	}

	adcfg = yy_config_parse(cfgpath, true);
	PJDLOG_ASSERT(adcfg != NULL);
	adist_config_dump(adcfg);

	if (proto_set("tls:certfile", adcfg->adc_certfile) == -1)
	pjdlog_exit(EX_TEMPFAIL, "Unable to set certfile path");
	if (proto_set("tls:keyfile", adcfg->adc_keyfile) == -1)
	pjdlog_exit(EX_TEMPFAIL, "Unable to set keyfile path");

	if (pidfile != NULL) {
	if (strlcpy(adcfg->adc_pidfile, pidfile,
	sizeof(adcfg->adc_pidfile)) >=
	sizeof(adcfg->adc_pidfile)) {
	pjdlog_exitx(EX_CONFIG, "Pidfile path is too long.");
	}
	}
	if (foreground && pidfile == NULL) {
	pfh = NULL;
	} else {
	pfh = pidfile_open(adcfg->adc_pidfile, 0600, &otherpid);
	if (pfh == NULL) {
	if (errno == EEXIST) {
	pjdlog_exitx(EX_TEMPFAIL,
	"Another auditdistd is already running, pid: %jd.",
	(intmax_t)otherpid);
	}
	/*
	* If we cannot create pidfile from other reasons,
	* only warn.
	*/
	pjdlog_errno(LOG_WARNING,
	"Unable to open or create pidfile %s",
	adcfg->adc_pidfile);
	}
	}

	/*
	* Restore default actions for interesting signals in case parent
	* process (like init(8)) decided to ignore some of them (like SIGHUP).
	*/
	PJDLOG_VERIFY(signal(SIGHUP, SIG_DFL) != SIG_ERR);
	PJDLOG_VERIFY(signal(SIGINT, SIG_DFL) != SIG_ERR);
	PJDLOG_VERIFY(signal(SIGTERM, SIG_DFL) != SIG_ERR);
	/*
	* Because SIGCHLD is ignored by default, setup dummy handler for it,
	* so we can mask it.
	*/
	PJDLOG_VERIFY(signal(SIGCHLD, dummy_sighandler) != SIG_ERR);

	PJDLOG_VERIFY(sigemptyset(&mask) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0);
	PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0);
	PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);

	/* Listen for remote connections. */
	TAILQ_FOREACH(lst, &adcfg->adc_listen, adl_next) {
	if (proto_server(lst->adl_addr, &lst->adl_conn) == -1) {
	KEEP_ERRNO((void)pidfile_remove(pfh));
	pjdlog_exit(EX_OSERR, "Unable to listen on address %s",
	lst->adl_addr);
	}
	}

	if (!foreground) {
	if (!launchd && daemon(0, 0) == -1) {
	KEEP_ERRNO((void)pidfile_remove(pfh));
	pjdlog_exit(EX_OSERR, "Unable to daemonize");
	}

	/* Start logging to syslog. */
	pjdlog_mode_set(PJDLOG_MODE_SYSLOG);
	}
	if (pfh != NULL) {
	/* Write PID to a file. */
	if (pidfile_write(pfh) < 0) {
	pjdlog_errno(LOG_WARNING,
	"Unable to write PID to a file");
	}
	}

	TAILQ_FOREACH(adhost, &adcfg->adc_hosts, adh_next) {
	if (adhost->adh_role == ADIST_ROLE_SENDER)
	adist_sender(adcfg, adhost);
	}

	main_loop();

	exit(0);
	}
	diff --git a/contrib/openbsm/bin/auditdistd/proto_tls.c b/contrib/openbsm/bin/auditdistd/proto_tls.c
	index ff251c5b0c81..31673084d5b5 100644
	--- a/contrib/openbsm/bin/auditdistd/proto_tls.c
	+++ b/contrib/openbsm/bin/auditdistd/proto_tls.c
	@@ -1,1074 +1,1074 @@
	/*-
	* Copyright (c) 2011 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Pawel Jakub Dawidek under sponsorship from
	* the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <config/config.h>

	#include <sys/param.h> /* MAXHOSTNAMELEN */
	#include <sys/socket.h>

	#include <arpa/inet.h>

	#include <netinet/in.h>
	#include <netinet/tcp.h>

	#include <errno.h>
	#include <fcntl.h>
	#include <netdb.h>
	#include <signal.h>
	#include <stdbool.h>
	#include <stdint.h>
	#include <stdio.h>
	#include <string.h>
	#include <unistd.h>

	#include <openssl/err.h>
	#include <openssl/ssl.h>

	#include <compat/compat.h>
	#ifndef HAVE_CLOSEFROM
	#include <compat/closefrom.h>
	#endif
	#ifndef HAVE_STRLCPY
	#include <compat/strlcpy.h>
	#endif

	#include "pjdlog.h"
	#include "proto_impl.h"
	#include "sandbox.h"
	#include "subr.h"

	#define TLS_CTX_MAGIC 0x715c7
	struct tls_ctx {
	int tls_magic;
	struct proto_conn *tls_sock;
	struct proto_conn *tls_tcp;
	char tls_laddr[256];
	char tls_raddr[256];
	int tls_side;
	#define TLS_SIDE_CLIENT 0
	#define TLS_SIDE_SERVER_LISTEN 1
	#define TLS_SIDE_SERVER_WORK 2
	bool tls_wait_called;
	};

	#define TLS_DEFAULT_TIMEOUT 30

	static int tls_connect_wait(void *ctx, int timeout);
	static void tls_close(void *ctx);

	static void
	block(int fd)
	{
	int flags;

	flags = fcntl(fd, F_GETFL);
	if (flags == -1)
	pjdlog_exit(EX_TEMPFAIL, "fcntl(F_GETFL) failed");
	flags &= ~O_NONBLOCK;
	if (fcntl(fd, F_SETFL, flags) == -1)
	pjdlog_exit(EX_TEMPFAIL, "fcntl(F_SETFL) failed");
	}

	static void
	nonblock(int fd)
	{
	int flags;

	flags = fcntl(fd, F_GETFL);
	if (flags == -1)
	pjdlog_exit(EX_TEMPFAIL, "fcntl(F_GETFL) failed");
	flags \|= O_NONBLOCK;
	if (fcntl(fd, F_SETFL, flags) == -1)
	pjdlog_exit(EX_TEMPFAIL, "fcntl(F_SETFL) failed");
	}

	static int
	wait_for_fd(int fd, int timeout)
	{
	struct timeval tv;
	fd_set fdset;
	int error, ret;

	error = 0;

	for (;;) {
	FD_ZERO(&fdset);
	FD_SET(fd, &fdset);

	tv.tv_sec = timeout;
	tv.tv_usec = 0;

	ret = select(fd + 1, NULL, &fdset, NULL,
	timeout == -1 ? NULL : &tv);
	if (ret == 0) {
	error = ETIMEDOUT;
	break;
	} else if (ret == -1) {
	if (errno == EINTR)
	continue;
	error = errno;
	break;
	}
	PJDLOG_ASSERT(ret > 0);
	PJDLOG_ASSERT(FD_ISSET(fd, &fdset));
	break;
	}

	return (error);
	}

	static void
	ssl_log_errors(void)
	{
	unsigned long error;

	while ((error = ERR_get_error()) != 0)
	pjdlog_error("SSL error: %s", ERR_error_string(error, NULL));
	}

	static int
	ssl_check_error(SSL *ssl, int ret)
	{
	int error;

	error = SSL_get_error(ssl, ret);

	switch (error) {
	case SSL_ERROR_NONE:
	return (0);
	case SSL_ERROR_WANT_READ:
	pjdlog_debug(2, "SSL_ERROR_WANT_READ");
	return (-1);
	case SSL_ERROR_WANT_WRITE:
	pjdlog_debug(2, "SSL_ERROR_WANT_WRITE");
	return (-1);
	case SSL_ERROR_ZERO_RETURN:
	pjdlog_exitx(EX_OK, "Connection closed.");
	case SSL_ERROR_SYSCALL:
	ssl_log_errors();
	pjdlog_exitx(EX_TEMPFAIL, "SSL I/O error.");
	case SSL_ERROR_SSL:
	ssl_log_errors();
	pjdlog_exitx(EX_TEMPFAIL, "SSL protocol error.");
	default:
	ssl_log_errors();
	pjdlog_exitx(EX_TEMPFAIL, "Unknown SSL error (%d).", error);
	}
	}

	static void
	tcp_recv_ssl_send(int recvfd, SSL *sendssl)
	{
	static unsigned char buf[65536];
	ssize_t tcpdone;
	int sendfd, ssldone;

	sendfd = SSL_get_fd(sendssl);
	PJDLOG_ASSERT(sendfd >= 0);
	pjdlog_debug(2, "%s: start %d -> %d", __func__, recvfd, sendfd);
	for (;;) {
	tcpdone = recv(recvfd, buf, sizeof(buf), 0);
	pjdlog_debug(2, "%s: recv() returned %zd", __func__, tcpdone);
	if (tcpdone == 0) {
	pjdlog_debug(1, "Connection terminated.");
	exit(0);
	} else if (tcpdone == -1) {
	if (errno == EINTR)
	continue;
	else if (errno == EAGAIN)
	break;
	pjdlog_exit(EX_TEMPFAIL, "recv() failed");
	}
	for (;;) {
	ssldone = SSL_write(sendssl, buf, (int)tcpdone);
	pjdlog_debug(2, "%s: send() returned %d", __func__,
	ssldone);
	if (ssl_check_error(sendssl, ssldone) == -1) {
	(void)wait_for_fd(sendfd, -1);
	continue;
	}
	PJDLOG_ASSERT(ssldone == tcpdone);
	break;
	}
	}
	pjdlog_debug(2, "%s: done %d -> %d", __func__, recvfd, sendfd);
	}

	static void
	ssl_recv_tcp_send(SSL *recvssl, int sendfd)
	{
	static unsigned char buf[65536];
	unsigned char *ptr;
	ssize_t tcpdone;
	size_t todo;
	int recvfd, ssldone;

	recvfd = SSL_get_fd(recvssl);
	PJDLOG_ASSERT(recvfd >= 0);
	pjdlog_debug(2, "%s: start %d -> %d", __func__, recvfd, sendfd);
	for (;;) {
	ssldone = SSL_read(recvssl, buf, sizeof(buf));
	pjdlog_debug(2, "%s: SSL_read() returned %d", __func__,
	ssldone);
	if (ssl_check_error(recvssl, ssldone) == -1)
	break;
	todo = (size_t)ssldone;
	ptr = buf;
	do {
	tcpdone = send(sendfd, ptr, todo, MSG_NOSIGNAL);
	pjdlog_debug(2, "%s: send() returned %zd", __func__,
	tcpdone);
	if (tcpdone == 0) {
	pjdlog_debug(1, "Connection terminated.");
	exit(0);
	} else if (tcpdone == -1) {
	if (errno == EINTR \|\| errno == ENOBUFS)
	continue;
	if (errno == EAGAIN) {
	(void)wait_for_fd(sendfd, -1);
	continue;
	}
	pjdlog_exit(EX_TEMPFAIL, "send() failed");
	}
	todo -= tcpdone;
	ptr += tcpdone;
	} while (todo > 0);
	}
	pjdlog_debug(2, "%s: done %d -> %d", __func__, recvfd, sendfd);
	}

	static void
	tls_loop(int sockfd, SSL *tcpssl)
	{
	fd_set fds;
	int maxfd, tcpfd;

	tcpfd = SSL_get_fd(tcpssl);
	PJDLOG_ASSERT(tcpfd >= 0);

	for (;;) {
	FD_ZERO(&fds);
	FD_SET(sockfd, &fds);
	FD_SET(tcpfd, &fds);
	maxfd = MAX(sockfd, tcpfd);

	PJDLOG_ASSERT(maxfd + 1 <= (int)FD_SETSIZE);
	if (select(maxfd + 1, &fds, NULL, NULL, NULL) == -1) {
	if (errno == EINTR)
	continue;
	pjdlog_exit(EX_TEMPFAIL, "select() failed");
	}
	if (FD_ISSET(sockfd, &fds))
	tcp_recv_ssl_send(sockfd, tcpssl);
	if (FD_ISSET(tcpfd, &fds))
	ssl_recv_tcp_send(tcpssl, sockfd);
	}
	}

	static void
	tls_certificate_verify(SSL ssl, const char fingerprint)
	{
	unsigned char md[EVP_MAX_MD_SIZE];
	char mdstr[sizeof("SHA256=") - 1 + EVP_MAX_MD_SIZE * 3];
	char *mdstrp;
	unsigned int i, mdsize;
	X509 *cert;

	if (fingerprint[0] == '\0') {
	pjdlog_debug(1, "No fingerprint verification requested.");
	return;
	}

	cert = SSL_get_peer_certificate(ssl);
	if (cert == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "No peer certificate received.");

	if (X509_digest(cert, EVP_sha256(), md, &mdsize) != 1)
	pjdlog_exitx(EX_TEMPFAIL, "X509_digest() failed.");
	PJDLOG_ASSERT(mdsize <= EVP_MAX_MD_SIZE);

	X509_free(cert);

	(void)strlcpy(mdstr, "SHA256=", sizeof(mdstr));
	mdstrp = mdstr + strlen(mdstr);
	for (i = 0; i < mdsize; i++) {
	PJDLOG_VERIFY(mdstrp + 3 <= mdstr + sizeof(mdstr));
	(void)sprintf(mdstrp, "%02hhX:", md[i]);
	mdstrp += 3;
	}
	/* Clear last colon. */
	mdstrp[-1] = '\0';
	if (strcasecmp(mdstr, fingerprint) != 0) {
	pjdlog_exitx(EX_NOPERM,
	"Finger print doesn't match. Received \"%s\", expected \"%s\"",
	mdstr, fingerprint);
	}
	}

	static void
	tls_exec_client(const char user, int startfd, const char srcaddr,
	const char dstaddr, const char fingerprint, const char *defport,
	int timeout, int debuglevel)
	{
	struct proto_conn *tcp;
	char saddr, daddr;
	SSL_CTX *sslctx;
	SSL *ssl;
	long ret;
	int sockfd, tcpfd;
	uint8_t connected;

	pjdlog_debug_set(debuglevel);
	pjdlog_prefix_set("[TLS sandbox] (client) ");
	#ifdef HAVE_SETPROCTITLE
	setproctitle("[TLS sandbox] (client) ");
	#endif
	proto_set("tcp:port", defport);

	sockfd = startfd;

	/* Change tls:// to tcp://. */
	if (srcaddr == NULL) {
	saddr = NULL;
	} else {
	saddr = strdup(srcaddr);
	if (saddr == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory.");
	bcopy("tcp://", saddr, 6);
	}
	daddr = strdup(dstaddr);
	if (daddr == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "Unable to allocate memory.");
	bcopy("tcp://", daddr, 6);

	/* Establish TCP connection. */
	if (proto_connect(saddr, daddr, timeout, &tcp) == -1)
	exit(EX_TEMPFAIL);

	SSL_load_error_strings();
	SSL_library_init();

	/*
	* TODO: On FreeBSD we could move this below sandbox() once libc and
	* libcrypto use sysctl kern.arandom to obtain random data
	* instead of /dev/urandom and friends.
	*/
	sslctx = SSL_CTX_new(TLS_client_method());
	if (sslctx == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "SSL_CTX_new() failed.");

	if (sandbox(user, true, "proto_tls client: %s", dstaddr) != 0)
	pjdlog_exitx(EX_CONFIG, "Unable to sandbox TLS client.");
	pjdlog_debug(1, "Privileges successfully dropped.");

	SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2 \| SSL_OP_NO_SSLv3);

	/* Load CA certs. */
	/* TODO */
	//SSL_CTX_load_verify_locations(sslctx, cacerts_file, NULL);

	ssl = SSL_new(sslctx);
	if (ssl == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "SSL_new() failed.");

	tcpfd = proto_descriptor(tcp);

	block(tcpfd);

	if (SSL_set_fd(ssl, tcpfd) != 1)
	pjdlog_exitx(EX_TEMPFAIL, "SSL_set_fd() failed.");

	ret = SSL_connect(ssl);
	ssl_check_error(ssl, (int)ret);

	nonblock(sockfd);
	nonblock(tcpfd);

	tls_certificate_verify(ssl, fingerprint);

	/*
	- * The following byte is send to make proto_connect_wait() to work.
	+ * The following byte is sent to make proto_connect_wait() work.
	*/
	connected = 1;
	for (;;) {
	switch (send(sockfd, &connected, sizeof(connected), 0)) {
	case -1:
	if (errno == EINTR \|\| errno == ENOBUFS)
	continue;
	if (errno == EAGAIN) {
	(void)wait_for_fd(sockfd, -1);
	continue;
	}
	pjdlog_exit(EX_TEMPFAIL, "send() failed");
	case 0:
	pjdlog_debug(1, "Connection terminated.");
	exit(0);
	case 1:
	break;
	}
	break;
	}

	tls_loop(sockfd, ssl);
	}

	static void
	tls_call_exec_client(struct proto_conn sock, const char srcaddr,
	const char *dstaddr, int timeout)
	{
	char timeoutstr, startfdstr, *debugstr;
	int startfd;

	/* Declare that we are receiver. */
	proto_recv(sock, NULL, 0);

	if (pjdlog_mode_get() == PJDLOG_MODE_STD)
	startfd = 3;
	else /* if (pjdlog_mode_get() == PJDLOG_MODE_SYSLOG) */
	startfd = 0;

	if (proto_descriptor(sock) != startfd) {
	/* Move socketpair descriptor to descriptor number startfd. */
	if (dup2(proto_descriptor(sock), startfd) == -1)
	pjdlog_exit(EX_OSERR, "dup2() failed");
	proto_close(sock);
	} else {
	/*
	- * The FD_CLOEXEC is cleared by dup2(2), so when we not
	+ * The FD_CLOEXEC is cleared by dup2(2), so when we do not
	* call it, we have to clear it by hand in case it is set.
	*/
	if (fcntl(startfd, F_SETFD, 0) == -1)
	pjdlog_exit(EX_OSERR, "fcntl() failed");
	}

	closefrom(startfd + 1);

	if (asprintf(&startfdstr, "%d", startfd) == -1)
	pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
	if (timeout == -1)
	timeout = TLS_DEFAULT_TIMEOUT;
	if (asprintf(&timeoutstr, "%d", timeout) == -1)
	pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
	if (asprintf(&debugstr, "%d", pjdlog_debug_get()) == -1)
	pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");

	execl(proto_get("execpath"), proto_get("execpath"), "proto", "tls",
	proto_get("user"), "client", startfdstr,
	srcaddr == NULL ? "" : srcaddr, dstaddr,
	proto_get("tls:fingerprint"), proto_get("tcp:port"), timeoutstr,
	debugstr, NULL);
	pjdlog_exit(EX_SOFTWARE, "execl() failed");
	}

	static int
	tls_connect(const char srcaddr, const char dstaddr, int timeout, void **ctxp)
	{
	struct tls_ctx *tlsctx;
	struct proto_conn *sock;
	pid_t pid;
	int error;

	PJDLOG_ASSERT(srcaddr == NULL \|\| srcaddr[0] != '\0');
	PJDLOG_ASSERT(dstaddr != NULL);
	PJDLOG_ASSERT(timeout >= -1);
	PJDLOG_ASSERT(ctxp != NULL);

	if (strncmp(dstaddr, "tls://", 6) != 0)
	return (-1);
	if (srcaddr != NULL && strncmp(srcaddr, "tls://", 6) != 0)
	return (-1);

	if (proto_connect(NULL, "socketpair://", -1, &sock) == -1)
	return (errno);

	#if 0
	/*
	* We use rfork() with the following flags to disable SIGCHLD
	* delivery upon the sandbox process exit.
	*/
	pid = rfork(RFFDG \| RFPROC \| RFTSIGZMB \| RFTSIGFLAGS(0));
	#else
	/*
	* We don't use rfork() to be able to log information about sandbox
	* process exiting.
	*/
	pid = fork();
	#endif
	switch (pid) {
	case -1:
	/* Failure. */
	error = errno;
	proto_close(sock);
	return (error);
	case 0:
	/* Child. */
	pjdlog_prefix_set("[TLS sandbox] (client) ");
	#ifdef HAVE_SETPROCTITLE
	setproctitle("[TLS sandbox] (client) ");
	#endif
	tls_call_exec_client(sock, srcaddr, dstaddr, timeout);
	/* NOTREACHED */
	default:
	/* Parent. */
	tlsctx = calloc(1, sizeof(*tlsctx));
	if (tlsctx == NULL) {
	error = errno;
	proto_close(sock);
	(void)kill(pid, SIGKILL);
	return (error);
	}
	proto_send(sock, NULL, 0);
	tlsctx->tls_sock = sock;
	tlsctx->tls_tcp = NULL;
	tlsctx->tls_side = TLS_SIDE_CLIENT;
	tlsctx->tls_wait_called = false;
	tlsctx->tls_magic = TLS_CTX_MAGIC;
	if (timeout >= 0) {
	error = tls_connect_wait(tlsctx, timeout);
	if (error != 0) {
	(void)kill(pid, SIGKILL);
	tls_close(tlsctx);
	return (error);
	}
	}
	*ctxp = tlsctx;
	return (0);
	}
	}

	static int
	tls_connect_wait(void *ctx, int timeout)
	{
	struct tls_ctx *tlsctx = ctx;
	int error, sockfd;
	uint8_t connected;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
	PJDLOG_ASSERT(tlsctx->tls_side == TLS_SIDE_CLIENT);
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);
	PJDLOG_ASSERT(!tlsctx->tls_wait_called);
	PJDLOG_ASSERT(timeout >= 0);

	sockfd = proto_descriptor(tlsctx->tls_sock);
	error = wait_for_fd(sockfd, timeout);
	if (error != 0)
	return (error);

	for (;;) {
	switch (recv(sockfd, &connected, sizeof(connected),
	MSG_WAITALL)) {
	case -1:
	if (errno == EINTR \|\| errno == ENOBUFS)
	continue;
	error = errno;
	break;
	case 0:
	pjdlog_debug(1, "Connection terminated.");
	error = ENOTCONN;
	break;
	case 1:
	tlsctx->tls_wait_called = true;
	break;
	}
	break;
	}

	return (error);
	}

	static int
	tls_server(const char lstaddr, void *ctxp)
	{
	struct proto_conn *tcp;
	struct tls_ctx *tlsctx;
	char *laddr;
	int error;

	if (strncmp(lstaddr, "tls://", 6) != 0)
	return (-1);

	tlsctx = malloc(sizeof(*tlsctx));
	if (tlsctx == NULL) {
	pjdlog_warning("Unable to allocate memory.");
	return (ENOMEM);
	}

	laddr = strdup(lstaddr);
	if (laddr == NULL) {
	free(tlsctx);
	pjdlog_warning("Unable to allocate memory.");
	return (ENOMEM);
	}
	bcopy("tcp://", laddr, 6);

	if (proto_server(laddr, &tcp) == -1) {
	error = errno;
	free(tlsctx);
	free(laddr);
	return (error);
	}
	free(laddr);

	tlsctx->tls_sock = NULL;
	tlsctx->tls_tcp = tcp;
	tlsctx->tls_side = TLS_SIDE_SERVER_LISTEN;
	tlsctx->tls_wait_called = true;
	tlsctx->tls_magic = TLS_CTX_MAGIC;
	*ctxp = tlsctx;

	return (0);
	}

	static void
	tls_exec_server(const char user, int startfd, const char privkey,
	const char *cert, int debuglevel)
	{
	SSL_CTX *sslctx;
	SSL *ssl;
	int sockfd, tcpfd, ret;

	pjdlog_debug_set(debuglevel);
	pjdlog_prefix_set("[TLS sandbox] (server) ");
	#ifdef HAVE_SETPROCTITLE
	setproctitle("[TLS sandbox] (server) ");
	#endif

	sockfd = startfd;
	tcpfd = startfd + 1;

	SSL_load_error_strings();
	SSL_library_init();

	sslctx = SSL_CTX_new(TLS_server_method());
	if (sslctx == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "SSL_CTX_new() failed.");

	SSL_CTX_set_options(sslctx, SSL_OP_NO_SSLv2 \| SSL_OP_NO_SSLv3);

	ssl = SSL_new(sslctx);
	if (ssl == NULL)
	pjdlog_exitx(EX_TEMPFAIL, "SSL_new() failed.");

	if (SSL_use_RSAPrivateKey_file(ssl, privkey, SSL_FILETYPE_PEM) != 1) {
	ssl_log_errors();
	pjdlog_exitx(EX_CONFIG,
	"SSL_use_RSAPrivateKey_file(%s) failed.", privkey);
	}

	if (SSL_use_certificate_file(ssl, cert, SSL_FILETYPE_PEM) != 1) {
	ssl_log_errors();
	pjdlog_exitx(EX_CONFIG, "SSL_use_certificate_file(%s) failed.",
	cert);
	}

	if (sandbox(user, true, "proto_tls server") != 0)
	pjdlog_exitx(EX_CONFIG, "Unable to sandbox TLS server.");
	pjdlog_debug(1, "Privileges successfully dropped.");

	nonblock(sockfd);
	nonblock(tcpfd);

	if (SSL_set_fd(ssl, tcpfd) != 1)
	pjdlog_exitx(EX_TEMPFAIL, "SSL_set_fd() failed.");

	ret = SSL_accept(ssl);
	ssl_check_error(ssl, ret);

	tls_loop(sockfd, ssl);
	}

	static void
	tls_call_exec_server(struct proto_conn sock, struct proto_conn tcp)
	{
	int startfd, sockfd, tcpfd, safefd;
	char startfdstr, debugstr;

	if (pjdlog_mode_get() == PJDLOG_MODE_STD)
	startfd = 3;
	else /* if (pjdlog_mode_get() == PJDLOG_MODE_SYSLOG) */
	startfd = 0;

	/* Declare that we are receiver. */
	proto_send(sock, NULL, 0);

	sockfd = proto_descriptor(sock);
	tcpfd = proto_descriptor(tcp);

	safefd = MAX(sockfd, tcpfd);
	safefd = MAX(safefd, startfd);
	safefd++;

	/* Move sockfd and tcpfd to safe numbers first. */
	if (dup2(sockfd, safefd) == -1)
	pjdlog_exit(EX_OSERR, "dup2() failed");
	proto_close(sock);
	sockfd = safefd;
	if (dup2(tcpfd, safefd + 1) == -1)
	pjdlog_exit(EX_OSERR, "dup2() failed");
	proto_close(tcp);
	tcpfd = safefd + 1;

	/* Move socketpair descriptor to descriptor number startfd. */
	if (dup2(sockfd, startfd) == -1)
	pjdlog_exit(EX_OSERR, "dup2() failed");
	(void)close(sockfd);
	/* Move tcp descriptor to descriptor number startfd + 1. */
	if (dup2(tcpfd, startfd + 1) == -1)
	pjdlog_exit(EX_OSERR, "dup2() failed");
	(void)close(tcpfd);

	closefrom(startfd + 2);

	/*
	* Even if FD_CLOEXEC was set on descriptors before dup2(), it should
	* have been cleared on dup2(), but better be safe than sorry.
	*/
	if (fcntl(startfd, F_SETFD, 0) == -1)
	pjdlog_exit(EX_OSERR, "fcntl() failed");
	if (fcntl(startfd + 1, F_SETFD, 0) == -1)
	pjdlog_exit(EX_OSERR, "fcntl() failed");

	if (asprintf(&startfdstr, "%d", startfd) == -1)
	pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");
	if (asprintf(&debugstr, "%d", pjdlog_debug_get()) == -1)
	pjdlog_exit(EX_TEMPFAIL, "asprintf() failed");

	execl(proto_get("execpath"), proto_get("execpath"), "proto", "tls",
	proto_get("user"), "server", startfdstr, proto_get("tls:keyfile"),
	proto_get("tls:certfile"), debugstr, NULL);
	pjdlog_exit(EX_SOFTWARE, "execl() failed");
	}

	static int
	tls_accept(void ctx, void *newctxp)
	{
	struct tls_ctx *tlsctx = ctx;
	struct tls_ctx *newtlsctx;
	struct proto_conn sock, tcp;
	pid_t pid;
	int error;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
	PJDLOG_ASSERT(tlsctx->tls_side == TLS_SIDE_SERVER_LISTEN);

	if (proto_connect(NULL, "socketpair://", -1, &sock) == -1)
	return (errno);

	/* Accept TCP connection. */
	if (proto_accept(tlsctx->tls_tcp, &tcp) == -1) {
	error = errno;
	proto_close(sock);
	return (error);
	}

	pid = fork();
	switch (pid) {
	case -1:
	/* Failure. */
	error = errno;
	proto_close(sock);
	return (error);
	case 0:
	/* Child. */
	pjdlog_prefix_set("[TLS sandbox] (server) ");
	#ifdef HAVE_SETPROCTITLE
	setproctitle("[TLS sandbox] (server) ");
	#endif
	/* Close listen socket. */
	proto_close(tlsctx->tls_tcp);
	tls_call_exec_server(sock, tcp);
	/* NOTREACHED */
	PJDLOG_ABORT("Unreachable.");
	default:
	/* Parent. */
	newtlsctx = calloc(1, sizeof(*tlsctx));
	if (newtlsctx == NULL) {
	error = errno;
	proto_close(sock);
	proto_close(tcp);
	(void)kill(pid, SIGKILL);
	return (error);
	}
	proto_local_address(tcp, newtlsctx->tls_laddr,
	sizeof(newtlsctx->tls_laddr));
	PJDLOG_ASSERT(strncmp(newtlsctx->tls_laddr, "tcp://", 6) == 0);
	bcopy("tls://", newtlsctx->tls_laddr, 6);
	*strrchr(newtlsctx->tls_laddr, ':') = '\0';
	proto_remote_address(tcp, newtlsctx->tls_raddr,
	sizeof(newtlsctx->tls_raddr));
	PJDLOG_ASSERT(strncmp(newtlsctx->tls_raddr, "tcp://", 6) == 0);
	bcopy("tls://", newtlsctx->tls_raddr, 6);
	*strrchr(newtlsctx->tls_raddr, ':') = '\0';
	proto_close(tcp);
	proto_recv(sock, NULL, 0);
	newtlsctx->tls_sock = sock;
	newtlsctx->tls_tcp = NULL;
	newtlsctx->tls_wait_called = true;
	newtlsctx->tls_side = TLS_SIDE_SERVER_WORK;
	newtlsctx->tls_magic = TLS_CTX_MAGIC;
	*newctxp = newtlsctx;
	return (0);
	}
	}

	static int
	tls_wrap(int fd, bool client, void **ctxp)
	{
	struct tls_ctx *tlsctx;
	struct proto_conn *sock;
	int error;

	tlsctx = calloc(1, sizeof(*tlsctx));
	if (tlsctx == NULL)
	return (errno);

	if (proto_wrap("socketpair", client, fd, &sock) == -1) {
	error = errno;
	free(tlsctx);
	return (error);
	}

	tlsctx->tls_sock = sock;
	tlsctx->tls_tcp = NULL;
	tlsctx->tls_wait_called = (client ? false : true);
	tlsctx->tls_side = (client ? TLS_SIDE_CLIENT : TLS_SIDE_SERVER_WORK);
	tlsctx->tls_magic = TLS_CTX_MAGIC;
	*ctxp = tlsctx;

	return (0);
	}

	static int
	tls_send(void ctx, const unsigned char data, size_t size, int fd)
	{
	struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
	PJDLOG_ASSERT(tlsctx->tls_side == TLS_SIDE_CLIENT \|\|
	tlsctx->tls_side == TLS_SIDE_SERVER_WORK);
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);
	PJDLOG_ASSERT(tlsctx->tls_wait_called);
	PJDLOG_ASSERT(fd == -1);

	if (proto_send(tlsctx->tls_sock, data, size) == -1)
	return (errno);

	return (0);
	}

	static int
	tls_recv(void ctx, unsigned char data, size_t size, int *fdp)
	{
	struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
	PJDLOG_ASSERT(tlsctx->tls_side == TLS_SIDE_CLIENT \|\|
	tlsctx->tls_side == TLS_SIDE_SERVER_WORK);
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);
	PJDLOG_ASSERT(tlsctx->tls_wait_called);
	PJDLOG_ASSERT(fdp == NULL);

	if (proto_recv(tlsctx->tls_sock, data, size) == -1)
	return (errno);

	return (0);
	}

	static int
	tls_descriptor(const void *ctx)
	{
	const struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);

	switch (tlsctx->tls_side) {
	case TLS_SIDE_CLIENT:
	case TLS_SIDE_SERVER_WORK:
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);

	return (proto_descriptor(tlsctx->tls_sock));
	case TLS_SIDE_SERVER_LISTEN:
	PJDLOG_ASSERT(tlsctx->tls_tcp != NULL);

	return (proto_descriptor(tlsctx->tls_tcp));
	default:
	PJDLOG_ABORT("Invalid side (%d).", tlsctx->tls_side);
	}
	}

	static bool
	tcp_address_match(const void ctx, const char addr)
	{
	const struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);

	return (strcmp(tlsctx->tls_raddr, addr) == 0);
	}

	static void
	tls_local_address(const void ctx, char addr, size_t size)
	{
	const struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
	PJDLOG_ASSERT(tlsctx->tls_wait_called);

	switch (tlsctx->tls_side) {
	case TLS_SIDE_CLIENT:
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);

	PJDLOG_VERIFY(strlcpy(addr, "tls://N/A", size) < size);
	break;
	case TLS_SIDE_SERVER_WORK:
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);

	PJDLOG_VERIFY(strlcpy(addr, tlsctx->tls_laddr, size) < size);
	break;
	case TLS_SIDE_SERVER_LISTEN:
	PJDLOG_ASSERT(tlsctx->tls_tcp != NULL);

	proto_local_address(tlsctx->tls_tcp, addr, size);
	PJDLOG_ASSERT(strncmp(addr, "tcp://", 6) == 0);
	/* Replace tcp:// prefix with tls:// */
	bcopy("tls://", addr, 6);
	break;
	default:
	PJDLOG_ABORT("Invalid side (%d).", tlsctx->tls_side);
	}
	}

	static void
	tls_remote_address(const void ctx, char addr, size_t size)
	{
	const struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);
	PJDLOG_ASSERT(tlsctx->tls_wait_called);

	switch (tlsctx->tls_side) {
	case TLS_SIDE_CLIENT:
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);

	PJDLOG_VERIFY(strlcpy(addr, "tls://N/A", size) < size);
	break;
	case TLS_SIDE_SERVER_WORK:
	PJDLOG_ASSERT(tlsctx->tls_sock != NULL);

	PJDLOG_VERIFY(strlcpy(addr, tlsctx->tls_raddr, size) < size);
	break;
	case TLS_SIDE_SERVER_LISTEN:
	PJDLOG_ASSERT(tlsctx->tls_tcp != NULL);

	proto_remote_address(tlsctx->tls_tcp, addr, size);
	PJDLOG_ASSERT(strncmp(addr, "tcp://", 6) == 0);
	/* Replace tcp:// prefix with tls:// */
	bcopy("tls://", addr, 6);
	break;
	default:
	PJDLOG_ABORT("Invalid side (%d).", tlsctx->tls_side);
	}
	}

	static void
	tls_close(void *ctx)
	{
	struct tls_ctx *tlsctx = ctx;

	PJDLOG_ASSERT(tlsctx != NULL);
	PJDLOG_ASSERT(tlsctx->tls_magic == TLS_CTX_MAGIC);

	if (tlsctx->tls_sock != NULL) {
	proto_close(tlsctx->tls_sock);
	tlsctx->tls_sock = NULL;
	}
	if (tlsctx->tls_tcp != NULL) {
	proto_close(tlsctx->tls_tcp);
	tlsctx->tls_tcp = NULL;
	}
	tlsctx->tls_side = 0;
	tlsctx->tls_magic = 0;
	free(tlsctx);
	}

	static int
	tls_exec(int argc, char *argv[])
	{

	PJDLOG_ASSERT(argc > 3);
	PJDLOG_ASSERT(strcmp(argv[0], "tls") == 0);

	pjdlog_init(atoi(argv[3]) == 0 ? PJDLOG_MODE_SYSLOG : PJDLOG_MODE_STD);

	if (strcmp(argv[2], "client") == 0) {
	if (argc != 10)
	return (EINVAL);
	tls_exec_client(argv[1], atoi(argv[3]),
	argv[4][0] == '\0' ? NULL : argv[4], argv[5], argv[6],
	argv[7], atoi(argv[8]), atoi(argv[9]));
	} else if (strcmp(argv[2], "server") == 0) {
	if (argc != 7)
	return (EINVAL);
	tls_exec_server(argv[1], atoi(argv[3]), argv[4], argv[5],
	atoi(argv[6]));
	}
	return (EINVAL);
	}

	static struct proto tls_proto = {
	.prt_name = "tls",
	.prt_connect = tls_connect,
	.prt_connect_wait = tls_connect_wait,
	.prt_server = tls_server,
	.prt_accept = tls_accept,
	.prt_wrap = tls_wrap,
	.prt_send = tls_send,
	.prt_recv = tls_recv,
	.prt_descriptor = tls_descriptor,
	.prt_address_match = tcp_address_match,
	.prt_local_address = tls_local_address,
	.prt_remote_address = tls_remote_address,
	.prt_close = tls_close,
	.prt_exec = tls_exec
	};

	static __constructor void
	tls_ctor(void)
	{

	proto_register(&tls_proto, false);
	}
	diff --git a/contrib/openbsm/libauditd/auditd_lib.c b/contrib/openbsm/libauditd/auditd_lib.c
	index 1e21adb39bb7..e6c1312bf49f 100644
	--- a/contrib/openbsm/libauditd/auditd_lib.c
	+++ b/contrib/openbsm/libauditd/auditd_lib.c
	@@ -1,1294 +1,1294 @@
	/*-
	* Copyright (c) 2008-2009 Apple Inc.
	* Copyright (c) 2016 Robert N. M. Watson
	* All rights reserved.
	*
	* Portions of this software were developed by BAE Systems, the University of
	* Cambridge Computer Laboratory, and Memorial University under DARPA/AFRL
	* contract FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent
	* Computing (TC) research program.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of Apple Inc. ("Apple") nor the names of
	* its contributors may be used to endorse or promote products derived
	* from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR
	* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
	* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/param.h>

	#include <config/config.h>

	#include <sys/dirent.h>
	#ifdef HAVE_FULL_QUEUE_H
	#include <sys/queue.h>
	#else /* !HAVE_FULL_QUEUE_H */
	#include <compat/queue.h>
	#endif /* !HAVE_FULL_QUEUE_H */
	#include <sys/mount.h>
	#include <sys/socket.h>

	#include <sys/stat.h>
	#include <sys/time.h>

	#include <netinet/in.h>

	#include <bsm/audit.h>
	#include <bsm/audit_uevents.h>
	#include <bsm/auditd_lib.h>
	#include <bsm/libbsm.h>

	#include <assert.h>
	#include <dirent.h>
	#include <err.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <stdio.h>
	#include <string.h>
	#include <stdlib.h>
	#include <time.h>
	#include <unistd.h>
	#include <netdb.h>

	#ifdef __APPLE__
	#include <notify.h>
	#ifndef __BSM_INTERNAL_NOTIFY_KEY
	#define __BSM_INTERNAL_NOTIFY_KEY "com.apple.audit.change"
	#endif /* __BSM_INTERNAL_NOTIFY_KEY */
	#endif /* __APPLE__ */

	/*
	* XXX This is temporary until this is moved to <bsm/audit.h> and shared with
	* the kernel.
	*/
	#ifndef AUDIT_HARD_LIMIT_FREE_BLOCKS
	#define AUDIT_HARD_LIMIT_FREE_BLOCKS 4
	#endif

	/*
	* Number of seconds to January 1, 2000
	*/
	#define JAN_01_2000 946598400

	struct dir_ent {
	char *dirname;
	uint8_t softlim;
	uint8_t hardlim;
	TAILQ_ENTRY(dir_ent) dirs;
	};

	static TAILQ_HEAD(, dir_ent) dir_q;

	struct audit_trail {
	time_t at_time;
	char *at_path;
	off_t at_size;

	TAILQ_ENTRY(audit_trail) at_trls;
	};

	static int auditd_minval = -1;
	static int auditd_dist = 0;

	static char auditd_host[MAXHOSTNAMELEN];
	static int auditd_hostlen = -1;

	static char *auditd_errmsg[] = {
	"no error", /* ADE_NOERR ( 0) */
	"could not parse audit_control(5) file", /* ADE_PARSE ( 1) */
	"auditon(2) failed", /* ADE_AUDITON ( 2) */
	"malloc(3) failed", /* ADE_NOMEM ( 3) */
	"all audit log directories over soft limit", /* ADE_SOFTLIM ( 4) */
	"all audit log directories over hard limit", /* ADE_HARDLIM ( 5) */
	"could not create file name string", /* ADE_STRERR ( 6) */
	"could not open audit record", /* ADE_AU_OPEN ( 7) */
	"could not close audit record", /* ADE_AU_CLOSE ( 8) */
	"could not set active audit session state", /* ADE_SETAUDIT ( 9) */
	"auditctl(2) failed (trail still swapped)", /* ADE_ACTL (10) */
	"auditctl(2) failed (trail not swapped)", /* ADE_ACTLERR (11) */
	"could not swap audit trail file", /* ADE_SWAPERR (12) */
	"could not rename crash recovery file", /* ADE_RENAME (13) */
	"could not read 'current' link file", /* ADE_READLINK (14) */
	"could not create 'current' link file", /* ADE_SYMLINK (15) */
	"invalid argument", /* ADE_INVAL (16) */
	"could not resolve hostname to address", /* ADE_GETADDR (17) */
	"address family not supported", /* ADE_ADDRFAM (18) */
	"error expiring audit trail files", /* ADE_EXPIRE (19) */
	};

	#define MAXERRCODE (sizeof(auditd_errmsg) / sizeof(auditd_errmsg[0]))

	#define NA_EVENT_STR_SIZE 128
	#define POL_STR_SIZE 128


	/*
	* Look up and return the error string for the given audit error code.
	*/
	const char *
	auditd_strerror(int errcode)
	{
	int idx = -errcode;

	if (idx < 0 \|\| idx > (int)MAXERRCODE)
	return ("Invalid auditd error code");

	return (auditd_errmsg[idx]);
	}


	/*
	* Free our local list of directory names and init list.
	*/
	static void
	free_dir_q(void)
	{
	struct dir_ent d1, d2;

	d1 = TAILQ_FIRST(&dir_q);
	while (d1 != NULL) {
	d2 = TAILQ_NEXT(d1, dirs);
	free(d1->dirname);
	free(d1);
	d1 = d2;
	}
	TAILQ_INIT(&dir_q);
	}

	/*
	* Concat the directory name to the given file name.
	* XXX We should affix the hostname also
	*/
	static char *
	affixdir(char name, struct dir_ent dirent)
	{
	char *fn = NULL;

	/*
	* Sanity check on file name.
	*/
	if (strlen(name) != FILENAME_LEN) {
	errno = EINVAL;
	return (NULL);
	}

	/*
	* If the host is set then also add the hostname to the filename.
	*/
	if (auditd_hostlen > 0)
	asprintf(&fn, "%s/%s.%s", dirent->dirname, name, auditd_host);
	else
	asprintf(&fn, "%s/%s", dirent->dirname, name);
	return (fn);
	}

	/*
	* Insert the directory entry in the list by the way they are ordered in
	* audit_control(5). Move the entries that are over the soft and hard limits
	* toward the tail.
	*/
	static void
	insert_orderly(struct dir_ent *denew)
	{
	struct dir_ent *dep;

	TAILQ_FOREACH(dep, &dir_q, dirs) {
	if (dep->softlim == 1 && denew->softlim == 0) {
	TAILQ_INSERT_BEFORE(dep, denew, dirs);
	return;
	}
	if (dep->hardlim == 1 && denew->hardlim == 0) {
	TAILQ_INSERT_BEFORE(dep, denew, dirs);
	return;
	}
	}
	TAILQ_INSERT_TAIL(&dir_q, denew, dirs);
	}

	/*
	* Get the min percentage of free blocks from audit_control(5) and that
	* value in the kernel. Return:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	*/
	int
	auditd_set_dist(void)
	{
	int ret;

	ret = getacdist();
	if (ret < 0)
	return (ADE_PARSE);

	auditd_dist = ret;

	return (ADE_NOERR);
	}

	/*
	* Get the host from audit_control(5) and set it in the audit kernel
	* information. Return:
	* ADE_NOERR on success.
	* ADE_PARSE error parsing audit_control(5).
	* ADE_AUDITON error getting/setting auditon(2) value.
	* ADE_GETADDR error getting address info for host.
	* ADE_ADDRFAM un-supported address family.
	*/
	int
	auditd_set_host(void)
	{
	struct sockaddr_in6 *sin6;
	struct sockaddr_in *sin;
	struct addrinfo *res;
	struct auditinfo_addr aia;
	int error, ret = ADE_NOERR;

	if ((getachost(auditd_host, sizeof(auditd_host)) != 0) \|\|
	((auditd_hostlen = strlen(auditd_host)) == 0)) {
	ret = ADE_PARSE;

	/*
	* To maintain reverse compatability with older audit_control
	* files, simply drop a warning if the host parameter has not
	* been set. However, we will explicitly disable the
	* generation of extended audit header by passing in a zeroed
	* termid structure.
	*/
	bzero(&aia, sizeof(aia));
	aia.ai_termid.at_type = AU_IPv4;
	error = audit_set_kaudit(&aia, sizeof(aia));
	if (error < 0 && errno != ENOSYS)
	ret = ADE_AUDITON;
	return (ret);
	}
	error = getaddrinfo(auditd_host, NULL, NULL, &res);
	if (error)
	return (ADE_GETADDR);
	switch (res->ai_family) {
	case PF_INET6:
	sin6 = (struct sockaddr_in6 *) res->ai_addr;
	bcopy(&sin6->sin6_addr.s6_addr,
	&aia.ai_termid.at_addr[0], sizeof(struct in6_addr));
	aia.ai_termid.at_type = AU_IPv6;
	break;

	case PF_INET:
	sin = (struct sockaddr_in *) res->ai_addr;
	bcopy(&sin->sin_addr.s_addr,
	&aia.ai_termid.at_addr[0], sizeof(struct in_addr));
	aia.ai_termid.at_type = AU_IPv4;
	break;

	default:
	/* Un-supported address family in host parameter. */
	errno = EAFNOSUPPORT;
	return (ADE_ADDRFAM);
	}

	if (audit_set_kaudit(&aia, sizeof(aia)) < 0)
	ret = ADE_AUDITON;

	return (ret);
	}

	/*
	* Get the min percentage of free blocks from audit_control(5) and that
	* value in the kernel. Return:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	* ADE_AUDITON error getting/setting auditon(2) value.
	*/
	int
	auditd_set_minfree(void)
	{
	au_qctrl_t qctrl;

	if (getacmin(&auditd_minval) != 0)
	return (ADE_PARSE);

	if (audit_get_qctrl(&qctrl, sizeof(qctrl)) != 0)
	return (ADE_AUDITON);

	if (qctrl.aq_minfree != auditd_minval) {
	qctrl.aq_minfree = auditd_minval;
	if (audit_set_qctrl(&qctrl, sizeof(qctrl)) != 0)
	return (ADE_AUDITON);
	}

	return (0);
	}

	/*
	* Convert a trailname into a timestamp (seconds). Return 0 if the conversion
	* was successful.
	*/
	static int
	trailname_to_tstamp(char fn, time_t tstamp)
	{
	struct tm tm;
	char ts[TIMESTAMP_LEN + 1];
	char *p;

	*tstamp = 0;

	/*
	* Get the ending time stamp.
	*/
	if ((p = strchr(fn, '.')) == NULL)
	return (1);
	strlcpy(ts, ++p, sizeof(ts));
	if (strlen(ts) != POSTFIX_LEN)
	return (1);

	bzero(&tm, sizeof(tm));

	/* seconds (0-60) */
	p = ts + POSTFIX_LEN - 2;
	tm.tm_sec = atol(p);
	if (tm.tm_sec < 0 \|\| tm.tm_sec > 60)
	return (1);

	/* minutes (0-59) */
	*p = '\0'; p -= 2;
	tm.tm_min = atol(p);
	if (tm.tm_min < 0 \|\| tm.tm_min > 59)
	return (1);

	/* hours (0 - 23) */
	*p = '\0'; p -= 2;
	tm.tm_hour = atol(p);
	if (tm.tm_hour < 0 \|\| tm.tm_hour > 23)
	return (1);

	/* day of month (1-31) */
	*p = '\0'; p -= 2;
	tm.tm_mday = atol(p);
	if (tm.tm_mday < 1 \|\| tm.tm_mday > 31)
	return (1);

	/* month (0 - 11) */
	*p = '\0'; p -= 2;
	tm.tm_mon = atol(p) - 1;
	if (tm.tm_mon < 0 \|\| tm.tm_mon > 11)
	return (1);

	/* year (year - 1900) */
	*p = '\0'; p -= 4;
	tm.tm_year = atol(p) - 1900;
	if (tm.tm_year < 0)
	return (1);

	*tstamp = timegm(&tm);

	return (0);
	}

	/*
	* Remove audit trails files according to the expiration conditions. Returns:
	* ADE_NOERR on success or there is nothing to do.
	* ADE_PARSE if error parsing audit_control(5).
	* ADE_NOMEM if could not allocate memory.
	* ADE_READLINK if could not read link file.
	* ADE_EXPIRE if there was an unexpected error.
	*/
	int
	auditd_expire_trails(int (warn_expired)(char ))
	{
	int andflg, len, ret = ADE_NOERR;
	size_t expire_size, total_size = 0L;
	time_t expire_age, oldest_time, current_time = time(NULL);
	struct dir_ent *traildir;
	struct audit_trail *at;
	char afnp, pn;
	TAILQ_HEAD(au_trls_head, audit_trail) head =
	TAILQ_HEAD_INITIALIZER(head);
	struct stat stbuf;
	char activefn[MAXPATHLEN];

	/*
	* Read the expiration conditions. If no conditions then return no
	* error.
	*/
	if (getacexpire(&andflg, &expire_age, &expire_size) < 0)
	return (ADE_PARSE);
	if (!expire_age && !expire_size)
	return (ADE_NOERR);

	/*
	* Read the 'current' trail file name. Trim off directory path.
	*/
	activefn[0] = '\0';
	len = readlink(AUDIT_CURRENT_LINK, activefn, MAXPATHLEN - 1);
	if (len < 0)
	return (ADE_READLINK);
	if ((afnp = strrchr(activefn, '/')) != NULL)
	afnp++;


	/*
	* Build tail queue of the trail files.
	*/
	TAILQ_FOREACH(traildir, &dir_q, dirs) {
	DIR *dirp;
	struct dirent *dp;

	dirp = opendir(traildir->dirname);
	while ((dp = readdir(dirp)) != NULL) {
	time_t tstamp = 0;
	struct audit_trail *new;

	/*
	* Quickly filter non-trail files.
	*/
	if (dp->d_namlen < FILENAME_LEN \|\|
	dp->d_name[POSTFIX_LEN] != '.')
	continue;

	if (asprintf(&pn, "%s/%s", traildir->dirname,
	dp->d_name) < 0) {
	ret = ADE_NOMEM;
	break;
	}

	if (stat(pn, &stbuf) < 0 \|\| !S_ISREG(stbuf.st_mode)) {
	free(pn);
	continue;
	}

	total_size += stbuf.st_size;

	/*
	* If this is the 'current' audit trail then
	* don't add it to the tail queue.
	*/
	if (NULL != afnp && strcmp(dp->d_name, afnp) == 0) {
	free(pn);
	continue;
	}

	/*
	* Get the ending time stamp encoded in the trail
	* name. If we can't read it or if it is older
	* than Jan 1, 2000 then use the mtime.
	*/
	if (trailname_to_tstamp(dp->d_name, &tstamp) != 0 \|\|
	tstamp < JAN_01_2000)
	tstamp = stbuf.st_mtime;

	/*
	* If the time stamp is older than Jan 1, 2000 then
	* update the mtime of the trail file to the current
	* time. This is so we don't prematurely remove a trail
	* file that was created while the system clock reset
	- * to the * "beginning of time" but later the system
	+ * to the "beginning of time" but later the system
	* clock is set to the correct current time.
	*/
	if (current_time >= JAN_01_2000 &&
	tstamp < JAN_01_2000) {
	struct timeval tv[2];

	tstamp = stbuf.st_mtime = current_time;
	TIMESPEC_TO_TIMEVAL(&tv[0],
	&stbuf.st_atimespec);
	TIMESPEC_TO_TIMEVAL(&tv[1],
	&stbuf.st_mtimespec);
	utimes(pn, tv);
	}

	/*
	* Allocate and populate the new entry.
	*/
	new = malloc(sizeof(*new));
	if (NULL == new) {
	free(pn);
	ret = ADE_NOMEM;
	break;
	}
	new->at_time = tstamp;
	new->at_size = stbuf.st_size;
	new->at_path = pn;

	/*
	* Check to see if we have a new head. Otherwise,
	* walk the tailq from the tail first and do a simple
	* insertion sort.
	*/
	if (TAILQ_EMPTY(&head) \|\|
	new->at_time <= TAILQ_FIRST(&head)->at_time) {
	TAILQ_INSERT_HEAD(&head, new, at_trls);
	continue;
	}

	TAILQ_FOREACH_REVERSE(at, &head, au_trls_head, at_trls)
	if (new->at_time >= at->at_time) {
	TAILQ_INSERT_AFTER(&head, at, new,
	at_trls);
	break;
	}

	}
	closedir(dirp);
	}

	oldest_time = current_time - expire_age;

	/*
	* Expire trail files, oldest (mtime) first, if the given
	* conditions are met.
	*/
	at = TAILQ_FIRST(&head);
	while (NULL != at) {
	struct audit_trail *at_next = TAILQ_NEXT(at, at_trls);

	if (andflg) {
	if ((expire_size && total_size > expire_size) &&
	(expire_age && at->at_time < oldest_time)) {
	if (warn_expired)
	(*warn_expired)(at->at_path);
	if (unlink(at->at_path) < 0)
	ret = ADE_EXPIRE;
	total_size -= at->at_size;
	}
	} else {
	if ((expire_size && total_size > expire_size) \|\|
	(expire_age && at->at_time < oldest_time)) {
	if (warn_expired)
	(*warn_expired)(at->at_path);
	if (unlink(at->at_path) < 0)
	ret = ADE_EXPIRE;
	total_size -= at->at_size;
	}
	}

	free(at->at_path);
	free(at);
	at = at_next;
	}

	return (ret);
	}

	/*
	* Parses the "dir" entry in audit_control(5) into an ordered list. Also, will
	* set the minfree and host values if not already set. Arguments include
	* function pointers to audit_warn functions for soft and hard limits. Returns:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	* ADE_AUDITON error getting/setting auditon(2) value,
	* ADE_NOMEM error allocating memory,
	* ADE_SOFTLIM if all the directories are over the soft limit,
	* ADE_HARDLIM if all the directories are over the hard limit,
	*/
	int
	auditd_read_dirs(int (warn_soft)(char ), int (warn_hard)(char ))
	{
	char cur_dir[MAXNAMLEN];
	struct dir_ent *dirent;
	struct statfs sfs;
	int err;
	char soft, hard;
	int tcnt = 0;
	int scnt = 0;
	int hcnt = 0;

	if (auditd_minval == -1 && (err = auditd_set_minfree()) != 0)
	return (err);

	if (auditd_hostlen == -1)
	auditd_set_host();

	/*
	* Init directory q. Force a re-read of the file the next time.
	*/
	free_dir_q();
	endac();

	/*
	* Read the list of directories into an ordered linked list
	* admin's preference, then those over soft limit and, finally,
	* those over the hard limit.
	*
	* XXX We should use the reentrant interfaces once they are
	* available.
	*/
	while (getacdir(cur_dir, MAXNAMLEN) >= 0) {
	if (statfs(cur_dir, &sfs) < 0)
	continue; /* XXX should warn */
	soft = (sfs.f_bfree < (sfs.f_blocks * auditd_minval / 100 )) ?
	1 : 0;
	hard = (sfs.f_bfree < AUDIT_HARD_LIMIT_FREE_BLOCKS) ? 1 : 0;
	if (soft) {
	if (warn_soft)
	(*warn_soft)(cur_dir);
	scnt++;
	}
	if (hard) {
	if (warn_hard)
	(*warn_hard)(cur_dir);
	hcnt++;
	}
	dirent = (struct dir_ent *) malloc(sizeof(struct dir_ent));
	if (dirent == NULL)
	return (ADE_NOMEM);
	dirent->softlim = soft;
	dirent->hardlim = hard;
	dirent->dirname = (char *) malloc(MAXNAMLEN);
	if (dirent->dirname == NULL) {
	free(dirent);
	return (ADE_NOMEM);
	}
	strlcpy(dirent->dirname, cur_dir, MAXNAMLEN);
	insert_orderly(dirent);
	tcnt++;
	}

	if (hcnt == tcnt)
	return (ADE_HARDLIM);
	if (scnt == tcnt)
	return (ADE_SOFTLIM);
	return (0);
	}

	void
	auditd_close_dirs(void)
	{
	free_dir_q();
	auditd_minval = -1;
	auditd_hostlen = -1;
	}


	/*
	* Process the audit event file, obtaining a class mapping for each event, and
	* set that mapping into the kernel. Return:
	* n number of event mappings that were successfully processed,
	* ADE_NOMEM if there was an error allocating memory.
	*
	* Historically, this code only set up the in-kernel class mapping. On
	* systems with an in-kernel event-to-name mapping, it also now installs that,
	* as it is iterating over the event list anyway. Failures there will be
	* ignored as not all kernels support the feature.
	*/
	int
	auditd_set_evcmap(void)
	{
	au_event_ent_t ev, *evp;
	au_evclass_map_t evc_map;
	au_evname_map_t evn_map;
	int ctr = 0;

	/*
	* XXX There's a risk here that the BSM library will return NULL
	* for an event when it can't properly map it to a class. In that
	* case, we will not process any events beyond the one that failed,
	* but should. We need a way to get a count of the events.
	*/
	ev.ae_name = (char *)malloc(AU_EVENT_NAME_MAX);
	ev.ae_desc = (char *)malloc(AU_EVENT_DESC_MAX);
	if (ev.ae_name == NULL \|\| ev.ae_desc == NULL) {
	if (ev.ae_name != NULL)
	free(ev.ae_name);
	return (ADE_NOMEM);
	}

	/*
	* XXXRW: Currently we have no way to remove mappings from the kernel
	* when they are removed from the file-based mappings.
	*/
	evp = &ev;
	setauevent();
	while ((evp = getauevent_r(evp)) != NULL) {
	/*
	* Set the event-to-name mapping entry. If there's not room
	* in the in-kernel string, then we skip the entry. Possibly
	* better than truncating...?
	*/
	if (strlcpy(evn_map.en_name, evp->ae_name,
	sizeof(evn_map.en_name)) < sizeof(evn_map.en_name)) {
	evn_map.en_number = evp->ae_number;
	(void)audit_set_event(&evn_map, sizeof(evn_map));
	}

	/*
	* Set the event-to-class mapping entry.
	*/
	evc_map.ec_number = evp->ae_number;
	evc_map.ec_class = evp->ae_class;
	if (audit_set_class(&evc_map, sizeof(evc_map)) == 0)
	ctr++;
	}
	endauevent();
	free(ev.ae_name);
	free(ev.ae_desc);

	return (ctr);
	}

	/*
	* Get the non-attributable event string and set the kernel mask. Return:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	* ADE_AUDITON error setting the mask using auditon(2).
	*/
	int
	auditd_set_namask(void)
	{
	au_mask_t aumask;
	char naeventstr[NA_EVENT_STR_SIZE];

	if (getacna(naeventstr, NA_EVENT_STR_SIZE) != 0 \|\|
	getauditflagsbin(naeventstr, &aumask) != 0)
	return (ADE_PARSE);

	if (audit_set_kmask(&aumask, sizeof(aumask)) != 0)
	return (ADE_AUDITON);

	return (ADE_NOERR);
	}

	/*
	* Set the audit control policy if a policy is configured in audit_control(5),
	* implement the policy. However, if one isn't defined or if there is an error
	* parsing the control file, set AUDIT_CNT to avoid leaving the system in a
	* fragile state. Return:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	* ADE_AUDITON error setting policy using auditon(2).
	*/
	int
	auditd_set_policy(void)
	{
	int policy;
	char polstr[POL_STR_SIZE];

	if (getacpol(polstr, POL_STR_SIZE) != 0 \|\|
	au_strtopol(polstr, &policy) != 0) {
	policy = AUDIT_CNT;
	if (audit_set_policy(&policy) != 0)
	return (ADE_AUDITON);
	return (ADE_PARSE);
	}

	if (audit_set_policy(&policy) != 0)
	return (ADE_AUDITON);

	return (ADE_NOERR);
	}

	/*
	* Set trail rotation size. Return:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	* ADE_AUDITON error setting file size using auditon(2).
	*/
	int
	auditd_set_fsize(void)
	{
	size_t filesz;
	au_fstat_t au_fstat;

	/*
	* Set trail rotation size.
	*/
	if (getacfilesz(&filesz) != 0)
	return (ADE_PARSE);

	bzero(&au_fstat, sizeof(au_fstat));
	au_fstat.af_filesz = filesz;
	if (audit_set_fsize(&au_fstat, sizeof(au_fstat)) != 0)
	return (ADE_AUDITON);

	return (ADE_NOERR);
	}

	/*
	* Set trail rotation size. Return:
	* ADE_NOERR on success,
	* ADE_PARSE error parsing audit_control(5),
	* ADE_AUDITON error setting queue size using auditon(2).
	*/
	int
	auditd_set_qsize(void)
	{
	int qsz;
	au_qctrl_t au_qctrl;

	/*
	* Set trail rotation size.
	*/
	if (getacqsize(&qsz) != 0)
	return (ADE_PARSE);

	if (audit_get_qctrl(&au_qctrl, sizeof(au_qctrl)) != 0)
	return (ADE_AUDITON);
	if (qsz != USE_DEFAULT_QSZ)
	au_qctrl.aq_hiwater = qsz;
	if (audit_set_qctrl(&au_qctrl, sizeof(au_qctrl)) != 0)
	return (ADE_AUDITON);

	return (ADE_NOERR);
	}

	static void
	inject_dist(const char fromname, char toname, size_t tonamesize)
	{
	char *ptr;

	ptr = strrchr(fromname, '/');
	assert(ptr != NULL);
	assert(ptr - fromname < (ssize_t)tonamesize);
	strlcpy(toname, fromname, ptr - fromname + 1);
	strlcat(toname, "/dist/", tonamesize);
	strlcat(toname, ptr + 1, tonamesize);
	}

	static int
	auditdist_link(const char *filename)
	{
	char fname[MAXPATHLEN];

	if (auditd_dist) {
	inject_dist(filename, fname, sizeof(fname));
	/* Ignore errors. */
	(void) link(filename, fname);
	}

	return (0);
	}

	int
	auditd_rename(const char fromname, const char toname)
	{
	char fname[MAXPATHLEN], tname[MAXPATHLEN];

	if (auditd_dist) {
	inject_dist(fromname, fname, sizeof(fname));
	inject_dist(toname, tname, sizeof(tname));
	/* Ignore errors. */
	(void) rename(fname, tname);
	}

	return (rename(fromname, toname));
	}

	/*
	* Create the new audit file with appropriate permissions and ownership.
	* Call auditctl(2) for this file.
	* Try to clean up if something goes wrong.
	* *errorp is modified only on auditctl(2) failure.
	*/
	static int
	open_trail(char fname, gid_t gid, int errorp)
	{
	int fd;

	/* XXXPJD: What should we do if the file already exists? */
	fd = open(fname, O_RDONLY \| O_CREAT, S_IRUSR);
	if (fd < 0)
	return (-1);
	if (fchown(fd, -1, gid) < 0 \|\| fchmod(fd, S_IRUSR \| S_IRGRP) < 0) {
	(void) close(fd);
	(void) unlink(fname);
	return (-1);
	}
	(void) close(fd);
	if (auditctl(fname) < 0) {
	*errorp = errno;
	(void) unlink(fname);
	return (-1);
	}
	(void) auditdist_link(fname);
	return (0);
	}

	/*
	* Create the new audit trail file, swap with existing audit file. Arguments
	* include timestamp for the filename, a pointer to a string for returning the
	* new file name, GID for trail file, and audit_warn function pointer for
	* 'getacdir()' errors. Returns:
	* ADE_NOERR on success,
	* ADE_STRERR if the file name string could not be created,
	* ADE_SWAPERR if the audit trail file could not be swapped,
	* ADE_ACTL if the auditctl(2) call failed but file swap still
	* successful.
	* ADE_ACTLERR if the auditctl(2) call failed and file swap failed.
	* ADE_SYMLINK if symlink(2) failed updating the current link.
	*/
	int
	auditd_swap_trail(char TS, char *newfile, gid_t gid,
	int (warn_getacdir)(char ))
	{
	char timestr[FILENAME_LEN + 1];
	char *fn;
	struct dir_ent *dirent;
	int saverrno = 0;

	if (strlen(TS) != TIMESTAMP_LEN \|\|
	snprintf(timestr, sizeof(timestr), "%s.%s", TS,
	NOT_TERMINATED) < 0) {
	errno = EINVAL;
	return (ADE_STRERR);
	}

	/* Try until we succeed. */
	TAILQ_FOREACH(dirent, &dir_q, dirs) {
	if (dirent->hardlim)
	continue;
	if ((fn = affixdir(timestr, dirent)) == NULL)
	return (ADE_STRERR);

	/*
	* Create the file and pass to the kernel if all went well.
	*/
	if (open_trail(fn, gid, &saverrno) == 0) {
	/* Success. */
	*newfile = fn;
	if (saverrno) {
	/*
	* auditctl() failed but still
	* successful. Return errno and "soft"
	* error.
	*/
	errno = saverrno;
	return (ADE_ACTL);
	}
	return (ADE_NOERR);
	}
	/*
	* auditctl failed setting log file. Try again.
	*/
	/*
	* Tell the administrator about lack of permissions for dir.
	*/
	if (warn_getacdir != NULL)
	(*warn_getacdir)(dirent->dirname);
	}
	if (saverrno) {
	errno = saverrno;
	return (ADE_ACTLERR);
	} else
	return (ADE_SWAPERR);
	}

	/*
	* Mask calling process from being audited. Returns:
	* ADE_NOERR on success,
	* ADE_SETAUDIT if setaudit(2) fails.
	*/
	#ifdef __APPLE__
	int
	auditd_prevent_audit(void)
	{
	auditinfo_addr_t aia;

	/*
	* To prevent event feedback cycles and avoid audit becoming stalled if
	* auditing is suspended we mask this processes events from being
	* audited. We allow the uid, tid, and mask fields to be implicitly
	* set to zero, but do set the audit session ID to the PID.
	*
	* XXXRW: Is there more to it than this?
	*/
	bzero(&aia, sizeof(aia));
	aia.ai_asid = AU_ASSIGN_ASID;
	aia.ai_termid.at_type = AU_IPv4;
	if (setaudit_addr(&aia, sizeof(aia)) != 0)
	return (ADE_SETAUDIT);
	return (ADE_NOERR);
	}
	#else
	int
	auditd_prevent_audit(void)
	{
	auditinfo_t ai;

	/*
	* To prevent event feedback cycles and avoid audit becoming stalled if
	* auditing is suspended we mask this processes events from being
	* audited. We allow the uid, tid, and mask fields to be implicitly
	* set to zero, but do set the audit session ID to the PID.
	*
	* XXXRW: Is there more to it than this?
	*/
	bzero(&ai, sizeof(ai));
	ai.ai_asid = getpid();
	if (setaudit(&ai) != 0)
	return (ADE_SETAUDIT);
	return (ADE_NOERR);
	}
	#endif /* !__APPLE__ */

	/*
	* Generate and submit audit record for audit startup or shutdown. The event
	* argument can be AUE_audit_recovery, AUE_audit_startup or
	* AUE_audit_shutdown. The path argument will add a path token, if not NULL.
	* Returns:
	* AUE_NOERR on success,
	* ADE_NOMEM if memory allocation fails,
	* ADE_AU_OPEN if au_open(3) fails,
	* ADE_AU_CLOSE if au_close(3) fails.
	*/
	int
	auditd_gen_record(int event, char *path)
	{
	int aufd;
	uid_t uid;
	pid_t pid;
	char *autext = NULL;
	token_t *tok;
	struct auditinfo_addr aia;

	if (event == AUE_audit_startup)
	asprintf(&autext, "%s::Audit startup", getprogname());
	else if (event == AUE_audit_shutdown)
	asprintf(&autext, "%s::Audit shutdown", getprogname());
	else if (event == AUE_audit_recovery)
	asprintf(&autext, "%s::Audit recovery", getprogname());
	else
	return (ADE_INVAL);
	if (autext == NULL)
	return (ADE_NOMEM);

	if ((aufd = au_open()) == -1) {
	free(autext);
	return (ADE_AU_OPEN);
	}
	bzero(&aia, sizeof(aia));
	uid = getuid(); pid = getpid();
	if ((tok = au_to_subject32_ex(uid, geteuid(), getegid(), uid, getgid(),
	pid, pid, &aia.ai_termid)) != NULL)
	au_write(aufd, tok);
	if ((tok = au_to_text(autext)) != NULL)
	au_write(aufd, tok);
	free(autext);
	if (path != NULL && (tok = au_to_path(path)) != NULL)
	au_write(aufd, tok);
	if ((tok = au_to_return32(0, 0)) != NULL)
	au_write(aufd, tok);
	if (au_close(aufd, 1, event) == -1)
	return (ADE_AU_CLOSE);

	return (ADE_NOERR);
	}

	/*
	* Check for a 'current' symlink and do crash recovery, if needed. Create a new
	* 'current' symlink. The argument 'curfile' is the file the 'current' symlink
	* should point to. Returns:
	* ADE_NOERR on success,
	* ADE_AU_OPEN if au_open(3) fails,
	* ADE_AU_CLOSE if au_close(3) fails.
	* ADE_RENAME if error renaming audit trail file,
	* ADE_READLINK if error reading the 'current' link,
	* ADE_SYMLINK if error creating 'current' link.
	*/
	int
	auditd_new_curlink(char *curfile)
	{
	int len, err;
	char *ptr;
	char *path = NULL;
	struct stat sb;
	char recoveredname[MAXPATHLEN];
	char newname[MAXPATHLEN];

	/*
	* Check to see if audit was shutdown properly. If not, clean up,
	* recover previous audit trail file, and generate audit record.
	*/
	len = readlink(AUDIT_CURRENT_LINK, recoveredname,
	sizeof(recoveredname) - 1);
	if (len > 0) {
	/* 'current' exist but is it pointing at a valid file? */
	recoveredname[len++] = '\0';
	if (stat(recoveredname, &sb) == 0) {
	/* Yes, rename it to a crash recovery file. */
	strlcpy(newname, recoveredname, sizeof(newname));

	if ((ptr = strstr(newname, NOT_TERMINATED)) != NULL) {
	memcpy(ptr, CRASH_RECOVERY, POSTFIX_LEN);
	if (auditd_rename(recoveredname, newname) != 0)
	return (ADE_RENAME);
	} else
	return (ADE_STRERR);

	path = newname;
	}

	/* 'current' symlink is (now) invalid so remove it. */
	(void) unlink(AUDIT_CURRENT_LINK);

	/* Note the crash recovery in current audit trail */
	err = auditd_gen_record(AUE_audit_recovery, path);
	if (err)
	return (err);
	}

	if (len < 0 && errno != ENOENT)
	return (ADE_READLINK);

	if (symlink(curfile, AUDIT_CURRENT_LINK) != 0)
	return (ADE_SYMLINK);

	return (0);
	}

	/*
	* Do just what we need to quickly start auditing. Assume no system logging or
	* notify. Return:
	* 0 on success,
	* -1 on failure.
	*/
	int
	audit_quick_start(void)
	{
	int err;
	char *newfile = NULL;
	time_t tt;
	char TS[TIMESTAMP_LEN + 1];
	int ret = 0;

	/*
	* Mask auditing of this process.
	*/
	if (auditd_prevent_audit() != 0)
	return (-1);

	/*
	* Read audit_control and get log directories.
	*/
	err = auditd_read_dirs(NULL, NULL);
	if (err != ADE_NOERR && err != ADE_SOFTLIM)
	return (-1);

	/*
	* Setup trail file distribution.
	*/
	(void) auditd_set_dist();

	/*
	* Create a new audit trail log.
	*/
	if (getTSstr(tt, TS, sizeof(TS)) != 0)
	return (-1);
	err = auditd_swap_trail(TS, &newfile, getgid(), NULL);
	if (err != ADE_NOERR && err != ADE_ACTL) {
	ret = -1;
	goto out;
	}

	/*
	* Add the current symlink and recover from crash, if needed.
	*/
	if (auditd_new_curlink(newfile) != 0) {
	ret = -1;
	goto out;
	}

	/*
	* At this point auditing has started so generate audit start-up record.
	*/
	if (auditd_gen_record(AUE_audit_startup, NULL) != 0) {
	ret = -1;
	goto out;
	}

	/*
	* Configure the audit controls.
	*/
	(void) auditd_set_evcmap();
	(void) auditd_set_namask();
	(void) auditd_set_policy();
	(void) auditd_set_fsize();
	(void) auditd_set_minfree();
	(void) auditd_set_host();

	out:
	if (newfile != NULL)
	free(newfile);

	return (ret);
	}

	/*
	* Shut down auditing quickly. Assumes that is only called on system shutdown.
	* Returns:
	* 0 on success,
	* -1 on failure.
	*/
	int
	audit_quick_stop(void)
	{
	int len;
	int cond;
	char *ptr;
	time_t tt;
	char oldname[MAXPATHLEN];
	char newname[MAXPATHLEN];
	char TS[TIMESTAMP_LEN + 1];

	/*
	* Auditing already disabled?
	*/
	if (audit_get_cond(&cond) != 0)
	return (-1);
	if (cond == AUC_NOAUDIT)
	return (0);

	/*
	* Generate audit shutdown record.
	*/
	(void) auditd_gen_record(AUE_audit_shutdown, NULL);

	/*
	* Shutdown auditing in the kernel.
	*/
	cond = AUC_DISABLED;
	if (audit_set_cond(&cond) != 0)
	return (-1);
	#ifdef __BSM_INTERNAL_NOTIFY_KEY
	notify_post(__BSM_INTERNAL_NOTIFY_KEY);
	#endif

	/*
	* Rename last audit trail and remove 'current' link.
	*/
	len = readlink(AUDIT_CURRENT_LINK, oldname, sizeof(oldname) - 1);
	if (len < 0)
	return (-1);
	oldname[len++] = '\0';

	if (getTSstr(tt, TS, sizeof(TS)) != 0)
	return (-1);

	strlcpy(newname, oldname, sizeof(newname));

	if ((ptr = strstr(newname, NOT_TERMINATED)) != NULL) {
	memcpy(ptr, TS, POSTFIX_LEN);
	if (auditd_rename(oldname, newname) != 0)
	return (-1);
	} else
	return (-1);

	(void) unlink(AUDIT_CURRENT_LINK);

	return (0);
	}
	diff --git a/contrib/openbsm/libbsm/au_control.3 b/contrib/openbsm/libbsm/au_control.3
	index fbf37b50b21c..c5a28f557222 100644
	--- a/contrib/openbsm/libbsm/au_control.3
	+++ b/contrib/openbsm/libbsm/au_control.3
	@@ -1,272 +1,272 @@
	.\"-
	.\" Copyright (c) 2005-2006 Robert N. M. Watson
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.Dd December 2, 2016
	.Dt AU_CONTROL 3
	.Os
	.Sh NAME
	.Nm setac ,
	.Nm endac ,
	.Nm getacdir ,
	.Nm getacdist ,
	.Nm getacexpire ,
	.Nm getacfilesz ,
	.Nm getacflg ,
	.Nm getachost ,
	.Nm getacmin ,
	.Nm getacna ,
	.Nm getacpol ,
	.Nm au_poltostr ,
	.Nm au_strtopol
	.Nd "look up information from the audit_control database"
	.Sh LIBRARY
	.Lb libbsm
	.Sh SYNOPSIS
	.In bsm/libbsm.h
	.Ft void
	.Fn setac void
	.Ft void
	.Fn endac void
	.Ft int
	.Fn getacdir "char *name" "int len"
	.Ft int
	.Fn getacdist "void"
	.Ft int
	.Fn getacexpire "int andflg" "time_t age" "size_t *size"
	.Ft int
	.Fn getacfilesz "size_t *size_val"
	.Ft int
	.Fn getacflg "char *auditstr" "int len"
	.Ft int
	.Fn getachost "char *auditstr" "int len"
	.Ft int
	.Fn getacmin "int *min_val"
	.Ft int
	.Fn getacna "char *auditstr" "int len"
	.Ft int
	.Fn getacpol "char *auditstr" "size_t len"
	.Ft int
	.Fn getacqsize "int *size_val"
	.Ft ssize_t
	.Fn au_poltostr "int policy" "size_t maxsize" "char *buf"
	.Ft int
	.Fn au_strtopol "const char polstr" "int policy"
	.Sh DESCRIPTION
	These interfaces may be used to look up information from the
	.Xr audit_control 5
	database, which contains various audit-related administrative parameters.
	.Pp
	The
	.Fn setac
	function
	resets the database iterator to the beginning of the database; see the
	.Sx BUGS
	section for more information.
	.Pp
	The
	.Fn endac
	function
	closes the
	.Xr audit_control 5
	database.
	.Pp
	The
	.Fn getacdir
	function
	returns the name of the directory where log data is stored via the passed
	character buffer
	.Fa name
	of length
	.Fa len .
	.Pp
	The
	.Fn getacdist
	function returns a value that allows to decide if trail files distribution is
	turned on or off.
	.Pp
	The
	.Fn getacexpire
	function
	returns the audit trail file expiration parameters in the passed
	.Vt int
	buffer
	.Fa andflg ,
	.Vt time_t
	buffer
	.Fa age
	and
	.Vt size_t
	buffer
	.Fa size .
	If the parameter is not specified in the
	.Xr audit_control 5
	file it is set to zero.
	.Pp
	The
	.Fn getacfilesz
	function
	returns the audit trail rotation size in the passed
	.Vt size_t
	buffer
	.Fa size_val .
	.Pp
	The
	.Fn getacflg
	function
	returns the audit system flags via the the passed character buffer
	.Fa auditstr
	of length
	.Fa len .
	.Pp
	The
	.Fn getachost
	function
	returns the local systems's audit host information via the the passed character
	buffer
	.Fa auditstr
	of length
	.Fa len .
	.Pp
	The
	.Fn getacmin
	function
	returns the minimum free disk space for the audit log target file system via
	the passed
	.Fa min_val
	variable.
	.Pp
	The
	.Fn getacna
	function
	returns the non-attributable flags via the passed character buffer
	.Fa auditstr
	of length
	.Fa len .
	.Pp
	The
	.Fn getacpol
	function
	returns the audit policy flags via the passed character buffer
	.Fa auditstr
	of length
	.Fa len .
	.Pp
	The
	.Fn getacqsize
	function returns the size of the audit post-commit queue in the passed
	.Fa size_val
	buffer.
	If the parameter is not specified in the
	.Xr audit_control 5
	file it is set to
	.Dv -1 ,
	indicating that the kernel's default queue size is being used.
	.Pp
	The
	.Fn au_poltostr
	function
	converts a numeric audit policy mask,
	.Fa policy ,
	to a string in the passed character buffer
	.Fa buf
	of lenth
	.Fa maxsize .
	.Pp
	The
	.Fn au_strtopol
	function
	converts an audit policy flags string,
	.Fa polstr ,
	to a numeric audit policy mask returned via
	.Fa policy .
	-.Sh RETURN VALULES
	+.Sh RETURN VALUES
	The
	.Fn getacfilesz ,
	.Fn getacdir ,
	.Fn getacexpire ,
	.Fn getacflg ,
	.Fn getachost ,
	.Fn getacmin ,
	.Fn getacna ,
	.Fn getacpol ,
	.Fn getacqsize ,
	and
	.Fn au_strtopol
	functions
	return 0 on success, or a negative value on failure, along with error
	information in
	.Va errno .
	.Pp
	The
	.Fn au_poltostr
	function
	returns a string length of 0 or more on success, or a negative value on
	if there is a failure.
	.Pp
	The
	.Fn getacdist
	function returns 1 if trail files distribution is turned on, 0 if it is turned
	off or negative value on failure.
	.Pp
	Functions that return a string value will return a failure if there is
	insufficient room in the passed character buffer for the full string.
	.Sh SEE ALSO
	.Xr libbsm 3 ,
	.Xr audit_control 5
	.Sh HISTORY
	The OpenBSM implementation was created by McAfee Research, the security
	division of McAfee Inc., under contract to Apple Computer, Inc., in 2004.
	It was subsequently adopted by the TrustedBSD Project as the foundation for
	the OpenBSM distribution.
	.Sh AUTHORS
	.An -nosplit
	This software was created by
	.An Robert Watson ,
	.An Wayne Salamon ,
	and
	.An Suresh Krishnaswamy
	for McAfee Research, the security research division of McAfee,
	Inc., under contract to Apple Computer, Inc.
	.Pp
	The Basic Security Module (BSM) interface to audit records and audit event
	stream format were defined by Sun Microsystems.
	.Sh BUGS
	These routines cannot currently distinguish between an entry not being found
	and an error accessing the database.
	The implementation should be changed to return an error via
	.Va errno
	when
	.Dv NULL
	is returned.
	.Pp
	There is no reason for the
	.Fn setac
	interface to be exposed as part of the public API, as it is called implicitly
	by other access functions and iteration is not supported.
	.Pp
	These interfaces inconsistently return various negative values depending on
	the failure mode, and do not always set
	.Va errno
	on failure.
	diff --git a/contrib/openbsm/libbsm/au_domain.3 b/contrib/openbsm/libbsm/au_domain.3
	index 2f16b3848151..9d3415fb3c30 100644
	--- a/contrib/openbsm/libbsm/au_domain.3
	+++ b/contrib/openbsm/libbsm/au_domain.3
	@@ -1,86 +1,86 @@
	.\"-
	.\" Copyright (c) 2008 Apple Inc.
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\" 3. Neither the name of Apple Inc. ("Apple") nor the names of
	.\" its contributors may be used to endorse or promote products derived
	.\" from this software without specific prior written permission.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR
	.\" ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
	.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	.\" POSSIBILITY OF SUCH DAMAGE.
	.\"
	.Dd December 28, 2008
	.Dt AU_BSM_TO_DOMAIN 3
	.Os
	.Sh NAME
	.Nm au_bsm_to_domain ,
	.Nm au_domain_to_bsm
	.Nd "convert between BSM and local protocol domains"
	.Sh LIBRARY
	.Lb libbsm
	.Sh SYNOPSIS
	.In bsm/libbsm.h
	.Ft int
	.Fn au_bsm_to_domain "u_short bsm_domain" "int *local_domainp"
	.Ft u_short
	.Fn au_domain_to_bsm "int local_domain"
	.Sh DESCRIPTION
	These interfaces may be used to convert between the local and BSM protocol
	domains.
	The
	.Fn au_bsm_to_domain
	function accepts a BSM domain,
	.Fa bsm_domain ,
	and converts it to a local domain, such as those passed to
	.Xr socket 2 ,
	that will be stored in the integer pointed to by
	.Fa local_domainp
	if successful.
	This call will fail if the BSM domain cannot be mapped into a local domain,
	which may occur if the socket token was generated on another operating
	system.
	.Pp
	The
	.Fn au_domain_to_bsm
	function accepts a local domain, and returns the BSM domain for it.
	This call cannot fail, and instead returns a BSM domain indicating to a later
	decoder that the domain could not be encoded.
	-.Sh RETURN VALULES
	+.Sh RETURN VALUES
	On success,
	.Fn au_bsm_to_domain
	returns 0 and a converted domain; on failure, it returns -1 but does not set
	.Xr errno 2 .
	.Sh SEE ALSO
	.Xr au_bsm_to_socket_type 3 ,
	.Xr au_socket_type_to_bsm 3 ,
	.Xr au_to_socket_ex 3 ,
	.Xr libbsm 3
	.Sh HISTORY
	.Fn au_bsm_to_domain
	and
	.Fn au_domain_to_bsm
	were introduced in OpenBSM 1.1.
	.Sh AUTHORS
	These functions were implemented by
	.An Robert Watson
	under contract to Apple Inc.
	.Pp
	The Basic Security Module (BSM) interface to audit records and audit event
	stream format were defined by Sun Microsystems.
	diff --git a/contrib/openbsm/libbsm/au_errno.3 b/contrib/openbsm/libbsm/au_errno.3
	index 93873cec7323..9a3d51db0123 100644
	--- a/contrib/openbsm/libbsm/au_errno.3
	+++ b/contrib/openbsm/libbsm/au_errno.3
	@@ -1,110 +1,110 @@
	.\"-
	.\" Copyright (c) 2008 Apple Inc.
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\" 3. Neither the name of Apple Inc. ("Apple") nor the names of
	.\" its contributors may be used to endorse or promote products derived
	.\" from this software without specific prior written permission.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR
	.\" ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
	.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	.\" POSSIBILITY OF SUCH DAMAGE.
	.\"
	.Dd December 8, 2008
	.Dt AU_BSM_TO_ERRNO 3
	.Os
	.Sh NAME
	.Nm au_bsm_to_errno ,
	.Nm au_errno_to_bsm ,
	.Nm au_strerror
	.Nd "convert between BSM and local error numbers"
	.Sh LIBRARY
	.Lb libbsm
	.Sh SYNOPSIS
	.In bsm/libbsm.h
	.Ft int
	.Fn au_bsm_to_errno "u_char bsm_error" "int *errorp"
	.Ft u_char
	.Fn au_errno_to_bsm "int error"
	.Ft const char *
	.Fn au_strerror "int bsm_error"
	.Sh DESCRIPTION
	These interfaces may be used to convert between the local (
	.Xr errno 2 )
	and BSM error number spaces found in BSM return tokens.
	.Pp
	The
	.Fn au_bsm_to_errno
	function accepts a BSM error value,
	.Fa bsm_error ,
	and converts it to an
	.Xr errno 2
	that will be stored in the integer pointed to by
	.Fa errorp
	if successful.
	This call will fail if the BSM error cannot be mapped into a local error
	number, which may occur if the return token was generated on another
	operating system.
	.Pp
	The
	.Fn au_errno_to_bsm
	function accepts a local
	.Xr errno 2
	value, and returns the BSM error number for it.
	This call cannot fail, and instead returns a BSM error number indicating to
	a later decoder that the error could not be encoded.
	.Pp
	The
	.Fn au_strerror
	function converts a BSM error value to a string, generally by converting first to a
	local error number and using the local
	.Xr strerror 3
	function, but will also work for errors that are not locally defined.
	-.Sh RETURN VALULES
	+.Sh RETURN VALUES
	On success,
	.Fn au_bsm_to_errno
	returns 0 and a converted error value; on failure, it returns -1 but does not
	set
	.Xr errno 2 .
	.Pp
	On success,
	.Fn au_strerror
	returns a pointer to an error string; on failure it will return
	.Dv NULL .
	.Sh SEE ALSO
	.Xr au_to_return 3 ,
	.Xr au_to_return32 3 ,
	.Xr au_to_return64 3 ,
	.Xr libbsm 3
	.Sh HISTORY
	.Fn au_bsm_to_errno
	and
	.Fn au_errno_to_bsm
	were introduced in OpenBSM 1.1.
	.Sh AUTHORS
	These functions were implemented by
	.An Robert Watson
	under contract to Apple Inc.
	.Pp
	The Basic Security Module (BSM) interface to audit records and audit event
	stream format were defined by Sun Microsystems.
	.Sh BUGS
	.Nm au_strerror
	is unable to provide localized strings for errors not available in the local
	operating system.
	diff --git a/contrib/openbsm/libbsm/au_socket_type.3 b/contrib/openbsm/libbsm/au_socket_type.3
	index 5668569f7b4d..54534b3fe80f 100644
	--- a/contrib/openbsm/libbsm/au_socket_type.3
	+++ b/contrib/openbsm/libbsm/au_socket_type.3
	@@ -1,91 +1,91 @@
	.\"-
	.\" Copyright (c) 2008 Apple Inc.
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\" 3. Neither the name of Apple Inc. ("Apple") nor the names of
	.\" its contributors may be used to endorse or promote products derived
	.\" from this software without specific prior written permission.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR
	.\" ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
	.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	.\" POSSIBILITY OF SUCH DAMAGE.
	.\"
	.Dd December 28, 2008
	.Dt AU_BSM_TO_SOCKET_TYPE 3
	.Os
	.Sh NAME
	.Nm au_bsm_to_socket_type ,
	.Nm au_socket_type_to_bsm
	.Nd "convert between BSM and local socket types"
	.Sh LIBRARY
	.Lb libbsm
	.Sh SYNOPSIS
	.In bsm/libbsm.h
	.Ft int
	.Fn au_bsm_to_socket_type "u_short bsm_socket_type" "int *local_socket_typep"
	.Ft u_short
	.Fn au_socket_type_to_bsm "int local_socket_type"
	.Sh DESCRIPTION
	These interfaces may be used to convert between the local and BSM socket
	types.
	The
	.Fn au_bsm_to_socket_type
	function accepts a BSM socket type,
	.Fa bsm_socket_type ,
	and converts it to a local socket type, such as those passed to
	.Xr socket 2 ,
	that will be stored in the integer pointed to by
	.Fa local_socket_typep
	if successful.
	This call will fail if the BSM socket type cannot be mapped into a local
	socket type, which may occur if the socket token was generated on another
	operating system.
	.Pp
	.Fn au_socket_type_to_bsm
	function accepts a local socket type, and returns the BSM socket type for it.
	This call cannot fail, and instead returns a BSM socket type indicating to a
	later decoder that the socket type could not be encoded.
	-.Sh RETURN VALULES
	+.Sh RETURN VALUES
	On success,
	.Fn au_bsm_to_socket_type
	returns 0 and a converted socket type; on failure, it returns -1 but does not
	set
	.Xr errno 2 .
	.Pp
	On success,
	.Fn au_strerror
	returns a pointer to an error string; on failure it will return
	.Dv NULL .
	.Sh SEE ALSO
	.Xr au_bsm_to_domain 3 ,
	.Xr au_domain_to_bsm 3 ,
	.Xr au_to_socket_ex 3 ,
	.Xr libbsm 3
	.Sh HISTORY
	.Fn au_bsm_to_socket_type
	and
	.Fn au_socket_type_to_bsm
	were introduced in OpenBSM 1.1.
	.Sh AUTHORS
	These functions were implemented by
	.An Robert Watson
	under contract to Apple Inc.
	.Pp
	The Basic Security Module (BSM) interface to audit records and audit event
	stream format were defined by Sun Microsystems.
	diff --git a/contrib/openbsm/man/audit.log.5 b/contrib/openbsm/man/audit.log.5
	index d85fdccb2bf0..a1db9981acd6 100644
	--- a/contrib/openbsm/man/audit.log.5
	+++ b/contrib/openbsm/man/audit.log.5
	@@ -1,670 +1,670 @@
	.\"-
	.\" Copyright (c) 2005-2006 Robert N. M. Watson
	.\" Copyright (c) 2008 Apple Inc.
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.Dd November 5, 2006
	.Dt AUDIT.LOG 5
	.Os
	.Sh NAME
	.Nm audit
	.Nd "Basic Security Module (BSM) file format"
	.Sh DESCRIPTION
	The
	.Nm
	file format is based on Sun's Basic Security Module (BSM) file format, a
	token-based record stream to represent system audit data.
	This file format is both flexible and extensible, able to describe a broad
	range of data types, and easily extended to describe new data types in a
	moderately backward and forward compatible way.
	.Pp
	BSM token streams typically begin and end with a
	.Dq file
	token, which provides time stamp and file name information for the stream;
	when processing a BSM token stream from a stream as opposed to a single file
	source, file tokens may be seen at any point between ordinary records
	identifying when particular parts of the stream begin and end.
	All other tokens will appear in the context of a complete BSM audit record,
	which begins with a
	.Dq header
	token, and ends with a
	.Dq trailer
	token, which describe the audit record.
	Between these two tokens will appear a variety of data tokens, such as
	process information, file path names, IPC object information, MAC labels,
	socket information, and so on.
	.Pp
	The BSM file format defines specific token orders for each record event type;
	however, some variation may occur depending on the operating system in use,
	what system options, such as mandatory access control, are present.
	.Pp
	This manual page documents the common token types and their binary format, and
	is intended for reference purposes only.
	It is recommended that application programmers use the
	.Xr libbsm 3
	interface to read and write tokens, rather than parsing or constructing
	records by hand.
	.Ss File Token
	The
	.Dq file
	token is used at the beginning and end of an audit log file to indicate
	when the audit log begins and ends.
	It includes a pathname so that, if concatenated together, original file
	boundaries are still observable, and gaps in the audit log can be identified.
	A
	.Dq file
	token can be created using
	.Xr au_to_file 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Seconds 4 bytes File time stamp"
	.It "Microseconds 4 bytes File time stamp"
	.It "File name length 2 bytes File name of audit trail"
	.It "File pathname N bytes + 1 NUL File name of audit trail"
	.El
	.Ss Header Token
	The
	.Dq header
	token is used to mark the beginning of a complete audit record, and includes
	the length of the total record in bytes, a version number for the record
	layout, the event type and subtype, and the time at which the event occurred.
	A 32-bit
	.Dq header
	token can be created using
	.Xr au_to_header32 3 ;
	a 64-bit
	.Dq header
	token can be created using
	.Xr au_to_header64 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Record Byte Count 4 bytes Number of bytes in record"
	-.It "Version Number 2 bytes Record version number"
	+.It "Version Number 1 byte Record version number"
	.It "Event Type 2 bytes Event type"
	.It "Event Modifier 2 bytes Event sub-type"
	.It "Seconds 4/8 bytes Record time stamp (32/64-bits)"
	.It "Nanoseconds 4/8 bytes Record time stamp (32/64-bits)"
	.El
	.Ss Expanded Header Token
	The
	.Dq expanded header
	token is an expanded version of the
	.Dq header
	token, with the addition of a machine IPv4 or IPv6 address.
	A 32-bit extended
	.Dq header
	token can be created using
	.Xr au_to_header32_ex 3 ;
	a 64-bit extended
	.Dq header
	token can be created using
	.Xr au_to_header64_ex 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Record Byte Count 4 bytes Number of bytes in record"
	-.It "Version Number 2 bytes Record version number"
	+.It "Version Number 1 byte Record version number"
	.It "Event Type 2 bytes Event type"
	.It "Event Modifier 2 bytes Event sub-type"
	.It "Address Type/Length 1 byte Host address type and length"
	.It "Machine Address 4/16 bytes IPv4 or IPv6 address"
	.It "Seconds 4/8 bytes Record time stamp (32/64-bits)"
	.It "Nanoseconds 4/8 bytes Record time stamp (32/64-bits)"
	.El
	.Ss Trailer Token
	The
	.Dq trailer
	terminates a BSM audit record, and contains a magic number,
	.Dv AUT_TRAILER_MAGIC
	and length that can be used to validate that the record was read properly.
	A
	.Dq trailer
	token can be created using
	.Xr au_to_trailer 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Trailer Magic 2 bytes Trailer magic number"
	.It "Record Byte Count 4 bytes Number of bytes in record"
	.El
	.Ss Arbitrary Data Token
	The
	.Dq arbitrary data
	token contains a byte stream of opaque (untyped) data.
	The size of the data is calculated as the size of each unit of data
	multiplied by the number of units of data.
	A
	.Dq How to print
	field is present to specify how to print the data, but interpretation of
	that field is not currently defined.
	An
	.Dq arbitrary data
	token can be created using
	.Xr au_to_data 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "How to Print 1 byte User-defined printing information"
	.It "Basic Unit 1 byte Size of a unit in bytes"
	.It "Unit Count 1 byte Number of units of data present"
	.It "Data Items Variable User data"
	.El
	.Ss in_addr Token
	The
	.Dq in_addr
	token holds a network byte order IPv4 address.
	An
	.Dq in_addr
	token can be created using
	.Xr au_to_in_addr 3
	for an IPv4 address.
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "IP Address 4 bytes IPv4 address"
	.El
	.Ss Expanded in_addr Token
	The
	.Dq in_addr_ex
	token holds a network byte order IPv4 or IPv6 address.
	An
	.Dq in_addr_ex
	token can be created using
	.Xr au_to_in_addr_ex 3
	for an IPv6 address.
	.Pp
	See the
	.Sx BUGS
	section for information on the storage of this token.
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "IP Address Type 1 byte Type of address"
	.It "IP Address 4/16 bytes IPv4 or IPv6 address"
	.El
	.Ss ip Token
	The
	.Dq ip
	token contains an IP packet header in network byte order.
	An
	.Dq ip
	token can be created using
	.Xr au_to_ip 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Version and IHL 1 byte Version and IP header length"
	.It "Type of Service 1 byte IP TOS field"
	.It "Length 2 bytes IP packet length in network byte order"
	.It "ID 2 bytes IP header ID for reassembly"
	.It "Offset 2 bytes IP fragment offset and flags, network byte order"
	.It "TTL 1 byte IP Time-to-Live"
	.It "Protocol 1 byte IP protocol number"
	.It "Checksum 2 bytes IP header checksum, network byte order"
	.It "Source Address 4 bytes IPv4 source address"
	.It "Destination Address 4 bytes IPv4 destination address"
	.El
	.Ss iport Token
	The
	.Dq iport
	token stores an IP port number in network byte order.
	An
	.Dq iport
	token can be created using
	.Xr au_to_iport 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Port Number 2 bytes Port number in network byte order"
	.El
	.Ss Path Token
	The
	.Dq path
	token contains a pathname.
	A
	.Dq path
	token can be created using
	.Xr au_to_path 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Path Length 2 bytes Length of path in bytes"
	.It "Path N bytes + 1 NUL Path name"
	.El
	.Ss path_attr Token
	The
	.Dq path_attr
	token contains a set of NUL-terminated path names.
	The
	.Xr libbsm 3
	API cannot currently create a
	.Dq path_attr
	token.
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Count 2 bytes Number of NUL-terminated string(s) in token"
	.It "Path Variable count NUL-terminated string(s)"
	.El
	.Ss Process Token
	The
	.Dq process
	token contains a description of the security properties of a process
	involved as the target of an auditable event, such as the destination for
	signal delivery.
	It should not be confused with the
	.Dq subject
	token, which describes the subject performing an auditable event.
	This includes both the traditional
	.Ux
	security properties, such as user IDs and group IDs, but also audit
	information such as the audit user ID and session.
	A
	.Dq process
	token can be created using
	.Xr au_to_process32 3
	or
	.Xr au_to_process64 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Audit ID 4 bytes Audit user ID"
	.It "Effective User ID 4 bytes Effective user ID"
	.It "Effective Group ID 4 bytes Effective group ID"
	.It "Real User ID 4 bytes Real user ID"
	.It "Real Group ID 4 bytes Real group ID"
	.It "Process ID 4 bytes Process ID"
	.It "Session ID 4 bytes Audit session ID"
	.It "Terminal Port ID 4/8 bytes Terminal port ID (32/64-bits)"
	.It "Terminal Machine Address 4 bytes IP address of machine"
	.El
	.Ss Expanded Process Token
	The
	.Dq expanded process
	token contains the contents of the
	.Dq process
	token, with the addition of a machine address type and variable length
	address storage capable of containing IPv6 addresses.
	An
	.Dq expanded process
	token can be created using
	.Xr au_to_process32_ex 3
	or
	.Xr au_to_process64_ex 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Audit ID 4 bytes Audit user ID"
	.It "Effective User ID 4 bytes Effective user ID"
	.It "Effective Group ID 4 bytes Effective group ID"
	.It "Real User ID 4 bytes Real user ID"
	.It "Real Group ID 4 bytes Real group ID"
	.It "Process ID 4 bytes Process ID"
	.It "Session ID 4 bytes Audit session ID"
	.It "Terminal Port ID 4/8 bytes Terminal port ID (32/64-bits)"
	-.It "Terminal Address Type/Length 1 byte Length of machine address"
	+.It "Terminal Address Type/Length 4 bytes Length of machine address"
	.It "Terminal Machine Address 4 bytes IPv4 or IPv6 address of machine"
	.El
	.Ss Return Token
	The
	.Dq return
	token contains a system call or library function return condition, including
	return value and error number associated with the global variable
	.Er errno .
	A
	.Dq return
	token can be created using
	.Xr au_to_return32 3
	or
	.Xr au_to_return64 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Error Number 1 byte Errno value, or 0 if undefined"
	.It "Return Value 4/8 bytes Return value (32/64-bits)"
	.El
	.Ss Subject Token
	The
	.Dq subject
	token contains information on the subject performing the operation described
	by an audit record, and includes similar information to that found in the
	.Dq process
	and
	.Dq expanded process
	tokens.
	However, those tokens are used where the process being described is the
	target of the operation, not the authorizing party.
	A
	.Dq subject
	token can be created using
	.Xr au_to_subject32 3
	and
	.Xr au_to_subject64 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Audit ID 4 bytes Audit user ID"
	.It "Effective User ID 4 bytes Effective user ID"
	.It "Effective Group ID 4 bytes Effective group ID"
	.It "Real User ID 4 bytes Real user ID"
	.It "Real Group ID 4 bytes Real group ID"
	.It "Process ID 4 bytes Process ID"
	.It "Session ID 4 bytes Audit session ID"
	.It "Terminal Port ID 4/8 bytes Terminal port ID (32/64-bits)"
	.It "Terminal Machine Address 4 bytes IP address of machine"
	.El
	.Ss Expanded Subject Token
	The
	.Dq expanded subject
	token consists of the same elements as the
	.Dq subject
	token, with the addition of type/length and variable size machine address
	information in the terminal ID.
	An
	.Dq expanded subject
	token can be created using
	.Xr au_to_subject32_ex 3
	or
	.Xr au_to_subject64_ex 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Audit ID 4 bytes Audit user ID"
	.It "Effective User ID 4 bytes Effective user ID"
	.It "Effective Group ID 4 bytes Effective group ID"
	.It "Real User ID 4 bytes Real user ID"
	.It "Real Group ID 4 bytes Real group ID"
	.It "Process ID 4 bytes Process ID"
	.It "Session ID 4 bytes Audit session ID"
	.It "Terminal Port ID 4/8 bytes Terminal port ID (32/64-bits)"
	.It "Terminal Address Type/Length 1 byte Length of machine address"
	.It "Terminal Machine Address 4 bytes IPv4 or IPv6 address of machine"
	.El
	.Ss System V IPC Token
	The
	.Dq System V IPC
	token contains the System V IPC message handle, semaphore handle or shared
	memory handle.
	A System V IPC token may be created using
	+.Xr au_to_ipc 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Object ID type 1 byte Object ID"
	.It "Object ID 4 bytes Object ID"
	.El
	.Ss Text Token
	The
	.Dq text
	token contains a single NUL-terminated text string.
	A
	.Dq text
	token may be created using
	.Xr au_to_text 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Text Length 2 bytes Length of text string including NUL"
	.It "Text N bytes + 1 NUL Text string including NUL"
	.El
	.Ss Attribute Token
	The
	.Dq attribute
	token describes the attributes of a file associated with the audit event.
	As files may be identified by 0, 1, or many path names, a path name is not
	included with the attribute block for a file; optional
	.Dq path
	tokens may also be present in an audit record indicating which path, if any,
	was used to reach the object.
	An
	.Dq attribute
	token can be created using
	.Xr au_to_attr32 3
	or
	.Xr au_to_attr64 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "File Access Mode 1 byte mode_t associated with file"
	.It "Owner User ID 4 bytes uid_t associated with file"
	.It "Owner Group ID 4 bytes gid_t associated with file"
	.It "File System ID 4 bytes fsid_t associated with file"
	.It "File System Node ID 8 bytes ino_t associated with file"
	.It "Device 4/8 bytes Device major/minor number (32/64-bit)"
	.El
	.Ss Groups Token
	The
	.Dq groups
	token contains a list of group IDs associated with the audit event.
	A
	.Dq groups
	token can be created using
	.Xr au_to_groups 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Number of Groups 2 bytes Number of groups in token"
	.It "Group List N * 4 bytes List of N group IDs"
	.El
	.Ss System V IPC Permission Token
	The
	.Dq System V IPC permission
	token contains a System V IPC access permissions.
	A System V IPC permission token may be created using
	.Xr au_to_ipc_perm 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It Li "Owner user ID" Ta "4 bytes" Ta "User ID of IPC owner"
	.It Li "Owner group ID" Ta "4 bytes" Ta "Group ID of IPC owner"
	.It Li "Creator user ID" Ta "4 bytes" Ta "User ID of IPC creator"
	.It Li "Creator group ID" Ta "4 bytes" Ta "Group ID of IPC creator"
	.It Li "Access mode" Ta "4 bytes" Ta "Access mode"
	.It Li "Sequence number" Ta "4 bytes" Ta "Sequence number"
	.It Li "Key" Ta "4 bytes" Ta "IPC key"
	.El
	.Ss Arg Token
	The
	.Dq arg
	token contains information about arguments of the system call.
	Depending on the size of the desired argument value, an Arg token may be
	created using
	.Xr au_to_arg32 3
	or
	.Xr au_to_arg64 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It Li "Argument ID" Ta "1 byte" Ta "Argument ID"
	.It Li "Argument value" Ta "4/8 bytes" Ta "Argument value"
	.It Li "Length" Ta "2 bytes" Ta "Length of the text"
	.It Li "Text" Ta "N bytes + 1 nul" Ta "The string including nul"
	.El
	.Ss exec_args Token
	The
	.Dq exec_args
	token contains information about arguments of the exec() system call.
	An exec_args token may be created using
	.Xr au_to_exec_args 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It Li "Count" Ta "4 bytes" Ta "Number of arguments"
	.It Li "Text" Ta "* bytes" Ta "Count nul-terminated strings"
	.El
	.Ss exec_env Token
	The
	.Dq exec_env
	token contains current environment variables to an exec() system call.
	An exec_args token may be created using
	.Xr au_to_exec_env 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It Li "Count ID" Ta "4 bytes" Ta "Number of variables"
	.It Li "Text" Ta "* bytes" Ta "Count nul-terminated strings"
	.El
	.Ss Exit Token
	The
	.Dq exit
	token contains process exit/return code information.
	An
	.Dq exit
	token can be created using
	.Xr au_to_exit 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Status 4 bytes Process status on exit"
	.It "Return Value 4 bytes Process return value on exit"
	.El
	.Ss Socket Token
	The
	.Dq socket
	token contains information about UNIX domain and Internet sockets.
	Each token has four or eight fields.
	Depending on the type of socket, a socket token may be created using
	.Xr au_to_sock_unix 3 ,
	.Xr au_to_sock_inet32 3
	or
	.Xr au_to_sock_inet128 3 .
	.Bl -column -offset 3n ".Sy Field Name Width XX" ".Sy XX Bytes XXXX" ".Sy Description"
	.It Sy "Field" Ta Sy Bytes Ta Sy Description
	.It Li "Token ID" Ta "1 byte" Ta "Token ID"
	.It Li "Socket family" Ta "2 bytes" Ta "Socket family"
	.It Li "Local port" Ta "2 bytes" Ta "Local port"
	.It Li "Socket address" Ta "4 bytes" Ta "Socket address"
	.El
	.Ss Expanded Socket Token
	The
	.Dq expanded socket
	token contains information about IPv4 and IPv6 sockets.
	A
	.Dq expanded socket
	token can be created using
	.Xr au_to_socket_ex 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It Li "Token ID" Ta "1 byte" Ta "Token ID"
	.It Li "Socket domain" Ta "2 bytes" Ta "Socket domain"
	.It Li "Socket type" Ta "2 bytes" Ta "Socket type"
	.It Li "Address type" Ta "2 byte" Ta "Address type (IPv4/IPv6)"
	.It Li "Local port" Ta "2 bytes" Ta "Local port"
	.It Li "Local IP address" Ta "4/16 bytes" Ta "Local IP address"
	.It Li "Remote port" Ta "2 bytes" Ta "Remote port"
	.It Li "Remote IP address" Ta "4/16 bytes" Ta "Remote IP address"
	.El
	.Ss Seq Token
	The
	.Dq seq
	token contains a unique and monotonically increasing audit event sequence ID.
	Due to the limited range of 32 bits, serial number arithmetic and caution
	should be used when comparing sequence numbers.
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Sequence Number 4 bytes Audit event sequence number"
	.El
	.Ss privilege Token
	The
	.Dq privilege
	token ...
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.El
	.Ss Use-of-auth Token
	The
	.Dq use-of-auth
	token ...
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.El
	.Ss Command Token
	The
	.Dq command
	token ...
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.El
	.Ss ACL Token
	The
	.Dq ACL
	token ...
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.El
	.Ss Zonename Token
	The
	.Dq zonename
	token holds a NUL-terminated string with the name of the zone or jail from
	which the record originated.
	A
	.Dq zonename
	token can be created using
	.Xr au_to_zonename 3 .
	.Bl -column -offset 3n ".No Terminal Address Type/Length" ".No N bytes + 1 NUL"
	.It Sy "Field Bytes Description"
	.It "Token ID 1 byte Token ID"
	.It "Zonename length 2 bytes Length of zonename string including NUL"
	.It "Zonename N bytes + 1 NUL Zonename string including NUL"
	.El
	.Sh SEE ALSO
	.Xr auditreduce 1 ,
	.Xr praudit 1 ,
	.Xr libbsm 3 ,
	.Xr audit 4 ,
	.Xr auditpipe 4 ,
	.Xr audit 8
	.Sh HISTORY
	The OpenBSM implementation was created by McAfee Research, the security
	division of McAfee Inc., under contract to Apple Computer Inc.\& in 2004.
	It was subsequently adopted by the TrustedBSD Project as the foundation for
	the OpenBSM distribution.
	.Sh AUTHORS
	The Basic Security Module (BSM) interface to audit records and audit event
	stream format were defined by Sun Microsystems.
	.Pp
	This manual page was written by
	.An Robert Watson Aq rwatson@FreeBSD.org .
	.Sh BUGS
	The
	.Dq How to print
	field in the
	.Dq arbitrary data
	token has undefined values.
	.Pp
	The
	.Dq in_addr
	and
	.Dq in_addr_ex
	token layout documented here appears to be in conflict with the
	.Xr libbsm 3
	implementation of
	.Xr au_to_in_addr_ex 3 .
	diff --git a/contrib/openbsm/man/getaudit.2 b/contrib/openbsm/man/getaudit.2
	index ae5843d45a25..8165c8819007 100644
	--- a/contrib/openbsm/man/getaudit.2
	+++ b/contrib/openbsm/man/getaudit.2
	@@ -1,186 +1,187 @@
	.\"-
	.\" Copyright (c) 2005 Robert N. M. Watson
	.\" Copyright (c) 2008 Apple Inc.
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	-.Dd October 19, 2008
	+.Dd March 14, 2018
	.Dt GETAUDIT 2
	.Os
	.Sh NAME
	.Nm getaudit ,
	.Nm getaudit_addr
	.Nd "retrieve audit session state"
	.Sh SYNOPSIS
	.In bsm/audit.h
	.Ft int
	.Fn getaudit "auditinfo_t *auditinfo"
	.Ft int
	.Fn getaudit_addr "auditinfo_addr_t *auditinfo_addr" "u_int length"
	.Sh DESCRIPTION
	The
	.Fn getaudit
	system call
	retrieves the active audit session state for the current process via the
	.Vt auditinfo_t
	pointed to by
	.Fa auditinfo .
	The
	.Fn getaudit_addr
	system call
	retrieves extended state via
	.Fa auditinfo_addr
	and
	.Fa length .
	.Pp
	The
	.Fa auditinfo_t
	data structure is defined as follows:
	.Bd -literal -offset indent
	struct auditinfo {
	au_id_t ai_auid; /* Audit user ID */
	au_mask_t ai_mask; /* Audit masks */
	au_tid_t ai_termid; /* Terminal ID */
	au_asid_t ai_asid; /* Audit session ID */
	+ au_asflgs_t ai_flags; /* Audit session flags. */
	};
	typedef struct auditinfo auditinfo_t;
	.Ed
	.Pp
	The
	.Fa ai_auid
	variable contains the audit identifier which is recorded in the audit log for
	each event the process caused.
	.Pp
	The
	.Fa au_mask_t
	data structure defines the bit mask for auditing successful and failed events
	out of the predefined list of event classes.
	It is defined as follows:
	.Bd -literal -offset indent
	struct au_mask {
	unsigned int am_success; /* success bits */
	unsigned int am_failure; /* failure bits */
	};
	typedef struct au_mask au_mask_t;
	.Ed
	.Pp
	The
	.Fa au_termid_t
	data structure defines the Terminal ID recorded with every event caused by the
	process.
	It is defined as follows:
	.Bd -literal -offset indent
	struct au_tid {
	dev_t port;
	u_int32_t machine;
	};
	typedef struct au_tid au_tid_t;
	.Ed
	.Pp
	The
	.Fa ai_asid
	variable contains the audit session ID which is recorded with every event
	caused by the process.
	.Pp
	The
	.Fn getaudit_addr
	system call
	uses the expanded
	.Fa auditinfo_addr_t
	data structure and supports Terminal IDs with larger addresses
	such as those used in IP version 6.
	It is defined as follows:
	.Bd -literal -offset indent
	struct auditinfo_addr {
	au_id_t ai_auid; /* Audit user ID. */
	au_mask_t ai_mask; /* Audit masks. */
	au_tid_addr_t ai_termid; /* Terminal ID. */
	au_asid_t ai_asid; /* Audit session ID. */
	};
	typedef struct auditinfo_addr auditinfo_addr_t;
	.Ed
	.Pp
	The
	.Fa au_tid_addr_t
	data structure which includes a larger address storage field and an additional
	field with the type of address stored:
	.Bd -literal -offset indent
	struct au_tid_addr {
	dev_t at_port;
	u_int32_t at_type;
	u_int32_t at_addr[4];
	};
	typedef struct au_tid_addr au_tid_addr_t;
	.Ed
	.Pp
	These system calls require an appropriate privilege to complete.
	.Sh RETURN VALUES
	.Rv -std getaudit getaudit_addr
	.Sh ERRORS
	The
	.Fn getaudit
	function will fail if:
	.Bl -tag -width Er
	.It Bq Er EFAULT
	A failure occurred while data transferred to or from
	the kernel failed.
	.It Bq Er EINVAL
	Illegal argument was passed by a system call.
	.It Bq Er EPERM
	The process does not have sufficient permission to complete
	the operation.
	.It Bq Er EOVERFLOW
	The
	.Fa length
	argument indicates an overflow condition will occur.
	.It Bq Er E2BIG
	The address is too big and, therefore,
	.Fn getaudit_addr
	should be used instead.
	.El
	.Sh SEE ALSO
	.Xr audit 2 ,
	.Xr auditon 2 ,
	.Xr getauid 2 ,
	.Xr setaudit 2 ,
	.Xr setauid 2 ,
	.Xr libbsm 3
	.Sh HISTORY
	The OpenBSM implementation was created by McAfee Research, the security
	division of McAfee Inc., under contract to Apple Computer Inc.\& in 2004.
	It was subsequently adopted by the TrustedBSD Project as the foundation for
	the OpenBSM distribution.
	.Sh AUTHORS
	.An -nosplit
	This software was created by McAfee Research, the security research division
	of McAfee, Inc., under contract to Apple Computer Inc.
	Additional authors include
	.An Wayne Salamon ,
	.An Robert Watson ,
	and SPARTA Inc.
	.Pp
	The Basic Security Module (BSM) interface to audit records and audit event
	stream format were defined by Sun Microsystems.
	.Pp
	This manual page was written by
	.An Robert Watson Aq rwatson@FreeBSD.org .
	diff --git a/contrib/openbsm/sys/bsm/audit.h b/contrib/openbsm/sys/bsm/audit.h
	index 73077b33bd53..26ac4cbf7584 100644
	--- a/contrib/openbsm/sys/bsm/audit.h
	+++ b/contrib/openbsm/sys/bsm/audit.h
	@@ -1,343 +1,343 @@
	/*-
	* Copyright (c) 2005-2009 Apple Inc.
	* Copyright (c) 2016 Robert N. M. Watson
	* All rights reserved.
	*
	* Portions of this software were developed by BAE Systems, the University of
	* Cambridge Computer Laboratory, and Memorial University under DARPA/AFRL
	* contract FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent
	* Computing (TC) research program.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of Apple Inc. ("Apple") nor the names of
	* its contributors may be used to endorse or promote products derived
	* from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	* DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#ifndef _BSM_AUDIT_H
	#define _BSM_AUDIT_H

	#include <sys/param.h>
	#include <sys/types.h>

	#define AUDIT_RECORD_MAGIC 0x828a0f1b
	#define MAX_AUDIT_RECORDS 20
	#define MAXAUDITDATA (0x8000 - 1)
	#define MAX_AUDIT_RECORD_SIZE MAXAUDITDATA
	#define MIN_AUDIT_FILE_SIZE (512 * 1024)

	/*
	- * Minimum noumber of free blocks on the filesystem containing the audit
	+ * Minimum number of free blocks on the filesystem containing the audit
	* log necessary to avoid a hard log rotation. DO NOT SET THIS VALUE TO 0
	* as the kernel does an unsigned compare, plus we want to leave a few blocks
	* free so userspace can terminate the log, etc.
	*/
	#define AUDIT_HARD_LIMIT_FREE_BLOCKS 4

	/*
	* Triggers for the audit daemon.
	*/
	#define AUDIT_TRIGGER_MIN 1
	#define AUDIT_TRIGGER_LOW_SPACE 1 /* Below low watermark. */
	#define AUDIT_TRIGGER_ROTATE_KERNEL 2 /* Kernel requests rotate. */
	#define AUDIT_TRIGGER_READ_FILE 3 /* Re-read config file. */
	#define AUDIT_TRIGGER_CLOSE_AND_DIE 4 /* Terminate audit. */
	#define AUDIT_TRIGGER_NO_SPACE 5 /* Below min free space. */
	#define AUDIT_TRIGGER_ROTATE_USER 6 /* User requests rotate. */
	#define AUDIT_TRIGGER_INITIALIZE 7 /* User initialize of auditd. */
	#define AUDIT_TRIGGER_EXPIRE_TRAILS 8 /* User expiration of trails. */
	#define AUDIT_TRIGGER_MAX 8

	/*
	* The special device filename (FreeBSD).
	*/
	#define AUDITDEV_FILENAME "audit"
	#define AUDIT_TRIGGER_FILE ("/dev/" AUDITDEV_FILENAME)

	/*
	* Pre-defined audit IDs
	*/
	#define AU_DEFAUDITID (uid_t)(-1)
	#define AU_DEFAUDITSID 0
	#define AU_ASSIGN_ASID -1

	/*
	* IPC types.
	*/
	#define AT_IPC_MSG ((u_char)1) /* Message IPC id. */
	#define AT_IPC_SEM ((u_char)2) /* Semaphore IPC id. */
	#define AT_IPC_SHM ((u_char)3) /* Shared mem IPC id. */

	/*
	* Audit conditions.
	*/
	#define AUC_UNSET 0
	#define AUC_AUDITING 1
	#define AUC_NOAUDIT 2
	#define AUC_DISABLED -1

	/*
	* auditon(2) commands.
	*/
	#define A_OLDGETPOLICY 2
	#define A_OLDSETPOLICY 3
	#define A_GETKMASK 4
	#define A_SETKMASK 5
	#define A_OLDGETQCTRL 6
	#define A_OLDSETQCTRL 7
	#define A_GETCWD 8
	#define A_GETCAR 9
	#define A_GETSTAT 12
	#define A_SETSTAT 13
	#define A_SETUMASK 14
	#define A_SETSMASK 15
	#define A_OLDGETCOND 20
	#define A_OLDSETCOND 21
	#define A_GETCLASS 22
	#define A_SETCLASS 23
	#define A_GETPINFO 24
	#define A_SETPMASK 25
	#define A_SETFSIZE 26
	#define A_GETFSIZE 27
	#define A_GETPINFO_ADDR 28
	#define A_GETKAUDIT 29
	#define A_SETKAUDIT 30
	#define A_SENDTRIGGER 31
	#define A_GETSINFO_ADDR 32
	#define A_GETPOLICY 33
	#define A_SETPOLICY 34
	#define A_GETQCTRL 35
	#define A_SETQCTRL 36
	#define A_GETCOND 37
	#define A_SETCOND 38
	#define A_GETEVENT 39 /* Get audit event-to-name mapping. */
	#define A_SETEVENT 40 /* Set audit event-to-name mapping. */

	/*
	* Audit policy controls.
	*/
	#define AUDIT_CNT 0x0001
	#define AUDIT_AHLT 0x0002
	#define AUDIT_ARGV 0x0004
	#define AUDIT_ARGE 0x0008
	#define AUDIT_SEQ 0x0010
	#define AUDIT_WINDATA 0x0020
	#define AUDIT_USER 0x0040
	#define AUDIT_GROUP 0x0080
	#define AUDIT_TRAIL 0x0100
	#define AUDIT_PATH 0x0200
	#define AUDIT_SCNT 0x0400
	#define AUDIT_PUBLIC 0x0800
	#define AUDIT_ZONENAME 0x1000
	#define AUDIT_PERZONE 0x2000

	/*
	* Default audit queue control parameters.
	*/
	#define AQ_HIWATER 100
	#define AQ_MAXHIGH 10000
	#define AQ_LOWATER 10
	#define AQ_BUFSZ MAXAUDITDATA
	#define AQ_MAXBUFSZ 1048576

	/*
	* Default minimum percentage free space on file system.
	*/
	#define AU_FS_MINFREE 20

	/*
	* Type definitions used indicating the length of variable length addresses
	* in tokens containing addresses, such as header fields.
	*/
	#define AU_IPv4 4
	#define AU_IPv6 16

	__BEGIN_DECLS

	typedef uid_t au_id_t;
	typedef pid_t au_asid_t;
	typedef u_int16_t au_event_t;
	typedef u_int16_t au_emod_t;
	typedef u_int32_t au_class_t;
	typedef u_int64_t au_asflgs_t __attribute__ ((aligned (8)));

	struct au_tid {
	dev_t port;
	u_int32_t machine;
	};
	typedef struct au_tid au_tid_t;

	struct au_tid_addr {
	dev_t at_port;
	u_int32_t at_type;
	u_int32_t at_addr[4];
	};
	typedef struct au_tid_addr au_tid_addr_t;

	struct au_mask {
	unsigned int am_success; /* Success bits. */
	unsigned int am_failure; /* Failure bits. */
	};
	typedef struct au_mask au_mask_t;

	struct auditinfo {
	au_id_t ai_auid; /* Audit user ID. */
	au_mask_t ai_mask; /* Audit masks. */
	au_tid_t ai_termid; /* Terminal ID. */
	au_asid_t ai_asid; /* Audit session ID. */
	};
	typedef struct auditinfo auditinfo_t;

	struct auditinfo_addr {
	au_id_t ai_auid; /* Audit user ID. */
	au_mask_t ai_mask; /* Audit masks. */
	au_tid_addr_t ai_termid; /* Terminal ID. */
	au_asid_t ai_asid; /* Audit session ID. */
	au_asflgs_t ai_flags; /* Audit session flags. */
	};
	typedef struct auditinfo_addr auditinfo_addr_t;

	struct auditpinfo {
	pid_t ap_pid; /* ID of target process. */
	au_id_t ap_auid; /* Audit user ID. */
	au_mask_t ap_mask; /* Audit masks. */
	au_tid_t ap_termid; /* Terminal ID. */
	au_asid_t ap_asid; /* Audit session ID. */
	};
	typedef struct auditpinfo auditpinfo_t;

	struct auditpinfo_addr {
	pid_t ap_pid; /* ID of target process. */
	au_id_t ap_auid; /* Audit user ID. */
	au_mask_t ap_mask; /* Audit masks. */
	au_tid_addr_t ap_termid; /* Terminal ID. */
	au_asid_t ap_asid; /* Audit session ID. */
	au_asflgs_t ap_flags; /* Audit session flags. */
	};
	typedef struct auditpinfo_addr auditpinfo_addr_t;

	struct au_session {
	auditinfo_addr_t as_aia_p; / Ptr to full audit info. */
	au_mask_t as_mask; /* Process Audit Masks. */
	};
	typedef struct au_session au_session_t;

	/*
	* Contents of token_t are opaque outside of libbsm.
	*/
	typedef struct au_token token_t;

	/*
	* Kernel audit queue control parameters:
	* Default: Maximum:
	- * aq_hiwater: AQ_HIWATER (100) AQ_MAXHIGH (10000)
	+ * aq_hiwater: AQ_HIWATER (100) AQ_MAXHIGH (10000)
	* aq_lowater: AQ_LOWATER (10) <aq_hiwater
	* aq_bufsz: AQ_BUFSZ (32767) AQ_MAXBUFSZ (1048576)
	- * aq_delay: 20 20000 (not used)
	+ * aq_delay: 20 20000 (not used)
	*/
	struct au_qctrl {
	int aq_hiwater; /* Max # of audit recs in queue when */
	- /* threads with new ARs get blocked. */
	+ /* threads with new ARs get blocked. */

	int aq_lowater; /* # of audit recs in queue when */
	/* blocked threads get unblocked. */

	int aq_bufsz; /* Max size of audit record for audit(2). */
	int aq_delay; /* Queue delay (not used). */
	int aq_minfree; /* Minimum filesystem percent free space. */
	};
	typedef struct au_qctrl au_qctrl_t;

	/*
	* Structure for the audit statistics.
	*/
	struct audit_stat {
	unsigned int as_version;
	unsigned int as_numevent;
	int as_generated;
	int as_nonattrib;
	int as_kernel;
	int as_audit;
	int as_auditctl;
	int as_enqueue;
	int as_written;
	int as_wblocked;
	int as_rblocked;
	int as_dropped;
	int as_totalsize;
	unsigned int as_memused;
	};
	typedef struct audit_stat au_stat_t;

	/*
	* Structure for the audit file statistics.
	*/
	struct audit_fstat {
	u_int64_t af_filesz;
	u_int64_t af_currsz;
	};
	typedef struct audit_fstat au_fstat_t;

	/*
	* Audit to event class mapping.
	*/
	struct au_evclass_map {
	au_event_t ec_number;
	au_class_t ec_class;
	};
	typedef struct au_evclass_map au_evclass_map_t;

	/*
	* Event-to-name mapping.
	*/
	#define EVNAMEMAP_NAME_SIZE 64
	struct au_evname_map {
	au_event_t en_number;
	char en_name[EVNAMEMAP_NAME_SIZE];
	};
	typedef struct au_evname_map au_evname_map_t;

	/*
	* Audit system calls.
	*/
	#if !defined(_KERNEL) && !defined(KERNEL)
	int audit(const void *, int);
	int auditon(int, void *, int);
	int auditctl(const char *);
	int getauid(au_id_t *);
	int setauid(const au_id_t *);
	int getaudit(struct auditinfo *);
	int setaudit(const struct auditinfo *);
	int getaudit_addr(struct auditinfo_addr *, int);
	int setaudit_addr(const struct auditinfo_addr *, int);

	#ifdef __APPLE_API_PRIVATE
	#include <mach/port.h>
	mach_port_name_t audit_session_self(void);
	au_asid_t audit_session_join(mach_port_name_t port);
	#endif /* __APPLE_API_PRIVATE */

	#endif /* defined(_KERNEL) \|\| defined(KERNEL) */

	__END_DECLS

	#endif /* !_BSM_AUDIT_H */
	diff --git a/crypto/openssh/FREEBSD-upgrade b/crypto/openssh/FREEBSD-upgrade
	index 6dcfae91fb1d..27e738bee2d4 100644
	--- a/crypto/openssh/FREEBSD-upgrade
	+++ b/crypto/openssh/FREEBSD-upgrade
	@@ -1,186 +1,179 @@
	FreeBSD maintainer's guide to OpenSSH-portable
	==============================================

	00) Make sure your mail spool has plenty of free space. It'll fill up
	pretty fast once you're done with this checklist.

	01) Download the latest OpenSSH-portable tarball and signature from
	OpenBSD (https://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/).

	02) Verify the signature:

	$ gpg --verify openssh-X.YpZ.tar.gz.asc

	03) Unpack the tarball in a suitable directory:

	$ tar xf openssh-X.YpZ.tar.gz

	04) Copy to the vendor directory:

	$ svn co svn+ssh://repo.freebsd.org/base/vendor-crypto/openssh/dist
	$ rsync --archive --delete openssh-X.YpZ/ dist/

	05) Take care of added / deleted files:

	$ svn rm $(svn stat dist \| awk '$1 == "!" { print $2 }')
	$ svn add --no-auto-props $(svn stat dist \| awk '$1 == "?" { print $2 }')

	06) Commit:

	$ svn commit -m "Vendor import of OpenSSH X.YpZ." dist

	07) Tag:

	$ svn copy -m "Tag OpenSSH X.YpZ." \
	svn+ssh://repo.freebsd.org/base/vendor-crypto/openssh/dist \
	svn+ssh://repo.freebsd.org/base/vendor-crypto/openssh/X.YpZ

	08) Check out head and run the pre-merge script, which strips our RCS
	tags from files that have them:

	$ svn co svn+ssh://repo.freebsd.org/base/head
	$ cd head/crypto/openssh
	$ sh freebsd-pre-merge.sh

	09) Merge from the vendor branch:

	$ svn merge -cNNNNNN \^/vendor-crypto/openssh/dist .

	A number of files have been deleted from FreeBSD's copy of ssh,
	including rendered man pages (which have a .0 extension). When
	svn prompts for these deleted files during the merge, choose 'r'
	(leaving them deleted).

	0A) Resolve conflicts. Remember to bump the version addendum in
	version.h, and update the default value in ssh{,d}_config and
	ssh{,d}_config.5.

	0B) Diff against the vendor branch:

	$ svn diff --no-diff-deleted --no-diff-added \
	--ignore-properties \^/vendor-crypto/openssh/X.YpZ .

	Files that have modifications relative to the vendor code, and
	only those files, must have the svn:keywords property set to
	FreeBSD=%H and be listed in the 'keywords' file created by the
	pre-merge script.

	0C) Run the post-merge script, which re-adds RCS tags to files that
	need them:

	$ sh freebsd-post-merge.sh

	0D) Run the configure script:

	$ sh freebsd-configure.sh

	0E) Review changes to config.h very carefully.

	Note that libwrap should not be defined in config.h; as of
	r311585 it is conditional on MK_TCP_WRAPPERS.

	0F) If source files have been added or removed, update the appropriate
	makefiles to reflect changes in the vendor's Makefile.in.

	10) Update ssh_namespace.h:

	$ sh freebsd-namespace.sh

	11) Build and install world, reboot, test. Pay particular attention
	to pam_ssh(8), which gropes inside libssh and will break if
	something significant changes or if ssh_namespace.h is out of
	whack.

	12) Commit, and hunker down for the inevitable storm of complaints.



	An overview of FreeBSD changes to OpenSSH-portable
	==================================================

	0) VersionAddendum

	The SSH protocol allows for a human-readable version string of up
	to 40 characters to be appended to the protocol version string.
	FreeBSD takes advantage of this to include a date indicating the
	"patch level", so people can easily determine whether their system
	is vulnerable when an OpenSSH advisory goes out. Some people,
	however, dislike advertising their patch level in the protocol
	handshake, so we've added a VersionAddendum configuration variable
	to allow them to change or disable it. Upstream added support for
	VersionAddendum on the server side, but we also support it on the
	client side.

	1) Modified server-side defaults

	We've modified some configuration defaults in sshd:

	- UsePAM defaults to "yes".
	- PermitRootLogin defaults to "no".
	- X11Forwarding defaults to "yes".
	- PasswordAuthentication defaults to "no".
	- VersionAddendum defaults to "FreeBSD-YYYYMMDD".
	- PrivilegeSeparation defaults to "sandbox".
	- UseDNS defaults to "yes".

	2) Modified client-side defaults

	We've modified some configuration defaults in ssh:

	- CheckHostIP defaults to "no".
	- VerifyHostKeyDNS defaults to "yes" if built with LDNS.
	- VersionAddendum defaults to "FreeBSD-YYYYMMDD".

	3) Canonic host names

	We've added code to ssh.c to canonicize the target host name after
	reading options but before trying to connect. This eliminates the
	usual problem with duplicate known_hosts entries.

	4) setusercontext() environment

	Our setusercontext(3) can set environment variables, which we must
	take care to transfer to the child's environment.

	5) TCP wrappers

	Support for TCP wrappers was removed in upstream 6.7p1. We've
	added it back by porting the 6.6p1 code forward.

	TCP wrappers support in sshd will be disabled in HEAD and will
	be removed from FreeBSD in the future.

	6) Agent client reference counting

	We've added code to ssh-agent.c to implement client reference
	counting; the agent will automatically exit when the last client
	disconnects.

	7) Class-based login restrictions

	We've added code to auth2.c to enforce the host.allow, host.deny,
	times.allow and times.deny login class capabilities.

	8) HPN

	We no longer have the HPN patches (adaptive buffer size for
	increased throughput on high-BxD links), but we recognize and
	ignore HPN-related configuration options to avoid breaking existing
	configurations.

	-9) AES-CBC
	-
	- The AES-CBC ciphers were removed from the server-side proposal list
	- in 6.7p1 due to theoretical weaknesses and the availability of
	- superior ciphers (including AES-CTR and AES-GCM). We have re-added
	- them for compatibility with third-party clients.
	-


	This port was brought to you by (in no particular order) DARPA, NAI
	Labs, ThinkSec, Nescafé, the Aberlour Glenlivet Distillery Co.,
	Suzanne Vega, and a Sanford's #69 Deluxe Marker.

	-- des@FreeBSD.org

	$FreeBSD$
	diff --git a/crypto/openssh/myproposal.h b/crypto/openssh/myproposal.h
	index d5a1b7a2fa24..27b4a15a1279 100644
	--- a/crypto/openssh/myproposal.h
	+++ b/crypto/openssh/myproposal.h
	@@ -1,211 +1,209 @@
	/* $OpenBSD: myproposal.h,v 1.57 2018/09/12 01:34:02 djm Exp $ */
	-/* $FreeBSD$ */

	/*
	* Copyright (c) 2000 Markus Friedl. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <openssl/opensslv.h>

	/* conditional algorithm support */

	#ifdef OPENSSL_HAS_ECC
	#ifdef OPENSSL_HAS_NISTP521
	# define KEX_ECDH_METHODS \
	"ecdh-sha2-nistp256," \
	"ecdh-sha2-nistp384," \
	"ecdh-sha2-nistp521,"
	# define HOSTKEY_ECDSA_CERT_METHODS \
	"ecdsa-sha2-nistp256-cert-v01@openssh.com," \
	"ecdsa-sha2-nistp384-cert-v01@openssh.com," \
	"ecdsa-sha2-nistp521-cert-v01@openssh.com,"
	# define HOSTKEY_ECDSA_METHODS \
	"ecdsa-sha2-nistp256," \
	"ecdsa-sha2-nistp384," \
	"ecdsa-sha2-nistp521,"
	#else
	# define KEX_ECDH_METHODS \
	"ecdh-sha2-nistp256," \
	"ecdh-sha2-nistp384,"
	# define HOSTKEY_ECDSA_CERT_METHODS \
	"ecdsa-sha2-nistp256-cert-v01@openssh.com," \
	"ecdsa-sha2-nistp384-cert-v01@openssh.com,"
	# define HOSTKEY_ECDSA_METHODS \
	"ecdsa-sha2-nistp256," \
	"ecdsa-sha2-nistp384,"
	#endif
	#else
	# define KEX_ECDH_METHODS
	# define HOSTKEY_ECDSA_CERT_METHODS
	# define HOSTKEY_ECDSA_METHODS
	#endif

	#ifdef OPENSSL_HAVE_EVPGCM
	# define AESGCM_CIPHER_MODES \
	",aes128-gcm@openssh.com,aes256-gcm@openssh.com"
	#else
	# define AESGCM_CIPHER_MODES
	#endif

	#ifdef HAVE_EVP_SHA256
	# define KEX_SHA2_METHODS \
	"diffie-hellman-group-exchange-sha256," \
	"diffie-hellman-group16-sha512," \
	"diffie-hellman-group18-sha512,"
	# define KEX_SHA2_GROUP14 \
	"diffie-hellman-group14-sha256,"
	#define SHA2_HMAC_MODES \
	"hmac-sha2-256," \
	"hmac-sha2-512,"
	#else
	# define KEX_SHA2_METHODS
	# define KEX_SHA2_GROUP14
	# define SHA2_HMAC_MODES
	#endif

	#ifdef WITH_OPENSSL
	# ifdef HAVE_EVP_SHA256
	# define KEX_CURVE25519_METHODS \
	"curve25519-sha256," \
	"curve25519-sha256@libssh.org,"
	# else
	# define KEX_CURVE25519_METHODS ""
	# endif
	#define KEX_COMMON_KEX \
	KEX_CURVE25519_METHODS \
	KEX_ECDH_METHODS \
	KEX_SHA2_METHODS

	#define KEX_SERVER_KEX KEX_COMMON_KEX \
	KEX_SHA2_GROUP14 \
	"diffie-hellman-group14-sha1" \

	#define KEX_CLIENT_KEX KEX_COMMON_KEX \
	"diffie-hellman-group-exchange-sha1," \
	KEX_SHA2_GROUP14 \
	"diffie-hellman-group14-sha1"

	#define KEX_DEFAULT_PK_ALG \
	HOSTKEY_ECDSA_CERT_METHODS \
	"ssh-ed25519-cert-v01@openssh.com," \
	"rsa-sha2-512-cert-v01@openssh.com," \
	"rsa-sha2-256-cert-v01@openssh.com," \
	"ssh-rsa-cert-v01@openssh.com," \
	HOSTKEY_ECDSA_METHODS \
	"ssh-ed25519," \
	"rsa-sha2-512," \
	"rsa-sha2-256," \
	"ssh-rsa"

	/* the actual algorithms */

	#define KEX_SERVER_ENCRYPT \
	"chacha20-poly1305@openssh.com," \
	"aes128-ctr,aes192-ctr,aes256-ctr" \
	- AESGCM_CIPHER_MODES \
	- ",aes128-cbc,aes192-cbc,aes256-cbc"
	+ AESGCM_CIPHER_MODES

	#define KEX_CLIENT_ENCRYPT KEX_SERVER_ENCRYPT

	#define KEX_SERVER_MAC \
	"umac-64-etm@openssh.com," \
	"umac-128-etm@openssh.com," \
	"hmac-sha2-256-etm@openssh.com," \
	"hmac-sha2-512-etm@openssh.com," \
	"hmac-sha1-etm@openssh.com," \
	"umac-64@openssh.com," \
	"umac-128@openssh.com," \
	"hmac-sha2-256," \
	"hmac-sha2-512," \
	"hmac-sha1"

	#define KEX_CLIENT_MAC KEX_SERVER_MAC

	/* Not a KEX value, but here so all the algorithm defaults are together */
	#define SSH_ALLOWED_CA_SIGALGS \
	"ecdsa-sha2-nistp256," \
	"ecdsa-sha2-nistp384," \
	"ecdsa-sha2-nistp521," \
	"ssh-ed25519," \
	"rsa-sha2-512," \
	"rsa-sha2-256," \
	"ssh-rsa"

	#else /* WITH_OPENSSL */

	#define KEX_SERVER_KEX \
	"curve25519-sha256," \
	"curve25519-sha256@libssh.org"
	#define KEX_DEFAULT_PK_ALG \
	"ssh-ed25519-cert-v01@openssh.com," \
	"ssh-ed25519"
	#define KEX_SERVER_ENCRYPT \
	"chacha20-poly1305@openssh.com," \
	"aes128-ctr,aes192-ctr,aes256-ctr"
	#define KEX_SERVER_MAC \
	"umac-64-etm@openssh.com," \
	"umac-128-etm@openssh.com," \
	"hmac-sha2-256-etm@openssh.com," \
	"hmac-sha2-512-etm@openssh.com," \
	"hmac-sha1-etm@openssh.com," \
	"umac-64@openssh.com," \
	"umac-128@openssh.com," \
	"hmac-sha2-256," \
	"hmac-sha2-512," \
	"hmac-sha1"

	#define KEX_CLIENT_KEX KEX_SERVER_KEX
	#define KEX_CLIENT_ENCRYPT KEX_SERVER_ENCRYPT
	#define KEX_CLIENT_MAC KEX_SERVER_MAC

	#define SSH_ALLOWED_CA_SIGALGS "ssh-ed25519"

	#endif /* WITH_OPENSSL */

	#define KEX_DEFAULT_COMP "none,zlib@openssh.com"
	#define KEX_DEFAULT_LANG ""

	#define KEX_CLIENT \
	KEX_CLIENT_KEX, \
	KEX_DEFAULT_PK_ALG, \
	KEX_CLIENT_ENCRYPT, \
	KEX_CLIENT_ENCRYPT, \
	KEX_CLIENT_MAC, \
	KEX_CLIENT_MAC, \
	KEX_DEFAULT_COMP, \
	KEX_DEFAULT_COMP, \
	KEX_DEFAULT_LANG, \
	KEX_DEFAULT_LANG

	#define KEX_SERVER \
	KEX_SERVER_KEX, \
	KEX_DEFAULT_PK_ALG, \
	KEX_SERVER_ENCRYPT, \
	KEX_SERVER_ENCRYPT, \
	KEX_SERVER_MAC, \
	KEX_SERVER_MAC, \
	KEX_DEFAULT_COMP, \
	KEX_DEFAULT_COMP, \
	KEX_DEFAULT_LANG, \
	KEX_DEFAULT_LANG

	diff --git a/crypto/openssh/servconf.c b/crypto/openssh/servconf.c
	index 5032fa6556d0..8b07e4b923ae 100644
	--- a/crypto/openssh/servconf.c
	+++ b/crypto/openssh/servconf.c
	@@ -1,2714 +1,2715 @@

	/* $OpenBSD: servconf.c,v 1.342 2018/09/20 23:40:16 djm Exp $ */
	/*
	* Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
	* All rights reserved
	*
	* As far as I am concerned, the code I have written for this software
	* can be used freely for any purpose. Any derived versions of this
	* software must be clearly marked as such, and if the derived work is
	* incompatible with the protocol description in the RFC file, it must be
	* called by a name other than "ssh" or "Secure Shell".
	*/

	#include "includes.h"
	__RCSID("$FreeBSD$");

	#include <sys/types.h>
	#include <sys/socket.h>
	#ifdef HAVE_SYS_SYSCTL_H
	#include <sys/sysctl.h>
	#endif

	#include <netinet/in.h>
	#include <netinet/in_systm.h>
	#include <netinet/ip.h>
	#ifdef HAVE_NET_ROUTE_H
	#include <net/route.h>
	#endif

	#include <ctype.h>
	#include <netdb.h>
	#include <pwd.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <signal.h>
	#include <unistd.h>
	#include <limits.h>
	#include <stdarg.h>
	#include <errno.h>
	#ifdef HAVE_UTIL_H
	#include <util.h>
	#endif

	#include "openbsd-compat/sys-queue.h"
	#include "xmalloc.h"
	#include "ssh.h"
	#include "log.h"
	#include "sshbuf.h"
	#include "misc.h"
	#include "servconf.h"
	#include "compat.h"
	#include "pathnames.h"
	#include "cipher.h"
	#include "sshkey.h"
	#include "kex.h"
	#include "mac.h"
	#include "match.h"
	#include "channels.h"
	#include "groupaccess.h"
	#include "canohost.h"
	#include "packet.h"
	#include "ssherr.h"
	#include "hostfile.h"
	#include "auth.h"
	#include "myproposal.h"
	#include "digest.h"
	#include "version.h"

	static void add_listen_addr(ServerOptions , const char ,
	const char *, int);
	static void add_one_listen_addr(ServerOptions , const char ,
	const char *, int);

	/* Use of privilege separation or not */
	extern int use_privsep;
	extern struct sshbuf *cfg;

	/* Initializes the server options to their default values. */

	void
	initialize_server_options(ServerOptions *options)
	{
	memset(options, 0, sizeof(*options));

	/* Portable-specific options */
	options->use_pam = -1;

	/* Standard Options */
	options->num_ports = 0;
	options->ports_from_cmdline = 0;
	options->queued_listen_addrs = NULL;
	options->num_queued_listens = 0;
	options->listen_addrs = NULL;
	options->num_listen_addrs = 0;
	options->address_family = -1;
	options->routing_domain = NULL;
	options->num_host_key_files = 0;
	options->num_host_cert_files = 0;
	options->host_key_agent = NULL;
	options->pid_file = NULL;
	options->login_grace_time = -1;
	options->permit_root_login = PERMIT_NOT_SET;
	options->ignore_rhosts = -1;
	options->ignore_user_known_hosts = -1;
	options->print_motd = -1;
	options->print_lastlog = -1;
	options->x11_forwarding = -1;
	options->x11_display_offset = -1;
	options->x11_use_localhost = -1;
	options->permit_tty = -1;
	options->permit_user_rc = -1;
	options->xauth_location = NULL;
	options->strict_modes = -1;
	options->tcp_keep_alive = -1;
	options->log_facility = SYSLOG_FACILITY_NOT_SET;
	options->log_level = SYSLOG_LEVEL_NOT_SET;
	options->hostbased_authentication = -1;
	options->hostbased_uses_name_from_packet_only = -1;
	options->hostbased_key_types = NULL;
	options->hostkeyalgorithms = NULL;
	options->pubkey_authentication = -1;
	options->pubkey_key_types = NULL;
	options->kerberos_authentication = -1;
	options->kerberos_or_local_passwd = -1;
	options->kerberos_ticket_cleanup = -1;
	options->kerberos_get_afs_token = -1;
	options->gss_authentication=-1;
	options->gss_cleanup_creds = -1;
	options->gss_strict_acceptor = -1;
	options->password_authentication = -1;
	options->kbd_interactive_authentication = -1;
	options->challenge_response_authentication = -1;
	options->permit_empty_passwd = -1;
	options->permit_user_env = -1;
	options->permit_user_env_whitelist = NULL;
	options->compression = -1;
	options->rekey_limit = -1;
	options->rekey_interval = -1;
	options->allow_tcp_forwarding = -1;
	options->allow_streamlocal_forwarding = -1;
	options->allow_agent_forwarding = -1;
	options->num_allow_users = 0;
	options->num_deny_users = 0;
	options->num_allow_groups = 0;
	options->num_deny_groups = 0;
	options->ciphers = NULL;
	options->macs = NULL;
	options->kex_algorithms = NULL;
	options->ca_sign_algorithms = NULL;
	options->fwd_opts.gateway_ports = -1;
	options->fwd_opts.streamlocal_bind_mask = (mode_t)-1;
	options->fwd_opts.streamlocal_bind_unlink = -1;
	options->num_subsystems = 0;
	options->max_startups_begin = -1;
	options->max_startups_rate = -1;
	options->max_startups = -1;
	options->max_authtries = -1;
	options->max_sessions = -1;
	options->banner = NULL;
	options->use_dns = -1;
	options->client_alive_interval = -1;
	options->client_alive_count_max = -1;
	options->num_authkeys_files = 0;
	options->num_accept_env = 0;
	options->num_setenv = 0;
	options->permit_tun = -1;
	options->permitted_opens = NULL;
	options->permitted_listens = NULL;
	options->adm_forced_command = NULL;
	options->chroot_directory = NULL;
	options->authorized_keys_command = NULL;
	options->authorized_keys_command_user = NULL;
	options->revoked_keys_file = NULL;
	options->trusted_user_ca_keys = NULL;
	options->authorized_principals_file = NULL;
	options->authorized_principals_command = NULL;
	options->authorized_principals_command_user = NULL;
	options->ip_qos_interactive = -1;
	options->ip_qos_bulk = -1;
	options->version_addendum = NULL;
	options->fingerprint_hash = -1;
	options->disable_forwarding = -1;
	options->expose_userauth_info = -1;
	options->use_blacklist = -1;
	}

	/* Returns 1 if a string option is unset or set to "none" or 0 otherwise. */
	static int
	option_clear_or_none(const char *o)
	{
	return o == NULL \|\| strcasecmp(o, "none") == 0;
	}

	static void
	assemble_algorithms(ServerOptions *o)
	{
	char all_cipher, all_mac, all_kex, all_key, *all_sig;
	int r;

	all_cipher = cipher_alg_list(',', 0);
	all_mac = mac_alg_list(',');
	all_kex = kex_alg_list(',');
	all_key = sshkey_alg_list(0, 0, 1, ',');
	all_sig = sshkey_alg_list(0, 1, 1, ',');
	#define ASSEMBLE(what, defaults, all) \
	do { \
	if ((r = kex_assemble_names(&o->what, defaults, all)) != 0) \
	fatal("%s: %s: %s", __func__, #what, ssh_err(r)); \
	} while (0)
	ASSEMBLE(ciphers, KEX_SERVER_ENCRYPT, all_cipher);
	ASSEMBLE(macs, KEX_SERVER_MAC, all_mac);
	ASSEMBLE(kex_algorithms, KEX_SERVER_KEX, all_kex);
	ASSEMBLE(hostkeyalgorithms, KEX_DEFAULT_PK_ALG, all_key);
	ASSEMBLE(hostbased_key_types, KEX_DEFAULT_PK_ALG, all_key);
	ASSEMBLE(pubkey_key_types, KEX_DEFAULT_PK_ALG, all_key);
	ASSEMBLE(ca_sign_algorithms, SSH_ALLOWED_CA_SIGALGS, all_sig);
	#undef ASSEMBLE
	free(all_cipher);
	free(all_mac);
	free(all_kex);
	free(all_key);
	free(all_sig);
	}

	static void
	array_append(const char file, const int line, const char directive,
	char **array, u_int lp, const char *s)
	{

	if (*lp >= INT_MAX)
	fatal("%s line %d: Too many %s entries", file, line, directive);

	array = xrecallocarray(array, lp, lp + 1, sizeof(**array));
	(array)[lp] = xstrdup(s);
	(*lp)++;
	}

	static const char *defaultkey = "[default]";

	void
	servconf_add_hostkey(const char *file, const int line,
	ServerOptions options, const char path)
	{
	char *apath = derelativise_path(path);

	if (file == defaultkey && access(path, R_OK) != 0)
	return;
	array_append(file, line, "HostKey",
	&options->host_key_files, &options->num_host_key_files, apath);
	free(apath);
	}

	void
	servconf_add_hostcert(const char *file, const int line,
	ServerOptions options, const char path)
	{
	char *apath = derelativise_path(path);

	array_append(file, line, "HostCertificate",
	&options->host_cert_files, &options->num_host_cert_files, apath);
	free(apath);
	}

	void
	fill_default_server_options(ServerOptions *options)
	{
	u_int i;

	/* Portable-specific options */
	if (options->use_pam == -1)
	options->use_pam = 1;

	/* Standard Options */
	if (options->num_host_key_files == 0) {
	/* fill default hostkeys for protocols */
	servconf_add_hostkey(defaultkey, 0, options,
	_PATH_HOST_RSA_KEY_FILE);
	servconf_add_hostkey(defaultkey, 0, options,
	_PATH_HOST_DSA_KEY_FILE);
	#ifdef OPENSSL_HAS_ECC
	servconf_add_hostkey(defaultkey, 0, options,
	_PATH_HOST_ECDSA_KEY_FILE);
	#endif
	servconf_add_hostkey(defaultkey, 0, options,
	_PATH_HOST_ED25519_KEY_FILE);
	#ifdef WITH_XMSS
	servconf_add_hostkey(defaultkey, 0, options,
	_PATH_HOST_XMSS_KEY_FILE);
	#endif /* WITH_XMSS */
	}
	if (options->num_host_key_files == 0)
	fatal("No host key files found");
	/* No certificates by default */
	if (options->num_ports == 0)
	options->ports[options->num_ports++] = SSH_DEFAULT_PORT;
	if (options->address_family == -1)
	options->address_family = AF_UNSPEC;
	if (options->listen_addrs == NULL)
	add_listen_addr(options, NULL, NULL, 0);
	if (options->pid_file == NULL)
	options->pid_file = xstrdup(_PATH_SSH_DAEMON_PID_FILE);
	if (options->login_grace_time == -1)
	options->login_grace_time = 120;
	if (options->permit_root_login == PERMIT_NOT_SET)
	options->permit_root_login = PERMIT_NO;
	if (options->ignore_rhosts == -1)
	options->ignore_rhosts = 1;
	if (options->ignore_user_known_hosts == -1)
	options->ignore_user_known_hosts = 0;
	if (options->print_motd == -1)
	options->print_motd = 1;
	if (options->print_lastlog == -1)
	options->print_lastlog = 1;
	if (options->x11_forwarding == -1)
	options->x11_forwarding = 1;
	if (options->x11_display_offset == -1)
	options->x11_display_offset = 10;
	if (options->x11_use_localhost == -1)
	options->x11_use_localhost = 1;
	if (options->xauth_location == NULL)
	options->xauth_location = xstrdup(_PATH_XAUTH);
	if (options->permit_tty == -1)
	options->permit_tty = 1;
	if (options->permit_user_rc == -1)
	options->permit_user_rc = 1;
	if (options->strict_modes == -1)
	options->strict_modes = 1;
	if (options->tcp_keep_alive == -1)
	options->tcp_keep_alive = 1;
	if (options->log_facility == SYSLOG_FACILITY_NOT_SET)
	options->log_facility = SYSLOG_FACILITY_AUTH;
	if (options->log_level == SYSLOG_LEVEL_NOT_SET)
	options->log_level = SYSLOG_LEVEL_INFO;
	if (options->hostbased_authentication == -1)
	options->hostbased_authentication = 0;
	if (options->hostbased_uses_name_from_packet_only == -1)
	options->hostbased_uses_name_from_packet_only = 0;
	if (options->pubkey_authentication == -1)
	options->pubkey_authentication = 1;
	if (options->kerberos_authentication == -1)
	options->kerberos_authentication = 0;
	if (options->kerberos_or_local_passwd == -1)
	options->kerberos_or_local_passwd = 1;
	if (options->kerberos_ticket_cleanup == -1)
	options->kerberos_ticket_cleanup = 1;
	if (options->kerberos_get_afs_token == -1)
	options->kerberos_get_afs_token = 0;
	if (options->gss_authentication == -1)
	options->gss_authentication = 0;
	if (options->gss_cleanup_creds == -1)
	options->gss_cleanup_creds = 1;
	if (options->gss_strict_acceptor == -1)
	options->gss_strict_acceptor = 1;
	if (options->password_authentication == -1)
	options->password_authentication = 0;
	if (options->kbd_interactive_authentication == -1)
	options->kbd_interactive_authentication = 0;
	if (options->challenge_response_authentication == -1)
	options->challenge_response_authentication = 1;
	if (options->permit_empty_passwd == -1)
	options->permit_empty_passwd = 0;
	if (options->permit_user_env == -1) {
	options->permit_user_env = 0;
	options->permit_user_env_whitelist = NULL;
	}
	if (options->compression == -1)
	options->compression = COMP_DELAYED;
	if (options->rekey_limit == -1)
	options->rekey_limit = 0;
	if (options->rekey_interval == -1)
	options->rekey_interval = 0;
	if (options->allow_tcp_forwarding == -1)
	options->allow_tcp_forwarding = FORWARD_ALLOW;
	if (options->allow_streamlocal_forwarding == -1)
	options->allow_streamlocal_forwarding = FORWARD_ALLOW;
	if (options->allow_agent_forwarding == -1)
	options->allow_agent_forwarding = 1;
	if (options->fwd_opts.gateway_ports == -1)
	options->fwd_opts.gateway_ports = 0;
	if (options->max_startups == -1)
	options->max_startups = 100;
	if (options->max_startups_rate == -1)
	options->max_startups_rate = 30; /* 30% */
	if (options->max_startups_begin == -1)
	options->max_startups_begin = 10;
	if (options->max_authtries == -1)
	options->max_authtries = DEFAULT_AUTH_FAIL_MAX;
	if (options->max_sessions == -1)
	options->max_sessions = DEFAULT_SESSIONS_MAX;
	if (options->use_dns == -1)
	options->use_dns = 1;
	if (options->client_alive_interval == -1)
	options->client_alive_interval = 0;
	if (options->client_alive_count_max == -1)
	options->client_alive_count_max = 3;
	if (options->num_authkeys_files == 0) {
	array_append(defaultkey, 0, "AuthorizedKeysFiles",
	&options->authorized_keys_files,
	&options->num_authkeys_files,
	_PATH_SSH_USER_PERMITTED_KEYS);
	array_append(defaultkey, 0, "AuthorizedKeysFiles",
	&options->authorized_keys_files,
	&options->num_authkeys_files,
	_PATH_SSH_USER_PERMITTED_KEYS2);
	}
	if (options->permit_tun == -1)
	options->permit_tun = SSH_TUNMODE_NO;
	if (options->ip_qos_interactive == -1)
	options->ip_qos_interactive = IPTOS_DSCP_AF21;
	if (options->ip_qos_bulk == -1)
	options->ip_qos_bulk = IPTOS_DSCP_CS1;
	if (options->version_addendum == NULL)
	options->version_addendum = xstrdup(SSH_VERSION_FREEBSD);
	if (options->fwd_opts.streamlocal_bind_mask == (mode_t)-1)
	options->fwd_opts.streamlocal_bind_mask = 0177;
	if (options->fwd_opts.streamlocal_bind_unlink == -1)
	options->fwd_opts.streamlocal_bind_unlink = 0;
	if (options->fingerprint_hash == -1)
	options->fingerprint_hash = SSH_FP_HASH_DEFAULT;
	if (options->disable_forwarding == -1)
	options->disable_forwarding = 0;
	if (options->expose_userauth_info == -1)
	options->expose_userauth_info = 0;
	if (options->use_blacklist == -1)
	options->use_blacklist = 0;

	assemble_algorithms(options);

	/* Turn privilege separation and sandboxing on by default */
	if (use_privsep == -1)
	use_privsep = PRIVSEP_ON;

	#define CLEAR_ON_NONE(v) \
	do { \
	if (option_clear_or_none(v)) { \
	free(v); \
	v = NULL; \
	} \
	} while(0)
	CLEAR_ON_NONE(options->pid_file);
	CLEAR_ON_NONE(options->xauth_location);
	CLEAR_ON_NONE(options->banner);
	CLEAR_ON_NONE(options->trusted_user_ca_keys);
	CLEAR_ON_NONE(options->revoked_keys_file);
	CLEAR_ON_NONE(options->authorized_principals_file);
	CLEAR_ON_NONE(options->adm_forced_command);
	CLEAR_ON_NONE(options->chroot_directory);
	CLEAR_ON_NONE(options->routing_domain);
	for (i = 0; i < options->num_host_key_files; i++)
	CLEAR_ON_NONE(options->host_key_files[i]);
	for (i = 0; i < options->num_host_cert_files; i++)
	CLEAR_ON_NONE(options->host_cert_files[i]);
	#undef CLEAR_ON_NONE

	/* Similar handling for AuthenticationMethods=any */
	if (options->num_auth_methods == 1 &&
	strcmp(options->auth_methods[0], "any") == 0) {
	free(options->auth_methods[0]);
	options->auth_methods[0] = NULL;
	options->num_auth_methods = 0;
	}

	#ifndef HAVE_MMAP
	if (use_privsep && options->compression == 1) {
	error("This platform does not support both privilege "
	"separation and compression");
	error("Compression disabled");
	options->compression = 0;
	}
	#endif

	}

	/* Keyword tokens. */
	typedef enum {
	sBadOption, /* == unknown option */
	/* Portable-specific options */
	sUsePAM,
	/* Standard Options */
	sPort, sHostKeyFile, sLoginGraceTime,
	sPermitRootLogin, sLogFacility, sLogLevel,
	sRhostsRSAAuthentication, sRSAAuthentication,
	sKerberosAuthentication, sKerberosOrLocalPasswd, sKerberosTicketCleanup,
	sKerberosGetAFSToken, sChallengeResponseAuthentication,
	sPasswordAuthentication, sKbdInteractiveAuthentication,
	sListenAddress, sAddressFamily,
	sPrintMotd, sPrintLastLog, sIgnoreRhosts,
	sX11Forwarding, sX11DisplayOffset, sX11UseLocalhost,
	sPermitTTY, sStrictModes, sEmptyPasswd, sTCPKeepAlive,
	sPermitUserEnvironment, sAllowTcpForwarding, sCompression,
	sRekeyLimit, sAllowUsers, sDenyUsers, sAllowGroups, sDenyGroups,
	sIgnoreUserKnownHosts, sCiphers, sMacs, sPidFile,
	sGatewayPorts, sPubkeyAuthentication, sPubkeyAcceptedKeyTypes,
	sXAuthLocation, sSubsystem, sMaxStartups, sMaxAuthTries, sMaxSessions,
	sBanner, sUseDNS, sHostbasedAuthentication,
	sHostbasedUsesNameFromPacketOnly, sHostbasedAcceptedKeyTypes,
	sHostKeyAlgorithms,
	sClientAliveInterval, sClientAliveCountMax, sAuthorizedKeysFile,
	sGssAuthentication, sGssCleanupCreds, sGssStrictAcceptor,
	sAcceptEnv, sSetEnv, sPermitTunnel,
	sMatch, sPermitOpen, sPermitListen, sForceCommand, sChrootDirectory,
	sUsePrivilegeSeparation, sAllowAgentForwarding,
	sHostCertificate,
	sRevokedKeys, sTrustedUserCAKeys, sAuthorizedPrincipalsFile,
	sAuthorizedPrincipalsCommand, sAuthorizedPrincipalsCommandUser,
	sKexAlgorithms, sCASignatureAlgorithms, sIPQoS, sVersionAddendum,
	sAuthorizedKeysCommand, sAuthorizedKeysCommandUser,
	sAuthenticationMethods, sHostKeyAgent, sPermitUserRC,
	sStreamLocalBindMask, sStreamLocalBindUnlink,
	sAllowStreamLocalForwarding, sFingerprintHash, sDisableForwarding,
	sExposeAuthInfo, sRDomain,
	sUseBlacklist,
	sDeprecated, sIgnore, sUnsupported
	} ServerOpCodes;

	#define SSHCFG_GLOBAL 0x01 /* allowed in main section of sshd_config */
	#define SSHCFG_MATCH 0x02 /* allowed inside a Match section */
	#define SSHCFG_ALL (SSHCFG_GLOBAL\|SSHCFG_MATCH)

	/* Textual representation of the tokens. */
	static struct {
	const char *name;
	ServerOpCodes opcode;
	u_int flags;
	} keywords[] = {
	/* Portable-specific options */
	#ifdef USE_PAM
	{ "usepam", sUsePAM, SSHCFG_GLOBAL },
	#else
	{ "usepam", sUnsupported, SSHCFG_GLOBAL },
	#endif
	{ "pamauthenticationviakbdint", sDeprecated, SSHCFG_GLOBAL },
	/* Standard Options */
	{ "port", sPort, SSHCFG_GLOBAL },
	{ "hostkey", sHostKeyFile, SSHCFG_GLOBAL },
	{ "hostdsakey", sHostKeyFile, SSHCFG_GLOBAL }, /* alias */
	{ "hostkeyagent", sHostKeyAgent, SSHCFG_GLOBAL },
	{ "pidfile", sPidFile, SSHCFG_GLOBAL },
	{ "serverkeybits", sDeprecated, SSHCFG_GLOBAL },
	{ "logingracetime", sLoginGraceTime, SSHCFG_GLOBAL },
	{ "keyregenerationinterval", sDeprecated, SSHCFG_GLOBAL },
	{ "permitrootlogin", sPermitRootLogin, SSHCFG_ALL },
	{ "syslogfacility", sLogFacility, SSHCFG_GLOBAL },
	{ "loglevel", sLogLevel, SSHCFG_ALL },
	{ "rhostsauthentication", sDeprecated, SSHCFG_GLOBAL },
	{ "rhostsrsaauthentication", sDeprecated, SSHCFG_ALL },
	{ "hostbasedauthentication", sHostbasedAuthentication, SSHCFG_ALL },
	{ "hostbasedusesnamefrompacketonly", sHostbasedUsesNameFromPacketOnly, SSHCFG_ALL },
	{ "hostbasedacceptedkeytypes", sHostbasedAcceptedKeyTypes, SSHCFG_ALL },
	{ "hostkeyalgorithms", sHostKeyAlgorithms, SSHCFG_GLOBAL },
	{ "rsaauthentication", sDeprecated, SSHCFG_ALL },
	{ "pubkeyauthentication", sPubkeyAuthentication, SSHCFG_ALL },
	{ "pubkeyacceptedkeytypes", sPubkeyAcceptedKeyTypes, SSHCFG_ALL },
	{ "dsaauthentication", sPubkeyAuthentication, SSHCFG_GLOBAL }, /* alias */
	#ifdef KRB5
	{ "kerberosauthentication", sKerberosAuthentication, SSHCFG_ALL },
	{ "kerberosorlocalpasswd", sKerberosOrLocalPasswd, SSHCFG_GLOBAL },
	{ "kerberosticketcleanup", sKerberosTicketCleanup, SSHCFG_GLOBAL },
	#ifdef USE_AFS
	{ "kerberosgetafstoken", sKerberosGetAFSToken, SSHCFG_GLOBAL },
	#else
	{ "kerberosgetafstoken", sUnsupported, SSHCFG_GLOBAL },
	#endif
	#else
	{ "kerberosauthentication", sUnsupported, SSHCFG_ALL },
	{ "kerberosorlocalpasswd", sUnsupported, SSHCFG_GLOBAL },
	{ "kerberosticketcleanup", sUnsupported, SSHCFG_GLOBAL },
	{ "kerberosgetafstoken", sUnsupported, SSHCFG_GLOBAL },
	#endif
	{ "kerberostgtpassing", sUnsupported, SSHCFG_GLOBAL },
	{ "afstokenpassing", sUnsupported, SSHCFG_GLOBAL },
	#ifdef GSSAPI
	{ "gssapiauthentication", sGssAuthentication, SSHCFG_ALL },
	{ "gssapicleanupcredentials", sGssCleanupCreds, SSHCFG_GLOBAL },
	{ "gssapistrictacceptorcheck", sGssStrictAcceptor, SSHCFG_GLOBAL },
	#else
	{ "gssapiauthentication", sUnsupported, SSHCFG_ALL },
	{ "gssapicleanupcredentials", sUnsupported, SSHCFG_GLOBAL },
	{ "gssapistrictacceptorcheck", sUnsupported, SSHCFG_GLOBAL },
	#endif
	{ "passwordauthentication", sPasswordAuthentication, SSHCFG_ALL },
	{ "kbdinteractiveauthentication", sKbdInteractiveAuthentication, SSHCFG_ALL },
	{ "challengeresponseauthentication", sChallengeResponseAuthentication, SSHCFG_GLOBAL },
	{ "skeyauthentication", sDeprecated, SSHCFG_GLOBAL },
	{ "checkmail", sDeprecated, SSHCFG_GLOBAL },
	{ "listenaddress", sListenAddress, SSHCFG_GLOBAL },
	{ "addressfamily", sAddressFamily, SSHCFG_GLOBAL },
	{ "printmotd", sPrintMotd, SSHCFG_GLOBAL },
	#ifdef DISABLE_LASTLOG
	{ "printlastlog", sUnsupported, SSHCFG_GLOBAL },
	#else
	{ "printlastlog", sPrintLastLog, SSHCFG_GLOBAL },
	#endif
	{ "ignorerhosts", sIgnoreRhosts, SSHCFG_GLOBAL },
	{ "ignoreuserknownhosts", sIgnoreUserKnownHosts, SSHCFG_GLOBAL },
	{ "x11forwarding", sX11Forwarding, SSHCFG_ALL },
	{ "x11displayoffset", sX11DisplayOffset, SSHCFG_ALL },
	{ "x11uselocalhost", sX11UseLocalhost, SSHCFG_ALL },
	{ "xauthlocation", sXAuthLocation, SSHCFG_GLOBAL },
	{ "strictmodes", sStrictModes, SSHCFG_GLOBAL },
	{ "permitemptypasswords", sEmptyPasswd, SSHCFG_ALL },
	{ "permituserenvironment", sPermitUserEnvironment, SSHCFG_GLOBAL },
	{ "uselogin", sDeprecated, SSHCFG_GLOBAL },
	{ "compression", sCompression, SSHCFG_GLOBAL },
	{ "rekeylimit", sRekeyLimit, SSHCFG_ALL },
	{ "tcpkeepalive", sTCPKeepAlive, SSHCFG_GLOBAL },
	{ "keepalive", sTCPKeepAlive, SSHCFG_GLOBAL }, /* obsolete alias */
	{ "allowtcpforwarding", sAllowTcpForwarding, SSHCFG_ALL },
	{ "allowagentforwarding", sAllowAgentForwarding, SSHCFG_ALL },
	{ "allowusers", sAllowUsers, SSHCFG_ALL },
	{ "denyusers", sDenyUsers, SSHCFG_ALL },
	{ "allowgroups", sAllowGroups, SSHCFG_ALL },
	{ "denygroups", sDenyGroups, SSHCFG_ALL },
	{ "ciphers", sCiphers, SSHCFG_GLOBAL },
	{ "macs", sMacs, SSHCFG_GLOBAL },
	{ "protocol", sIgnore, SSHCFG_GLOBAL },
	{ "gatewayports", sGatewayPorts, SSHCFG_ALL },
	{ "subsystem", sSubsystem, SSHCFG_GLOBAL },
	{ "maxstartups", sMaxStartups, SSHCFG_GLOBAL },
	{ "maxauthtries", sMaxAuthTries, SSHCFG_ALL },
	{ "maxsessions", sMaxSessions, SSHCFG_ALL },
	{ "banner", sBanner, SSHCFG_ALL },
	{ "usedns", sUseDNS, SSHCFG_GLOBAL },
	{ "verifyreversemapping", sDeprecated, SSHCFG_GLOBAL },
	{ "reversemappingcheck", sDeprecated, SSHCFG_GLOBAL },
	{ "clientaliveinterval", sClientAliveInterval, SSHCFG_ALL },
	{ "clientalivecountmax", sClientAliveCountMax, SSHCFG_ALL },
	{ "authorizedkeysfile", sAuthorizedKeysFile, SSHCFG_ALL },
	{ "authorizedkeysfile2", sDeprecated, SSHCFG_ALL },
	{ "useprivilegeseparation", sDeprecated, SSHCFG_GLOBAL},
	{ "acceptenv", sAcceptEnv, SSHCFG_ALL },
	{ "setenv", sSetEnv, SSHCFG_ALL },
	{ "permittunnel", sPermitTunnel, SSHCFG_ALL },
	{ "permittty", sPermitTTY, SSHCFG_ALL },
	{ "permituserrc", sPermitUserRC, SSHCFG_ALL },
	{ "match", sMatch, SSHCFG_ALL },
	{ "permitopen", sPermitOpen, SSHCFG_ALL },
	{ "permitlisten", sPermitListen, SSHCFG_ALL },
	{ "forcecommand", sForceCommand, SSHCFG_ALL },
	{ "chrootdirectory", sChrootDirectory, SSHCFG_ALL },
	{ "hostcertificate", sHostCertificate, SSHCFG_GLOBAL },
	{ "revokedkeys", sRevokedKeys, SSHCFG_ALL },
	{ "trustedusercakeys", sTrustedUserCAKeys, SSHCFG_ALL },
	{ "authorizedprincipalsfile", sAuthorizedPrincipalsFile, SSHCFG_ALL },
	{ "kexalgorithms", sKexAlgorithms, SSHCFG_GLOBAL },
	{ "ipqos", sIPQoS, SSHCFG_ALL },
	{ "authorizedkeyscommand", sAuthorizedKeysCommand, SSHCFG_ALL },
	{ "authorizedkeyscommanduser", sAuthorizedKeysCommandUser, SSHCFG_ALL },
	{ "authorizedprincipalscommand", sAuthorizedPrincipalsCommand, SSHCFG_ALL },
	{ "authorizedprincipalscommanduser", sAuthorizedPrincipalsCommandUser, SSHCFG_ALL },
	{ "versionaddendum", sVersionAddendum, SSHCFG_GLOBAL },
	{ "authenticationmethods", sAuthenticationMethods, SSHCFG_ALL },
	{ "streamlocalbindmask", sStreamLocalBindMask, SSHCFG_ALL },
	{ "streamlocalbindunlink", sStreamLocalBindUnlink, SSHCFG_ALL },
	{ "allowstreamlocalforwarding", sAllowStreamLocalForwarding, SSHCFG_ALL },
	{ "fingerprinthash", sFingerprintHash, SSHCFG_GLOBAL },
	{ "disableforwarding", sDisableForwarding, SSHCFG_ALL },
	{ "exposeauthinfo", sExposeAuthInfo, SSHCFG_ALL },
	{ "rdomain", sRDomain, SSHCFG_ALL },
	{ "casignaturealgorithms", sCASignatureAlgorithms, SSHCFG_ALL },
	{ "useblacklist", sUseBlacklist, SSHCFG_GLOBAL },
	+ { "useblocklist", sUseBlacklist, SSHCFG_GLOBAL }, /* alias */
	{ "noneenabled", sUnsupported, SSHCFG_ALL },
	{ "hpndisabled", sDeprecated, SSHCFG_ALL },
	{ "hpnbuffersize", sDeprecated, SSHCFG_ALL },
	{ "tcprcvbufpoll", sDeprecated, SSHCFG_ALL },
	{ NULL, sBadOption, 0 }
	};

	static struct {
	int val;
	char *text;
	} tunmode_desc[] = {
	{ SSH_TUNMODE_NO, "no" },
	{ SSH_TUNMODE_POINTOPOINT, "point-to-point" },
	{ SSH_TUNMODE_ETHERNET, "ethernet" },
	{ SSH_TUNMODE_YES, "yes" },
	{ -1, NULL }
	};

	/* Returns an opcode name from its number */

	static const char *
	lookup_opcode_name(ServerOpCodes code)
	{
	u_int i;

	for (i = 0; keywords[i].name != NULL; i++)
	if (keywords[i].opcode == code)
	return(keywords[i].name);
	return "UNKNOWN";
	}


	/*
	* Returns the number of the token pointed to by cp or sBadOption.
	*/

	static ServerOpCodes
	parse_token(const char cp, const char filename,
	int linenum, u_int *flags)
	{
	u_int i;

	for (i = 0; keywords[i].name; i++)
	if (strcasecmp(cp, keywords[i].name) == 0) {
	*flags = keywords[i].flags;
	return keywords[i].opcode;
	}

	error("%s: line %d: Bad configuration option: %s",
	filename, linenum, cp);
	return sBadOption;
	}

	char *
	derelativise_path(const char *path)
	{
	char expanded, ret, cwd[PATH_MAX];

	if (strcasecmp(path, "none") == 0)
	return xstrdup("none");
	expanded = tilde_expand_filename(path, getuid());
	if (*expanded == '/')
	return expanded;
	if (getcwd(cwd, sizeof(cwd)) == NULL)
	fatal("%s: getcwd: %s", __func__, strerror(errno));
	xasprintf(&ret, "%s/%s", cwd, expanded);
	free(expanded);
	return ret;
	}

	static void
	add_listen_addr(ServerOptions options, const char addr,
	const char *rdomain, int port)
	{
	u_int i;

	if (port > 0)
	add_one_listen_addr(options, addr, rdomain, port);
	else {
	for (i = 0; i < options->num_ports; i++) {
	add_one_listen_addr(options, addr, rdomain,
	options->ports[i]);
	}
	}
	}

	static void
	add_one_listen_addr(ServerOptions options, const char addr,
	const char *rdomain, int port)
	{
	struct addrinfo hints, ai, aitop;
	char strport[NI_MAXSERV];
	int gaierr;
	u_int i;

	/* Find listen_addrs entry for this rdomain */
	for (i = 0; i < options->num_listen_addrs; i++) {
	if (rdomain == NULL && options->listen_addrs[i].rdomain == NULL)
	break;
	if (rdomain == NULL \|\| options->listen_addrs[i].rdomain == NULL)
	continue;
	if (strcmp(rdomain, options->listen_addrs[i].rdomain) == 0)
	break;
	}
	if (i >= options->num_listen_addrs) {
	/* No entry for this rdomain; allocate one */
	if (i >= INT_MAX)
	fatal("%s: too many listen addresses", __func__);
	options->listen_addrs = xrecallocarray(options->listen_addrs,
	options->num_listen_addrs, options->num_listen_addrs + 1,
	sizeof(*options->listen_addrs));
	i = options->num_listen_addrs++;
	if (rdomain != NULL)
	options->listen_addrs[i].rdomain = xstrdup(rdomain);
	}
	/* options->listen_addrs[i] points to the addresses for this rdomain */

	memset(&hints, 0, sizeof(hints));
	hints.ai_family = options->address_family;
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_flags = (addr == NULL) ? AI_PASSIVE : 0;
	snprintf(strport, sizeof strport, "%d", port);
	if ((gaierr = getaddrinfo(addr, strport, &hints, &aitop)) != 0)
	fatal("bad addr or host: %s (%s)",
	addr ? addr : "<NULL>",
	ssh_gai_strerror(gaierr));
	for (ai = aitop; ai->ai_next; ai = ai->ai_next)
	;
	ai->ai_next = options->listen_addrs[i].addrs;
	options->listen_addrs[i].addrs = aitop;
	}

	/* Returns nonzero if the routing domain name is valid */
	static int
	valid_rdomain(const char *name)
	{
	#if defined(HAVE_SYS_VALID_RDOMAIN)
	return sys_valid_rdomain(name);
	#elif defined(__OpenBSD__)
	const char *errstr;
	long long num;
	struct rt_tableinfo info;
	int mib[6];
	size_t miblen = sizeof(mib);

	if (name == NULL)
	return 1;

	num = strtonum(name, 0, 255, &errstr);
	if (errstr != NULL)
	return 0;

	/* Check whether the table actually exists */
	memset(mib, 0, sizeof(mib));
	mib[0] = CTL_NET;
	mib[1] = PF_ROUTE;
	mib[4] = NET_RT_TABLE;
	mib[5] = (int)num;
	if (sysctl(mib, 6, &info, &miblen, NULL, 0) == -1)
	return 0;

	return 1;
	#else /* defined(__OpenBSD__) */
	error("Routing domains are not supported on this platform");
	return 0;
	#endif
	}

	/*
	* Queue a ListenAddress to be processed once we have all of the Ports
	* and AddressFamily options.
	*/
	static void
	queue_listen_addr(ServerOptions options, const char addr,
	const char *rdomain, int port)
	{
	struct queued_listenaddr *qla;

	options->queued_listen_addrs = xrecallocarray(
	options->queued_listen_addrs,
	options->num_queued_listens, options->num_queued_listens + 1,
	sizeof(*options->queued_listen_addrs));
	qla = &options->queued_listen_addrs[options->num_queued_listens++];
	qla->addr = xstrdup(addr);
	qla->port = port;
	qla->rdomain = rdomain == NULL ? NULL : xstrdup(rdomain);
	}

	/*
	* Process queued (text) ListenAddress entries.
	*/
	static void
	process_queued_listen_addrs(ServerOptions *options)
	{
	u_int i;
	struct queued_listenaddr *qla;

	if (options->num_ports == 0)
	options->ports[options->num_ports++] = SSH_DEFAULT_PORT;
	if (options->address_family == -1)
	options->address_family = AF_UNSPEC;

	for (i = 0; i < options->num_queued_listens; i++) {
	qla = &options->queued_listen_addrs[i];
	add_listen_addr(options, qla->addr, qla->rdomain, qla->port);
	free(qla->addr);
	free(qla->rdomain);
	}
	free(options->queued_listen_addrs);
	options->queued_listen_addrs = NULL;
	options->num_queued_listens = 0;
	}

	/*
	* Inform channels layer of permitopen options for a single forwarding
	* direction (local/remote).
	*/
	static void
	process_permitopen_list(struct ssh *ssh, ServerOpCodes opcode,
	char **opens, u_int num_opens)
	{
	u_int i;
	int port;
	char host, arg, *oarg;
	int where = opcode == sPermitOpen ? FORWARD_LOCAL : FORWARD_REMOTE;
	const char *what = lookup_opcode_name(opcode);

	channel_clear_permission(ssh, FORWARD_ADM, where);
	if (num_opens == 0)
	return; /* permit any */

	/* handle keywords: "any" / "none" */
	if (num_opens == 1 && strcmp(opens[0], "any") == 0)
	return;
	if (num_opens == 1 && strcmp(opens[0], "none") == 0) {
	channel_disable_admin(ssh, where);
	return;
	}
	/* Otherwise treat it as a list of permitted host:port */
	for (i = 0; i < num_opens; i++) {
	oarg = arg = xstrdup(opens[i]);
	host = hpdelim(&arg);
	if (host == NULL)
	fatal("%s: missing host in %s", __func__, what);
	host = cleanhostname(host);
	if (arg == NULL \|\| ((port = permitopen_port(arg)) < 0))
	fatal("%s: bad port number in %s", __func__, what);
	/* Send it to channels layer */
	channel_add_permission(ssh, FORWARD_ADM,
	where, host, port);
	free(oarg);
	}
	}

	/*
	* Inform channels layer of permitopen options from configuration.
	*/
	void
	process_permitopen(struct ssh ssh, ServerOptions options)
	{
	process_permitopen_list(ssh, sPermitOpen,
	options->permitted_opens, options->num_permitted_opens);
	process_permitopen_list(ssh, sPermitListen,
	options->permitted_listens,
	options->num_permitted_listens);
	}

	struct connection_info *
	get_connection_info(int populate, int use_dns)
	{
	struct ssh ssh = active_state; / XXX */
	static struct connection_info ci;

	if (!populate)
	return &ci;
	ci.host = auth_get_canonical_hostname(ssh, use_dns);
	ci.address = ssh_remote_ipaddr(ssh);
	ci.laddress = ssh_local_ipaddr(ssh);
	ci.lport = ssh_local_port(ssh);
	ci.rdomain = ssh_packet_rdomain_in(ssh);
	return &ci;
	}

	/*
	* The strategy for the Match blocks is that the config file is parsed twice.
	*
	* The first time is at startup. activep is initialized to 1 and the
	* directives in the global context are processed and acted on. Hitting a
	* Match directive unsets activep and the directives inside the block are
	* checked for syntax only.
	*
	* The second time is after a connection has been established but before
	* authentication. activep is initialized to 2 and global config directives
	* are ignored since they have already been processed. If the criteria in a
	* Match block is met, activep is set and the subsequent directives
	* processed and actioned until EOF or another Match block unsets it. Any
	* options set are copied into the main server config.
	*
	* Potential additions/improvements:
	* - Add Match support for pre-kex directives, eg. Ciphers.
	*
	* - Add a Tag directive (idea from David Leonard) ala pf, eg:
	* Match Address 192.168.0.*
	* Tag trusted
	* Match Group wheel
	* Tag trusted
	* Match Tag trusted
	* AllowTcpForwarding yes
	* GatewayPorts clientspecified
	* [...]
	*
	* - Add a PermittedChannelRequests directive
	* Match Group shell
	* PermittedChannelRequests session,forwarded-tcpip
	*/

	static int
	match_cfg_line_group(const char grps, int line, const char user)
	{
	int result = 0;
	struct passwd *pw;

	if (user == NULL)
	goto out;

	if ((pw = getpwnam(user)) == NULL) {
	debug("Can't match group at line %d because user %.100s does "
	"not exist", line, user);
	} else if (ga_init(pw->pw_name, pw->pw_gid) == 0) {
	debug("Can't Match group because user %.100s not in any group "
	"at line %d", user, line);
	} else if (ga_match_pattern_list(grps) != 1) {
	debug("user %.100s does not match group list %.100s at line %d",
	user, grps, line);
	} else {
	debug("user %.100s matched group list %.100s at line %d", user,
	grps, line);
	result = 1;
	}
	out:
	ga_free();
	return result;
	}

	static void
	match_test_missing_fatal(const char criteria, const char attrib)
	{
	fatal("'Match %s' in configuration but '%s' not in connection "
	"test specification.", criteria, attrib);
	}

	/*
	* All of the attributes on a single Match line are ANDed together, so we need
	* to check every attribute and set the result to zero if any attribute does
	* not match.
	*/
	static int
	match_cfg_line(char *condition, int line, struct connection_info ci)
	{
	int result = 1, attributes = 0, port;
	char arg, attrib, cp = condition;

	if (ci == NULL)
	debug3("checking syntax for 'Match %s'", cp);
	else
	debug3("checking match for '%s' user %s host %s addr %s "
	"laddr %s lport %d", cp, ci->user ? ci->user : "(null)",
	ci->host ? ci->host : "(null)",
	ci->address ? ci->address : "(null)",
	ci->laddress ? ci->laddress : "(null)", ci->lport);

	while ((attrib = strdelim(&cp)) && *attrib != '\0') {
	attributes++;
	if (strcasecmp(attrib, "all") == 0) {
	if (attributes != 1 \|\|
	((arg = strdelim(&cp)) != NULL && *arg != '\0')) {
	error("'all' cannot be combined with other "
	"Match attributes");
	return -1;
	}
	*condition = cp;
	return 1;
	}
	if ((arg = strdelim(&cp)) == NULL \|\| *arg == '\0') {
	error("Missing Match criteria for %s", attrib);
	return -1;
	}
	if (strcasecmp(attrib, "user") == 0) {
	if (ci == NULL) {
	result = 0;
	continue;
	}
	if (ci->user == NULL)
	match_test_missing_fatal("User", "user");
	if (match_pattern_list(ci->user, arg, 0) != 1)
	result = 0;
	else
	debug("user %.100s matched 'User %.100s' at "
	"line %d", ci->user, arg, line);
	} else if (strcasecmp(attrib, "group") == 0) {
	if (ci == NULL) {
	result = 0;
	continue;
	}
	if (ci->user == NULL)
	match_test_missing_fatal("Group", "user");
	switch (match_cfg_line_group(arg, line, ci->user)) {
	case -1:
	return -1;
	case 0:
	result = 0;
	}
	} else if (strcasecmp(attrib, "host") == 0) {
	if (ci == NULL) {
	result = 0;
	continue;
	}
	if (ci->host == NULL)
	match_test_missing_fatal("Host", "host");
	if (match_hostname(ci->host, arg) != 1)
	result = 0;
	else
	debug("connection from %.100s matched 'Host "
	"%.100s' at line %d", ci->host, arg, line);
	} else if (strcasecmp(attrib, "address") == 0) {
	if (ci == NULL) {
	result = 0;
	continue;
	}
	if (ci->address == NULL)
	match_test_missing_fatal("Address", "addr");
	switch (addr_match_list(ci->address, arg)) {
	case 1:
	debug("connection from %.100s matched 'Address "
	"%.100s' at line %d", ci->address, arg, line);
	break;
	case 0:
	case -1:
	result = 0;
	break;
	case -2:
	return -1;
	}
	} else if (strcasecmp(attrib, "localaddress") == 0){
	if (ci == NULL) {
	result = 0;
	continue;
	}
	if (ci->laddress == NULL)
	match_test_missing_fatal("LocalAddress",
	"laddr");
	switch (addr_match_list(ci->laddress, arg)) {
	case 1:
	debug("connection from %.100s matched "
	"'LocalAddress %.100s' at line %d",
	ci->laddress, arg, line);
	break;
	case 0:
	case -1:
	result = 0;
	break;
	case -2:
	return -1;
	}
	} else if (strcasecmp(attrib, "localport") == 0) {
	if ((port = a2port(arg)) == -1) {
	error("Invalid LocalPort '%s' on Match line",
	arg);
	return -1;
	}
	if (ci == NULL) {
	result = 0;
	continue;
	}
	if (ci->lport == 0)
	match_test_missing_fatal("LocalPort", "lport");
	/* TODO support port lists */
	if (port == ci->lport)
	debug("connection from %.100s matched "
	"'LocalPort %d' at line %d",
	ci->laddress, port, line);
	else
	result = 0;
	} else if (strcasecmp(attrib, "rdomain") == 0) {
	if (ci == NULL \|\| ci->rdomain == NULL) {
	result = 0;
	continue;
	}
	if (match_pattern_list(ci->rdomain, arg, 0) != 1)
	result = 0;
	else
	debug("user %.100s matched 'RDomain %.100s' at "
	"line %d", ci->rdomain, arg, line);
	} else {
	error("Unsupported Match attribute %s", attrib);
	return -1;
	}
	}
	if (attributes == 0) {
	error("One or more attributes required for Match");
	return -1;
	}
	if (ci != NULL)
	debug3("match %sfound", result ? "" : "not ");
	*condition = cp;
	return result;
	}

	#define WHITESPACE " \t\r\n"

	/* Multistate option parsing */
	struct multistate {
	char *key;
	int value;
	};
	static const struct multistate multistate_flag[] = {
	{ "yes", 1 },
	{ "no", 0 },
	{ NULL, -1 }
	};
	static const struct multistate multistate_addressfamily[] = {
	{ "inet", AF_INET },
	{ "inet6", AF_INET6 },
	{ "any", AF_UNSPEC },
	{ NULL, -1 }
	};
	static const struct multistate multistate_permitrootlogin[] = {
	{ "without-password", PERMIT_NO_PASSWD },
	{ "prohibit-password", PERMIT_NO_PASSWD },
	{ "forced-commands-only", PERMIT_FORCED_ONLY },
	{ "yes", PERMIT_YES },
	{ "no", PERMIT_NO },
	{ NULL, -1 }
	};
	static const struct multistate multistate_compression[] = {
	{ "yes", COMP_DELAYED },
	{ "delayed", COMP_DELAYED },
	{ "no", COMP_NONE },
	{ NULL, -1 }
	};
	static const struct multistate multistate_gatewayports[] = {
	{ "clientspecified", 2 },
	{ "yes", 1 },
	{ "no", 0 },
	{ NULL, -1 }
	};
	static const struct multistate multistate_tcpfwd[] = {
	{ "yes", FORWARD_ALLOW },
	{ "all", FORWARD_ALLOW },
	{ "no", FORWARD_DENY },
	{ "remote", FORWARD_REMOTE },
	{ "local", FORWARD_LOCAL },
	{ NULL, -1 }
	};

	int
	process_server_config_line(ServerOptions options, char line,
	const char filename, int linenum, int activep,
	struct connection_info *connectinfo)
	{
	char cp, chararrayptr, charptr, arg, arg2, *p;
	int cmdline = 0, *intptr, value, value2, n, port;
	SyslogFacility *log_facility_ptr;
	LogLevel *log_level_ptr;
	ServerOpCodes opcode;
	u_int i, *uintptr, uvalue, flags = 0;
	size_t len;
	long long val64;
	const struct multistate *multistate_ptr;
	const char *errstr;

	/* Strip trailing whitespace. Allow \f (form feed) at EOL only */
	if ((len = strlen(line)) == 0)
	return 0;
	for (len--; len > 0; len--) {
	if (strchr(WHITESPACE "\f", line[len]) == NULL)
	break;
	line[len] = '\0';
	}

	cp = line;
	if ((arg = strdelim(&cp)) == NULL)
	return 0;
	/* Ignore leading whitespace */
	if (*arg == '\0')
	arg = strdelim(&cp);
	if (!arg \|\| !arg \|\| arg == '#')
	return 0;
	intptr = NULL;
	charptr = NULL;
	opcode = parse_token(arg, filename, linenum, &flags);

	if (activep == NULL) { /* We are processing a command line directive */
	cmdline = 1;
	activep = &cmdline;
	}
	if (*activep && opcode != sMatch)
	debug3("%s:%d setting %s %s", filename, linenum, arg, cp);
	if (*activep == 0 && !(flags & SSHCFG_MATCH)) {
	if (connectinfo == NULL) {
	fatal("%s line %d: Directive '%s' is not allowed "
	"within a Match block", filename, linenum, arg);
	} else { /* this is a directive we have already processed */
	while (arg)
	arg = strdelim(&cp);
	return 0;
	}
	}

	switch (opcode) {
	/* Portable-specific options */
	case sUsePAM:
	intptr = &options->use_pam;
	goto parse_flag;

	/* Standard Options */
	case sBadOption:
	return -1;
	case sPort:
	/* ignore ports from configfile if cmdline specifies ports */
	if (options->ports_from_cmdline)
	return 0;
	if (options->num_ports >= MAX_PORTS)
	fatal("%s line %d: too many ports.",
	filename, linenum);
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing port number.",
	filename, linenum);
	options->ports[options->num_ports++] = a2port(arg);
	if (options->ports[options->num_ports-1] <= 0)
	fatal("%s line %d: Badly formatted port number.",
	filename, linenum);
	break;

	case sLoginGraceTime:
	intptr = &options->login_grace_time;
	parse_time:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing time value.",
	filename, linenum);
	if ((value = convtime(arg)) == -1)
	fatal("%s line %d: invalid time value.",
	filename, linenum);
	if (activep && intptr == -1)
	*intptr = value;
	break;

	case sListenAddress:
	arg = strdelim(&cp);
	if (arg == NULL \|\| *arg == '\0')
	fatal("%s line %d: missing address",
	filename, linenum);
	/* check for bare IPv6 address: no "[]" and 2 or more ":" */
	if (strchr(arg, '[') == NULL && (p = strchr(arg, ':')) != NULL
	&& strchr(p+1, ':') != NULL) {
	port = 0;
	p = arg;
	} else {
	p = hpdelim(&arg);
	if (p == NULL)
	fatal("%s line %d: bad address:port usage",
	filename, linenum);
	p = cleanhostname(p);
	if (arg == NULL)
	port = 0;
	else if ((port = a2port(arg)) <= 0)
	fatal("%s line %d: bad port number",
	filename, linenum);
	}
	/* Optional routing table */
	arg2 = NULL;
	if ((arg = strdelim(&cp)) != NULL) {
	if (strcmp(arg, "rdomain") != 0 \|\|
	(arg2 = strdelim(&cp)) == NULL)
	fatal("%s line %d: bad ListenAddress syntax",
	filename, linenum);
	if (!valid_rdomain(arg2))
	fatal("%s line %d: bad routing domain",
	filename, linenum);
	}

	queue_listen_addr(options, p, arg2, port);

	break;

	case sAddressFamily:
	intptr = &options->address_family;
	multistate_ptr = multistate_addressfamily;
	parse_multistate:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing argument.",
	filename, linenum);
	value = -1;
	for (i = 0; multistate_ptr[i].key != NULL; i++) {
	if (strcasecmp(arg, multistate_ptr[i].key) == 0) {
	value = multistate_ptr[i].value;
	break;
	}
	}
	if (value == -1)
	fatal("%s line %d: unsupported option \"%s\".",
	filename, linenum, arg);
	if (activep && intptr == -1)
	*intptr = value;
	break;

	case sHostKeyFile:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing file name.",
	filename, linenum);
	if (*activep)
	servconf_add_hostkey(filename, linenum, options, arg);
	break;

	case sHostKeyAgent:
	charptr = &options->host_key_agent;
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing socket name.",
	filename, linenum);
	if (activep && charptr == NULL)
	*charptr = !strcmp(arg, SSH_AUTHSOCKET_ENV_NAME) ?
	xstrdup(arg) : derelativise_path(arg);
	break;

	case sHostCertificate:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing file name.",
	filename, linenum);
	if (*activep)
	servconf_add_hostcert(filename, linenum, options, arg);
	break;

	case sPidFile:
	charptr = &options->pid_file;
	parse_filename:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing file name.",
	filename, linenum);
	if (activep && charptr == NULL) {
	*charptr = derelativise_path(arg);
	/* increase optional counter */
	if (intptr != NULL)
	intptr = intptr + 1;
	}
	break;

	case sPermitRootLogin:
	intptr = &options->permit_root_login;
	multistate_ptr = multistate_permitrootlogin;
	goto parse_multistate;

	case sIgnoreRhosts:
	intptr = &options->ignore_rhosts;
	parse_flag:
	multistate_ptr = multistate_flag;
	goto parse_multistate;

	case sIgnoreUserKnownHosts:
	intptr = &options->ignore_user_known_hosts;
	goto parse_flag;

	case sHostbasedAuthentication:
	intptr = &options->hostbased_authentication;
	goto parse_flag;

	case sHostbasedUsesNameFromPacketOnly:
	intptr = &options->hostbased_uses_name_from_packet_only;
	goto parse_flag;

	case sHostbasedAcceptedKeyTypes:
	charptr = &options->hostbased_key_types;
	parse_keytypes:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing argument.",
	filename, linenum);
	if (*arg != '-' &&
	!sshkey_names_valid2(*arg == '+' ? arg + 1 : arg, 1))
	fatal("%s line %d: Bad key types '%s'.",
	filename, linenum, arg ? arg : "<NONE>");
	if (activep && charptr == NULL)
	*charptr = xstrdup(arg);
	break;

	case sHostKeyAlgorithms:
	charptr = &options->hostkeyalgorithms;
	goto parse_keytypes;

	case sCASignatureAlgorithms:
	charptr = &options->ca_sign_algorithms;
	goto parse_keytypes;

	case sPubkeyAuthentication:
	intptr = &options->pubkey_authentication;
	goto parse_flag;

	case sPubkeyAcceptedKeyTypes:
	charptr = &options->pubkey_key_types;
	goto parse_keytypes;

	case sKerberosAuthentication:
	intptr = &options->kerberos_authentication;
	goto parse_flag;

	case sKerberosOrLocalPasswd:
	intptr = &options->kerberos_or_local_passwd;
	goto parse_flag;

	case sKerberosTicketCleanup:
	intptr = &options->kerberos_ticket_cleanup;
	goto parse_flag;

	case sKerberosGetAFSToken:
	intptr = &options->kerberos_get_afs_token;
	goto parse_flag;

	case sGssAuthentication:
	intptr = &options->gss_authentication;
	goto parse_flag;

	case sGssCleanupCreds:
	intptr = &options->gss_cleanup_creds;
	goto parse_flag;

	case sGssStrictAcceptor:
	intptr = &options->gss_strict_acceptor;
	goto parse_flag;

	case sPasswordAuthentication:
	intptr = &options->password_authentication;
	goto parse_flag;

	case sKbdInteractiveAuthentication:
	intptr = &options->kbd_interactive_authentication;
	goto parse_flag;

	case sChallengeResponseAuthentication:
	intptr = &options->challenge_response_authentication;
	goto parse_flag;

	case sPrintMotd:
	intptr = &options->print_motd;
	goto parse_flag;

	case sPrintLastLog:
	intptr = &options->print_lastlog;
	goto parse_flag;

	case sX11Forwarding:
	intptr = &options->x11_forwarding;
	goto parse_flag;

	case sX11DisplayOffset:
	intptr = &options->x11_display_offset;
	parse_int:
	arg = strdelim(&cp);
	if ((errstr = atoi_err(arg, &value)) != NULL)
	fatal("%s line %d: integer value %s.",
	filename, linenum, errstr);
	if (activep && intptr == -1)
	*intptr = value;
	break;

	case sX11UseLocalhost:
	intptr = &options->x11_use_localhost;
	goto parse_flag;

	case sXAuthLocation:
	charptr = &options->xauth_location;
	goto parse_filename;

	case sPermitTTY:
	intptr = &options->permit_tty;
	goto parse_flag;

	case sPermitUserRC:
	intptr = &options->permit_user_rc;
	goto parse_flag;

	case sStrictModes:
	intptr = &options->strict_modes;
	goto parse_flag;

	case sTCPKeepAlive:
	intptr = &options->tcp_keep_alive;
	goto parse_flag;

	case sEmptyPasswd:
	intptr = &options->permit_empty_passwd;
	goto parse_flag;

	case sPermitUserEnvironment:
	intptr = &options->permit_user_env;
	charptr = &options->permit_user_env_whitelist;
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing argument.",
	filename, linenum);
	value = 0;
	p = NULL;
	if (strcmp(arg, "yes") == 0)
	value = 1;
	else if (strcmp(arg, "no") == 0)
	value = 0;
	else {
	/* Pattern-list specified */
	value = 1;
	p = xstrdup(arg);
	}
	if (activep && intptr == -1) {
	*intptr = value;
	*charptr = p;
	p = NULL;
	}
	free(p);
	break;

	case sCompression:
	intptr = &options->compression;
	multistate_ptr = multistate_compression;
	goto parse_multistate;

	case sRekeyLimit:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%.200s line %d: Missing argument.", filename,
	linenum);
	if (strcmp(arg, "default") == 0) {
	val64 = 0;
	} else {
	if (scan_scaled(arg, &val64) == -1)
	fatal("%.200s line %d: Bad number '%s': %s",
	filename, linenum, arg, strerror(errno));
	if (val64 != 0 && val64 < 16)
	fatal("%.200s line %d: RekeyLimit too small",
	filename, linenum);
	}
	if (*activep && options->rekey_limit == -1)
	options->rekey_limit = val64;
	if (cp != NULL) { /* optional rekey interval present */
	if (strcmp(cp, "none") == 0) {
	(void)strdelim(&cp); /* discard */
	break;
	}
	intptr = &options->rekey_interval;
	goto parse_time;
	}
	break;

	case sGatewayPorts:
	intptr = &options->fwd_opts.gateway_ports;
	multistate_ptr = multistate_gatewayports;
	goto parse_multistate;

	case sUseDNS:
	intptr = &options->use_dns;
	goto parse_flag;

	case sLogFacility:
	log_facility_ptr = &options->log_facility;
	arg = strdelim(&cp);
	value = log_facility_number(arg);
	if (value == SYSLOG_FACILITY_NOT_SET)
	fatal("%.200s line %d: unsupported log facility '%s'",
	filename, linenum, arg ? arg : "<NONE>");
	if (*log_facility_ptr == -1)
	*log_facility_ptr = (SyslogFacility) value;
	break;

	case sLogLevel:
	log_level_ptr = &options->log_level;
	arg = strdelim(&cp);
	value = log_level_number(arg);
	if (value == SYSLOG_LEVEL_NOT_SET)
	fatal("%.200s line %d: unsupported log level '%s'",
	filename, linenum, arg ? arg : "<NONE>");
	if (activep && log_level_ptr == -1)
	*log_level_ptr = (LogLevel) value;
	break;

	case sAllowTcpForwarding:
	intptr = &options->allow_tcp_forwarding;
	multistate_ptr = multistate_tcpfwd;
	goto parse_multistate;

	case sAllowStreamLocalForwarding:
	intptr = &options->allow_streamlocal_forwarding;
	multistate_ptr = multistate_tcpfwd;
	goto parse_multistate;

	case sAllowAgentForwarding:
	intptr = &options->allow_agent_forwarding;
	goto parse_flag;

	case sDisableForwarding:
	intptr = &options->disable_forwarding;
	goto parse_flag;

	case sAllowUsers:
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	if (match_user(NULL, NULL, NULL, arg) == -1)
	fatal("%s line %d: invalid AllowUsers pattern: "
	"\"%.100s\"", filename, linenum, arg);
	if (!*activep)
	continue;
	array_append(filename, linenum, "AllowUsers",
	&options->allow_users, &options->num_allow_users,
	arg);
	}
	break;

	case sDenyUsers:
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	if (match_user(NULL, NULL, NULL, arg) == -1)
	fatal("%s line %d: invalid DenyUsers pattern: "
	"\"%.100s\"", filename, linenum, arg);
	if (!*activep)
	continue;
	array_append(filename, linenum, "DenyUsers",
	&options->deny_users, &options->num_deny_users,
	arg);
	}
	break;

	case sAllowGroups:
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	if (!*activep)
	continue;
	array_append(filename, linenum, "AllowGroups",
	&options->allow_groups, &options->num_allow_groups,
	arg);
	}
	break;

	case sDenyGroups:
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	if (!*activep)
	continue;
	array_append(filename, linenum, "DenyGroups",
	&options->deny_groups, &options->num_deny_groups,
	arg);
	}
	break;

	case sCiphers:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing argument.", filename, linenum);
	if (arg != '-' && !ciphers_valid(arg == '+' ? arg + 1 : arg))
	fatal("%s line %d: Bad SSH2 cipher spec '%s'.",
	filename, linenum, arg ? arg : "<NONE>");
	if (options->ciphers == NULL)
	options->ciphers = xstrdup(arg);
	break;

	case sMacs:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing argument.", filename, linenum);
	if (arg != '-' && !mac_valid(arg == '+' ? arg + 1 : arg))
	fatal("%s line %d: Bad SSH2 mac spec '%s'.",
	filename, linenum, arg ? arg : "<NONE>");
	if (options->macs == NULL)
	options->macs = xstrdup(arg);
	break;

	case sKexAlgorithms:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing argument.",
	filename, linenum);
	if (*arg != '-' &&
	!kex_names_valid(*arg == '+' ? arg + 1 : arg))
	fatal("%s line %d: Bad SSH2 KexAlgorithms '%s'.",
	filename, linenum, arg ? arg : "<NONE>");
	if (options->kex_algorithms == NULL)
	options->kex_algorithms = xstrdup(arg);
	break;

	case sSubsystem:
	if (options->num_subsystems >= MAX_SUBSYSTEMS) {
	fatal("%s line %d: too many subsystems defined.",
	filename, linenum);
	}
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing subsystem name.",
	filename, linenum);
	if (!*activep) {
	arg = strdelim(&cp);
	break;
	}
	for (i = 0; i < options->num_subsystems; i++)
	if (strcmp(arg, options->subsystem_name[i]) == 0)
	fatal("%s line %d: Subsystem '%s' already defined.",
	filename, linenum, arg);
	options->subsystem_name[options->num_subsystems] = xstrdup(arg);
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing subsystem command.",
	filename, linenum);
	options->subsystem_command[options->num_subsystems] = xstrdup(arg);

	/* Collect arguments (separate to executable) */
	p = xstrdup(arg);
	len = strlen(p) + 1;
	while ((arg = strdelim(&cp)) != NULL && *arg != '\0') {
	len += 1 + strlen(arg);
	p = xreallocarray(p, 1, len);
	strlcat(p, " ", len);
	strlcat(p, arg, len);
	}
	options->subsystem_args[options->num_subsystems] = p;
	options->num_subsystems++;
	break;

	case sMaxStartups:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing MaxStartups spec.",
	filename, linenum);
	if ((n = sscanf(arg, "%d:%d:%d",
	&options->max_startups_begin,
	&options->max_startups_rate,
	&options->max_startups)) == 3) {
	if (options->max_startups_begin >
	options->max_startups \|\|
	options->max_startups_rate > 100 \|\|
	options->max_startups_rate < 1)
	fatal("%s line %d: Illegal MaxStartups spec.",
	filename, linenum);
	} else if (n != 1)
	fatal("%s line %d: Illegal MaxStartups spec.",
	filename, linenum);
	else
	options->max_startups = options->max_startups_begin;
	break;

	case sMaxAuthTries:
	intptr = &options->max_authtries;
	goto parse_int;

	case sMaxSessions:
	intptr = &options->max_sessions;
	goto parse_int;

	case sBanner:
	charptr = &options->banner;
	goto parse_filename;

	/*
	* These options can contain %X options expanded at
	* connect time, so that you can specify paths like:
	*
	* AuthorizedKeysFile /etc/ssh_keys/%u
	*/
	case sAuthorizedKeysFile:
	if (*activep && options->num_authkeys_files == 0) {
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	arg = tilde_expand_filename(arg, getuid());
	array_append(filename, linenum,
	"AuthorizedKeysFile",
	&options->authorized_keys_files,
	&options->num_authkeys_files, arg);
	free(arg);
	}
	}
	return 0;

	case sAuthorizedPrincipalsFile:
	charptr = &options->authorized_principals_file;
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing file name.",
	filename, linenum);
	if (activep && charptr == NULL) {
	*charptr = tilde_expand_filename(arg, getuid());
	/* increase optional counter */
	if (intptr != NULL)
	intptr = intptr + 1;
	}
	break;

	case sClientAliveInterval:
	intptr = &options->client_alive_interval;
	goto parse_time;

	case sClientAliveCountMax:
	intptr = &options->client_alive_count_max;
	goto parse_int;

	case sAcceptEnv:
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	if (strchr(arg, '=') != NULL)
	fatal("%s line %d: Invalid environment name.",
	filename, linenum);
	if (!*activep)
	continue;
	array_append(filename, linenum, "AcceptEnv",
	&options->accept_env, &options->num_accept_env,
	arg);
	}
	break;

	case sSetEnv:
	uvalue = options->num_setenv;
	while ((arg = strdelimw(&cp)) && *arg != '\0') {
	if (strchr(arg, '=') == NULL)
	fatal("%s line %d: Invalid environment.",
	filename, linenum);
	if (!*activep \|\| uvalue != 0)
	continue;
	array_append(filename, linenum, "SetEnv",
	&options->setenv, &options->num_setenv, arg);
	}
	break;

	case sPermitTunnel:
	intptr = &options->permit_tun;
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: Missing yes/point-to-point/"
	"ethernet/no argument.", filename, linenum);
	value = -1;
	for (i = 0; tunmode_desc[i].val != -1; i++)
	if (strcmp(tunmode_desc[i].text, arg) == 0) {
	value = tunmode_desc[i].val;
	break;
	}
	if (value == -1)
	fatal("%s line %d: Bad yes/point-to-point/ethernet/"
	"no argument: %s", filename, linenum, arg);
	if (activep && intptr == -1)
	*intptr = value;
	break;

	case sMatch:
	if (cmdline)
	fatal("Match directive not supported as a command-line "
	"option");
	value = match_cfg_line(&cp, linenum, connectinfo);
	if (value < 0)
	fatal("%s line %d: Bad Match condition", filename,
	linenum);
	*activep = value;
	break;

	case sPermitListen:
	case sPermitOpen:
	if (opcode == sPermitListen) {
	uintptr = &options->num_permitted_listens;
	chararrayptr = &options->permitted_listens;
	} else {
	uintptr = &options->num_permitted_opens;
	chararrayptr = &options->permitted_opens;
	}
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing %s specification",
	filename, linenum, lookup_opcode_name(opcode));
	uvalue = uintptr; / modified later */
	if (strcmp(arg, "any") == 0 \|\| strcmp(arg, "none") == 0) {
	if (*activep && uvalue == 0) {
	*uintptr = 1;
	*chararrayptr = xcalloc(1,
	sizeof(**chararrayptr));
	(*chararrayptr)[0] = xstrdup(arg);
	}
	break;
	}
	for (; arg != NULL && *arg != '\0'; arg = strdelim(&cp)) {
	if (opcode == sPermitListen &&
	strchr(arg, ':') == NULL) {
	/*
	* Allow bare port number for PermitListen
	* to indicate a wildcard listen host.
	*/
	xasprintf(&arg2, "*:%s", arg);
	} else {
	arg2 = xstrdup(arg);
	p = hpdelim(&arg);
	if (p == NULL) {
	fatal("%s line %d: missing host in %s",
	filename, linenum,
	lookup_opcode_name(opcode));
	}
	p = cleanhostname(p);
	}
	if (arg == NULL \|\|
	((port = permitopen_port(arg)) < 0)) {
	fatal("%s line %d: bad port number in %s",
	filename, linenum,
	lookup_opcode_name(opcode));
	}
	if (*activep && uvalue == 0) {
	array_append(filename, linenum,
	lookup_opcode_name(opcode),
	chararrayptr, uintptr, arg2);
	}
	free(arg2);
	}
	break;

	case sForceCommand:
	if (cp == NULL \|\| *cp == '\0')
	fatal("%.200s line %d: Missing argument.", filename,
	linenum);
	len = strspn(cp, WHITESPACE);
	if (*activep && options->adm_forced_command == NULL)
	options->adm_forced_command = xstrdup(cp + len);
	return 0;

	case sChrootDirectory:
	charptr = &options->chroot_directory;

	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing file name.",
	filename, linenum);
	if (activep && charptr == NULL)
	*charptr = xstrdup(arg);
	break;

	case sTrustedUserCAKeys:
	charptr = &options->trusted_user_ca_keys;
	goto parse_filename;

	case sRevokedKeys:
	charptr = &options->revoked_keys_file;
	goto parse_filename;

	case sIPQoS:
	arg = strdelim(&cp);
	if ((value = parse_ipqos(arg)) == -1)
	fatal("%s line %d: Bad IPQoS value: %s",
	filename, linenum, arg);
	arg = strdelim(&cp);
	if (arg == NULL)
	value2 = value;
	else if ((value2 = parse_ipqos(arg)) == -1)
	fatal("%s line %d: Bad IPQoS value: %s",
	filename, linenum, arg);
	if (*activep) {
	options->ip_qos_interactive = value;
	options->ip_qos_bulk = value2;
	}
	break;

	case sVersionAddendum:
	if (cp == NULL \|\| *cp == '\0')
	fatal("%.200s line %d: Missing argument.", filename,
	linenum);
	len = strspn(cp, WHITESPACE);
	if (*activep && options->version_addendum == NULL) {
	if (strcasecmp(cp + len, "none") == 0)
	options->version_addendum = xstrdup("");
	else if (strchr(cp + len, '\r') != NULL)
	fatal("%.200s line %d: Invalid argument",
	filename, linenum);
	else
	options->version_addendum = xstrdup(cp + len);
	}
	return 0;

	case sAuthorizedKeysCommand:
	if (cp == NULL)
	fatal("%.200s line %d: Missing argument.", filename,
	linenum);
	len = strspn(cp, WHITESPACE);
	if (*activep && options->authorized_keys_command == NULL) {
	if (cp[len] != '/' && strcasecmp(cp + len, "none") != 0)
	fatal("%.200s line %d: AuthorizedKeysCommand "
	"must be an absolute path",
	filename, linenum);
	options->authorized_keys_command = xstrdup(cp + len);
	}
	return 0;

	case sAuthorizedKeysCommandUser:
	charptr = &options->authorized_keys_command_user;

	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing AuthorizedKeysCommandUser "
	"argument.", filename, linenum);
	if (activep && charptr == NULL)
	*charptr = xstrdup(arg);
	break;

	case sAuthorizedPrincipalsCommand:
	if (cp == NULL)
	fatal("%.200s line %d: Missing argument.", filename,
	linenum);
	len = strspn(cp, WHITESPACE);
	if (*activep &&
	options->authorized_principals_command == NULL) {
	if (cp[len] != '/' && strcasecmp(cp + len, "none") != 0)
	fatal("%.200s line %d: "
	"AuthorizedPrincipalsCommand must be "
	"an absolute path", filename, linenum);
	options->authorized_principals_command =
	xstrdup(cp + len);
	}
	return 0;

	case sAuthorizedPrincipalsCommandUser:
	charptr = &options->authorized_principals_command_user;

	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing "
	"AuthorizedPrincipalsCommandUser argument.",
	filename, linenum);
	if (activep && charptr == NULL)
	*charptr = xstrdup(arg);
	break;

	case sAuthenticationMethods:
	if (options->num_auth_methods == 0) {
	value = 0; /* seen "any" pseudo-method */
	value2 = 0; /* successfully parsed any method */
	while ((arg = strdelim(&cp)) && *arg != '\0') {
	if (strcmp(arg, "any") == 0) {
	if (options->num_auth_methods > 0) {
	fatal("%s line %d: \"any\" "
	"must appear alone in "
	"AuthenticationMethods",
	filename, linenum);
	}
	value = 1;
	} else if (value) {
	fatal("%s line %d: \"any\" must appear "
	"alone in AuthenticationMethods",
	filename, linenum);
	} else if (auth2_methods_valid(arg, 0) != 0) {
	fatal("%s line %d: invalid "
	"authentication method list.",
	filename, linenum);
	}
	value2 = 1;
	if (!*activep)
	continue;
	array_append(filename, linenum,
	"AuthenticationMethods",
	&options->auth_methods,
	&options->num_auth_methods, arg);
	}
	if (value2 == 0) {
	fatal("%s line %d: no AuthenticationMethods "
	"specified", filename, linenum);
	}
	}
	return 0;

	case sStreamLocalBindMask:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%s line %d: missing StreamLocalBindMask "
	"argument.", filename, linenum);
	/* Parse mode in octal format */
	value = strtol(arg, &p, 8);
	if (arg == p \|\| value < 0 \|\| value > 0777)
	fatal("%s line %d: Bad mask.", filename, linenum);
	if (*activep)
	options->fwd_opts.streamlocal_bind_mask = (mode_t)value;
	break;

	case sStreamLocalBindUnlink:
	intptr = &options->fwd_opts.streamlocal_bind_unlink;
	goto parse_flag;

	case sFingerprintHash:
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%.200s line %d: Missing argument.",
	filename, linenum);
	if ((value = ssh_digest_alg_by_name(arg)) == -1)
	fatal("%.200s line %d: Invalid hash algorithm \"%s\".",
	filename, linenum, arg);
	if (*activep)
	options->fingerprint_hash = value;
	break;

	case sExposeAuthInfo:
	intptr = &options->expose_userauth_info;
	goto parse_flag;

	case sRDomain:
	charptr = &options->routing_domain;
	arg = strdelim(&cp);
	if (!arg \|\| *arg == '\0')
	fatal("%.200s line %d: Missing argument.",
	filename, linenum);
	if (strcasecmp(arg, "none") != 0 && strcmp(arg, "%D") != 0 &&
	!valid_rdomain(arg))
	fatal("%s line %d: bad routing domain",
	filename, linenum);
	if (activep && charptr == NULL)
	*charptr = xstrdup(arg);
	break;

	case sUseBlacklist:
	intptr = &options->use_blacklist;
	goto parse_flag;

	case sDeprecated:
	case sIgnore:
	case sUnsupported:
	do_log2(opcode == sIgnore ?
	SYSLOG_LEVEL_DEBUG2 : SYSLOG_LEVEL_INFO,
	"%s line %d: %s option %s", filename, linenum,
	opcode == sUnsupported ? "Unsupported" : "Deprecated", arg);
	while (arg)
	arg = strdelim(&cp);
	break;

	default:
	fatal("%s line %d: Missing handler for opcode %s (%d)",
	filename, linenum, arg, opcode);
	}
	if ((arg = strdelim(&cp)) != NULL && *arg != '\0')
	fatal("%s line %d: garbage at end of line; \"%.200s\".",
	filename, linenum, arg);
	return 0;
	}

	/* Reads the server configuration file. */

	void
	load_server_config(const char filename, struct sshbuf conf)
	{
	char line = NULL, cp;
	size_t linesize = 0;
	FILE *f;
	int r, lineno = 0;

	debug2("%s: filename %s", __func__, filename);
	if ((f = fopen(filename, "r")) == NULL) {
	perror(filename);
	exit(1);
	}
	sshbuf_reset(conf);
	while (getline(&line, &linesize, f) != -1) {
	lineno++;
	/*
	* Trim out comments and strip whitespace
	* NB - preserve newlines, they are needed to reproduce
	* line numbers later for error messages
	*/
	if ((cp = strchr(line, '#')) != NULL)
	memcpy(cp, "\n", 2);
	cp = line + strspn(line, " \t\r");
	if ((r = sshbuf_put(conf, cp, strlen(cp))) != 0)
	fatal("%s: buffer error: %s", __func__, ssh_err(r));
	}
	free(line);
	if ((r = sshbuf_put_u8(conf, 0)) != 0)
	fatal("%s: buffer error: %s", __func__, ssh_err(r));
	fclose(f);
	debug2("%s: done config len = %zu", __func__, sshbuf_len(conf));
	}

	void
	parse_server_match_config(ServerOptions *options,
	struct connection_info *connectinfo)
	{
	ServerOptions mo;

	initialize_server_options(&mo);
	parse_server_config(&mo, "reprocess config", cfg, connectinfo);
	copy_set_server_options(options, &mo, 0);
	}

	int parse_server_match_testspec(struct connection_info ci, char spec)
	{
	char *p;

	while ((p = strsep(&spec, ",")) && *p != '\0') {
	if (strncmp(p, "addr=", 5) == 0) {
	ci->address = xstrdup(p + 5);
	} else if (strncmp(p, "host=", 5) == 0) {
	ci->host = xstrdup(p + 5);
	} else if (strncmp(p, "user=", 5) == 0) {
	ci->user = xstrdup(p + 5);
	} else if (strncmp(p, "laddr=", 6) == 0) {
	ci->laddress = xstrdup(p + 6);
	} else if (strncmp(p, "rdomain=", 8) == 0) {
	ci->rdomain = xstrdup(p + 8);
	} else if (strncmp(p, "lport=", 6) == 0) {
	ci->lport = a2port(p + 6);
	if (ci->lport == -1) {
	fprintf(stderr, "Invalid port '%s' in test mode"
	" specification %s\n", p+6, p);
	return -1;
	}
	} else {
	fprintf(stderr, "Invalid test mode specification %s\n",
	p);
	return -1;
	}
	}
	return 0;
	}

	/*
	* Copy any supported values that are set.
	*
	* If the preauth flag is set, we do not bother copying the string or
	* array values that are not used pre-authentication, because any that we
	* do use must be explicitly sent in mm_getpwnamallow().
	*/
	void
	copy_set_server_options(ServerOptions dst, ServerOptions src, int preauth)
	{
	#define M_CP_INTOPT(n) do {\
	if (src->n != -1) \
	dst->n = src->n; \
	} while (0)

	M_CP_INTOPT(password_authentication);
	M_CP_INTOPT(gss_authentication);
	M_CP_INTOPT(pubkey_authentication);
	M_CP_INTOPT(kerberos_authentication);
	M_CP_INTOPT(hostbased_authentication);
	M_CP_INTOPT(hostbased_uses_name_from_packet_only);
	M_CP_INTOPT(kbd_interactive_authentication);
	M_CP_INTOPT(permit_root_login);
	M_CP_INTOPT(permit_empty_passwd);

	M_CP_INTOPT(allow_tcp_forwarding);
	M_CP_INTOPT(allow_streamlocal_forwarding);
	M_CP_INTOPT(allow_agent_forwarding);
	M_CP_INTOPT(disable_forwarding);
	M_CP_INTOPT(expose_userauth_info);
	M_CP_INTOPT(permit_tun);
	M_CP_INTOPT(fwd_opts.gateway_ports);
	M_CP_INTOPT(fwd_opts.streamlocal_bind_unlink);
	M_CP_INTOPT(x11_display_offset);
	M_CP_INTOPT(x11_forwarding);
	M_CP_INTOPT(x11_use_localhost);
	M_CP_INTOPT(permit_tty);
	M_CP_INTOPT(permit_user_rc);
	M_CP_INTOPT(max_sessions);
	M_CP_INTOPT(max_authtries);
	M_CP_INTOPT(client_alive_count_max);
	M_CP_INTOPT(client_alive_interval);
	M_CP_INTOPT(ip_qos_interactive);
	M_CP_INTOPT(ip_qos_bulk);
	M_CP_INTOPT(rekey_limit);
	M_CP_INTOPT(rekey_interval);
	M_CP_INTOPT(log_level);

	/*
	* The bind_mask is a mode_t that may be unsigned, so we can't use
	* M_CP_INTOPT - it does a signed comparison that causes compiler
	* warnings.
	*/
	if (src->fwd_opts.streamlocal_bind_mask != (mode_t)-1) {
	dst->fwd_opts.streamlocal_bind_mask =
	src->fwd_opts.streamlocal_bind_mask;
	}

	/* M_CP_STROPT and M_CP_STRARRAYOPT should not appear before here */
	#define M_CP_STROPT(n) do {\
	if (src->n != NULL && dst->n != src->n) { \
	free(dst->n); \
	dst->n = src->n; \
	} \
	} while(0)
	#define M_CP_STRARRAYOPT(s, num_s) do {\
	u_int i; \
	if (src->num_s != 0) { \
	for (i = 0; i < dst->num_s; i++) \
	free(dst->s[i]); \
	free(dst->s); \
	dst->s = xcalloc(src->num_s, sizeof(*dst->s)); \
	for (i = 0; i < src->num_s; i++) \
	dst->s[i] = xstrdup(src->s[i]); \
	dst->num_s = src->num_s; \
	} \
	} while(0)

	/* See comment in servconf.h */
	COPY_MATCH_STRING_OPTS();

	/* Arguments that accept '+...' need to be expanded */
	assemble_algorithms(dst);

	/*
	* The only things that should be below this point are string options
	* which are only used after authentication.
	*/
	if (preauth)
	return;

	/* These options may be "none" to clear a global setting */
	M_CP_STROPT(adm_forced_command);
	if (option_clear_or_none(dst->adm_forced_command)) {
	free(dst->adm_forced_command);
	dst->adm_forced_command = NULL;
	}
	M_CP_STROPT(chroot_directory);
	if (option_clear_or_none(dst->chroot_directory)) {
	free(dst->chroot_directory);
	dst->chroot_directory = NULL;
	}
	}

	#undef M_CP_INTOPT
	#undef M_CP_STROPT
	#undef M_CP_STRARRAYOPT

	void
	parse_server_config(ServerOptions options, const char filename,
	struct sshbuf conf, struct connection_info connectinfo)
	{
	int active, linenum, bad_options = 0;
	char cp, obuf, *cbuf;

	debug2("%s: config %s len %zu", __func__, filename, sshbuf_len(conf));

	if ((obuf = cbuf = sshbuf_dup_string(conf)) == NULL)
	fatal("%s: sshbuf_dup_string failed", __func__);
	active = connectinfo ? 0 : 1;
	linenum = 1;
	while ((cp = strsep(&cbuf, "\n")) != NULL) {
	if (process_server_config_line(options, cp, filename,
	linenum++, &active, connectinfo) != 0)
	bad_options++;
	}
	free(obuf);
	if (bad_options > 0)
	fatal("%s: terminating, %d bad configuration options",
	filename, bad_options);
	process_queued_listen_addrs(options);
	}

	static const char *
	fmt_multistate_int(int val, const struct multistate *m)
	{
	u_int i;

	for (i = 0; m[i].key != NULL; i++) {
	if (m[i].value == val)
	return m[i].key;
	}
	return "UNKNOWN";
	}

	static const char *
	fmt_intarg(ServerOpCodes code, int val)
	{
	if (val == -1)
	return "unset";
	switch (code) {
	case sAddressFamily:
	return fmt_multistate_int(val, multistate_addressfamily);
	case sPermitRootLogin:
	return fmt_multistate_int(val, multistate_permitrootlogin);
	case sGatewayPorts:
	return fmt_multistate_int(val, multistate_gatewayports);
	case sCompression:
	return fmt_multistate_int(val, multistate_compression);
	case sAllowTcpForwarding:
	return fmt_multistate_int(val, multistate_tcpfwd);
	case sAllowStreamLocalForwarding:
	return fmt_multistate_int(val, multistate_tcpfwd);
	case sFingerprintHash:
	return ssh_digest_alg_name(val);
	default:
	switch (val) {
	case 0:
	return "no";
	case 1:
	return "yes";
	default:
	return "UNKNOWN";
	}
	}
	}

	static void
	dump_cfg_int(ServerOpCodes code, int val)
	{
	printf("%s %d\n", lookup_opcode_name(code), val);
	}

	static void
	dump_cfg_oct(ServerOpCodes code, int val)
	{
	printf("%s 0%o\n", lookup_opcode_name(code), val);
	}

	static void
	dump_cfg_fmtint(ServerOpCodes code, int val)
	{
	printf("%s %s\n", lookup_opcode_name(code), fmt_intarg(code, val));
	}

	static void
	dump_cfg_string(ServerOpCodes code, const char *val)
	{
	printf("%s %s\n", lookup_opcode_name(code),
	val == NULL ? "none" : val);
	}

	static void
	dump_cfg_strarray(ServerOpCodes code, u_int count, char **vals)
	{
	u_int i;

	for (i = 0; i < count; i++)
	printf("%s %s\n", lookup_opcode_name(code), vals[i]);
	}

	static void
	dump_cfg_strarray_oneline(ServerOpCodes code, u_int count, char **vals)
	{
	u_int i;

	if (count <= 0 && code != sAuthenticationMethods)
	return;
	printf("%s", lookup_opcode_name(code));
	for (i = 0; i < count; i++)
	printf(" %s", vals[i]);
	if (code == sAuthenticationMethods && count == 0)
	printf(" any");
	printf("\n");
	}

	static char *
	format_listen_addrs(struct listenaddr *la)
	{
	int r;
	struct addrinfo *ai;
	char addr[NI_MAXHOST], port[NI_MAXSERV];
	char laddr1 = xstrdup(""), laddr2 = NULL;

	/*
	* ListenAddress must be after Port. add_one_listen_addr pushes
	* addresses onto a stack, so to maintain ordering we need to
	* print these in reverse order.
	*/
	for (ai = la->addrs; ai; ai = ai->ai_next) {
	if ((r = getnameinfo(ai->ai_addr, ai->ai_addrlen, addr,
	sizeof(addr), port, sizeof(port),
	NI_NUMERICHOST\|NI_NUMERICSERV)) != 0) {
	error("getnameinfo: %.100s", ssh_gai_strerror(r));
	continue;
	}
	laddr2 = laddr1;
	if (ai->ai_family == AF_INET6) {
	xasprintf(&laddr1, "listenaddress [%s]:%s%s%s\n%s",
	addr, port,
	la->rdomain == NULL ? "" : " rdomain ",
	la->rdomain == NULL ? "" : la->rdomain,
	laddr2);
	} else {
	xasprintf(&laddr1, "listenaddress %s:%s%s%s\n%s",
	addr, port,
	la->rdomain == NULL ? "" : " rdomain ",
	la->rdomain == NULL ? "" : la->rdomain,
	laddr2);
	}
	free(laddr2);
	}
	return laddr1;
	}

	void
	dump_config(ServerOptions *o)
	{
	char *s;
	u_int i;

	/* these are usually at the top of the config */
	for (i = 0; i < o->num_ports; i++)
	printf("port %d\n", o->ports[i]);
	dump_cfg_fmtint(sAddressFamily, o->address_family);

	for (i = 0; i < o->num_listen_addrs; i++) {
	s = format_listen_addrs(&o->listen_addrs[i]);
	printf("%s", s);
	free(s);
	}

	/* integer arguments */
	#ifdef USE_PAM
	dump_cfg_fmtint(sUsePAM, o->use_pam);
	#endif
	dump_cfg_int(sLoginGraceTime, o->login_grace_time);
	dump_cfg_int(sX11DisplayOffset, o->x11_display_offset);
	dump_cfg_int(sMaxAuthTries, o->max_authtries);
	dump_cfg_int(sMaxSessions, o->max_sessions);
	dump_cfg_int(sClientAliveInterval, o->client_alive_interval);
	dump_cfg_int(sClientAliveCountMax, o->client_alive_count_max);
	dump_cfg_oct(sStreamLocalBindMask, o->fwd_opts.streamlocal_bind_mask);

	/* formatted integer arguments */
	dump_cfg_fmtint(sPermitRootLogin, o->permit_root_login);
	dump_cfg_fmtint(sIgnoreRhosts, o->ignore_rhosts);
	dump_cfg_fmtint(sIgnoreUserKnownHosts, o->ignore_user_known_hosts);
	dump_cfg_fmtint(sHostbasedAuthentication, o->hostbased_authentication);
	dump_cfg_fmtint(sHostbasedUsesNameFromPacketOnly,
	o->hostbased_uses_name_from_packet_only);
	dump_cfg_fmtint(sPubkeyAuthentication, o->pubkey_authentication);
	#ifdef KRB5
	dump_cfg_fmtint(sKerberosAuthentication, o->kerberos_authentication);
	dump_cfg_fmtint(sKerberosOrLocalPasswd, o->kerberos_or_local_passwd);
	dump_cfg_fmtint(sKerberosTicketCleanup, o->kerberos_ticket_cleanup);
	# ifdef USE_AFS
	dump_cfg_fmtint(sKerberosGetAFSToken, o->kerberos_get_afs_token);
	# endif
	#endif
	#ifdef GSSAPI
	dump_cfg_fmtint(sGssAuthentication, o->gss_authentication);
	dump_cfg_fmtint(sGssCleanupCreds, o->gss_cleanup_creds);
	#endif
	dump_cfg_fmtint(sPasswordAuthentication, o->password_authentication);
	dump_cfg_fmtint(sKbdInteractiveAuthentication,
	o->kbd_interactive_authentication);
	dump_cfg_fmtint(sChallengeResponseAuthentication,
	o->challenge_response_authentication);
	dump_cfg_fmtint(sPrintMotd, o->print_motd);
	#ifndef DISABLE_LASTLOG
	dump_cfg_fmtint(sPrintLastLog, o->print_lastlog);
	#endif
	dump_cfg_fmtint(sX11Forwarding, o->x11_forwarding);
	dump_cfg_fmtint(sX11UseLocalhost, o->x11_use_localhost);
	dump_cfg_fmtint(sPermitTTY, o->permit_tty);
	dump_cfg_fmtint(sPermitUserRC, o->permit_user_rc);
	dump_cfg_fmtint(sStrictModes, o->strict_modes);
	dump_cfg_fmtint(sTCPKeepAlive, o->tcp_keep_alive);
	dump_cfg_fmtint(sEmptyPasswd, o->permit_empty_passwd);
	dump_cfg_fmtint(sCompression, o->compression);
	dump_cfg_fmtint(sGatewayPorts, o->fwd_opts.gateway_ports);
	dump_cfg_fmtint(sUseDNS, o->use_dns);
	dump_cfg_fmtint(sAllowTcpForwarding, o->allow_tcp_forwarding);
	dump_cfg_fmtint(sAllowAgentForwarding, o->allow_agent_forwarding);
	dump_cfg_fmtint(sDisableForwarding, o->disable_forwarding);
	dump_cfg_fmtint(sAllowStreamLocalForwarding, o->allow_streamlocal_forwarding);
	dump_cfg_fmtint(sStreamLocalBindUnlink, o->fwd_opts.streamlocal_bind_unlink);
	dump_cfg_fmtint(sFingerprintHash, o->fingerprint_hash);
	dump_cfg_fmtint(sExposeAuthInfo, o->expose_userauth_info);
	dump_cfg_fmtint(sUseBlacklist, o->use_blacklist);

	/* string arguments */
	dump_cfg_string(sPidFile, o->pid_file);
	dump_cfg_string(sXAuthLocation, o->xauth_location);
	dump_cfg_string(sCiphers, o->ciphers ? o->ciphers : KEX_SERVER_ENCRYPT);
	dump_cfg_string(sMacs, o->macs ? o->macs : KEX_SERVER_MAC);
	dump_cfg_string(sBanner, o->banner);
	dump_cfg_string(sForceCommand, o->adm_forced_command);
	dump_cfg_string(sChrootDirectory, o->chroot_directory);
	dump_cfg_string(sTrustedUserCAKeys, o->trusted_user_ca_keys);
	dump_cfg_string(sRevokedKeys, o->revoked_keys_file);
	dump_cfg_string(sAuthorizedPrincipalsFile,
	o->authorized_principals_file);
	dump_cfg_string(sVersionAddendum, *o->version_addendum == '\0'
	? "none" : o->version_addendum);
	dump_cfg_string(sAuthorizedKeysCommand, o->authorized_keys_command);
	dump_cfg_string(sAuthorizedKeysCommandUser, o->authorized_keys_command_user);
	dump_cfg_string(sAuthorizedPrincipalsCommand, o->authorized_principals_command);
	dump_cfg_string(sAuthorizedPrincipalsCommandUser, o->authorized_principals_command_user);
	dump_cfg_string(sHostKeyAgent, o->host_key_agent);
	dump_cfg_string(sKexAlgorithms,
	o->kex_algorithms ? o->kex_algorithms : KEX_SERVER_KEX);
	dump_cfg_string(sCASignatureAlgorithms, o->ca_sign_algorithms ?
	o->ca_sign_algorithms : SSH_ALLOWED_CA_SIGALGS);
	dump_cfg_string(sHostbasedAcceptedKeyTypes, o->hostbased_key_types ?
	o->hostbased_key_types : KEX_DEFAULT_PK_ALG);
	dump_cfg_string(sHostKeyAlgorithms, o->hostkeyalgorithms ?
	o->hostkeyalgorithms : KEX_DEFAULT_PK_ALG);
	dump_cfg_string(sPubkeyAcceptedKeyTypes, o->pubkey_key_types ?
	o->pubkey_key_types : KEX_DEFAULT_PK_ALG);
	dump_cfg_string(sRDomain, o->routing_domain);

	/* string arguments requiring a lookup */
	dump_cfg_string(sLogLevel, log_level_name(o->log_level));
	dump_cfg_string(sLogFacility, log_facility_name(o->log_facility));

	/* string array arguments */
	dump_cfg_strarray_oneline(sAuthorizedKeysFile, o->num_authkeys_files,
	o->authorized_keys_files);
	dump_cfg_strarray(sHostKeyFile, o->num_host_key_files,
	o->host_key_files);
	dump_cfg_strarray(sHostCertificate, o->num_host_cert_files,
	o->host_cert_files);
	dump_cfg_strarray(sAllowUsers, o->num_allow_users, o->allow_users);
	dump_cfg_strarray(sDenyUsers, o->num_deny_users, o->deny_users);
	dump_cfg_strarray(sAllowGroups, o->num_allow_groups, o->allow_groups);
	dump_cfg_strarray(sDenyGroups, o->num_deny_groups, o->deny_groups);
	dump_cfg_strarray(sAcceptEnv, o->num_accept_env, o->accept_env);
	dump_cfg_strarray(sSetEnv, o->num_setenv, o->setenv);
	dump_cfg_strarray_oneline(sAuthenticationMethods,
	o->num_auth_methods, o->auth_methods);

	/* other arguments */
	for (i = 0; i < o->num_subsystems; i++)
	printf("subsystem %s %s\n", o->subsystem_name[i],
	o->subsystem_args[i]);

	printf("maxstartups %d:%d:%d\n", o->max_startups_begin,
	o->max_startups_rate, o->max_startups);

	s = NULL;
	for (i = 0; tunmode_desc[i].val != -1; i++) {
	if (tunmode_desc[i].val == o->permit_tun) {
	s = tunmode_desc[i].text;
	break;
	}
	}
	dump_cfg_string(sPermitTunnel, s);

	printf("ipqos %s ", iptos2str(o->ip_qos_interactive));
	printf("%s\n", iptos2str(o->ip_qos_bulk));

	printf("rekeylimit %llu %d\n", (unsigned long long)o->rekey_limit,
	o->rekey_interval);

	printf("permitopen");
	if (o->num_permitted_opens == 0)
	printf(" any");
	else {
	for (i = 0; i < o->num_permitted_opens; i++)
	printf(" %s", o->permitted_opens[i]);
	}
	printf("\n");
	printf("permitlisten");
	if (o->num_permitted_listens == 0)
	printf(" any");
	else {
	for (i = 0; i < o->num_permitted_listens; i++)
	printf(" %s", o->permitted_listens[i]);
	}
	printf("\n");

	if (o->permit_user_env_whitelist == NULL) {
	dump_cfg_fmtint(sPermitUserEnvironment, o->permit_user_env);
	} else {
	printf("permituserenvironment %s\n",
	o->permit_user_env_whitelist);
	}

	}
	diff --git a/crypto/openssh/sshd_config.5 b/crypto/openssh/sshd_config.5
	index 8176078319d1..d28622d984a1 100644
	--- a/crypto/openssh/sshd_config.5
	+++ b/crypto/openssh/sshd_config.5
	@@ -1,1847 +1,1851 @@
	.\"
	.\" Author: Tatu Ylonen <ylo@cs.hut.fi>
	.\" Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
	.\" All rights reserved
	.\"
	.\" As far as I am concerned, the code I have written for this software
	.\" can be used freely for any purpose. Any derived versions of this
	.\" software must be clearly marked as such, and if the derived work is
	.\" incompatible with the protocol description in the RFC file, it must be
	.\" called by a name other than "ssh" or "Secure Shell".
	.\"
	.\" Copyright (c) 1999,2000 Markus Friedl. All rights reserved.
	.\" Copyright (c) 1999 Aaron Campbell. All rights reserved.
	.\" Copyright (c) 1999 Theo de Raadt. All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	.\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	.\"
	.\" $OpenBSD: sshd_config.5,v 1.282 2018/09/20 03:28:06 djm Exp $
	.\" $FreeBSD$
	-.Dd $Mdocdate: September 20 2018 $
	+.Dd $Mdocdate: July 28 2020 $
	.Dt SSHD_CONFIG 5
	.Os
	.Sh NAME
	.Nm sshd_config
	.Nd OpenSSH SSH daemon configuration file
	.Sh DESCRIPTION
	.Xr sshd 8
	reads configuration data from
	.Pa /etc/ssh/sshd_config
	(or the file specified with
	.Fl f
	on the command line).
	The file contains keyword-argument pairs, one per line.
	For each keyword, the first obtained value will be used.
	Lines starting with
	.Ql #
	and empty lines are interpreted as comments.
	Arguments may optionally be enclosed in double quotes
	.Pq \&"
	in order to represent arguments containing spaces.
	.Pp
	The possible
	keywords and their meanings are as follows (note that
	keywords are case-insensitive and arguments are case-sensitive):
	.Bl -tag -width Ds
	.It Cm AcceptEnv
	Specifies what environment variables sent by the client will be copied into
	the session's
	.Xr environ 7 .
	See
	.Cm SendEnv
	and
	.Cm SetEnv
	in
	.Xr ssh_config 5
	for how to configure the client.
	The
	.Ev TERM
	environment variable is always accepted whenever the client
	requests a pseudo-terminal as it is required by the protocol.
	Variables are specified by name, which may contain the wildcard characters
	.Ql *
	and
	.Ql \&? .
	Multiple environment variables may be separated by whitespace or spread
	across multiple
	.Cm AcceptEnv
	directives.
	Be warned that some environment variables could be used to bypass restricted
	user environments.
	For this reason, care should be taken in the use of this directive.
	The default is not to accept any environment variables.
	.It Cm AddressFamily
	Specifies which address family should be used by
	.Xr sshd 8 .
	Valid arguments are
	.Cm any
	(the default),
	.Cm inet
	(use IPv4 only), or
	.Cm inet6
	(use IPv6 only).
	.It Cm AllowAgentForwarding
	Specifies whether
	.Xr ssh-agent 1
	forwarding is permitted.
	The default is
	.Cm yes .
	Note that disabling agent forwarding does not improve security
	unless users are also denied shell access, as they can always install
	their own forwarders.
	.It Cm AllowGroups
	This keyword can be followed by a list of group name patterns, separated
	by spaces.
	If specified, login is allowed only for users whose primary
	group or supplementary group list matches one of the patterns.
	Only group names are valid; a numerical group ID is not recognized.
	By default, login is allowed for all groups.
	The allow/deny directives are processed in the following order:
	.Cm DenyUsers ,
	.Cm AllowUsers ,
	.Cm DenyGroups ,
	and finally
	.Cm AllowGroups .
	.Pp
	See PATTERNS in
	.Xr ssh_config 5
	for more information on patterns.
	.It Cm AllowStreamLocalForwarding
	Specifies whether StreamLocal (Unix-domain socket) forwarding is permitted.
	The available options are
	.Cm yes
	(the default)
	or
	.Cm all
	to allow StreamLocal forwarding,
	.Cm no
	to prevent all StreamLocal forwarding,
	.Cm local
	to allow local (from the perspective of
	.Xr ssh 1 )
	forwarding only or
	.Cm remote
	to allow remote forwarding only.
	Note that disabling StreamLocal forwarding does not improve security unless
	users are also denied shell access, as they can always install their
	own forwarders.
	.It Cm AllowTcpForwarding
	Specifies whether TCP forwarding is permitted.
	The available options are
	.Cm yes
	(the default)
	or
	.Cm all
	to allow TCP forwarding,
	.Cm no
	to prevent all TCP forwarding,
	.Cm local
	to allow local (from the perspective of
	.Xr ssh 1 )
	forwarding only or
	.Cm remote
	to allow remote forwarding only.
	Note that disabling TCP forwarding does not improve security unless
	users are also denied shell access, as they can always install their
	own forwarders.
	.It Cm AllowUsers
	This keyword can be followed by a list of user name patterns, separated
	by spaces.
	If specified, login is allowed only for user names that
	match one of the patterns.
	Only user names are valid; a numerical user ID is not recognized.
	By default, login is allowed for all users.
	If the pattern takes the form USER@HOST then USER and HOST
	are separately checked, restricting logins to particular
	users from particular hosts.
	HOST criteria may additionally contain addresses to match in CIDR
	address/masklen format.
	The allow/deny directives are processed in the following order:
	.Cm DenyUsers ,
	.Cm AllowUsers ,
	.Cm DenyGroups ,
	and finally
	.Cm AllowGroups .
	.Pp
	See PATTERNS in
	.Xr ssh_config 5
	for more information on patterns.
	.It Cm AuthenticationMethods
	Specifies the authentication methods that must be successfully completed
	for a user to be granted access.
	This option must be followed by one or more lists of comma-separated
	authentication method names, or by the single string
	.Cm any
	to indicate the default behaviour of accepting any single authentication
	method.
	If the default is overridden, then successful authentication requires
	completion of every method in at least one of these lists.
	.Pp
	For example,
	.Qq publickey,password publickey,keyboard-interactive
	would require the user to complete public key authentication, followed by
	either password or keyboard interactive authentication.
	Only methods that are next in one or more lists are offered at each stage,
	so for this example it would not be possible to attempt password or
	keyboard-interactive authentication before public key.
	.Pp
	For keyboard interactive authentication it is also possible to
	restrict authentication to a specific device by appending a
	colon followed by the device identifier
	.Cm bsdauth
	or
	.Cm pam .
	depending on the server configuration.
	For example,
	.Qq keyboard-interactive:bsdauth
	would restrict keyboard interactive authentication to the
	.Cm bsdauth
	device.
	.Pp
	If the publickey method is listed more than once,
	.Xr sshd 8
	verifies that keys that have been used successfully are not reused for
	subsequent authentications.
	For example,
	.Qq publickey,publickey
	requires successful authentication using two different public keys.
	.Pp
	Note that each authentication method listed should also be explicitly enabled
	in the configuration.
	.Pp
	The available authentication methods are:
	.Qq gssapi-with-mic ,
	.Qq hostbased ,
	.Qq keyboard-interactive ,
	.Qq none
	(used for access to password-less accounts when
	.Cm PermitEmptyPasswords
	is enabled),
	.Qq password
	and
	.Qq publickey .
	.It Cm AuthorizedKeysCommand
	Specifies a program to be used to look up the user's public keys.
	The program must be owned by root, not writable by group or others and
	specified by an absolute path.
	Arguments to
	.Cm AuthorizedKeysCommand
	accept the tokens described in the
	.Sx TOKENS
	section.
	If no arguments are specified then the username of the target user is used.
	.Pp
	The program should produce on standard output zero or
	more lines of authorized_keys output (see
	.Sx AUTHORIZED_KEYS
	in
	.Xr sshd 8 ) .
	If a key supplied by
	.Cm AuthorizedKeysCommand
	does not successfully authenticate
	and authorize the user then public key authentication continues using the usual
	.Cm AuthorizedKeysFile
	files.
	By default, no
	.Cm AuthorizedKeysCommand
	is run.
	.It Cm AuthorizedKeysCommandUser
	Specifies the user under whose account the
	.Cm AuthorizedKeysCommand
	is run.
	It is recommended to use a dedicated user that has no other role on the host
	than running authorized keys commands.
	If
	.Cm AuthorizedKeysCommand
	is specified but
	.Cm AuthorizedKeysCommandUser
	is not, then
	.Xr sshd 8
	will refuse to start.
	.It Cm AuthorizedKeysFile
	Specifies the file that contains the public keys used for user authentication.
	The format is described in the
	.Sx AUTHORIZED_KEYS FILE FORMAT
	section of
	.Xr sshd 8 .
	Arguments to
	.Cm AuthorizedKeysFile
	accept the tokens described in the
	.Sx TOKENS
	section.
	After expansion,
	.Cm AuthorizedKeysFile
	is taken to be an absolute path or one relative to the user's home
	directory.
	Multiple files may be listed, separated by whitespace.
	Alternately this option may be set to
	.Cm none
	to skip checking for user keys in files.
	The default is
	.Qq .ssh/authorized_keys .ssh/authorized_keys2 .
	.It Cm AuthorizedPrincipalsCommand
	Specifies a program to be used to generate the list of allowed
	certificate principals as per
	.Cm AuthorizedPrincipalsFile .
	The program must be owned by root, not writable by group or others and
	specified by an absolute path.
	Arguments to
	.Cm AuthorizedPrincipalsCommand
	accept the tokens described in the
	.Sx TOKENS
	section.
	If no arguments are specified then the username of the target user is used.
	.Pp
	The program should produce on standard output zero or
	more lines of
	.Cm AuthorizedPrincipalsFile
	output.
	If either
	.Cm AuthorizedPrincipalsCommand
	or
	.Cm AuthorizedPrincipalsFile
	is specified, then certificates offered by the client for authentication
	must contain a principal that is listed.
	By default, no
	.Cm AuthorizedPrincipalsCommand
	is run.
	.It Cm AuthorizedPrincipalsCommandUser
	Specifies the user under whose account the
	.Cm AuthorizedPrincipalsCommand
	is run.
	It is recommended to use a dedicated user that has no other role on the host
	than running authorized principals commands.
	If
	.Cm AuthorizedPrincipalsCommand
	is specified but
	.Cm AuthorizedPrincipalsCommandUser
	is not, then
	.Xr sshd 8
	will refuse to start.
	.It Cm AuthorizedPrincipalsFile
	Specifies a file that lists principal names that are accepted for
	certificate authentication.
	When using certificates signed by a key listed in
	.Cm TrustedUserCAKeys ,
	this file lists names, one of which must appear in the certificate for it
	to be accepted for authentication.
	Names are listed one per line preceded by key options (as described in
	.Sx AUTHORIZED_KEYS FILE FORMAT
	in
	.Xr sshd 8 ) .
	Empty lines and comments starting with
	.Ql #
	are ignored.
	.Pp
	Arguments to
	.Cm AuthorizedPrincipalsFile
	accept the tokens described in the
	.Sx TOKENS
	section.
	After expansion,
	.Cm AuthorizedPrincipalsFile
	is taken to be an absolute path or one relative to the user's home directory.
	The default is
	.Cm none ,
	i.e. not to use a principals file \(en in this case, the username
	of the user must appear in a certificate's principals list for it to be
	accepted.
	.Pp
	Note that
	.Cm AuthorizedPrincipalsFile
	is only used when authentication proceeds using a CA listed in
	.Cm TrustedUserCAKeys
	and is not consulted for certification authorities trusted via
	.Pa ~/.ssh/authorized_keys ,
	though the
	.Cm principals=
	key option offers a similar facility (see
	.Xr sshd 8
	for details).
	.It Cm Banner
	The contents of the specified file are sent to the remote user before
	authentication is allowed.
	If the argument is
	.Cm none
	then no banner is displayed.
	By default, no banner is displayed.
	.It Cm CASignatureAlgorithms
	Specifies which algorithms are allowed for signing of certificates
	by certificate authorities (CAs).
	The default is:
	.Bd -literal -offset indent
	ecdsa-sha2-nistp256.ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,
	ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa
	.Ed
	.Pp
	Certificates signed using other algorithms will not be accepted for
	public key or host-based authentication.
	.It Cm ChallengeResponseAuthentication
	Specifies whether challenge-response authentication is allowed (e.g. via
	PAM or through authentication styles supported in
	.Xr login.conf 5 )
	The default is
	.Cm yes .
	.It Cm ChrootDirectory
	Specifies the pathname of a directory to
	.Xr chroot 2
	to after authentication.
	At session startup
	.Xr sshd 8
	checks that all components of the pathname are root-owned directories
	which are not writable by any other user or group.
	After the chroot,
	.Xr sshd 8
	changes the working directory to the user's home directory.
	Arguments to
	.Cm ChrootDirectory
	accept the tokens described in the
	.Sx TOKENS
	section.
	.Pp
	The
	.Cm ChrootDirectory
	must contain the necessary files and directories to support the
	user's session.
	For an interactive session this requires at least a shell, typically
	.Xr sh 1 ,
	and basic
	.Pa /dev
	nodes such as
	.Xr null 4 ,
	.Xr zero 4 ,
	.Xr stdin 4 ,
	.Xr stdout 4 ,
	.Xr stderr 4 ,
	and
	.Xr tty 4
	devices.
	For file transfer sessions using SFTP
	no additional configuration of the environment is necessary if the in-process
	sftp-server is used,
	though sessions which use logging may require
	.Pa /dev/log
	inside the chroot directory on some operating systems (see
	.Xr sftp-server 8
	for details).
	.Pp
	For safety, it is very important that the directory hierarchy be
	prevented from modification by other processes on the system (especially
	those outside the jail).
	Misconfiguration can lead to unsafe environments which
	.Xr sshd 8
	cannot detect.
	.Pp
	The default is
	.Cm none ,
	indicating not to
	.Xr chroot 2 .
	.It Cm Ciphers
	Specifies the ciphers allowed.
	Multiple ciphers must be comma-separated.
	If the specified value begins with a
	.Sq +
	character, then the specified ciphers will be appended to the default set
	instead of replacing them.
	If the specified value begins with a
	.Sq -
	character, then the specified ciphers (including wildcards) will be removed
	from the default set instead of replacing them.
	.Pp
	The supported ciphers are:
	.Pp
	.Bl -item -compact -offset indent
	.It
	3des-cbc
	.It
	aes128-cbc
	.It
	aes192-cbc
	.It
	aes256-cbc
	.It
	aes128-ctr
	.It
	aes192-ctr
	.It
	aes256-ctr
	.It
	aes128-gcm@openssh.com
	.It
	aes256-gcm@openssh.com
	.It
	chacha20-poly1305@openssh.com
	.El
	.Pp
	The default is:
	.Bd -literal -offset indent
	chacha20-poly1305@openssh.com,
	aes128-ctr,aes192-ctr,aes256-ctr,
	-aes128-gcm@openssh.com,aes256-gcm@openssh.com,
	-aes128-cbc,aes192-cbc,aes256-cbc
	+aes128-gcm@openssh.com,aes256-gcm@openssh.com
	.Ed
	.Pp
	The list of available ciphers may also be obtained using
	.Qq ssh -Q cipher .
	.It Cm ClientAliveCountMax
	Sets the number of client alive messages which may be sent without
	.Xr sshd 8
	receiving any messages back from the client.
	If this threshold is reached while client alive messages are being sent,
	sshd will disconnect the client, terminating the session.
	It is important to note that the use of client alive messages is very
	different from
	.Cm TCPKeepAlive .
	The client alive messages are sent through the encrypted channel
	and therefore will not be spoofable.
	The TCP keepalive option enabled by
	.Cm TCPKeepAlive
	is spoofable.
	The client alive mechanism is valuable when the client or
	server depend on knowing when a connection has become inactive.
	.Pp
	The default value is 3.
	If
	.Cm ClientAliveInterval
	is set to 15, and
	.Cm ClientAliveCountMax
	is left at the default, unresponsive SSH clients
	will be disconnected after approximately 45 seconds.
	.It Cm ClientAliveInterval
	Sets a timeout interval in seconds after which if no data has been received
	from the client,
	.Xr sshd 8
	will send a message through the encrypted
	channel to request a response from the client.
	The default
	is 0, indicating that these messages will not be sent to the client.
	.It Cm Compression
	Specifies whether compression is enabled after
	the user has authenticated successfully.
	The argument must be
	.Cm yes ,
	.Cm delayed
	(a legacy synonym for
	.Cm yes )
	or
	.Cm no .
	The default is
	.Cm yes .
	.It Cm DenyGroups
	This keyword can be followed by a list of group name patterns, separated
	by spaces.
	Login is disallowed for users whose primary group or supplementary
	group list matches one of the patterns.
	Only group names are valid; a numerical group ID is not recognized.
	By default, login is allowed for all groups.
	The allow/deny directives are processed in the following order:
	.Cm DenyUsers ,
	.Cm AllowUsers ,
	.Cm DenyGroups ,
	and finally
	.Cm AllowGroups .
	.Pp
	See PATTERNS in
	.Xr ssh_config 5
	for more information on patterns.
	.It Cm DenyUsers
	This keyword can be followed by a list of user name patterns, separated
	by spaces.
	Login is disallowed for user names that match one of the patterns.
	Only user names are valid; a numerical user ID is not recognized.
	By default, login is allowed for all users.
	If the pattern takes the form USER@HOST then USER and HOST
	are separately checked, restricting logins to particular
	users from particular hosts.
	HOST criteria may additionally contain addresses to match in CIDR
	address/masklen format.
	The allow/deny directives are processed in the following order:
	.Cm DenyUsers ,
	.Cm AllowUsers ,
	.Cm DenyGroups ,
	and finally
	.Cm AllowGroups .
	.Pp
	See PATTERNS in
	.Xr ssh_config 5
	for more information on patterns.
	.It Cm DisableForwarding
	Disables all forwarding features, including X11,
	.Xr ssh-agent 1 ,
	TCP and StreamLocal.
	This option overrides all other forwarding-related options and may
	simplify restricted configurations.
	.It Cm ExposeAuthInfo
	Writes a temporary file containing a list of authentication methods and
	public credentials (e.g. keys) used to authenticate the user.
	The location of the file is exposed to the user session through the
	.Ev SSH_USER_AUTH
	environment variable.
	The default is
	.Cm no .
	.It Cm FingerprintHash
	Specifies the hash algorithm used when logging key fingerprints.
	Valid options are:
	.Cm md5
	and
	.Cm sha256 .
	The default is
	.Cm sha256 .
	.It Cm ForceCommand
	Forces the execution of the command specified by
	.Cm ForceCommand ,
	ignoring any command supplied by the client and
	.Pa ~/.ssh/rc
	if present.
	The command is invoked by using the user's login shell with the -c option.
	This applies to shell, command, or subsystem execution.
	It is most useful inside a
	.Cm Match
	block.
	The command originally supplied by the client is available in the
	.Ev SSH_ORIGINAL_COMMAND
	environment variable.
	Specifying a command of
	.Cm internal-sftp
	will force the use of an in-process SFTP server that requires no support
	files when used with
	.Cm ChrootDirectory .
	The default is
	.Cm none .
	.It Cm GatewayPorts
	Specifies whether remote hosts are allowed to connect to ports
	forwarded for the client.
	By default,
	.Xr sshd 8
	binds remote port forwardings to the loopback address.
	This prevents other remote hosts from connecting to forwarded ports.
	.Cm GatewayPorts
	can be used to specify that sshd
	should allow remote port forwardings to bind to non-loopback addresses, thus
	allowing other hosts to connect.
	The argument may be
	.Cm no
	to force remote port forwardings to be available to the local host only,
	.Cm yes
	to force remote port forwardings to bind to the wildcard address, or
	.Cm clientspecified
	to allow the client to select the address to which the forwarding is bound.
	The default is
	.Cm no .
	.It Cm GSSAPIAuthentication
	Specifies whether user authentication based on GSSAPI is allowed.
	The default is
	.Cm no .
	.It Cm GSSAPICleanupCredentials
	Specifies whether to automatically destroy the user's credentials cache
	on logout.
	The default is
	.Cm yes .
	.It Cm GSSAPIStrictAcceptorCheck
	Determines whether to be strict about the identity of the GSSAPI acceptor
	a client authenticates against.
	If set to
	.Cm yes
	then the client must authenticate against the host
	service on the current hostname.
	If set to
	.Cm no
	then the client may authenticate against any service key stored in the
	machine's default store.
	This facility is provided to assist with operation on multi homed machines.
	The default is
	.Cm yes .
	.It Cm HostbasedAcceptedKeyTypes
	Specifies the key types that will be accepted for hostbased authentication
	as a list of comma-separated patterns.
	Alternately if the specified value begins with a
	.Sq +
	character, then the specified key types will be appended to the default set
	instead of replacing them.
	If the specified value begins with a
	.Sq -
	character, then the specified key types (including wildcards) will be removed
	from the default set instead of replacing them.
	The default for this option is:
	.Bd -literal -offset 3n
	ecdsa-sha2-nistp256-cert-v01@openssh.com,
	ecdsa-sha2-nistp384-cert-v01@openssh.com,
	ecdsa-sha2-nistp521-cert-v01@openssh.com,
	ssh-ed25519-cert-v01@openssh.com,
	rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,
	ssh-rsa-cert-v01@openssh.com,
	ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,
	ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa
	.Ed
	.Pp
	The list of available key types may also be obtained using
	.Qq ssh -Q key .
	.It Cm HostbasedAuthentication
	Specifies whether rhosts or /etc/hosts.equiv authentication together
	with successful public key client host authentication is allowed
	(host-based authentication).
	The default is
	.Cm no .
	.It Cm HostbasedUsesNameFromPacketOnly
	Specifies whether or not the server will attempt to perform a reverse
	name lookup when matching the name in the
	.Pa ~/.shosts ,
	.Pa ~/.rhosts ,
	and
	.Pa /etc/hosts.equiv
	files during
	.Cm HostbasedAuthentication .
	A setting of
	.Cm yes
	means that
	.Xr sshd 8
	uses the name supplied by the client rather than
	attempting to resolve the name from the TCP connection itself.
	The default is
	.Cm no .
	.It Cm HostCertificate
	Specifies a file containing a public host certificate.
	The certificate's public key must match a private host key already specified
	by
	.Cm HostKey .
	The default behaviour of
	.Xr sshd 8
	is not to load any certificates.
	.It Cm HostKey
	Specifies a file containing a private host key
	used by SSH.
	The defaults are
	.Pa /etc/ssh/ssh_host_ecdsa_key ,
	.Pa /etc/ssh/ssh_host_ed25519_key
	and
	.Pa /etc/ssh/ssh_host_rsa_key .
	.Pp
	Note that
	.Xr sshd 8
	will refuse to use a file if it is group/world-accessible
	and that the
	.Cm HostKeyAlgorithms
	option restricts which of the keys are actually used by
	.Xr sshd 8 .
	.Pp
	It is possible to have multiple host key files.
	It is also possible to specify public host key files instead.
	In this case operations on the private key will be delegated
	to an
	.Xr ssh-agent 1 .
	.It Cm HostKeyAgent
	Identifies the UNIX-domain socket used to communicate
	with an agent that has access to the private host keys.
	If the string
	.Qq SSH_AUTH_SOCK
	is specified, the location of the socket will be read from the
	.Ev SSH_AUTH_SOCK
	environment variable.
	.It Cm HostKeyAlgorithms
	Specifies the host key algorithms
	that the server offers.
	The default for this option is:
	.Bd -literal -offset 3n
	ecdsa-sha2-nistp256-cert-v01@openssh.com,
	ecdsa-sha2-nistp384-cert-v01@openssh.com,
	ecdsa-sha2-nistp521-cert-v01@openssh.com,
	ssh-ed25519-cert-v01@openssh.com,
	rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,
	ssh-rsa-cert-v01@openssh.com,
	ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,
	ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa
	.Ed
	.Pp
	The list of available key types may also be obtained using
	.Qq ssh -Q key .
	.It Cm IgnoreRhosts
	Specifies that
	.Pa .rhosts
	and
	.Pa .shosts
	files will not be used in
	.Cm HostbasedAuthentication .
	.Pp
	.Pa /etc/hosts.equiv
	and
	.Pa /etc/ssh/shosts.equiv
	are still used.
	The default is
	.Cm yes .
	.It Cm IgnoreUserKnownHosts
	Specifies whether
	.Xr sshd 8
	should ignore the user's
	.Pa ~/.ssh/known_hosts
	during
	.Cm HostbasedAuthentication
	and use only the system-wide known hosts file
	.Pa /etc/ssh/known_hosts .
	The default is
	.Cm no .
	.It Cm IPQoS
	Specifies the IPv4 type-of-service or DSCP class for the connection.
	Accepted values are
	.Cm af11 ,
	.Cm af12 ,
	.Cm af13 ,
	.Cm af21 ,
	.Cm af22 ,
	.Cm af23 ,
	.Cm af31 ,
	.Cm af32 ,
	.Cm af33 ,
	.Cm af41 ,
	.Cm af42 ,
	.Cm af43 ,
	.Cm cs0 ,
	.Cm cs1 ,
	.Cm cs2 ,
	.Cm cs3 ,
	.Cm cs4 ,
	.Cm cs5 ,
	.Cm cs6 ,
	.Cm cs7 ,
	.Cm ef ,
	.Cm lowdelay ,
	.Cm throughput ,
	.Cm reliability ,
	a numeric value, or
	.Cm none
	to use the operating system default.
	This option may take one or two arguments, separated by whitespace.
	If one argument is specified, it is used as the packet class unconditionally.
	If two values are specified, the first is automatically selected for
	interactive sessions and the second for non-interactive sessions.
	The default is
	.Cm af21
	(Low-Latency Data)
	for interactive sessions and
	.Cm cs1
	(Lower Effort)
	for non-interactive sessions.
	.It Cm KbdInteractiveAuthentication
	Specifies whether to allow keyboard-interactive authentication.
	The argument to this keyword must be
	.Cm yes
	or
	.Cm no .
	The default is to use whatever value
	.Cm ChallengeResponseAuthentication
	is set to
	(by default
	.Cm yes ) .
	.It Cm KerberosAuthentication
	Specifies whether the password provided by the user for
	.Cm PasswordAuthentication
	will be validated through the Kerberos KDC.
	To use this option, the server needs a
	Kerberos servtab which allows the verification of the KDC's identity.
	The default is
	.Cm no .
	.It Cm KerberosGetAFSToken
	If AFS is active and the user has a Kerberos 5 TGT, attempt to acquire
	an AFS token before accessing the user's home directory.
	The default is
	.Cm no .
	.It Cm KerberosOrLocalPasswd
	If password authentication through Kerberos fails then
	the password will be validated via any additional local mechanism
	such as
	.Pa /etc/passwd .
	The default is
	.Cm yes .
	.It Cm KerberosTicketCleanup
	Specifies whether to automatically destroy the user's ticket cache
	file on logout.
	The default is
	.Cm yes .
	.It Cm KexAlgorithms
	Specifies the available KEX (Key Exchange) algorithms.
	Multiple algorithms must be comma-separated.
	Alternately if the specified value begins with a
	.Sq +
	character, then the specified methods will be appended to the default set
	instead of replacing them.
	If the specified value begins with a
	.Sq -
	character, then the specified methods (including wildcards) will be removed
	from the default set instead of replacing them.
	The supported algorithms are:
	.Pp
	.Bl -item -compact -offset indent
	.It
	curve25519-sha256
	.It
	curve25519-sha256@libssh.org
	.It
	diffie-hellman-group1-sha1
	.It
	diffie-hellman-group14-sha1
	.It
	diffie-hellman-group14-sha256
	.It
	diffie-hellman-group16-sha512
	.It
	diffie-hellman-group18-sha512
	.It
	diffie-hellman-group-exchange-sha1
	.It
	diffie-hellman-group-exchange-sha256
	.It
	ecdh-sha2-nistp256
	.It
	ecdh-sha2-nistp384
	.It
	ecdh-sha2-nistp521
	.El
	.Pp
	The default is:
	.Bd -literal -offset indent
	curve25519-sha256,curve25519-sha256@libssh.org,
	ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,
	diffie-hellman-group-exchange-sha256,
	diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,
	diffie-hellman-group14-sha256,diffie-hellman-group14-sha1
	.Ed
	.Pp
	The list of available key exchange algorithms may also be obtained using
	.Qq ssh -Q kex .
	.It Cm ListenAddress
	Specifies the local addresses
	.Xr sshd 8
	should listen on.
	The following forms may be used:
	.Pp
	.Bl -item -offset indent -compact
	.It
	.Cm ListenAddress
	.Sm off
	.Ar hostname \| address
	.Sm on
	.Op Cm rdomain Ar domain
	.It
	.Cm ListenAddress
	.Sm off
	.Ar hostname : port
	.Sm on
	.Op Cm rdomain Ar domain
	.It
	.Cm ListenAddress
	.Sm off
	.Ar IPv4_address : port
	.Sm on
	.Op Cm rdomain Ar domain
	.It
	.Cm ListenAddress
	.Sm off
	.Oo Ar hostname \| address Oc : Ar port
	.Sm on
	.Op Cm rdomain Ar domain
	.El
	.Pp
	The optional
	.Cm rdomain
	qualifier requests
	.Xr sshd 8
	listen in an explicit routing domain.
	If
	.Ar port
	is not specified,
	sshd will listen on the address and all
	.Cm Port
	options specified.
	The default is to listen on all local addresses on the current default
	routing domain.
	Multiple
	.Cm ListenAddress
	options are permitted.
	For more information on routing domains, see
	.Xr rdomain 4 .
	.It Cm LoginGraceTime
	The server disconnects after this time if the user has not
	successfully logged in.
	If the value is 0, there is no time limit.
	The default is 120 seconds.
	.It Cm LogLevel
	Gives the verbosity level that is used when logging messages from
	.Xr sshd 8 .
	The possible values are:
	QUIET, FATAL, ERROR, INFO, VERBOSE, DEBUG, DEBUG1, DEBUG2, and DEBUG3.
	The default is INFO.
	DEBUG and DEBUG1 are equivalent.
	DEBUG2 and DEBUG3 each specify higher levels of debugging output.
	Logging with a DEBUG level violates the privacy of users and is not recommended.
	.It Cm MACs
	Specifies the available MAC (message authentication code) algorithms.
	The MAC algorithm is used for data integrity protection.
	Multiple algorithms must be comma-separated.
	If the specified value begins with a
	.Sq +
	character, then the specified algorithms will be appended to the default set
	instead of replacing them.
	If the specified value begins with a
	.Sq -
	character, then the specified algorithms (including wildcards) will be removed
	from the default set instead of replacing them.
	.Pp
	The algorithms that contain
	.Qq -etm
	calculate the MAC after encryption (encrypt-then-mac).
	These are considered safer and their use recommended.
	The supported MACs are:
	.Pp
	.Bl -item -compact -offset indent
	.It
	hmac-md5
	.It
	hmac-md5-96
	.It
	hmac-sha1
	.It
	hmac-sha1-96
	.It
	hmac-sha2-256
	.It
	hmac-sha2-512
	.It
	umac-64@openssh.com
	.It
	umac-128@openssh.com
	.It
	hmac-md5-etm@openssh.com
	.It
	hmac-md5-96-etm@openssh.com
	.It
	hmac-sha1-etm@openssh.com
	.It
	hmac-sha1-96-etm@openssh.com
	.It
	hmac-sha2-256-etm@openssh.com
	.It
	hmac-sha2-512-etm@openssh.com
	.It
	umac-64-etm@openssh.com
	.It
	umac-128-etm@openssh.com
	.El
	.Pp
	The default is:
	.Bd -literal -offset indent
	umac-64-etm@openssh.com,umac-128-etm@openssh.com,
	hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,
	hmac-sha1-etm@openssh.com,
	umac-64@openssh.com,umac-128@openssh.com,
	hmac-sha2-256,hmac-sha2-512,hmac-sha1
	.Ed
	.Pp
	The list of available MAC algorithms may also be obtained using
	.Qq ssh -Q mac .
	.It Cm Match
	Introduces a conditional block.
	If all of the criteria on the
	.Cm Match
	line are satisfied, the keywords on the following lines override those
	set in the global section of the config file, until either another
	.Cm Match
	line or the end of the file.
	If a keyword appears in multiple
	.Cm Match
	blocks that are satisfied, only the first instance of the keyword is
	applied.
	.Pp
	The arguments to
	.Cm Match
	are one or more criteria-pattern pairs or the single token
	.Cm All
	which matches all criteria.
	The available criteria are
	.Cm User ,
	.Cm Group ,
	.Cm Host ,
	.Cm LocalAddress ,
	.Cm LocalPort ,
	.Cm RDomain ,
	and
	.Cm Address
	(with
	.Cm RDomain
	representing the
	.Xr rdomain 4
	on which the connection was received.)
	.Pp
	The match patterns may consist of single entries or comma-separated
	lists and may use the wildcard and negation operators described in the
	.Sx PATTERNS
	section of
	.Xr ssh_config 5 .
	.Pp
	The patterns in an
	.Cm Address
	criteria may additionally contain addresses to match in CIDR
	address/masklen format,
	such as 192.0.2.0/24 or 2001:db8::/32.
	Note that the mask length provided must be consistent with the address -
	it is an error to specify a mask length that is too long for the address
	or one with bits set in this host portion of the address.
	For example, 192.0.2.0/33 and 192.0.2.0/8, respectively.
	.Pp
	Only a subset of keywords may be used on the lines following a
	.Cm Match
	keyword.
	Available keywords are
	.Cm AcceptEnv ,
	.Cm AllowAgentForwarding ,
	.Cm AllowGroups ,
	.Cm AllowStreamLocalForwarding ,
	.Cm AllowTcpForwarding ,
	.Cm AllowUsers ,
	.Cm AuthenticationMethods ,
	.Cm AuthorizedKeysCommand ,
	.Cm AuthorizedKeysCommandUser ,
	.Cm AuthorizedKeysFile ,
	.Cm AuthorizedPrincipalsCommand ,
	.Cm AuthorizedPrincipalsCommandUser ,
	.Cm AuthorizedPrincipalsFile ,
	.Cm Banner ,
	.Cm ChrootDirectory ,
	.Cm ClientAliveCountMax ,
	.Cm ClientAliveInterval ,
	.Cm DenyGroups ,
	.Cm DenyUsers ,
	.Cm ForceCommand ,
	.Cm GatewayPorts ,
	.Cm GSSAPIAuthentication ,
	.Cm HostbasedAcceptedKeyTypes ,
	.Cm HostbasedAuthentication ,
	.Cm HostbasedUsesNameFromPacketOnly ,
	.Cm IPQoS ,
	.Cm KbdInteractiveAuthentication ,
	.Cm KerberosAuthentication ,
	.Cm LogLevel ,
	.Cm MaxAuthTries ,
	.Cm MaxSessions ,
	.Cm PasswordAuthentication ,
	.Cm PermitEmptyPasswords ,
	.Cm PermitListen ,
	.Cm PermitOpen ,
	.Cm PermitRootLogin ,
	.Cm PermitTTY ,
	.Cm PermitTunnel ,
	.Cm PermitUserRC ,
	.Cm PubkeyAcceptedKeyTypes ,
	.Cm PubkeyAuthentication ,
	.Cm RekeyLimit ,
	.Cm RevokedKeys ,
	.Cm RDomain ,
	.Cm SetEnv ,
	.Cm StreamLocalBindMask ,
	.Cm StreamLocalBindUnlink ,
	.Cm TrustedUserCAKeys ,
	.Cm X11DisplayOffset ,
	.Cm X11Forwarding
	and
	.Cm X11UseLocalHost .
	.It Cm MaxAuthTries
	Specifies the maximum number of authentication attempts permitted per
	connection.
	Once the number of failures reaches half this value,
	additional failures are logged.
	The default is 6.
	.It Cm MaxSessions
	Specifies the maximum number of open shell, login or subsystem (e.g. sftp)
	sessions permitted per network connection.
	Multiple sessions may be established by clients that support connection
	multiplexing.
	Setting
	.Cm MaxSessions
	to 1 will effectively disable session multiplexing, whereas setting it to 0
	will prevent all shell, login and subsystem sessions while still permitting
	forwarding.
	The default is 10.
	.It Cm MaxStartups
	Specifies the maximum number of concurrent unauthenticated connections to the
	SSH daemon.
	Additional connections will be dropped until authentication succeeds or the
	.Cm LoginGraceTime
	expires for a connection.
	The default is 10:30:100.
	.Pp
	Alternatively, random early drop can be enabled by specifying
	the three colon separated values
	start:rate:full (e.g. "10:30:60").
	.Xr sshd 8
	will refuse connection attempts with a probability of rate/100 (30%)
	if there are currently start (10) unauthenticated connections.
	The probability increases linearly and all connection attempts
	are refused if the number of unauthenticated connections reaches full (60).
	.It Cm PasswordAuthentication
	Specifies whether password authentication is allowed.
	See also
	.Cm UsePAM .
	The default is
	.Cm no .
	.It Cm PermitEmptyPasswords
	When password authentication is allowed, it specifies whether the
	server allows login to accounts with empty password strings.
	The default is
	.Cm no .
	.It Cm PermitListen
	Specifies the addresses/ports on which a remote TCP port forwarding may listen.
	The listen specification must be one of the following forms:
	.Pp
	.Bl -item -offset indent -compact
	.It
	.Cm PermitListen
	.Sm off
	.Ar port
	.Sm on
	.It
	.Cm PermitListen
	.Sm off
	.Ar host : port
	.Sm on
	.El
	.Pp
	Multiple permissions may be specified by separating them with whitespace.
	An argument of
	.Cm any
	can be used to remove all restrictions and permit any listen requests.
	An argument of
	.Cm none
	can be used to prohibit all listen requests.
	The host name may contain wildcards as described in the PATTERNS section in
	.Xr ssh_config 5 .
	The wildcard
	.Sq *
	can also be used in place of a port number to allow all ports.
	By default all port forwarding listen requests are permitted.
	Note that the
	.Cm GatewayPorts
	option may further restrict which addresses may be listened on.
	Note also that
	.Xr ssh 1
	will request a listen host of
	.Dq localhost
	if no listen host was specifically requested, and this this name is
	treated differently to explicit localhost addresses of
	.Dq 127.0.0.1
	and
	.Dq ::1 .
	.It Cm PermitOpen
	Specifies the destinations to which TCP port forwarding is permitted.
	The forwarding specification must be one of the following forms:
	.Pp
	.Bl -item -offset indent -compact
	.It
	.Cm PermitOpen
	.Sm off
	.Ar host : port
	.Sm on
	.It
	.Cm PermitOpen
	.Sm off
	.Ar IPv4_addr : port
	.Sm on
	.It
	.Cm PermitOpen
	.Sm off
	.Ar \&[ IPv6_addr \&] : port
	.Sm on
	.El
	.Pp
	Multiple forwards may be specified by separating them with whitespace.
	An argument of
	.Cm any
	can be used to remove all restrictions and permit any forwarding requests.
	An argument of
	.Cm none
	can be used to prohibit all forwarding requests.
	The wildcard
	.Sq *
	can be used for host or port to allow all hosts or ports, respectively.
	By default all port forwarding requests are permitted.
	.It Cm PermitRootLogin
	Specifies whether root can log in using
	.Xr ssh 1 .
	The argument must be
	.Cm yes ,
	.Cm prohibit-password ,
	.Cm forced-commands-only ,
	or
	.Cm no .
	The default is
	.Cm no .
	Note that if
	.Cm ChallengeResponseAuthentication
	and
	.Cm UsePAM
	are both
	.Cm yes ,
	this setting may be overridden by the PAM policy.
	.Pp
	If this option is set to
	.Cm prohibit-password
	(or its deprecated alias,
	.Cm without-password ) ,
	password and keyboard-interactive authentication are disabled for root.
	.Pp
	If this option is set to
	.Cm forced-commands-only ,
	root login with public key authentication will be allowed,
	but only if the
	.Ar command
	option has been specified
	(which may be useful for taking remote backups even if root login is
	normally not allowed).
	All other authentication methods are disabled for root.
	.Pp
	If this option is set to
	.Cm no ,
	root is not allowed to log in.
	.It Cm PermitTTY
	Specifies whether
	.Xr pty 4
	allocation is permitted.
	The default is
	.Cm yes .
	.It Cm PermitTunnel
	Specifies whether
	.Xr tun 4
	device forwarding is allowed.
	The argument must be
	.Cm yes ,
	.Cm point-to-point
	(layer 3),
	.Cm ethernet
	(layer 2), or
	.Cm no .
	Specifying
	.Cm yes
	permits both
	.Cm point-to-point
	and
	.Cm ethernet .
	The default is
	.Cm no .
	.Pp
	Independent of this setting, the permissions of the selected
	.Xr tun 4
	device must allow access to the user.
	.It Cm PermitUserEnvironment
	Specifies whether
	.Pa ~/.ssh/environment
	and
	.Cm environment=
	options in
	.Pa ~/.ssh/authorized_keys
	are processed by
	.Xr sshd 8 .
	Valid options are
	.Cm yes ,
	.Cm no
	or a pattern-list specifying which environment variable names to accept
	(for example
	.Qq LANG,LC_* ) .
	The default is
	.Cm no .
	Enabling environment processing may enable users to bypass access
	restrictions in some configurations using mechanisms such as
	.Ev LD_PRELOAD .
	.It Cm PermitUserRC
	Specifies whether any
	.Pa ~/.ssh/rc
	file is executed.
	The default is
	.Cm yes .
	.It Cm PidFile
	Specifies the file that contains the process ID of the
	SSH daemon, or
	.Cm none
	to not write one.
	The default is
	.Pa /var/run/sshd.pid .
	.It Cm Port
	Specifies the port number that
	.Xr sshd 8
	listens on.
	The default is 22.
	Multiple options of this type are permitted.
	See also
	.Cm ListenAddress .
	.It Cm PrintLastLog
	Specifies whether
	.Xr sshd 8
	should print the date and time of the last user login when a user logs
	in interactively.
	The default is
	.Cm yes .
	.It Cm PrintMotd
	Specifies whether
	.Xr sshd 8
	should print
	.Pa /etc/motd
	when a user logs in interactively.
	(On some systems it is also printed by the shell,
	.Pa /etc/profile ,
	or equivalent.)
	The default is
	.Cm yes .
	.It Cm PubkeyAcceptedKeyTypes
	Specifies the key types that will be accepted for public key authentication
	as a list of comma-separated patterns.
	Alternately if the specified value begins with a
	.Sq +
	character, then the specified key types will be appended to the default set
	instead of replacing them.
	If the specified value begins with a
	.Sq -
	character, then the specified key types (including wildcards) will be removed
	from the default set instead of replacing them.
	The default for this option is:
	.Bd -literal -offset 3n
	ecdsa-sha2-nistp256-cert-v01@openssh.com,
	ecdsa-sha2-nistp384-cert-v01@openssh.com,
	ecdsa-sha2-nistp521-cert-v01@openssh.com,
	ssh-ed25519-cert-v01@openssh.com,
	rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,
	ssh-rsa-cert-v01@openssh.com,
	ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,
	ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa
	.Ed
	.Pp
	The list of available key types may also be obtained using
	.Qq ssh -Q key .
	.It Cm PubkeyAuthentication
	Specifies whether public key authentication is allowed.
	The default is
	.Cm yes .
	.It Cm RekeyLimit
	Specifies the maximum amount of data that may be transmitted before the
	session key is renegotiated, optionally followed a maximum amount of
	time that may pass before the session key is renegotiated.
	The first argument is specified in bytes and may have a suffix of
	.Sq K ,
	.Sq M ,
	or
	.Sq G
	to indicate Kilobytes, Megabytes, or Gigabytes, respectively.
	The default is between
	.Sq 1G
	and
	.Sq 4G ,
	depending on the cipher.
	The optional second value is specified in seconds and may use any of the
	units documented in the
	.Sx TIME FORMATS
	section.
	The default value for
	.Cm RekeyLimit
	is
	.Cm default none ,
	which means that rekeying is performed after the cipher's default amount
	of data has been sent or received and no time based rekeying is done.
	.It Cm RevokedKeys
	Specifies revoked public keys file, or
	.Cm none
	to not use one.
	Keys listed in this file will be refused for public key authentication.
	Note that if this file is not readable, then public key authentication will
	be refused for all users.
	Keys may be specified as a text file, listing one public key per line, or as
	an OpenSSH Key Revocation List (KRL) as generated by
	.Xr ssh-keygen 1 .
	For more information on KRLs, see the KEY REVOCATION LISTS section in
	.Xr ssh-keygen 1 .
	.It Cm RDomain
	Specifies an explicit routing domain that is applied after authentication
	has completed.
	The user session, as well and any forwarded or listening IP sockets,
	will be bound to this
	.Xr rdomain 4 .
	If the routing domain is set to
	.Cm \&%D ,
	then the domain in which the incoming connection was received will be applied.
	.It Cm SetEnv
	Specifies one or more environment variables to set in child sessions started
	by
	.Xr sshd 8
	as
	.Dq NAME=VALUE .
	The environment value may be quoted (e.g. if it contains whitespace
	characters).
	Environment variables set by
	.Cm SetEnv
	override the default environment and any variables specified by the user
	via
	.Cm AcceptEnv
	or
	.Cm PermitUserEnvironment .
	.It Cm StreamLocalBindMask
	Sets the octal file creation mode mask
	.Pq umask
	used when creating a Unix-domain socket file for local or remote
	port forwarding.
	This option is only used for port forwarding to a Unix-domain socket file.
	.Pp
	The default value is 0177, which creates a Unix-domain socket file that is
	readable and writable only by the owner.
	Note that not all operating systems honor the file mode on Unix-domain
	socket files.
	.It Cm StreamLocalBindUnlink
	Specifies whether to remove an existing Unix-domain socket file for local
	or remote port forwarding before creating a new one.
	If the socket file already exists and
	.Cm StreamLocalBindUnlink
	is not enabled,
	.Nm sshd
	will be unable to forward the port to the Unix-domain socket file.
	This option is only used for port forwarding to a Unix-domain socket file.
	.Pp
	The argument must be
	.Cm yes
	or
	.Cm no .
	The default is
	.Cm no .
	.It Cm StrictModes
	Specifies whether
	.Xr sshd 8
	should check file modes and ownership of the
	user's files and home directory before accepting login.
	This is normally desirable because novices sometimes accidentally leave their
	directory or files world-writable.
	The default is
	.Cm yes .
	Note that this does not apply to
	.Cm ChrootDirectory ,
	whose permissions and ownership are checked unconditionally.
	.It Cm Subsystem
	Configures an external subsystem (e.g. file transfer daemon).
	Arguments should be a subsystem name and a command (with optional arguments)
	to execute upon subsystem request.
	.Pp
	The command
	.Cm sftp-server
	implements the SFTP file transfer subsystem.
	.Pp
	Alternately the name
	.Cm internal-sftp
	implements an in-process SFTP server.
	This may simplify configurations using
	.Cm ChrootDirectory
	to force a different filesystem root on clients.
	.Pp
	By default no subsystems are defined.
	.It Cm SyslogFacility
	Gives the facility code that is used when logging messages from
	.Xr sshd 8 .
	The possible values are: DAEMON, USER, AUTH, LOCAL0, LOCAL1, LOCAL2,
	LOCAL3, LOCAL4, LOCAL5, LOCAL6, LOCAL7.
	The default is AUTH.
	.It Cm TCPKeepAlive
	Specifies whether the system should send TCP keepalive messages to the
	other side.
	If they are sent, death of the connection or crash of one
	of the machines will be properly noticed.
	However, this means that
	connections will die if the route is down temporarily, and some people
	find it annoying.
	On the other hand, if TCP keepalives are not sent,
	sessions may hang indefinitely on the server, leaving
	.Qq ghost
	users and consuming server resources.
	.Pp
	The default is
	.Cm yes
	(to send TCP keepalive messages), and the server will notice
	if the network goes down or the client host crashes.
	This avoids infinitely hanging sessions.
	.Pp
	To disable TCP keepalive messages, the value should be set to
	.Cm no .
	.It Cm TrustedUserCAKeys
	Specifies a file containing public keys of certificate authorities that are
	trusted to sign user certificates for authentication, or
	.Cm none
	to not use one.
	Keys are listed one per line; empty lines and comments starting with
	.Ql #
	are allowed.
	If a certificate is presented for authentication and has its signing CA key
	listed in this file, then it may be used for authentication for any user
	listed in the certificate's principals list.
	Note that certificates that lack a list of principals will not be permitted
	for authentication using
	.Cm TrustedUserCAKeys .
	For more details on certificates, see the CERTIFICATES section in
	.Xr ssh-keygen 1 .
	.It Cm UseBlacklist
	Specifies whether
	.Xr sshd 8
	attempts to send authentication success and failure messages
	to the
	.Xr blacklistd 8
	daemon.
	The default is
	.Cm no .
	+For forward compatibility with an upcoming
	+.Xr blacklistd
	+rename, the
	+.Cm UseBlocklist
	+alias can be used instead.
	.It Cm UseDNS
	Specifies whether
	.Xr sshd 8
	should look up the remote host name, and to check that
	the resolved host name for the remote IP address maps back to the
	very same IP address.
	.Pp
	If this option is set to
	.Cm no ,
	then only addresses and not host names may be used in
	.Pa ~/.ssh/authorized_keys
	.Cm from
	and
	.Nm
	.Cm Match
	.Cm Host
	directives.
	The default is
	.Dq yes .
	.It Cm UsePAM
	Enables the Pluggable Authentication Module interface.
	If set to
	.Cm yes
	this will enable PAM authentication using
	.Cm ChallengeResponseAuthentication
	and
	.Cm PasswordAuthentication
	in addition to PAM account and session module processing for all
	authentication types.
	.Pp
	Because PAM challenge-response authentication usually serves an equivalent
	role to password authentication, you should disable either
	.Cm PasswordAuthentication
	or
	.Cm ChallengeResponseAuthentication.
	.Pp
	If
	.Cm UsePAM
	is enabled, you will not be able to run
	.Xr sshd 8
	as a non-root user.
	The default is
	.Cm yes .
	.It Cm VersionAddendum
	Optionally specifies additional text to append to the SSH protocol banner
	sent by the server upon connection.
	The default is
	.Qq FreeBSD-20180909 .
	The value
	.Cm none
	may be used to disable this.
	.It Cm X11DisplayOffset
	Specifies the first display number available for
	.Xr sshd 8 Ns 's
	X11 forwarding.
	This prevents sshd from interfering with real X11 servers.
	The default is 10.
	.It Cm X11Forwarding
	Specifies whether X11 forwarding is permitted.
	The argument must be
	.Cm yes
	or
	.Cm no .
	The default is
	.Cm yes .
	.Pp
	When X11 forwarding is enabled, there may be additional exposure to
	the server and to client displays if the
	.Xr sshd 8
	proxy display is configured to listen on the wildcard address (see
	.Cm X11UseLocalhost ) ,
	though this is not the default.
	Additionally, the authentication spoofing and authentication data
	verification and substitution occur on the client side.
	The security risk of using X11 forwarding is that the client's X11
	display server may be exposed to attack when the SSH client requests
	forwarding (see the warnings for
	.Cm ForwardX11
	in
	.Xr ssh_config 5 ) .
	A system administrator may have a stance in which they want to
	protect clients that may expose themselves to attack by unwittingly
	requesting X11 forwarding, which can warrant a
	.Cm no
	setting.
	.Pp
	Note that disabling X11 forwarding does not prevent users from
	forwarding X11 traffic, as users can always install their own forwarders.
	.It Cm X11UseLocalhost
	Specifies whether
	.Xr sshd 8
	should bind the X11 forwarding server to the loopback address or to
	the wildcard address.
	By default,
	sshd binds the forwarding server to the loopback address and sets the
	hostname part of the
	.Ev DISPLAY
	environment variable to
	.Cm localhost .
	This prevents remote hosts from connecting to the proxy display.
	However, some older X11 clients may not function with this
	configuration.
	.Cm X11UseLocalhost
	may be set to
	.Cm no
	to specify that the forwarding server should be bound to the wildcard
	address.
	The argument must be
	.Cm yes
	or
	.Cm no .
	The default is
	.Cm yes .
	.It Cm XAuthLocation
	Specifies the full pathname of the
	.Xr xauth 1
	program, or
	.Cm none
	to not use one.
	The default is
	.Pa /usr/local/bin/xauth .
	.El
	.Sh TIME FORMATS
	.Xr sshd 8
	command-line arguments and configuration file options that specify time
	may be expressed using a sequence of the form:
	.Sm off
	.Ar time Op Ar qualifier ,
	.Sm on
	where
	.Ar time
	is a positive integer value and
	.Ar qualifier
	is one of the following:
	.Pp
	.Bl -tag -width Ds -compact -offset indent
	.It Aq Cm none
	seconds
	.It Cm s \| Cm S
	seconds
	.It Cm m \| Cm M
	minutes
	.It Cm h \| Cm H
	hours
	.It Cm d \| Cm D
	days
	.It Cm w \| Cm W
	weeks
	.El
	.Pp
	Each member of the sequence is added together to calculate
	the total time value.
	.Pp
	Time format examples:
	.Pp
	.Bl -tag -width Ds -compact -offset indent
	.It 600
	600 seconds (10 minutes)
	.It 10m
	10 minutes
	.It 1h30m
	1 hour 30 minutes (90 minutes)
	.El
	.Sh TOKENS
	Arguments to some keywords can make use of tokens,
	which are expanded at runtime:
	.Pp
	.Bl -tag -width XXXX -offset indent -compact
	.It %%
	A literal
	.Sq % .
	.It \&%D
	The routing domain in which the incoming connection was received.
	.It %F
	The fingerprint of the CA key.
	.It %f
	The fingerprint of the key or certificate.
	.It %h
	The home directory of the user.
	.It %i
	The key ID in the certificate.
	.It %K
	The base64-encoded CA key.
	.It %k
	The base64-encoded key or certificate for authentication.
	.It %s
	The serial number of the certificate.
	.It \&%T
	The type of the CA key.
	.It %t
	The key or certificate type.
	.It \&%U
	The numeric user ID of the target user.
	.It %u
	The username.
	.El
	.Pp
	.Cm AuthorizedKeysCommand
	accepts the tokens %%, %f, %h, %k, %t, %U, and %u.
	.Pp
	.Cm AuthorizedKeysFile
	accepts the tokens %%, %h, %U, and %u.
	.Pp
	.Cm AuthorizedPrincipalsCommand
	accepts the tokens %%, %F, %f, %h, %i, %K, %k, %s, %T, %t, %U, and %u.
	.Pp
	.Cm AuthorizedPrincipalsFile
	accepts the tokens %%, %h, %U, and %u.
	.Pp
	.Cm ChrootDirectory
	accepts the tokens %%, %h, %U, and %u.
	.Pp
	.Cm RoutingDomain
	accepts the token %D.
	.Sh FILES
	.Bl -tag -width Ds
	.It Pa /etc/ssh/sshd_config
	Contains configuration data for
	.Xr sshd 8 .
	This file should be writable by root only, but it is recommended
	(though not necessary) that it be world-readable.
	.El
	.Sh SEE ALSO
	.Xr sftp-server 8 ,
	.Xr sshd 8
	.Sh AUTHORS
	.An -nosplit
	OpenSSH is a derivative of the original and free
	ssh 1.2.12 release by
	.An Tatu Ylonen .
	.An Aaron Campbell , Bob Beck , Markus Friedl , Niels Provos ,
	.An Theo de Raadt
	and
	.An Dug Song
	removed many bugs, re-added newer features and
	created OpenSSH.
	.An Markus Friedl
	contributed the support for SSH protocol versions 1.5 and 2.0.
	.An Niels Provos
	and
	.An Markus Friedl
	contributed support for privilege separation.
	diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
	index 3c7350ea24e1..e3164c7d76ba 100644
	--- a/etc/mtree/BSD.usr.dist
	+++ b/etc/mtree/BSD.usr.dist
	@@ -1,1280 +1,1276 @@
	# $FreeBSD$
	#
	# Please see the file src/etc/mtree/README before making changes to this file.
	#

	/set type=dir uname=root gname=wheel mode=0755
	.
	bin
	..
	include
	private
	bsdstat
	..
	event1
	..
	gmock
	internal
	custom
	..
	..
	..
	gtest
	internal
	custom
	..
	..
	..
	sqlite3
	..
	ucl
	..
	zstd
	..
	..
	..
	lib
	aout
	..
	clang
	10.0.1
	include
	cuda_wrappers
	..
	fuzzer
	..
	openmp_wrappers
	..
	ppc_wrappers
	..
	profile
	..
	sanitizer
	..
	xray
	..
	..
	lib
	freebsd
	..
	..
	..
	..
	compat
	aout
	..
	..
	dtrace
	..
	engines
	..
	i18n
	..
	libxo
	encoder
	..
	..
	..
	libdata
	gcc
	..
	ldscripts
	..
	pkgconfig
	..
	..
	libexec
	bsdconfig
	020.docsinstall
	include
	..
	..
	030.packages
	include
	..
	..
	040.password
	include
	..
	..
	050.diskmgmt
	include
	..
	..
	070.usermgmt
	include
	..
	..
	080.console
	include
	..
	..
	090.timezone
	include
	..
	..
	110.mouse
	include
	..
	..
	120.networking
	include
	..
	..
	130.security
	include
	..
	..
	140.startup
	include
	..
	..
	150.ttys
	include
	..
	..
	dot
	include
	..
	..
	include
	..
	includes
	include
	..
	..
	..
	bsdinstall
	..
	dwatch
	..
	hyperv
	..
	lpr
	ru
	..
	..
	sendmail
	..
	sm.bin
	..
	..
	local
	..
	obj nochange
	..
	sbin
	..
	share
	atf
	..
	bsdconfig
	media
	..
	networking
	..
	packages
	..
	password
	..
	startup
	..
	timezone
	..
	usermgmt
	..
	..
	calendar
	de_AT.ISO_8859-15
	..
	de_DE.ISO8859-1
	..
	fr_FR.ISO8859-1
	..
	hr_HR.ISO8859-2
	..
	hu_HU.ISO8859-2
	..
	pt_BR.ISO8859-1
	..
	pt_BR.UTF-8
	..
	ru_RU.KOI8-R
	..
	ru_RU.UTF-8
	..
	uk_UA.KOI8-U
	..
	..
	certs
	blacklisted tags=package=caroot
	..
	trusted tags=package=caroot
	..
	..
	dict
	..
	doc
	IPv6
	..
	atf
	..
	kyua
	..
	legal
	..
	llvm
	clang
	..
	..
	ncurses
	..
	ntp
	drivers
	icons
	..
	scripts
	..
	..
	hints
	..
	icons
	..
	pic
	..
	scripts
	..
	..
	pjdfstest
	..
	..
	dtrace
	..
	examples
	BSD_daemon
	..
	FreeBSD_version
	..
	IPv6
	..
	bhyve
	..
	bootforth
	..
	bsdconfig
	..
	csh
	..
	diskless
	..
	dma
	..
	drivers
	..
	dwatch
	..
	etc
	defaults
	..
	..
	find_interface
	..
	hast
	..
	hostapd
	..
	indent
	..
	ipfilter
	..
	ipfw
	..
	jails
	..
	kld
	cdev
	module
	..
	test
	..
	..
	dyn_sysctl
	..
	firmware
	fwconsumer
	..
	fwimage
	..
	..
	khelp
	..
	syscall
	module
	..
	test
	..
	..
	..
	kyua
	..
	libusb20
	..
	libvgl
	..
	mdoc
	..
	netgraph
	..
	perfmon
	..
	pf
	..
	ppi
	..
	ppp
	..
	printing
	..
	scsi_target
	..
	ses
	getencstat
	..
	sesd
	..
	setencstat
	..
	setobjstat
	..
	srcs
	..
	..
	smbfs
	print
	..
	..
	sunrpc
	dir
	..
	msg
	..
	sort
	..
	..
	tcsh
	..
	uefisign
	..
	ypldap
	..
	..
	firmware
	..
	games
	fortune
	..
	..
	i18n
	csmapper
	APPLE
	..
	AST
	..
	BIG5
	..
	CNS
	..
	CP
	..
	EBCDIC
	..
	GB
	..
	GEORGIAN
	..
	ISO-8859
	..
	ISO646
	..
	JIS
	..
	KAZAKH
	..
	KOI
	..
	KS
	..
	MISC
	..
	TCVN
	..
	..
	esdb
	APPLE
	..
	AST
	..
	BIG5
	..
	CP
	..
	DEC
	..
	EBCDIC
	..
	EUC
	..
	GB
	..
	GEORGIAN
	..
	ISO-2022
	..
	ISO-8859
	..
	ISO646
	..
	KAZAKH
	..
	KOI
	..
	MISC
	..
	TCVN
	..
	UTF
	..
	..
	..
	keys
	pkg
	revoked tags=package=runtime
	..
	trusted tags=package=runtime
	..
	..
	..
	kyua
	misc
	..
	store
	..
	..
	locale
	af_ZA.ISO8859-1
	..
	af_ZA.ISO8859-15
	..
	af_ZA.UTF-8
	..
	ar_AE.UTF-8
	..
	ar_EG.UTF-8
	..
	ar_JO.UTF-8
	..
	ar_MA.UTF-8
	..
	ar_QA.UTF-8
	..
	ar_SA.UTF-8
	..
	am_ET.UTF-8
	..
	be_BY.CP1131
	..
	be_BY.CP1251
	..
	be_BY.ISO8859-5
	..
	be_BY.UTF-8
	..
	bg_BG.CP1251
	..
	bg_BG.UTF-8
	..
	ca_AD.ISO8859-1
	..
	ca_AD.ISO8859-15
	..
	ca_ES.ISO8859-1
	..
	ca_ES.ISO8859-15
	..
	ca_FR.ISO8859-1
	..
	ca_FR.ISO8859-15
	..
	ca_IT.ISO8859-1
	..
	ca_IT.ISO8859-15
	..
	ca_AD.UTF-8
	..
	ca_ES.UTF-8
	..
	ca_FR.UTF-8
	..
	ca_IT.UTF-8
	..
	cs_CZ.ISO8859-2
	..
	cs_CZ.UTF-8
	..
	da_DK.ISO8859-1
	..
	da_DK.ISO8859-15
	..
	da_DK.UTF-8
	..
	de_AT.ISO8859-1
	..
	de_AT.ISO8859-15
	..
	de_AT.UTF-8
	..
	de_CH.ISO8859-1
	..
	de_CH.ISO8859-15
	..
	de_CH.UTF-8
	..
	de_DE.ISO8859-1
	..
	de_DE.ISO8859-15
	..
	de_DE.UTF-8
	..
	el_GR.ISO8859-7
	..
	el_GR.UTF-8
	..
	en_AU.ISO8859-1
	..
	en_AU.ISO8859-15
	..
	en_AU.US-ASCII
	..
	en_AU.UTF-8
	..
	en_CA.ISO8859-1
	..
	en_CA.ISO8859-15
	..
	en_CA.US-ASCII
	..
	en_CA.UTF-8
	..
	en_GB.ISO8859-1
	..
	en_GB.ISO8859-15
	..
	en_GB.US-ASCII
	..
	en_GB.UTF-8
	..
	en_HK.ISO8859-1
	..
	en_HK.UTF-8
	..
	en_IE.ISO8859-1
	..
	en_IE.ISO8859-15
	..
	en_IE.UTF-8
	..
	en_NZ.ISO8859-1
	..
	en_NZ.ISO8859-15
	..
	en_NZ.US-ASCII
	..
	en_NZ.UTF-8
	..
	en_PH.UTF-8
	..
	en_SG.ISO8859-1
	..
	en_SG.UTF-8
	..
	en_US.ISO8859-1
	..
	en_US.ISO8859-15
	..
	en_US.US-ASCII
	..
	en_US.UTF-8
	..
	en_ZA.ISO8859-1
	..
	en_ZA.ISO8859-15
	..
	en_ZA.US-ASCII
	..
	en_ZA.UTF-8
	..
	es_AR.ISO8859-1
	..
	es_AR.UTF-8
	..
	es_CR.UTF-8
	..
	es_ES.ISO8859-1
	..
	es_ES.ISO8859-15
	..
	es_ES.UTF-8
	..
	es_MX.ISO8859-1
	..
	es_MX.UTF-8
	..
	et_EE.ISO8859-1
	..
	et_EE.ISO8859-15
	..
	et_EE.UTF-8
	..
	eu_ES.ISO8859-1
	..
	eu_ES.ISO8859-15
	..
	eu_ES.UTF-8
	..
	fi_FI.ISO8859-1
	..
	fi_FI.ISO8859-15
	..
	fi_FI.UTF-8
	..
	fr_BE.ISO8859-1
	..
	fr_BE.ISO8859-15
	..
	fr_BE.UTF-8
	..
	fr_CA.ISO8859-1
	..
	fr_CA.ISO8859-15
	..
	fr_CA.UTF-8
	..
	fr_CH.ISO8859-1
	..
	fr_CH.ISO8859-15
	..
	fr_CH.UTF-8
	..
	fr_FR.ISO8859-1
	..
	fr_FR.ISO8859-15
	..
	fr_FR.UTF-8
	..
	ga_IE.UTF-8
	..
	he_IL.UTF-8
	..
	hi_IN.ISCII-DEV
	..
	hi_IN.UTF-8
	..
	hr_HR.ISO8859-2
	..
	hr_HR.UTF-8
	..
	hu_HU.ISO8859-2
	..
	hu_HU.UTF-8
	..
	hy_AM.ARMSCII-8
	..
	hy_AM.UTF-8
	..
	is_IS.ISO8859-1
	..
	is_IS.ISO8859-15
	..
	is_IS.UTF-8
	..
	it_CH.ISO8859-1
	..
	it_CH.ISO8859-15
	..
	it_CH.UTF-8
	..
	it_IT.ISO8859-1
	..
	it_IT.ISO8859-15
	..
	it_IT.UTF-8
	..
	ja_JP.SJIS
	..
	ja_JP.UTF-8
	..
	ja_JP.eucJP
	..
	kk_KZ.UTF-8
	..
	ko_KR.CP949
	..
	ko_KR.UTF-8
	..
	ko_KR.eucKR
	..
	lt_LT.ISO8859-13
	..
	lt_LT.UTF-8
	..
	lv_LV.ISO8859-13
	..
	lv_LV.UTF-8
	..
	mn_MN.UTF-8
	..
	nb_NO.ISO8859-1
	..
	nb_NO.ISO8859-15
	..
	nb_NO.UTF-8
	..
	nl_BE.ISO8859-1
	..
	nl_BE.ISO8859-15
	..
	nl_BE.UTF-8
	..
	nl_NL.ISO8859-1
	..
	nl_NL.ISO8859-15
	..
	nl_NL.UTF-8
	..
	nn_NO.ISO8859-1
	..
	nn_NO.ISO8859-15
	..
	nn_NO.UTF-8
	..
	pl_PL.ISO8859-2
	..
	pl_PL.UTF-8
	..
	pt_BR.ISO8859-1
	..
	pt_BR.UTF-8
	..
	pt_PT.ISO8859-1
	..
	pt_PT.ISO8859-15
	..
	pt_PT.UTF-8
	..
	ro_RO.ISO8859-2
	..
	ro_RO.UTF-8
	..
	ru_RU.CP1251
	..
	ru_RU.CP866
	..
	ru_RU.ISO8859-5
	..
	ru_RU.KOI8-R
	..
	ru_RU.UTF-8
	..
	se_FI.UTF-8
	..
	se_NO.UTF-8
	..
	sk_SK.ISO8859-2
	..
	sk_SK.UTF-8
	..
	sl_SI.ISO8859-2
	..
	sl_SI.UTF-8
	..
	sr_RS.ISO8859-5
	..
	sr_RS.UTF-8
	..
	sr_RS.ISO8859-2
	..
	sr_RS.UTF-8@latin
	..
	sv_FI.ISO8859-1
	..
	sv_FI.ISO8859-15
	..
	sv_FI.UTF-8
	..
	sv_SE.ISO8859-1
	..
	sv_SE.ISO8859-15
	..
	sv_SE.UTF-8
	..
	tr_TR.ISO8859-9
	..
	tr_TR.UTF-8
	..
	uk_UA.CP1251
	..
	uk_UA.ISO8859-5
	..
	uk_UA.KOI8-U
	..
	uk_UA.UTF-8
	..
	zh_CN.GB18030
	..
	zh_CN.GB2312
	..
	zh_CN.GBK
	..
	zh_CN.eucCN
	..
	zh_CN.UTF-8
	..
	zh_HK.UTF-8
	..
	zh_TW.Big5
	..
	zh_TW.UTF-8
	..
	..
	man
	man1
	..
	man2
	..
	man3
	..
	man4
	aarch64
	..
	amd64
	..
	arm
	..
	i386
	..
	powerpc
	..
	- sparc64
	- ..
	..
	man5
	..
	man6
	..
	man7
	..
	man8
	amd64
	..
	i386
	..
	powerpc
	..
	- sparc64
	- ..
	..
	man9
	..
	..
	misc
	fonts
	..
	..
	mk
	..
	nls
	C
	..
	af_ZA.ISO8859-1
	..
	af_ZA.ISO8859-15
	..
	af_ZA.UTF-8
	..
	am_ET.UTF-8
	..
	be_BY.CP1131
	..
	be_BY.CP1251
	..
	be_BY.ISO8859-5
	..
	be_BY.UTF-8
	..
	bg_BG.CP1251
	..
	bg_BG.UTF-8
	..
	ca_ES.ISO8859-1
	..
	ca_ES.ISO8859-15
	..
	ca_ES.UTF-8
	..
	cs_CZ.ISO8859-2
	..
	cs_CZ.UTF-8
	..
	da_DK.ISO8859-1
	..
	da_DK.ISO8859-15
	..
	da_DK.UTF-8
	..
	de_AT.ISO8859-1
	..
	de_AT.ISO8859-15
	..
	de_AT.UTF-8
	..
	de_CH.ISO8859-1
	..
	de_CH.ISO8859-15
	..
	de_CH.UTF-8
	..
	de_DE.ISO8859-1
	..
	de_DE.ISO8859-15
	..
	de_DE.UTF-8
	..
	el_GR.ISO8859-7
	..
	el_GR.UTF-8
	..
	en_AU.ISO8859-1
	..
	en_AU.ISO8859-15
	..
	en_AU.US-ASCII
	..
	en_AU.UTF-8
	..
	en_CA.ISO8859-1
	..
	en_CA.ISO8859-15
	..
	en_CA.US-ASCII
	..
	en_CA.UTF-8
	..
	en_GB.ISO8859-1
	..
	en_GB.ISO8859-15
	..
	en_GB.US-ASCII
	..
	en_GB.UTF-8
	..
	en_IE.UTF-8
	..
	en_NZ.ISO8859-1
	..
	en_NZ.ISO8859-15
	..
	en_NZ.US-ASCII
	..
	en_NZ.UTF-8
	..
	en_US.ISO8859-1
	..
	en_US.ISO8859-15
	..
	en_US.UTF-8
	..
	es_ES.ISO8859-1
	..
	es_ES.ISO8859-15
	..
	es_ES.UTF-8
	..
	et_EE.ISO8859-15
	..
	et_EE.UTF-8
	..
	fi_FI.ISO8859-1
	..
	fi_FI.ISO8859-15
	..
	fi_FI.UTF-8
	..
	fr_BE.ISO8859-1
	..
	fr_BE.ISO8859-15
	..
	fr_BE.UTF-8
	..
	fr_CA.ISO8859-1
	..
	fr_CA.ISO8859-15
	..
	fr_CA.UTF-8
	..
	fr_CH.ISO8859-1
	..
	fr_CH.ISO8859-15
	..
	fr_CH.UTF-8
	..
	fr_FR.ISO8859-1
	..
	fr_FR.ISO8859-15
	..
	fr_FR.UTF-8
	..
	gl_ES.ISO8859-1
	..
	he_IL.UTF-8
	..
	hi_IN.ISCII-DEV
	..
	hr_HR.ISO8859-2
	..
	hr_HR.UTF-8
	..
	hu_HU.ISO8859-2
	..
	hu_HU.UTF-8
	..
	hy_AM.ARMSCII-8
	..
	hy_AM.UTF-8
	..
	is_IS.ISO8859-1
	..
	is_IS.ISO8859-15
	..
	is_IS.UTF-8
	..
	it_CH.ISO8859-1
	..
	it_CH.ISO8859-15
	..
	it_CH.UTF-8
	..
	it_IT.ISO8859-1
	..
	it_IT.ISO8859-15
	..
	it_IT.UTF-8
	..
	ja_JP.SJIS
	..
	ja_JP.UTF-8
	..
	ja_JP.eucJP
	..
	kk_KZ.PT154
	..
	kk_KZ.UTF-8
	..
	ko_KR.CP949
	..
	ko_KR.UTF-8
	..
	ko_KR.eucKR
	..
	lt_LT.ISO8859-13
	..
	lt_LT.UTF-8
	..
	lv_LV.ISO8859-13
	..
	lv_LV.UTF-8
	..
	mn_MN.UTF-8
	..
	nl_BE.ISO8859-1
	..
	nl_BE.ISO8859-15
	..
	nl_BE.UTF-8
	..
	nl_NL.ISO8859-1
	..
	nl_NL.ISO8859-15
	..
	nl_NL.UTF-8
	..
	no_NO.ISO8859-1
	..
	no_NO.ISO8859-15
	..
	no_NO.UTF-8
	..
	pl_PL.ISO8859-2
	..
	pl_PL.UTF-8
	..
	pt_BR.ISO8859-1
	..
	pt_BR.UTF-8
	..
	pt_PT.ISO8859-1
	..
	pt_PT.ISO8859-15
	..
	pt_PT.UTF-8
	..
	ro_RO.ISO8859-2
	..
	ro_RO.UTF-8
	..
	ru_RU.CP1251
	..
	ru_RU.CP866
	..
	ru_RU.ISO8859-5
	..
	ru_RU.KOI8-R
	..
	ru_RU.UTF-8
	..
	sk_SK.ISO8859-2
	..
	sk_SK.UTF-8
	..
	sl_SI.ISO8859-2
	..
	sl_SI.UTF-8
	..
	sr_YU.ISO8859-2
	..
	sr_YU.ISO8859-5
	..
	sr_YU.UTF-8
	..
	sv_SE.ISO8859-1
	..
	sv_SE.ISO8859-15
	..
	sv_SE.UTF-8
	..
	tr_TR.ISO8859-9
	..
	tr_TR.UTF-8
	..
	uk_UA.ISO8859-5
	..
	uk_UA.KOI8-U
	..
	uk_UA.UTF-8
	..
	zh_CN.GB18030
	..
	zh_CN.GB2312
	..
	zh_CN.GBK
	..
	zh_CN.UTF-8
	..
	zh_CN.eucCN
	..
	zh_HK.UTF-8
	..
	zh_TW.UTF-8
	..
	..
	openssl
	man
	man1
	..
	man3
	..
	man5
	..
	man7
	..
	..
	..
	security
	..
	sendmail
	..
	skel
	..
	snmp
	defs
	..
	mibs
	..
	..
	syscons
	fonts
	..
	keymaps
	..
	scrnmaps
	..
	..
	tabset
	..
	vi
	catalog
	..
	..
	vt
	fonts
	..
	keymaps
	..
	..
	zoneinfo
	Africa
	..
	America
	Argentina
	..
	Indiana
	..
	Kentucky
	..
	North_Dakota
	..
	..
	Antarctica
	..
	Arctic
	..
	Asia
	..
	Atlantic
	..
	Australia
	..
	Brazil
	..
	Canada
	..
	Chile
	..
	Etc
	..
	Europe
	..
	Indian
	..
	Mexico
	..
	Pacific
	..
	SystemV
	..
	US
	..
	..
	..
	src nochange
	..
	..
	diff --git a/include/regex.h b/include/regex.h
	index 6b0838a9bd97..3bea3df4f3d4 100644
	--- a/include/regex.h
	+++ b/include/regex.h
	@@ -1,118 +1,119 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1992 Henry Spencer.
	* Copyright (c) 1992, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* Henry Spencer of the University of Toronto.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)regex.h 8.2 (Berkeley) 1/3/94
	* $FreeBSD$
	*/

	#ifndef _REGEX_H_
	#define _REGEX_H_

	#include <sys/cdefs.h>
	#include <sys/_types.h>

	/* types */
	typedef __off_t regoff_t;

	#ifndef _SIZE_T_DECLARED
	typedef __size_t size_t;
	#define _SIZE_T_DECLARED
	#endif

	typedef struct {
	int re_magic;
	size_t re_nsub; /* number of parenthesized subexpressions */
	const char re_endp; / end pointer for REG_PEND */
	struct re_guts re_g; / none of your business :-) */
	} regex_t;

	typedef struct {
	regoff_t rm_so; /* start of match */
	regoff_t rm_eo; /* end of match */
	} regmatch_t;

	/* regcomp() flags */
	#define REG_BASIC 0000
	#define REG_EXTENDED 0001
	#define REG_ICASE 0002
	#define REG_NOSUB 0004
	#define REG_NEWLINE 0010
	#define REG_NOSPEC 0020
	#define REG_PEND 0040
	#define REG_DUMP 0200
	+#define REG_POSIX 0400 /* only POSIX-compliant regex (libregex) */

	/* regerror() flags */
	#define REG_ENOSYS (-1)
	#define REG_NOMATCH 1
	#define REG_BADPAT 2
	#define REG_ECOLLATE 3
	#define REG_ECTYPE 4
	#define REG_EESCAPE 5
	#define REG_ESUBREG 6
	#define REG_EBRACK 7
	#define REG_EPAREN 8
	#define REG_EBRACE 9
	#define REG_BADBR 10
	#define REG_ERANGE 11
	#define REG_ESPACE 12
	#define REG_BADRPT 13
	#define REG_EMPTY 14
	#define REG_ASSERT 15
	#define REG_INVARG 16
	#define REG_ILLSEQ 17
	#define REG_ATOI 255 /* convert name to number (!) */
	#define REG_ITOA 0400 /* convert number to name (!) */

	/* regexec() flags */
	#define REG_NOTBOL 00001
	#define REG_NOTEOL 00002
	#define REG_STARTEND 00004
	#define REG_TRACE 00400 /* tracing of execution */
	#define REG_LARGE 01000 /* force large representation */
	#define REG_BACKR 02000 /* force use of backref code */

	__BEGIN_DECLS
	int regcomp(regex_t * __restrict, const char * __restrict, int);
	size_t regerror(int, const regex_t * __restrict, char * __restrict, size_t);
	/*
	* XXX forth parameter should be `regmatch_t [__restrict]', but isn't because
	* of a bug in GCC 3.2 (when -std=c99 is specified) which perceives this as a
	* syntax error.
	*/
	int regexec(const regex_t * __restrict, const char * __restrict, size_t,
	regmatch_t * __restrict, int);
	void regfree(regex_t *);
	__END_DECLS

	#endif /* !_REGEX_H_ */
	diff --git a/lib/libc/regex/Symbol.map b/lib/libc/regex/Symbol.map
	index 5821f62b6430..1da7b81ba11b 100644
	--- a/lib/libc/regex/Symbol.map
	+++ b/lib/libc/regex/Symbol.map
	@@ -1,10 +1,13 @@
	/*
	* $FreeBSD$
	*/

	FBSD_1.0 {
	- regcomp;
	regerror;
	regexec;
	regfree;
	};
	+
	+FBSD_1.6 {
	+ regcomp;
	+};
	diff --git a/lib/libc/regex/regcomp.c b/lib/libc/regex/regcomp.c
	index 5e772c21d381..28bad13ac365 100644
	--- a/lib/libc/regex/regcomp.c
	+++ b/lib/libc/regex/regcomp.c
	@@ -1,1981 +1,2066 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1992, 1993, 1994 Henry Spencer.
	* Copyright (c) 1992, 1993, 1994
	* The Regents of the University of California. All rights reserved.
	*
	* Copyright (c) 2011 The FreeBSD Foundation
	* All rights reserved.
	* Portions of this software were developed by David Chisnall
	* under sponsorship from the FreeBSD Foundation.
	*
	* This code is derived from software contributed to Berkeley by
	* Henry Spencer.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)regcomp.c 8.5 (Berkeley) 3/20/94
	*/

	#if defined(LIBC_SCCS) && !defined(lint)
	static char sccsid[] = "@(#)regcomp.c 8.5 (Berkeley) 3/20/94";
	#endif /* LIBC_SCCS and not lint */
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/types.h>
	#include <stdio.h>
	#include <string.h>
	#include <ctype.h>
	#include <limits.h>
	#include <stdlib.h>
	#include <regex.h>
	#include <stdbool.h>
	#include <wchar.h>
	#include <wctype.h>

	#ifndef LIBREGEX
	#include "collate.h"
	#endif

	#include "utils.h"
	#include "regex2.h"

	#include "cname.h"

	/*
	* Branching context, used to keep track of branch state for all of the branch-
	* aware functions. In addition to keeping track of branch positions for the
	* p_branch_* functions, we use this to simplify some clumsiness in BREs for
	* detection of whether ^ is acting as an anchor or being used erroneously and
	* also for whether we're in a sub-expression or not.
	*/
	struct branchc {
	sopno start;
	sopno back;
	sopno fwd;

	int nbranch;
	int nchain;
	bool outer;
	bool terminate;
	};

	/*
	* parse structure, passed up and down to avoid global variables and
	* other clumsinesses
	*/
	struct parse {
	const char next; / next character in RE */
	const char end; / end of string (-> NUL normally) */
	int error; /* has an error been seen? */
	sop strip; / malloced strip */
	sopno ssize; /* malloced strip size (allocated) */
	sopno slen; /* malloced strip length (used) */
	int ncsalloc; /* number of csets allocated */
	struct re_guts *g;
	# define NPAREN 10 /* we need to remember () 1-9 for back refs */
	sopno pbegin[NPAREN]; /* -> ( ([0] unused) */
	sopno pend[NPAREN]; /* -> ) ([0] unused) */
	bool allowbranch; /* can this expression branch? */
	bool bre; /* convenience; is this a BRE? */
	+ int pflags; /* other parsing flags -- legacy escapes? */
	bool (parse_expr)(struct parse , struct branchc *);
	void (pre_parse)(struct parse , struct branchc *);
	void (post_parse)(struct parse , struct branchc *);
	};

	+#define PFLAG_LEGACY_ESC 0x00000001
	+
	/* ========= begin header generated by ./mkh ========= */
	#ifdef __cplusplus
	extern "C" {
	#endif

	/* === regcomp.c === */
	static bool p_ere_exp(struct parse p, struct branchc bc);
	static void p_str(struct parse *p);
	static int p_branch_eat_delim(struct parse p, struct branchc bc);
	static void p_branch_ins_offset(struct parse p, struct branchc bc);
	static void p_branch_fix_tail(struct parse p, struct branchc bc);
	static bool p_branch_empty(struct parse p, struct branchc bc);
	static bool p_branch_do(struct parse p, struct branchc bc);
	static void p_bre_pre_parse(struct parse p, struct branchc bc);
	static void p_bre_post_parse(struct parse p, struct branchc bc);
	static void p_re(struct parse *p, int end1, int end2);
	static bool p_simp_re(struct parse p, struct branchc bc);
	static int p_count(struct parse *p);
	static void p_bracket(struct parse *p);
	static int p_range_cmp(wchar_t c1, wchar_t c2);
	static void p_b_term(struct parse p, cset cs);
	static void p_b_cclass(struct parse p, cset cs);
	static void p_b_eclass(struct parse p, cset cs);
	static wint_t p_b_symbol(struct parse *p);
	static wint_t p_b_coll_elem(struct parse *p, wint_t endc);
	+static bool may_escape(struct parse *p, const wint_t ch);
	static wint_t othercase(wint_t ch);
	static void bothcases(struct parse *p, wint_t ch);
	static void ordinary(struct parse *p, wint_t ch);
	static void nonnewline(struct parse *p);
	static void repeat(struct parse *p, sopno start, int from, int to);
	static int seterr(struct parse *p, int e);
	static cset allocset(struct parse p);
	static void freeset(struct parse p, cset cs);
	static void CHadd(struct parse p, cset cs, wint_t ch);
	static void CHaddrange(struct parse p, cset cs, wint_t min, wint_t max);
	static void CHaddtype(struct parse p, cset cs, wctype_t wct);
	static wint_t singleton(cset *cs);
	static sopno dupl(struct parse *p, sopno start, sopno finish);
	static void doemit(struct parse *p, sop op, size_t opnd);
	static void doinsert(struct parse *p, sop op, size_t opnd, sopno pos);
	static void dofwd(struct parse *p, sopno pos, sop value);
	static int enlarge(struct parse *p, sopno size);
	static void stripsnug(struct parse p, struct re_guts g);
	static void findmust(struct parse p, struct re_guts g);
	static int altoffset(sop *scan, int offset);
	static void computejumps(struct parse p, struct re_guts g);
	static void computematchjumps(struct parse p, struct re_guts g);
	static sopno pluscount(struct parse p, struct re_guts g);
	static wint_t wgetnext(struct parse *p);

	#ifdef __cplusplus
	}
	#endif
	/* ========= end header generated by ./mkh ========= */

	static char nuls[10]; /* place to point scanner in event of error */

	/*
	* macros for use with parse structure
	* BEWARE: these know that the parse structure is named `p' !!!
	*/
	#define PEEK() (*p->next)
	#define PEEK2() (*(p->next+1))
	#define MORE() (p->next < p->end)
	#define MORE2() (p->next+1 < p->end)
	#define SEE(c) (MORE() && PEEK() == (c))
	#define SEETWO(a, b) (MORE() && MORE2() && PEEK() == (a) && PEEK2() == (b))
	#define SEESPEC(a) (p->bre ? SEETWO('\\', a) : SEE(a))
	#define EAT(c) ((SEE(c)) ? (NEXT(), 1) : 0)
	#define EATTWO(a, b) ((SEETWO(a, b)) ? (NEXT2(), 1) : 0)
	#define NEXT() (p->next++)
	#define NEXT2() (p->next += 2)
	#define NEXTn(n) (p->next += (n))
	#define GETNEXT() (*p->next++)
	#define WGETNEXT() wgetnext(p)
	#define SETERROR(e) seterr(p, (e))
	#define REQUIRE(co, e) ((co) \|\| SETERROR(e))
	#define MUSTSEE(c, e) (REQUIRE(MORE() && PEEK() == (c), e))
	#define MUSTEAT(c, e) (REQUIRE(MORE() && GETNEXT() == (c), e))
	#define MUSTNOTSEE(c, e) (REQUIRE(!MORE() \|\| PEEK() != (c), e))
	#define EMIT(op, sopnd) doemit(p, (sop)(op), (size_t)(sopnd))
	#define INSERT(op, pos) doinsert(p, (sop)(op), HERE()-(pos)+1, pos)
	#define AHEAD(pos) dofwd(p, pos, HERE()-(pos))
	#define ASTERN(sop, pos) EMIT(sop, HERE()-pos)
	#define HERE() (p->slen)
	#define THERE() (p->slen - 1)
	#define THERETHERE() (p->slen - 2)
	#define DROP(n) (p->slen -= (n))

	/* Macro used by computejump()/computematchjump() */
	#define MIN(a,b) ((a)<(b)?(a):(b))

	-/*
	- - regcomp - interface for parser and compilation
	- = extern int regcomp(regex_t , const char , int);
	- = #define REG_BASIC 0000
	- = #define REG_EXTENDED 0001
	- = #define REG_ICASE 0002
	- = #define REG_NOSUB 0004
	- = #define REG_NEWLINE 0010
	- = #define REG_NOSPEC 0020
	- = #define REG_PEND 0040
	- = #define REG_DUMP 0200
	- */
	-int /* 0 success, otherwise REG_something */
	-regcomp(regex_t * __restrict preg,
	+static int /* 0 success, otherwise REG_something */
	+regcomp_internal(regex_t * __restrict preg,
	const char * __restrict pattern,
	- int cflags)
	+ int cflags, int pflags)
	{
	struct parse pa;
	struct re_guts *g;
	struct parse *p = &pa;
	int i;
	size_t len;
	size_t maxlen;
	#ifdef REDEBUG
	# define GOODFLAGS(f) (f)
	#else
	# define GOODFLAGS(f) ((f)&~REG_DUMP)
	#endif

	cflags = GOODFLAGS(cflags);
	if ((cflags&REG_EXTENDED) && (cflags&REG_NOSPEC))
	return(REG_INVARG);

	if (cflags&REG_PEND) {
	if (preg->re_endp < pattern)
	return(REG_INVARG);
	len = preg->re_endp - pattern;
	} else
	len = strlen(pattern);

	/* do the mallocs early so failure handling is easy */
	g = (struct re_guts *)malloc(sizeof(struct re_guts));
	if (g == NULL)
	return(REG_ESPACE);
	/*
	* Limit the pattern space to avoid a 32-bit overflow on buffer
	* extension. Also avoid any signed overflow in case of conversion
	* so make the real limit based on a 31-bit overflow.
	*
	* Likely not applicable on 64-bit systems but handle the case
	* generically (who are we to stop people from using ~715MB+
	* patterns?).
	*/
	maxlen = ((size_t)-1 >> 1) / sizeof(sop) * 2 / 3;
	if (len >= maxlen) {
	free((char *)g);
	return(REG_ESPACE);
	}
	p->ssize = len/(size_t)2(size_t)3 + (size_t)1; / ugh */
	assert(p->ssize >= len);

	p->strip = (sop )malloc(p->ssize sizeof(sop));
	p->slen = 0;
	if (p->strip == NULL) {
	free((char *)g);
	return(REG_ESPACE);
	}

	/* set things up */
	p->g = g;
	p->next = pattern; /* convenience; we do not modify it */
	p->end = p->next + len;
	p->error = 0;
	p->ncsalloc = 0;
	+ p->pflags = pflags;
	for (i = 0; i < NPAREN; i++) {
	p->pbegin[i] = 0;
	p->pend[i] = 0;
	}
	if (cflags & REG_EXTENDED) {
	p->allowbranch = true;
	p->bre = false;
	p->parse_expr = p_ere_exp;
	p->pre_parse = NULL;
	p->post_parse = NULL;
	} else {
	p->allowbranch = false;
	p->bre = true;
	p->parse_expr = p_simp_re;
	p->pre_parse = p_bre_pre_parse;
	p->post_parse = p_bre_post_parse;
	}
	g->sets = NULL;
	g->ncsets = 0;
	g->cflags = cflags;
	g->iflags = 0;
	g->nbol = 0;
	g->neol = 0;
	g->must = NULL;
	g->moffset = -1;
	g->charjump = NULL;
	g->matchjump = NULL;
	g->mlen = 0;
	g->nsub = 0;
	g->backrefs = 0;

	/* do it */
	EMIT(OEND, 0);
	g->firststate = THERE();
	if (cflags & REG_NOSPEC)
	p_str(p);
	else
	p_re(p, OUT, OUT);
	EMIT(OEND, 0);
	g->laststate = THERE();

	/* tidy up loose ends and fill things in */
	stripsnug(p, g);
	findmust(p, g);
	/* only use Boyer-Moore algorithm if the pattern is bigger
	* than three characters
	*/
	if(g->mlen > 3) {
	computejumps(p, g);
	computematchjumps(p, g);
	if(g->matchjump == NULL && g->charjump != NULL) {
	free(g->charjump);
	g->charjump = NULL;
	}
	}
	g->nplus = pluscount(p, g);
	g->magic = MAGIC2;
	preg->re_nsub = g->nsub;
	preg->re_g = g;
	preg->re_magic = MAGIC1;
	#ifndef REDEBUG
	/* not debugging, so can't rely on the assert() in regexec() */
	if (g->iflags&BAD)
	SETERROR(REG_ASSERT);
	#endif

	/* win or lose, we're done */
	if (p->error != 0) /* lose */
	regfree(preg);
	return(p->error);
	}

	+/*
	+ - regcomp - interface for parser and compilation
	+ = extern int regcomp(regex_t , const char , int);
	+ = #define REG_BASIC 0000
	+ = #define REG_EXTENDED 0001
	+ = #define REG_ICASE 0002
	+ = #define REG_NOSUB 0004
	+ = #define REG_NEWLINE 0010
	+ = #define REG_NOSPEC 0020
	+ = #define REG_PEND 0040
	+ = #define REG_DUMP 0200
	+ */
	+int /* 0 success, otherwise REG_something */
	+regcomp(regex_t * __restrict preg,
	+ const char * __restrict pattern,
	+ int cflags)
	+{
	+
	+ return (regcomp_internal(preg, pattern, cflags, 0));
	+}
	+
	+#ifndef LIBREGEX
	+/*
	+ * Legacy interface that requires more lax escaping behavior.
	+ */
	+int
	+freebsd12_regcomp(regex_t * __restrict preg,
	+ const char * __restrict pattern,
	+ int cflags, int pflags)
	+{
	+
	+ return (regcomp_internal(preg, pattern, cflags, PFLAG_LEGACY_ESC));
	+}
	+
	+__sym_compat(regcomp, freebsd12_regcomp, FBSD_1.0);
	+#endif /* !LIBREGEX */
	+
	/*
	- p_ere_exp - parse one subERE, an atom possibly followed by a repetition op,
	- return whether we should terminate or not
	== static bool p_ere_exp(struct parse *p);
	*/
	static bool
	p_ere_exp(struct parse p, struct branchc bc)
	{
	char c;
	wint_t wc;
	sopno pos;
	int count;
	int count2;
	sopno subno;
	int wascaret = 0;

	(void)bc;
	assert(MORE()); /* caller should have ensured this */
	c = GETNEXT();

	pos = HERE();
	switch (c) {
	case '(':
	(void)REQUIRE(MORE(), REG_EPAREN);
	p->g->nsub++;
	subno = p->g->nsub;
	if (subno < NPAREN)
	p->pbegin[subno] = HERE();
	EMIT(OLPAREN, subno);
	if (!SEE(')'))
	p_re(p, ')', IGN);
	if (subno < NPAREN) {
	p->pend[subno] = HERE();
	assert(p->pend[subno] != 0);
	}
	EMIT(ORPAREN, subno);
	(void)MUSTEAT(')', REG_EPAREN);
	break;
	#ifndef POSIX_MISTAKE
	case ')': /* happens only if no current unmatched ( */
	/*
	* You may ask, why the ifndef? Because I didn't notice
	* this until slightly too late for 1003.2, and none of the
	* other 1003.2 regular-expression reviewers noticed it at
	* all. So an unmatched ) is legal POSIX, at least until
	* we can get it fixed.
	*/
	SETERROR(REG_EPAREN);
	break;
	#endif
	case '^':
	EMIT(OBOL, 0);
	p->g->iflags \|= USEBOL;
	p->g->nbol++;
	wascaret = 1;
	break;
	case '$':
	EMIT(OEOL, 0);
	p->g->iflags \|= USEEOL;
	p->g->neol++;
	break;
	case '\|':
	SETERROR(REG_EMPTY);
	break;
	case '*':
	case '+':
	case '?':
	case '{':
	SETERROR(REG_BADRPT);
	break;
	case '.':
	if (p->g->cflags&REG_NEWLINE)
	nonnewline(p);
	else
	EMIT(OANY, 0);
	break;
	case '[':
	p_bracket(p);
	break;
	case '\\':
	(void)REQUIRE(MORE(), REG_EESCAPE);
	wc = WGETNEXT();
	switch (wc) {
	case '<':
	EMIT(OBOW, 0);
	break;
	case '>':
	EMIT(OEOW, 0);
	break;
	default:
	- ordinary(p, wc);
	+ if (may_escape(p, wc))
	+ ordinary(p, wc);
	+ else
	+ SETERROR(REG_EESCAPE);
	break;
	}
	break;
	default:
	if (p->error != 0)
	return (false);
	p->next--;
	wc = WGETNEXT();
	ordinary(p, wc);
	break;
	}

	if (!MORE())
	return (false);
	c = PEEK();
	/* we call { a repetition if followed by a digit */
	if (!( c == '*' \|\| c == '+' \|\| c == '?' \|\| c == '{'))
	return (false); /* no repetition, we're done */
	else if (c == '{')
	(void)REQUIRE(MORE2() && \
	(isdigit((uch)PEEK2()) \|\| PEEK2() == ','), REG_BADRPT);
	NEXT();

	(void)REQUIRE(!wascaret, REG_BADRPT);
	switch (c) {
	case '': / implemented as +? */
	/* this case does not require the (y\|) trick, noKLUDGE */
	INSERT(OPLUS_, pos);
	ASTERN(O_PLUS, pos);
	INSERT(OQUEST_, pos);
	ASTERN(O_QUEST, pos);
	break;
	case '+':
	INSERT(OPLUS_, pos);
	ASTERN(O_PLUS, pos);
	break;
	case '?':
	/* KLUDGE: emit y? as (y\|) until subtle bug gets fixed */
	INSERT(OCH_, pos); /* offset slightly wrong */
	ASTERN(OOR1, pos); /* this one's right */
	AHEAD(pos); /* fix the OCH_ */
	EMIT(OOR2, 0); /* offset very wrong... */
	AHEAD(THERE()); /* ...so fix it */
	ASTERN(O_CH, THERETHERE());
	break;
	case '{':
	count = p_count(p);
	if (EAT(',')) {
	if (isdigit((uch)PEEK())) {
	count2 = p_count(p);
	(void)REQUIRE(count <= count2, REG_BADBR);
	} else /* single number with comma */
	count2 = INFINITY;
	} else /* just a single number */
	count2 = count;
	repeat(p, pos, count, count2);
	if (!EAT('}')) { /* error heuristics */
	while (MORE() && PEEK() != '}')
	NEXT();
	(void)REQUIRE(MORE(), REG_EBRACE);
	SETERROR(REG_BADBR);
	}
	break;
	}

	if (!MORE())
	return (false);
	c = PEEK();
	if (!( c == '*' \|\| c == '+' \|\| c == '?' \|\|
	(c == '{' && MORE2() && isdigit((uch)PEEK2())) ) )
	return (false);
	SETERROR(REG_BADRPT);
	return (false);
	}

	/*
	- p_str - string (no metacharacters) "parser"
	== static void p_str(struct parse *p);
	*/
	static void
	p_str(struct parse *p)
	{
	(void)REQUIRE(MORE(), REG_EMPTY);
	while (MORE())
	ordinary(p, WGETNEXT());
	}

	/*
	* Eat consecutive branch delimiters for the kind of expression that we are
	* parsing, return the number of delimiters that we ate.
	*/
	static int
	p_branch_eat_delim(struct parse p, struct branchc bc)
	{
	int nskip;

	(void)bc;
	nskip = 0;
	while (EAT('\|'))
	++nskip;
	return (nskip);
	}

	/*
	* Insert necessary branch book-keeping operations. This emits a
	* bogus 'next' offset, since we still have more to parse
	*/
	static void
	p_branch_ins_offset(struct parse p, struct branchc bc)
	{

	if (bc->nbranch == 0) {
	INSERT(OCH_, bc->start); /* offset is wrong */
	bc->fwd = bc->start;
	bc->back = bc->start;
	}

	ASTERN(OOR1, bc->back);
	bc->back = THERE();
	AHEAD(bc->fwd); /* fix previous offset */
	bc->fwd = HERE();
	EMIT(OOR2, 0); /* offset is very wrong */
	++bc->nbranch;
	}

	/*
	* Fix the offset of the tail branch, if we actually had any branches.
	* This is to correct the bogus placeholder offset that we use.
	*/
	static void
	p_branch_fix_tail(struct parse p, struct branchc bc)
	{

	/* Fix bogus offset at the tail if we actually have branches */
	if (bc->nbranch > 0) {
	AHEAD(bc->fwd);
	ASTERN(O_CH, bc->back);
	}
	}

	/*
	* Signal to the parser that an empty branch has been encountered; this will,
	* in the future, be used to allow for more permissive behavior with empty
	* branches. The return value should indicate whether parsing may continue
	* or not.
	*/
	static bool
	p_branch_empty(struct parse p, struct branchc bc)
	{

	(void)bc;
	SETERROR(REG_EMPTY);
	return (false);
	}

	/*
	* Take care of any branching requirements. This includes inserting the
	* appropriate branching instructions as well as eating all of the branch
	* delimiters until we either run out of pattern or need to parse more pattern.
	*/
	static bool
	p_branch_do(struct parse p, struct branchc bc)
	{
	int ate = 0;

	ate = p_branch_eat_delim(p, bc);
	if (ate == 0)
	return (false);
	else if ((ate > 1 \|\| (bc->outer && !MORE())) && !p_branch_empty(p, bc))
	/*
	* Halt parsing only if we have an empty branch and p_branch_empty
	* indicates that we must not continue. In the future, this will not
	* necessarily be an error.
	*/
	return (false);
	p_branch_ins_offset(p, bc);

	return (true);
	}

	static void
	p_bre_pre_parse(struct parse p, struct branchc bc)
	{

	(void) bc;
	/*
	* Does not move cleanly into expression parser because of
	* ordinary interpration of * at the beginning position of
	* an expression.
	*/
	if (EAT('^')) {
	EMIT(OBOL, 0);
	p->g->iflags \|= USEBOL;
	p->g->nbol++;
	}
	}

	static void
	p_bre_post_parse(struct parse p, struct branchc bc)
	{

	/* Expression is terminating due to EOL token */
	if (bc->terminate) {
	DROP(1);
	EMIT(OEOL, 0);
	p->g->iflags \|= USEEOL;
	p->g->neol++;
	}
	}

	/*
	- p_re - Top level parser, concatenation and BRE anchoring
	== static void p_re(struct parse *p, int end1, int end2);
	* Giving end1 as OUT essentially eliminates the end1/end2 check.
	*
	* This implementation is a bit of a kludge, in that a trailing $ is first
	* taken as an ordinary character and then revised to be an anchor.
	* The amount of lookahead needed to avoid this kludge is excessive.
	*/
	static void
	p_re(struct parse *p,
	int end1, /* first terminating character */
	int end2) /* second terminating character; ignored for EREs */
	{
	struct branchc bc;

	bc.nbranch = 0;
	if (end1 == OUT && end2 == OUT)
	bc.outer = true;
	else
	bc.outer = false;
	#define SEEEND() (!p->bre ? SEE(end1) : SEETWO(end1, end2))
	for (;;) {
	bc.start = HERE();
	bc.nchain = 0;
	bc.terminate = false;
	if (p->pre_parse != NULL)
	p->pre_parse(p, &bc);
	while (MORE() && (!p->allowbranch \|\| !SEESPEC('\|')) && !SEEEND()) {
	bc.terminate = p->parse_expr(p, &bc);
	++bc.nchain;
	}
	if (p->post_parse != NULL)
	p->post_parse(p, &bc);
	(void) REQUIRE(HERE() != bc.start, REG_EMPTY);
	if (!p->allowbranch)
	break;
	/*
	* p_branch_do's return value indicates whether we should
	* continue parsing or not. This is both for correctness and
	* a slight optimization, because it will check if we've
	* encountered an empty branch or the end of the string
	* immediately following a branch delimiter.
	*/
	if (!p_branch_do(p, &bc))
	break;
	}
	#undef SEE_END
	if (p->allowbranch)
	p_branch_fix_tail(p, &bc);
	assert(!MORE() \|\| SEE(end1));
	}

	/*
	- p_simp_re - parse a simple RE, an atom possibly followed by a repetition
	== static bool p_simp_re(struct parse p, struct branchc bc);
	*/
	static bool /* was the simple RE an unbackslashed $? */
	p_simp_re(struct parse p, struct branchc bc)
	{
	int c;
	int count;
	int count2;
	sopno pos;
	int i;
	wint_t wc;
	sopno subno;
	# define BACKSL (1<<CHAR_BIT)

	pos = HERE(); /* repetition op, if any, covers from here */

	assert(MORE()); /* caller should have ensured this */
	c = GETNEXT();
	if (c == '\\') {
	(void)REQUIRE(MORE(), REG_EESCAPE);
	c = BACKSL \| GETNEXT();
	}
	switch (c) {
	case '.':
	if (p->g->cflags&REG_NEWLINE)
	nonnewline(p);
	else
	EMIT(OANY, 0);
	break;
	case '[':
	p_bracket(p);
	break;
	case BACKSL\|'<':
	EMIT(OBOW, 0);
	break;
	case BACKSL\|'>':
	EMIT(OEOW, 0);
	break;
	case BACKSL\|'{':
	SETERROR(REG_BADRPT);
	break;
	case BACKSL\|'(':
	p->g->nsub++;
	subno = p->g->nsub;
	if (subno < NPAREN)
	p->pbegin[subno] = HERE();
	EMIT(OLPAREN, subno);
	/* the MORE here is an error heuristic */
	if (MORE() && !SEETWO('\\', ')'))
	p_re(p, '\\', ')');
	if (subno < NPAREN) {
	p->pend[subno] = HERE();
	assert(p->pend[subno] != 0);
	}
	EMIT(ORPAREN, subno);
	(void)REQUIRE(EATTWO('\\', ')'), REG_EPAREN);
	break;
	case BACKSL\|')': /* should not get here -- must be user */
	SETERROR(REG_EPAREN);
	break;
	case BACKSL\|'1':
	case BACKSL\|'2':
	case BACKSL\|'3':
	case BACKSL\|'4':
	case BACKSL\|'5':
	case BACKSL\|'6':
	case BACKSL\|'7':
	case BACKSL\|'8':
	case BACKSL\|'9':
	i = (c&~BACKSL) - '0';
	assert(i < NPAREN);
	if (p->pend[i] != 0) {
	assert(i <= p->g->nsub);
	EMIT(OBACK_, i);
	assert(p->pbegin[i] != 0);
	assert(OP(p->strip[p->pbegin[i]]) == OLPAREN);
	assert(OP(p->strip[p->pend[i]]) == ORPAREN);
	(void) dupl(p, p->pbegin[i]+1, p->pend[i]);
	EMIT(O_BACK, i);
	} else
	SETERROR(REG_ESUBREG);
	p->g->backrefs = 1;
	break;
	case '*':
	/*
	* Ordinary if used as the first character beyond BOL anchor of
	* a (sub-)expression, counts as a bad repetition operator if it
	* appears otherwise.
	*/
	(void)REQUIRE(bc->nchain == 0, REG_BADRPT);
	/* FALLTHROUGH */
	default:
	if (p->error != 0)
	return (false); /* Definitely not $... */
	p->next--;
	wc = WGETNEXT();
	- ordinary(p, wc);
	+ if ((c & BACKSL) == 0 \|\| may_escape(p, wc))
	+ ordinary(p, wc);
	+ else
	+ SETERROR(REG_EESCAPE);
	break;
	}

	if (EAT('')) { / implemented as +? */
	/* this case does not require the (y\|) trick, noKLUDGE */
	INSERT(OPLUS_, pos);
	ASTERN(O_PLUS, pos);
	INSERT(OQUEST_, pos);
	ASTERN(O_QUEST, pos);
	} else if (EATTWO('\\', '{')) {
	count = p_count(p);
	if (EAT(',')) {
	if (MORE() && isdigit((uch)PEEK())) {
	count2 = p_count(p);
	(void)REQUIRE(count <= count2, REG_BADBR);
	} else /* single number with comma */
	count2 = INFINITY;
	} else /* just a single number */
	count2 = count;
	repeat(p, pos, count, count2);
	if (!EATTWO('\\', '}')) { /* error heuristics */
	while (MORE() && !SEETWO('\\', '}'))
	NEXT();
	(void)REQUIRE(MORE(), REG_EBRACE);
	SETERROR(REG_BADBR);
	}
	} else if (c == '$') /* $ (but not \$) ends it */
	return (true);

	return (false);
	}

	/*
	- p_count - parse a repetition count
	== static int p_count(struct parse *p);
	*/
	static int /* the value */
	p_count(struct parse *p)
	{
	int count = 0;
	int ndigits = 0;

	while (MORE() && isdigit((uch)PEEK()) && count <= DUPMAX) {
	count = count*10 + (GETNEXT() - '0');
	ndigits++;
	}

	(void)REQUIRE(ndigits > 0 && count <= DUPMAX, REG_BADBR);
	return(count);
	}

	/*
	- p_bracket - parse a bracketed character list
	== static void p_bracket(struct parse *p);
	*/
	static void
	p_bracket(struct parse *p)
	{
	cset *cs;
	wint_t ch;

	/* Dept of Truly Sickening Special-Case Kludges */
	if (p->next + 5 < p->end && strncmp(p->next, "[:<:]]", 6) == 0) {
	EMIT(OBOW, 0);
	NEXTn(6);
	return;
	}
	if (p->next + 5 < p->end && strncmp(p->next, "[:>:]]", 6) == 0) {
	EMIT(OEOW, 0);
	NEXTn(6);
	return;
	}

	if ((cs = allocset(p)) == NULL)
	return;

	if (p->g->cflags&REG_ICASE)
	cs->icase = 1;
	if (EAT('^'))
	cs->invert = 1;
	if (EAT(']'))
	CHadd(p, cs, ']');
	else if (EAT('-'))
	CHadd(p, cs, '-');
	while (MORE() && PEEK() != ']' && !SEETWO('-', ']'))
	p_b_term(p, cs);
	if (EAT('-'))
	CHadd(p, cs, '-');
	(void)MUSTEAT(']', REG_EBRACK);

	if (p->error != 0) /* don't mess things up further */
	return;

	if (cs->invert && p->g->cflags&REG_NEWLINE)
	cs->bmp['\n' >> 3] \|= 1 << ('\n' & 7);

	if ((ch = singleton(cs)) != OUT) { /* optimize singleton sets */
	ordinary(p, ch);
	freeset(p, cs);
	} else
	EMIT(OANYOF, (int)(cs - p->g->sets));
	}

	static int
	p_range_cmp(wchar_t c1, wchar_t c2)
	{
	#ifndef LIBREGEX
	return __wcollate_range_cmp(c1, c2);
	#else
	/* Copied from libc/collate __wcollate_range_cmp */
	wchar_t s1[2], s2[2];

	s1[0] = c1;
	s1[1] = L'\0';
	s2[0] = c2;
	s2[1] = L'\0';
	return (wcscoll(s1, s2));
	#endif
	}

	/*
	- p_b_term - parse one term of a bracketed character list
	== static void p_b_term(struct parse p, cset cs);
	*/
	static void
	p_b_term(struct parse p, cset cs)
	{
	char c;
	wint_t start, finish;
	wint_t i;
	#ifndef LIBREGEX
	struct xlocale_collate *table =
	(struct xlocale_collate*)__get_locale()->components[XLC_COLLATE];
	#endif
	/* classify what we've got */
	switch ((MORE()) ? PEEK() : '\0') {
	case '[':
	c = (MORE2()) ? PEEK2() : '\0';
	break;
	case '-':
	SETERROR(REG_ERANGE);
	return; /* NOTE RETURN */
	default:
	c = '\0';
	break;
	}

	switch (c) {
	case ':': /* character class */
	NEXT2();
	(void)REQUIRE(MORE(), REG_EBRACK);
	c = PEEK();
	(void)REQUIRE(c != '-' && c != ']', REG_ECTYPE);
	p_b_cclass(p, cs);
	(void)REQUIRE(MORE(), REG_EBRACK);
	(void)REQUIRE(EATTWO(':', ']'), REG_ECTYPE);
	break;
	case '=': /* equivalence class */
	NEXT2();
	(void)REQUIRE(MORE(), REG_EBRACK);
	c = PEEK();
	(void)REQUIRE(c != '-' && c != ']', REG_ECOLLATE);
	p_b_eclass(p, cs);
	(void)REQUIRE(MORE(), REG_EBRACK);
	(void)REQUIRE(EATTWO('=', ']'), REG_ECOLLATE);
	break;
	default: /* symbol, ordinary character, or range */
	start = p_b_symbol(p);
	if (SEE('-') && MORE2() && PEEK2() != ']') {
	/* range */
	NEXT();
	if (EAT('-'))
	finish = '-';
	else
	finish = p_b_symbol(p);
	} else
	finish = start;
	if (start == finish)
	CHadd(p, cs, start);
	else {
	#ifndef LIBREGEX
	if (table->__collate_load_error \|\| MB_CUR_MAX > 1) {
	#else
	if (MB_CUR_MAX > 1) {
	#endif
	(void)REQUIRE(start <= finish, REG_ERANGE);
	CHaddrange(p, cs, start, finish);
	} else {
	(void)REQUIRE(p_range_cmp(start, finish) <= 0, REG_ERANGE);
	for (i = 0; i <= UCHAR_MAX; i++) {
	if (p_range_cmp(start, i) <= 0 &&
	p_range_cmp(i, finish) <= 0 )
	CHadd(p, cs, i);
	}
	}
	}
	break;
	}
	}

	/*
	- p_b_cclass - parse a character-class name and deal with it
	== static void p_b_cclass(struct parse p, cset cs);
	*/
	static void
	p_b_cclass(struct parse p, cset cs)
	{
	const char *sp = p->next;
	size_t len;
	wctype_t wct;
	char clname[16];

	while (MORE() && isalpha((uch)PEEK()))
	NEXT();
	len = p->next - sp;
	if (len >= sizeof(clname) - 1) {
	SETERROR(REG_ECTYPE);
	return;
	}
	memcpy(clname, sp, len);
	clname[len] = '\0';
	if ((wct = wctype(clname)) == 0) {
	SETERROR(REG_ECTYPE);
	return;
	}
	CHaddtype(p, cs, wct);
	}

	/*
	- p_b_eclass - parse an equivalence-class name and deal with it
	== static void p_b_eclass(struct parse p, cset cs);
	*
	* This implementation is incomplete. xxx
	*/
	static void
	p_b_eclass(struct parse p, cset cs)
	{
	wint_t c;

	c = p_b_coll_elem(p, '=');
	CHadd(p, cs, c);
	}

	/*
	- p_b_symbol - parse a character or [..]ed multicharacter collating symbol
	== static wint_t p_b_symbol(struct parse *p);
	*/
	static wint_t /* value of symbol */
	p_b_symbol(struct parse *p)
	{
	wint_t value;

	(void)REQUIRE(MORE(), REG_EBRACK);
	if (!EATTWO('[', '.'))
	return(WGETNEXT());

	/* collating symbol */
	value = p_b_coll_elem(p, '.');
	(void)REQUIRE(EATTWO('.', ']'), REG_ECOLLATE);
	return(value);
	}

	/*
	- p_b_coll_elem - parse a collating-element name and look it up
	== static wint_t p_b_coll_elem(struct parse *p, wint_t endc);
	*/
	static wint_t /* value of collating element */
	p_b_coll_elem(struct parse *p,
	wint_t endc) /* name ended by endc,']' */
	{
	const char *sp = p->next;
	struct cname *cp;
	mbstate_t mbs;
	wchar_t wc;
	size_t clen, len;

	while (MORE() && !SEETWO(endc, ']'))
	NEXT();
	if (!MORE()) {
	SETERROR(REG_EBRACK);
	return(0);
	}
	len = p->next - sp;
	for (cp = cnames; cp->name != NULL; cp++)
	if (strncmp(cp->name, sp, len) == 0 && strlen(cp->name) == len)
	return(cp->code); /* known name */
	memset(&mbs, 0, sizeof(mbs));
	if ((clen = mbrtowc(&wc, sp, len, &mbs)) == len)
	return (wc); /* single character */
	else if (clen == (size_t)-1 \|\| clen == (size_t)-2)
	SETERROR(REG_ILLSEQ);
	else
	SETERROR(REG_ECOLLATE); /* neither */
	return(0);
	}

	+/*
	+ - may_escape - determine whether 'ch' is escape-able in the current context
	+ == static int may_escape(struct parse *p, const wint_t ch)
	+ */
	+static bool
	+may_escape(struct parse *p, const wint_t ch)
	+{
	+
	+ if ((p->pflags & PFLAG_LEGACY_ESC) != 0)
	+ return (true);
	+ if (isalpha(ch) \|\| ch == '\'' \|\| ch == '`')
	+ return (false);
	+ return (true);
	+#ifdef NOTYET
	+ /*
	+ * Build a whitelist of characters that may be escaped to produce an
	+ * ordinary in the current context. This assumes that these have not
	+ * been otherwise interpreted as a special character. Escaping an
	+ * ordinary character yields undefined results according to
	+ * IEEE 1003.1-2008. Some extensions (notably, some GNU extensions) take
	+ * advantage of this and use escaped ordinary characters to provide
	+ * special meaning, e.g. \b, \B, \w, \W, \s, \S.
	+ */
	+ switch(ch) {
	+ case '\|':
	+ case '+':
	+ case '?':
	+ /* The above characters may not be escaped in BREs */
	+ if (!(p->g->cflags&REG_EXTENDED))
	+ return (false);
	+ /* Fallthrough */
	+ case '(':
	+ case ')':
	+ case '{':
	+ case '}':
	+ case '.':
	+ case '[':
	+ case ']':
	+ case '\\':
	+ case '*':
	+ case '^':
	+ case '$':
	+ return (true);
	+ default:
	+ return (false);
	+ }
	+#endif
	+}
	+
	/*
	- othercase - return the case counterpart of an alphabetic
	== static wint_t othercase(wint_t ch);
	*/
	static wint_t /* if no counterpart, return ch */
	othercase(wint_t ch)
	{
	assert(iswalpha(ch));
	if (iswupper(ch))
	return(towlower(ch));
	else if (iswlower(ch))
	return(towupper(ch));
	else /* peculiar, but could happen */
	return(ch);
	}

	/*
	- bothcases - emit a dualcase version of a two-case character
	== static void bothcases(struct parse *p, wint_t ch);
	*
	* Boy, is this implementation ever a kludge...
	*/
	static void
	bothcases(struct parse *p, wint_t ch)
	{
	const char *oldnext = p->next;
	const char *oldend = p->end;
	char bracket[3 + MB_LEN_MAX];
	size_t n;
	mbstate_t mbs;

	assert(othercase(ch) != ch); /* p_bracket() would recurse */
	p->next = bracket;
	memset(&mbs, 0, sizeof(mbs));
	n = wcrtomb(bracket, ch, &mbs);
	assert(n != (size_t)-1);
	bracket[n] = ']';
	bracket[n + 1] = '\0';
	p->end = bracket+n+1;
	p_bracket(p);
	assert(p->next == p->end);
	p->next = oldnext;
	p->end = oldend;
	}

	/*
	- ordinary - emit an ordinary character
	== static void ordinary(struct parse *p, wint_t ch);
	*/
	static void
	ordinary(struct parse *p, wint_t ch)
	{
	cset *cs;

	if ((p->g->cflags&REG_ICASE) && iswalpha(ch) && othercase(ch) != ch)
	bothcases(p, ch);
	else if ((ch & OPDMASK) == ch)
	EMIT(OCHAR, ch);
	else {
	/*
	* Kludge: character is too big to fit into an OCHAR operand.
	* Emit a singleton set.
	*/
	if ((cs = allocset(p)) == NULL)
	return;
	CHadd(p, cs, ch);
	EMIT(OANYOF, (int)(cs - p->g->sets));
	}
	}

	/*
	- nonnewline - emit REG_NEWLINE version of OANY
	== static void nonnewline(struct parse *p);
	*
	* Boy, is this implementation ever a kludge...
	*/
	static void
	nonnewline(struct parse *p)
	{
	const char *oldnext = p->next;
	const char *oldend = p->end;
	char bracket[4];

	p->next = bracket;
	p->end = bracket+3;
	bracket[0] = '^';
	bracket[1] = '\n';
	bracket[2] = ']';
	bracket[3] = '\0';
	p_bracket(p);
	assert(p->next == bracket+3);
	p->next = oldnext;
	p->end = oldend;
	}

	/*
	- repeat - generate code for a bounded repetition, recursively if needed
	== static void repeat(struct parse *p, sopno start, int from, int to);
	*/
	static void
	repeat(struct parse *p,
	sopno start, /* operand from here to end of strip */
	int from, /* repeated from this number */
	int to) /* to this number of times (maybe INFINITY) */
	{
	sopno finish = HERE();
	# define N 2
	# define INF 3
	# define REP(f, t) ((f)*8 + (t))
	# define MAP(n) (((n) <= 1) ? (n) : ((n) == INFINITY) ? INF : N)
	sopno copy;

	if (p->error != 0) /* head off possible runaway recursion */
	return;

	assert(from <= to);

	switch (REP(MAP(from), MAP(to))) {
	case REP(0, 0): /* must be user doing this */
	DROP(finish-start); /* drop the operand */
	break;
	case REP(0, 1): /* as x{1,1}? */
	case REP(0, N): /* as x{1,n}? */
	case REP(0, INF): /* as x{1,}? */
	/* KLUDGE: emit y? as (y\|) until subtle bug gets fixed */
	INSERT(OCH_, start); /* offset is wrong... */
	repeat(p, start+1, 1, to);
	ASTERN(OOR1, start);
	AHEAD(start); /* ... fix it */
	EMIT(OOR2, 0);
	AHEAD(THERE());
	ASTERN(O_CH, THERETHERE());
	break;
	case REP(1, 1): /* trivial case */
	/* done */
	break;
	case REP(1, N): /* as x?x{1,n-1} */
	/* KLUDGE: emit y? as (y\|) until subtle bug gets fixed */
	INSERT(OCH_, start);
	ASTERN(OOR1, start);
	AHEAD(start);
	EMIT(OOR2, 0); /* offset very wrong... */
	AHEAD(THERE()); /* ...so fix it */
	ASTERN(O_CH, THERETHERE());
	copy = dupl(p, start+1, finish+1);
	assert(copy == finish+4);
	repeat(p, copy, 1, to-1);
	break;
	case REP(1, INF): /* as x+ */
	INSERT(OPLUS_, start);
	ASTERN(O_PLUS, start);
	break;
	case REP(N, N): /* as xx{m-1,n-1} */
	copy = dupl(p, start, finish);
	repeat(p, copy, from-1, to-1);
	break;
	case REP(N, INF): /* as xx{n-1,INF} */
	copy = dupl(p, start, finish);
	repeat(p, copy, from-1, to);
	break;
	default: /* "can't happen" */
	SETERROR(REG_ASSERT); /* just in case */
	break;
	}
	}

	/*
	- wgetnext - helper function for WGETNEXT() macro. Gets the next wide
	- character from the parse struct, signals a REG_ILLSEQ error if the
	- character can't be converted. Returns the number of bytes consumed.
	*/
	static wint_t
	wgetnext(struct parse *p)
	{
	mbstate_t mbs;
	wchar_t wc;
	size_t n;

	memset(&mbs, 0, sizeof(mbs));
	n = mbrtowc(&wc, p->next, p->end - p->next, &mbs);
	if (n == (size_t)-1 \|\| n == (size_t)-2) {
	SETERROR(REG_ILLSEQ);
	return (0);
	}
	if (n == 0)
	n = 1;
	p->next += n;
	return (wc);
	}

	/*
	- seterr - set an error condition
	== static int seterr(struct parse *p, int e);
	*/
	static int /* useless but makes type checking happy */
	seterr(struct parse *p, int e)
	{
	if (p->error == 0) /* keep earliest error condition */
	p->error = e;
	p->next = nuls; /* try to bring things to a halt */
	p->end = nuls;
	return(0); /* make the return value well-defined */
	}

	/*
	- allocset - allocate a set of characters for []
	== static cset allocset(struct parse p);
	*/
	static cset *
	allocset(struct parse *p)
	{
	cset cs, ncs;

	ncs = reallocarray(p->g->sets, p->g->ncsets + 1, sizeof(*ncs));
	if (ncs == NULL) {
	SETERROR(REG_ESPACE);
	return (NULL);
	}
	p->g->sets = ncs;
	cs = &p->g->sets[p->g->ncsets++];
	memset(cs, 0, sizeof(*cs));

	return(cs);
	}

	/*
	- freeset - free a now-unused set
	== static void freeset(struct parse p, cset cs);
	*/
	static void
	freeset(struct parse p, cset cs)
	{
	cset *top = &p->g->sets[p->g->ncsets];

	free(cs->wides);
	free(cs->ranges);
	free(cs->types);
	memset(cs, 0, sizeof(*cs));
	if (cs == top-1) /* recover only the easy case */
	p->g->ncsets--;
	}

	/*
	- singleton - Determine whether a set contains only one character,
	- returning it if so, otherwise returning OUT.
	*/
	static wint_t
	singleton(cset *cs)
	{
	wint_t i, s, n;

	for (i = n = 0; i < NC; i++)
	if (CHIN(cs, i)) {
	n++;
	s = i;
	}
	if (n == 1)
	return (s);
	if (cs->nwides == 1 && cs->nranges == 0 && cs->ntypes == 0 &&
	cs->icase == 0)
	return (cs->wides[0]);
	/* Don't bother handling the other cases. */
	return (OUT);
	}

	/*
	- CHadd - add character to character set.
	*/
	static void
	CHadd(struct parse p, cset cs, wint_t ch)
	{
	wint_t nch, *newwides;
	assert(ch >= 0);
	if (ch < NC)
	cs->bmp[ch >> 3] \|= 1 << (ch & 7);
	else {
	newwides = reallocarray(cs->wides, cs->nwides + 1,
	sizeof(*cs->wides));
	if (newwides == NULL) {
	SETERROR(REG_ESPACE);
	return;
	}
	cs->wides = newwides;
	cs->wides[cs->nwides++] = ch;
	}
	if (cs->icase) {
	if ((nch = towlower(ch)) < NC)
	cs->bmp[nch >> 3] \|= 1 << (nch & 7);
	if ((nch = towupper(ch)) < NC)
	cs->bmp[nch >> 3] \|= 1 << (nch & 7);
	}
	}

	/*
	- CHaddrange - add all characters in the range [min,max] to a character set.
	*/
	static void
	CHaddrange(struct parse p, cset cs, wint_t min, wint_t max)
	{
	crange *newranges;

	for (; min < NC && min <= max; min++)
	CHadd(p, cs, min);
	if (min >= max)
	return;
	newranges = reallocarray(cs->ranges, cs->nranges + 1,
	sizeof(*cs->ranges));
	if (newranges == NULL) {
	SETERROR(REG_ESPACE);
	return;
	}
	cs->ranges = newranges;
	cs->ranges[cs->nranges].min = min;
	cs->ranges[cs->nranges].max = max;
	cs->nranges++;
	}

	/*
	- CHaddtype - add all characters of a certain type to a character set.
	*/
	static void
	CHaddtype(struct parse p, cset cs, wctype_t wct)
	{
	wint_t i;
	wctype_t *newtypes;

	for (i = 0; i < NC; i++)
	if (iswctype(i, wct))
	CHadd(p, cs, i);
	newtypes = reallocarray(cs->types, cs->ntypes + 1,
	sizeof(*cs->types));
	if (newtypes == NULL) {
	SETERROR(REG_ESPACE);
	return;
	}
	cs->types = newtypes;
	cs->types[cs->ntypes++] = wct;
	}

	/*
	- dupl - emit a duplicate of a bunch of sops
	== static sopno dupl(struct parse *p, sopno start, sopno finish);
	*/
	static sopno /* start of duplicate */
	dupl(struct parse *p,
	sopno start, /* from here */
	sopno finish) /* to this less one */
	{
	sopno ret = HERE();
	sopno len = finish - start;

	assert(finish >= start);
	if (len == 0)
	return(ret);
	if (!enlarge(p, p->ssize + len)) /* this many unexpected additions */
	return(ret);
	(void) memcpy((char *)(p->strip + p->slen),
	(char )(p->strip + start), (size_t)lensizeof(sop));
	p->slen += len;
	return(ret);
	}

	/*
	- doemit - emit a strip operator
	== static void doemit(struct parse *p, sop op, size_t opnd);
	*
	* It might seem better to implement this as a macro with a function as
	* hard-case backup, but it's just too big and messy unless there are
	* some changes to the data structures. Maybe later.
	*/
	static void
	doemit(struct parse *p, sop op, size_t opnd)
	{
	/* avoid making error situations worse */
	if (p->error != 0)
	return;

	/* deal with oversize operands ("can't happen", more or less) */
	assert(opnd < 1<<OPSHIFT);

	/* deal with undersized strip */
	if (p->slen >= p->ssize)
	if (!enlarge(p, (p->ssize+1) / 2 * 3)) /* +50% */
	return;

	/* finally, it's all reduced to the easy case */
	p->strip[p->slen++] = SOP(op, opnd);
	}

	/*
	- doinsert - insert a sop into the strip
	== static void doinsert(struct parse *p, sop op, size_t opnd, sopno pos);
	*/
	static void
	doinsert(struct parse *p, sop op, size_t opnd, sopno pos)
	{
	sopno sn;
	sop s;
	int i;

	/* avoid making error situations worse */
	if (p->error != 0)
	return;

	sn = HERE();
	EMIT(op, opnd); /* do checks, ensure space */
	assert(HERE() == sn+1);
	s = p->strip[sn];

	/* adjust paren pointers */
	assert(pos > 0);
	for (i = 1; i < NPAREN; i++) {
	if (p->pbegin[i] >= pos) {
	p->pbegin[i]++;
	}
	if (p->pend[i] >= pos) {
	p->pend[i]++;
	}
	}

	memmove((char )&p->strip[pos+1], (char )&p->strip[pos],
	(HERE()-pos-1)*sizeof(sop));
	p->strip[pos] = s;
	}

	/*
	- dofwd - complete a forward reference
	== static void dofwd(struct parse *p, sopno pos, sop value);
	*/
	static void
	dofwd(struct parse *p, sopno pos, sop value)
	{
	/* avoid making error situations worse */
	if (p->error != 0)
	return;

	assert(value < 1<<OPSHIFT);
	p->strip[pos] = OP(p->strip[pos]) \| value;
	}

	/*
	- enlarge - enlarge the strip
	== static int enlarge(struct parse *p, sopno size);
	*/
	static int
	enlarge(struct parse *p, sopno size)
	{
	sop *sp;

	if (p->ssize >= size)
	return 1;

	sp = reallocarray(p->strip, size, sizeof(sop));
	if (sp == NULL) {
	SETERROR(REG_ESPACE);
	return 0;
	}
	p->strip = sp;
	p->ssize = size;
	return 1;
	}

	/*
	- stripsnug - compact the strip
	== static void stripsnug(struct parse p, struct re_guts g);
	*/
	static void
	stripsnug(struct parse p, struct re_guts g)
	{
	g->nstates = p->slen;
	g->strip = reallocarray((char *)p->strip, p->slen, sizeof(sop));
	if (g->strip == NULL) {
	SETERROR(REG_ESPACE);
	g->strip = p->strip;
	}
	}

	/*
	- findmust - fill in must and mlen with longest mandatory literal string
	== static void findmust(struct parse p, struct re_guts g);
	*
	* This algorithm could do fancy things like analyzing the operands of \|
	* for common subsequences. Someday. This code is simple and finds most
	* of the interesting cases.
	*
	* Note that must and mlen got initialized during setup.
	*/
	static void
	findmust(struct parse p, struct re_guts g)
	{
	sop *scan;
	sop *start = NULL;
	sop *newstart = NULL;
	sopno newlen;
	sop s;
	char *cp;
	int offset;
	char buf[MB_LEN_MAX];
	size_t clen;
	mbstate_t mbs;

	/* avoid making error situations worse */
	if (p->error != 0)
	return;

	/*
	* It's not generally safe to do a ``char'' substring search on
	* multibyte character strings, but it's safe for at least
	* UTF-8 (see RFC 3629).
	*/
	if (MB_CUR_MAX > 1 &&
	strcmp(_CurrentRuneLocale->__encoding, "UTF-8") != 0)
	return;

	/* find the longest OCHAR sequence in strip */
	newlen = 0;
	offset = 0;
	g->moffset = 0;
	scan = g->strip + 1;
	do {
	s = *scan++;
	switch (OP(s)) {
	case OCHAR: /* sequence member */
	if (newlen == 0) { /* new sequence */
	memset(&mbs, 0, sizeof(mbs));
	newstart = scan - 1;
	}
	clen = wcrtomb(buf, OPND(s), &mbs);
	if (clen == (size_t)-1)
	goto toohard;
	newlen += clen;
	break;
	case OPLUS_: /* things that don't break one */
	case OLPAREN:
	case ORPAREN:
	break;
	case OQUEST_: /* things that must be skipped */
	case OCH_:
	offset = altoffset(scan, offset);
	scan--;
	do {
	scan += OPND(s);
	s = *scan;
	/* assert() interferes w debug printouts */
	if (OP(s) != (sop)O_QUEST &&
	OP(s) != (sop)O_CH && OP(s) != (sop)OOR2) {
	g->iflags \|= BAD;
	return;
	}
	} while (OP(s) != (sop)O_QUEST && OP(s) != (sop)O_CH);
	/* FALLTHROUGH */
	case OBOW: /* things that break a sequence */
	case OEOW:
	case OBOL:
	case OEOL:
	case O_QUEST:
	case O_CH:
	case OEND:
	if (newlen > (sopno)g->mlen) { /* ends one */
	start = newstart;
	g->mlen = newlen;
	if (offset > -1) {
	g->moffset += offset;
	offset = newlen;
	} else
	g->moffset = offset;
	} else {
	if (offset > -1)
	offset += newlen;
	}
	newlen = 0;
	break;
	case OANY:
	if (newlen > (sopno)g->mlen) { /* ends one */
	start = newstart;
	g->mlen = newlen;
	if (offset > -1) {
	g->moffset += offset;
	offset = newlen;
	} else
	g->moffset = offset;
	} else {
	if (offset > -1)
	offset += newlen;
	}
	if (offset > -1)
	offset++;
	newlen = 0;
	break;
	case OANYOF: /* may or may not invalidate offset */
	/* First, everything as OANY */
	if (newlen > (sopno)g->mlen) { /* ends one */
	start = newstart;
	g->mlen = newlen;
	if (offset > -1) {
	g->moffset += offset;
	offset = newlen;
	} else
	g->moffset = offset;
	} else {
	if (offset > -1)
	offset += newlen;
	}
	if (offset > -1)
	offset++;
	newlen = 0;
	break;
	toohard:
	default:
	/* Anything here makes it impossible or too hard
	* to calculate the offset -- so we give up;
	* save the last known good offset, in case the
	* must sequence doesn't occur later.
	*/
	if (newlen > (sopno)g->mlen) { /* ends one */
	start = newstart;
	g->mlen = newlen;
	if (offset > -1)
	g->moffset += offset;
	else
	g->moffset = offset;
	}
	offset = -1;
	newlen = 0;
	break;
	}
	} while (OP(s) != OEND);

	if (g->mlen == 0) { /* there isn't one */
	g->moffset = -1;
	return;
	}

	/* turn it into a character string */
	g->must = malloc((size_t)g->mlen + 1);
	if (g->must == NULL) { /* argh; just forget it */
	g->mlen = 0;
	g->moffset = -1;
	return;
	}
	cp = g->must;
	scan = start;
	memset(&mbs, 0, sizeof(mbs));
	while (cp < g->must + g->mlen) {
	while (OP(s = *scan++) != OCHAR)
	continue;
	clen = wcrtomb(cp, OPND(s), &mbs);
	assert(clen != (size_t)-1);
	cp += clen;
	}
	assert(cp == g->must + g->mlen);
	cp++ = '\0'; / just on general principles */
	}

	/*
	- altoffset - choose biggest offset among multiple choices
	== static int altoffset(sop *scan, int offset);
	*
	* Compute, recursively if necessary, the largest offset among multiple
	* re paths.
	*/
	static int
	altoffset(sop *scan, int offset)
	{
	int largest;
	int try;
	sop s;

	/* If we gave up already on offsets, return */
	if (offset == -1)
	return -1;

	largest = 0;
	try = 0;
	s = *scan++;
	while (OP(s) != (sop)O_QUEST && OP(s) != (sop)O_CH) {
	switch (OP(s)) {
	case OOR1:
	if (try > largest)
	largest = try;
	try = 0;
	break;
	case OQUEST_:
	case OCH_:
	try = altoffset(scan, try);
	if (try == -1)
	return -1;
	scan--;
	do {
	scan += OPND(s);
	s = *scan;
	if (OP(s) != (sop)O_QUEST &&
	OP(s) != (sop)O_CH && OP(s) != (sop)OOR2)
	return -1;
	} while (OP(s) != (sop)O_QUEST && OP(s) != (sop)O_CH);
	/* We must skip to the next position, or we'll
	* leave altoffset() too early.
	*/
	scan++;
	break;
	case OANYOF:
	case OCHAR:
	case OANY:
	try++;
	case OBOW:
	case OEOW:
	case OLPAREN:
	case ORPAREN:
	case OOR2:
	break;
	default:
	try = -1;
	break;
	}
	if (try == -1)
	return -1;
	s = *scan++;
	}

	if (try > largest)
	largest = try;

	return largest+offset;
	}

	/*
	- computejumps - compute char jumps for BM scan
	== static void computejumps(struct parse p, struct re_guts g);
	*
	* This algorithm assumes g->must exists and is has size greater than
	* zero. It's based on the algorithm found on Computer Algorithms by
	* Sara Baase.
	*
	* A char jump is the number of characters one needs to jump based on
	* the value of the character from the text that was mismatched.
	*/
	static void
	computejumps(struct parse p, struct re_guts g)
	{
	int ch;
	int mindex;

	/* Avoid making errors worse */
	if (p->error != 0)
	return;

	g->charjump = (int )malloc((NC_MAX + 1) sizeof(int));
	if (g->charjump == NULL) /* Not a fatal error */
	return;
	/* Adjust for signed chars, if necessary */
	g->charjump = &g->charjump[-(CHAR_MIN)];

	/* If the character does not exist in the pattern, the jump
	* is equal to the number of characters in the pattern.
	*/
	for (ch = CHAR_MIN; ch < (CHAR_MAX + 1); ch++)
	g->charjump[ch] = g->mlen;

	/* If the character does exist, compute the jump that would
	* take us to the last character in the pattern equal to it
	* (notice that we match right to left, so that last character
	* is the first one that would be matched).
	*/
	for (mindex = 0; mindex < g->mlen; mindex++)
	g->charjump[(int)g->must[mindex]] = g->mlen - mindex - 1;
	}

	/*
	- computematchjumps - compute match jumps for BM scan
	== static void computematchjumps(struct parse p, struct re_guts g);
	*
	* This algorithm assumes g->must exists and is has size greater than
	* zero. It's based on the algorithm found on Computer Algorithms by
	* Sara Baase.
	*
	* A match jump is the number of characters one needs to advance based
	* on the already-matched suffix.
	* Notice that all values here are minus (g->mlen-1), because of the way
	* the search algorithm works.
	*/
	static void
	computematchjumps(struct parse p, struct re_guts g)
	{
	int mindex; /* General "must" iterator */
	int suffix; /* Keeps track of matching suffix */
	int ssuffix; /* Keeps track of suffixes' suffix */
	int* pmatches; /* pmatches[k] points to the next i
	* such that i+1...mlen is a substring
	* of k+1...k+mlen-i-1
	*/

	/* Avoid making errors worse */
	if (p->error != 0)
	return;

	pmatches = (int) malloc(g->mlen sizeof(int));
	if (pmatches == NULL) {
	g->matchjump = NULL;
	return;
	}

	g->matchjump = (int) malloc(g->mlen sizeof(int));
	if (g->matchjump == NULL) { /* Not a fatal error */
	free(pmatches);
	return;
	}

	/* Set maximum possible jump for each character in the pattern */
	for (mindex = 0; mindex < g->mlen; mindex++)
	g->matchjump[mindex] = 2*g->mlen - mindex - 1;

	/* Compute pmatches[] */
	for (mindex = g->mlen - 1, suffix = g->mlen; mindex >= 0;
	mindex--, suffix--) {
	pmatches[mindex] = suffix;

	/* If a mismatch is found, interrupting the substring,
	* compute the matchjump for that position. If no
	* mismatch is found, then a text substring mismatched
	* against the suffix will also mismatch against the
	* substring.
	*/
	while (suffix < g->mlen
	&& g->must[mindex] != g->must[suffix]) {
	g->matchjump[suffix] = MIN(g->matchjump[suffix],
	g->mlen - mindex - 1);
	suffix = pmatches[suffix];
	}
	}

	/* Compute the matchjump up to the last substring found to jump
	* to the beginning of the largest must pattern prefix matching
	* it's own suffix.
	*/
	for (mindex = 0; mindex <= suffix; mindex++)
	g->matchjump[mindex] = MIN(g->matchjump[mindex],
	g->mlen + suffix - mindex);

	ssuffix = pmatches[suffix];
	while (suffix < g->mlen) {
	while (suffix <= ssuffix && suffix < g->mlen) {
	g->matchjump[suffix] = MIN(g->matchjump[suffix],
	g->mlen + ssuffix - suffix);
	suffix++;
	}
	if (suffix < g->mlen)
	ssuffix = pmatches[ssuffix];
	}

	free(pmatches);
	}

	/*
	- pluscount - count + nesting
	== static sopno pluscount(struct parse p, struct re_guts g);
	*/
	static sopno /* nesting depth */
	pluscount(struct parse p, struct re_guts g)
	{
	sop *scan;
	sop s;
	sopno plusnest = 0;
	sopno maxnest = 0;

	if (p->error != 0)
	return(0); /* there may not be an OEND */

	scan = g->strip + 1;
	do {
	s = *scan++;
	switch (OP(s)) {
	case OPLUS_:
	plusnest++;
	break;
	case O_PLUS:
	if (plusnest > maxnest)
	maxnest = plusnest;
	plusnest--;
	break;
	}
	} while (OP(s) != OEND);
	if (plusnest != 0)
	g->iflags \|= BAD;
	return(maxnest);
	}
	diff --git a/lib/libpmc/libpmc_pmu_util.c b/lib/libpmc/libpmc_pmu_util.c
	index 11949a3ad3cd..d652573a883f 100644
	--- a/lib/libpmc/libpmc_pmu_util.c
	+++ b/lib/libpmc/libpmc_pmu_util.c
	@@ -1,638 +1,628 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2018, Matthew Macy
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*
	*/

	#include <sys/types.h>
	#include <sys/errno.h>
	+#include <sys/pmc.h>
	#include <sys/sysctl.h>
	#include <stddef.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <regex.h>
	#include <string.h>
	#include <pmc.h>
	#include <pmclog.h>
	#include <assert.h>
	#include <libpmcstat.h>
	#include "pmu-events/pmu-events.h"

	#if defined(__amd64__) \|\| defined(__i386__)
	struct pmu_alias {
	const char *pa_alias;
	const char *pa_name;
	};

	typedef enum {
	PMU_INVALID,
	PMU_INTEL,
	PMU_AMD,
	} pmu_mfr_t;

	static struct pmu_alias pmu_intel_alias_table[] = {
	{"UNHALTED_CORE_CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"},
	{"UNHALTED-CORE-CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"},
	{"LLC_MISSES", "LONGEST_LAT_CACHE.MISS"},
	{"LLC-MISSES", "LONGEST_LAT_CACHE.MISS"},
	{"LLC_REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"},
	{"LLC-REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"},
	{"LLC_MISS_RHITM", "mem_load_l3_miss_retired.remote_hitm"},
	{"LLC-MISS-RHITM", "mem_load_l3_miss_retired.remote_hitm"},
	{"RESOURCE_STALL", "RESOURCE_STALLS.ANY"},
	{"RESOURCE_STALLS_ANY", "RESOURCE_STALLS.ANY"},
	{"BRANCH_INSTRUCTION_RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"},
	{"BRANCH-INSTRUCTION-RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"},
	{"BRANCH_MISSES_RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"},
	{"BRANCH-MISSES-RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"},
	{"cycles", "tsc-tsc"},
	{"unhalted-cycles", "CPU_CLK_UNHALTED.THREAD_P_ANY"},
	{"instructions", "inst-retired.any_p"},
	{"branch-mispredicts", "br_misp_retired.all_branches"},
	{"branches", "br_inst_retired.all_branches"},
	{"interrupts", "hw_interrupts.received"},
	{"ic-misses", "frontend_retired.l1i_miss"},
	{NULL, NULL},
	};

	static struct pmu_alias pmu_amd_alias_table[] = {
	{"UNHALTED_CORE_CYCLES", "ls_not_halted_cyc"},
	{"UNHALTED-CORE-CYCLES", "ls_not_halted_cyc"},
	{NULL, NULL},
	};


	static pmu_mfr_t
	pmu_events_mfr(void)
	{
	- char *buf;
	- size_t s;
	+ char buf[PMC_CPUID_LEN];
	+ size_t s = sizeof(buf);
	pmu_mfr_t mfr;

	- if (sysctlbyname("kern.hwpmc.cpuid", (void *)NULL, &s,
	- (void *)NULL, 0) == -1)
	- return (PMU_INVALID);
	- if ((buf = malloc(s + 1)) == NULL)
	- return (PMU_INVALID);
	if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
	- (void *)NULL, 0) == -1) {
	- free(buf);
	+ (void *)NULL, 0) == -1)
	return (PMU_INVALID);
	- }
	if (strcasestr(buf, "AuthenticAMD") != NULL \|\|
	strcasestr(buf, "HygonGenuine") != NULL)
	mfr = PMU_AMD;
	else if (strcasestr(buf, "GenuineIntel") != NULL)
	mfr = PMU_INTEL;
	else
	mfr = PMU_INVALID;
	- free(buf);
	return (mfr);
	}

	/*
	* The Intel fixed mode counters are:
	* "inst_retired.any",
	* "cpu_clk_unhalted.thread",
	* "cpu_clk_unhalted.thread_any",
	* "cpu_clk_unhalted.ref_tsc",
	*
	*/

	static const char *
	pmu_alias_get(const char *name)
	{
	pmu_mfr_t mfr;
	struct pmu_alias *pa;
	struct pmu_alias *pmu_alias_table;

	if ((mfr = pmu_events_mfr()) == PMU_INVALID)
	return (name);
	if (mfr == PMU_AMD)
	pmu_alias_table = pmu_amd_alias_table;
	else if (mfr == PMU_INTEL)
	pmu_alias_table = pmu_intel_alias_table;
	else
	return (name);

	for (pa = pmu_alias_table; pa->pa_alias != NULL; pa++)
	if (strcasecmp(name, pa->pa_alias) == 0)
	return (pa->pa_name);

	return (name);
	}

	struct pmu_event_desc {
	uint64_t ped_period;
	uint64_t ped_offcore_rsp;
	uint64_t ped_l3_thread;
	uint64_t ped_l3_slice;
	uint32_t ped_event;
	uint32_t ped_frontend;
	uint32_t ped_ldlat;
	uint32_t ped_config1;
	int16_t ped_umask;
	uint8_t ped_cmask;
	uint8_t ped_any;
	uint8_t ped_inv;
	uint8_t ped_edge;
	uint8_t ped_fc_mask;
	uint8_t ped_ch_mask;
	};

	static const struct pmu_events_map *
	pmu_events_map_get(const char *cpuid)
	{
	regex_t re;
	regmatch_t pmatch[1];
	- size_t s;
	- char buf[64];
	+ char buf[PMC_CPUID_LEN];
	+ size_t s = sizeof(buf);
	int match;
	const struct pmu_events_map *pme;

	if (cpuid != NULL) {
	- memcpy(buf, cpuid, 64);
	+ strlcpy(buf, cpuid, s);
	} else {
	- if (sysctlbyname("kern.hwpmc.cpuid", (void *)NULL, &s,
	- (void *)NULL, 0) == -1)
	- return (NULL);
	if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
	(void *)NULL, 0) == -1)
	return (NULL);
	}
	for (pme = pmu_events_map; pme->cpuid != NULL; pme++) {
	if (regcomp(&re, pme->cpuid, REG_EXTENDED) != 0) {
	printf("regex '%s' failed to compile, ignoring\n",
	pme->cpuid);
	continue;
	}
	match = regexec(&re, buf, 1, pmatch, 0);
	regfree(&re);
	if (match == 0) {
	if (pmatch[0].rm_so == 0 && (buf[pmatch[0].rm_eo] == 0
	\|\| buf[pmatch[0].rm_eo] == '-'))
	return (pme);
	}
	}
	return (NULL);
	}

	static const struct pmu_event *
	pmu_event_get(const char cpuid, const char event_name, int *idx)
	{
	const struct pmu_events_map *pme;
	const struct pmu_event *pe;
	int i;

	if ((pme = pmu_events_map_get(cpuid)) == NULL)
	return (NULL);
	for (i = 0, pe = pme->table; pe->name \|\| pe->desc \|\| pe->event; pe++, i++) {
	if (pe->name == NULL)
	continue;
	if (strcasecmp(pe->name, event_name) == 0) {
	if (idx)
	*idx = i;
	return (pe);
	}
	}
	return (NULL);
	}

	int
	pmc_pmu_idx_get_by_event(const char cpuid, const char event)
	{
	int idx;
	const char *realname;

	realname = pmu_alias_get(event);
	if (pmu_event_get(cpuid, realname, &idx) == NULL)
	return (-1);
	return (idx);
	}

	const char *
	pmc_pmu_event_get_by_idx(const char *cpuid, int idx)
	{
	const struct pmu_events_map *pme;

	if ((pme = pmu_events_map_get(cpuid)) == NULL)
	return (NULL);
	assert(pme->table[idx].name);
	return (pme->table[idx].name);
	}

	static int
	pmu_parse_event(struct pmu_event_desc ped, const char eventin)
	{
	char *event;
	char kvp, key, value, r;
	char *debug;

	if ((event = strdup(eventin)) == NULL)
	return (ENOMEM);
	r = event;
	bzero(ped, sizeof(*ped));
	ped->ped_period = DEFAULT_SAMPLE_COUNT;
	ped->ped_umask = -1;
	while ((kvp = strsep(&event, ",")) != NULL) {
	key = strsep(&kvp, "=");
	if (key == NULL)
	abort();
	value = kvp;
	if (strcmp(key, "umask") == 0)
	ped->ped_umask = strtol(value, NULL, 16);
	else if (strcmp(key, "event") == 0)
	ped->ped_event = strtol(value, NULL, 16);
	else if (strcmp(key, "period") == 0)
	ped->ped_period = strtol(value, NULL, 10);
	else if (strcmp(key, "offcore_rsp") == 0)
	ped->ped_offcore_rsp = strtol(value, NULL, 16);
	else if (strcmp(key, "any") == 0)
	ped->ped_any = strtol(value, NULL, 10);
	else if (strcmp(key, "cmask") == 0)
	ped->ped_cmask = strtol(value, NULL, 10);
	else if (strcmp(key, "inv") == 0)
	ped->ped_inv = strtol(value, NULL, 10);
	else if (strcmp(key, "edge") == 0)
	ped->ped_edge = strtol(value, NULL, 10);
	else if (strcmp(key, "frontend") == 0)
	ped->ped_frontend = strtol(value, NULL, 16);
	else if (strcmp(key, "ldlat") == 0)
	ped->ped_ldlat = strtol(value, NULL, 16);
	else if (strcmp(key, "fc_mask") == 0)
	ped->ped_fc_mask = strtol(value, NULL, 16);
	else if (strcmp(key, "ch_mask") == 0)
	ped->ped_ch_mask = strtol(value, NULL, 16);
	else if (strcmp(key, "config1") == 0)
	ped->ped_config1 = strtol(value, NULL, 16);
	else if (strcmp(key, "l3_thread_mask") == 0)
	ped->ped_l3_thread = strtol(value, NULL, 16);
	else if (strcmp(key, "l3_slice_mask") == 0)
	ped->ped_l3_slice = strtol(value, NULL, 16);
	else {
	debug = getenv("PMUDEBUG");
	if (debug != NULL && strcmp(debug, "true") == 0 && value != NULL)
	printf("unrecognized kvpair: %s:%s\n", key, value);
	}
	}
	free(r);
	return (0);
	}

	uint64_t
	pmc_pmu_sample_rate_get(const char *event_name)
	{
	const struct pmu_event *pe;
	struct pmu_event_desc ped;

	event_name = pmu_alias_get(event_name);
	if ((pe = pmu_event_get(NULL, event_name, NULL)) == NULL)
	return (DEFAULT_SAMPLE_COUNT);
	if (pe->alias && (pe = pmu_event_get(NULL, pe->alias, NULL)) == NULL)
	return (DEFAULT_SAMPLE_COUNT);
	if (pe->event == NULL)
	return (DEFAULT_SAMPLE_COUNT);
	if (pmu_parse_event(&ped, pe->event))
	return (DEFAULT_SAMPLE_COUNT);
	return (ped.ped_period);
	}

	int
	pmc_pmu_enabled(void)
	{

	return (pmu_events_map_get(NULL) != NULL);
	}

	void
	pmc_pmu_print_counters(const char *event_name)
	{
	const struct pmu_events_map *pme;
	const struct pmu_event *pe;
	struct pmu_event_desc ped;
	char *debug;
	int do_debug;

	debug = getenv("PMUDEBUG");
	do_debug = 0;

	if (debug != NULL && strcmp(debug, "true") == 0)
	do_debug = 1;
	if ((pme = pmu_events_map_get(NULL)) == NULL)
	return;
	for (pe = pme->table; pe->name \|\| pe->desc \|\| pe->event; pe++) {
	if (pe->name == NULL)
	continue;
	if (event_name != NULL && strcasestr(pe->name, event_name) == NULL)
	continue;
	printf("\t%s\n", pe->name);
	if (do_debug)
	pmu_parse_event(&ped, pe->event);
	}
	}

	void
	pmc_pmu_print_counter_desc(const char *ev)
	{
	const struct pmu_events_map *pme;
	const struct pmu_event *pe;

	if ((pme = pmu_events_map_get(NULL)) == NULL)
	return;
	for (pe = pme->table; pe->name \|\| pe->desc \|\| pe->event; pe++) {
	if (pe->name == NULL)
	continue;
	if (strcasestr(pe->name, ev) != NULL &&
	pe->desc != NULL)
	printf("%s:\t%s\n", pe->name, pe->desc);
	}
	}

	void
	pmc_pmu_print_counter_desc_long(const char *ev)
	{
	const struct pmu_events_map *pme;
	const struct pmu_event *pe;

	if ((pme = pmu_events_map_get(NULL)) == NULL)
	return;
	for (pe = pme->table; pe->name \|\| pe->desc \|\| pe->event; pe++) {
	if (pe->name == NULL)
	continue;
	if (strcasestr(pe->name, ev) != NULL) {
	if (pe->long_desc != NULL)
	printf("%s:\n%s\n", pe->name, pe->long_desc);
	else if (pe->desc != NULL)
	printf("%s:\t%s\n", pe->name, pe->desc);
	}
	}
	}

	void
	pmc_pmu_print_counter_full(const char *ev)
	{
	const struct pmu_events_map *pme;
	const struct pmu_event *pe;

	if ((pme = pmu_events_map_get(NULL)) == NULL)
	return;
	for (pe = pme->table; pe->name \|\| pe->desc \|\| pe->event; pe++) {
	if (pe->name == NULL)
	continue;
	if (strcasestr(pe->name, ev) == NULL)
	continue;
	printf("name: %s\n", pe->name);
	if (pe->long_desc != NULL)
	printf("desc: %s\n", pe->long_desc);
	else if (pe->desc != NULL)
	printf("desc: %s\n", pe->desc);
	if (pe->event != NULL)
	printf("event: %s\n", pe->event);
	if (pe->topic != NULL)
	printf("topic: %s\n", pe->topic);
	if (pe->pmu != NULL)
	printf("pmu: %s\n", pe->pmu);
	if (pe->unit != NULL)
	printf("unit: %s\n", pe->unit);
	if (pe->perpkg != NULL)
	printf("perpkg: %s\n", pe->perpkg);
	if (pe->metric_expr != NULL)
	printf("metric_expr: %s\n", pe->metric_expr);
	if (pe->metric_name != NULL)
	printf("metric_name: %s\n", pe->metric_name);
	if (pe->metric_group != NULL)
	printf("metric_group: %s\n", pe->metric_group);
	}
	}

	static int
	pmc_pmu_amd_pmcallocate(const char event_name, struct pmc_op_pmcallocate pm,
	struct pmu_event_desc *ped)
	{
	struct pmc_md_amd_op_pmcallocate *amd;
	const struct pmu_event *pe;
	int idx = -1;

	amd = &pm->pm_md.pm_amd;
	if (ped->ped_umask > 0) {
	pm->pm_caps \|= PMC_CAP_QUALIFIER;
	amd->pm_amd_config \|= AMD_PMC_TO_UNITMASK(ped->ped_umask);
	}
	pm->pm_class = PMC_CLASS_K8;
	pe = pmu_event_get(NULL, event_name, &idx);

	if (strcmp("l3cache", pe->topic) == 0){
	amd->pm_amd_config \|= AMD_PMC_TO_EVENTMASK(ped->ped_event);
	amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_L3_CACHE;
	amd->pm_amd_config \|= AMD_PMC_TO_L3SLICE(ped->ped_l3_slice);
	amd->pm_amd_config \|= AMD_PMC_TO_L3CORE(ped->ped_l3_thread);
	}
	else if (strcmp("data fabric", pe->topic) == 0){

	amd->pm_amd_config \|= AMD_PMC_TO_EVENTMASK_DF(ped->ped_event);
	amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_DATA_FABRIC;
	}
	else{
	amd->pm_amd_config \|= AMD_PMC_TO_EVENTMASK(ped->ped_event);
	amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_CORE;
	if ((pm->pm_caps & (PMC_CAP_USER\|PMC_CAP_SYSTEM)) == 0 \|\|
	(pm->pm_caps & (PMC_CAP_USER\|PMC_CAP_SYSTEM)) ==
	(PMC_CAP_USER\|PMC_CAP_SYSTEM))
	amd->pm_amd_config \|= (AMD_PMC_USR \| AMD_PMC_OS);
	else if (pm->pm_caps & PMC_CAP_USER)
	amd->pm_amd_config \|= AMD_PMC_USR;
	else if (pm->pm_caps & PMC_CAP_SYSTEM)
	amd->pm_amd_config \|= AMD_PMC_OS;
	if (ped->ped_edge)
	amd->pm_amd_config \|= AMD_PMC_EDGE;
	if (ped->ped_inv)
	amd->pm_amd_config \|= AMD_PMC_EDGE;
	if (pm->pm_caps & PMC_CAP_INTERRUPT)
	amd->pm_amd_config \|= AMD_PMC_INT;
	}
	return (0);
	}

	static int
	pmc_pmu_intel_pmcallocate(const char event_name, struct pmc_op_pmcallocate pm,
	struct pmu_event_desc *ped)
	{
	struct pmc_md_iap_op_pmcallocate *iap;
	int isfixed;

	isfixed = 0;
	iap = &pm->pm_md.pm_iap;
	if (strcasestr(event_name, "UNC_") == event_name \|\|
	strcasestr(event_name, "uncore") != NULL) {
	pm->pm_class = PMC_CLASS_UCP;
	pm->pm_caps \|= PMC_CAP_QUALIFIER;
	} else if ((ped->ped_umask == -1) \|\|
	(ped->ped_event == 0x0 && ped->ped_umask == 0x3)) {
	pm->pm_class = PMC_CLASS_IAF;
	} else {
	pm->pm_class = PMC_CLASS_IAP;
	pm->pm_caps \|= PMC_CAP_QUALIFIER;
	}
	iap->pm_iap_config \|= IAP_EVSEL(ped->ped_event);
	if (ped->ped_umask > 0)
	iap->pm_iap_config \|= IAP_UMASK(ped->ped_umask);
	iap->pm_iap_config \|= IAP_CMASK(ped->ped_cmask);
	iap->pm_iap_rsp = ped->ped_offcore_rsp;

	if ((pm->pm_caps & (PMC_CAP_USER\|PMC_CAP_SYSTEM)) == 0 \|\|
	(pm->pm_caps & (PMC_CAP_USER\|PMC_CAP_SYSTEM)) ==
	(PMC_CAP_USER\|PMC_CAP_SYSTEM))
	iap->pm_iap_config \|= (IAP_USR \| IAP_OS);
	else if (pm->pm_caps & PMC_CAP_USER)
	iap->pm_iap_config \|= IAP_USR;
	else if (pm->pm_caps & PMC_CAP_SYSTEM)
	iap->pm_iap_config \|= IAP_OS;
	if (ped->ped_edge)
	iap->pm_iap_config \|= IAP_EDGE;
	if (ped->ped_any)
	iap->pm_iap_config \|= IAP_ANY;
	if (ped->ped_inv)
	iap->pm_iap_config \|= IAP_EDGE;
	if (pm->pm_caps & PMC_CAP_INTERRUPT)
	iap->pm_iap_config \|= IAP_INT;
	return (0);
	}

	int
	pmc_pmu_pmcallocate(const char event_name, struct pmc_op_pmcallocate pm)
	{
	const struct pmu_event *pe;
	struct pmu_event_desc ped;
	pmu_mfr_t mfr;
	int idx = -1;

	if ((mfr = pmu_events_mfr()) == PMU_INVALID)
	return (ENOENT);

	bzero(&pm->pm_md, sizeof(pm->pm_md));
	pm->pm_caps \|= (PMC_CAP_READ \| PMC_CAP_WRITE);
	event_name = pmu_alias_get(event_name);
	if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL)
	return (ENOENT);
	if (pe->alias && (pe = pmu_event_get(NULL, pe->alias, &idx)) == NULL)
	return (ENOENT);
	assert(idx >= 0);
	pm->pm_ev = idx;

	if (pe->event == NULL)
	return (ENOENT);
	if (pmu_parse_event(&ped, pe->event))
	return (ENOENT);

	if (mfr == PMU_INTEL)
	return (pmc_pmu_intel_pmcallocate(event_name, pm, &ped));
	else
	return (pmc_pmu_amd_pmcallocate(event_name, pm, &ped));
	}

	/*
	* Ultimately rely on AMD calling theirs the same
	*/
	static const char *stat_mode_cntrs[] = {
	"cpu_clk_unhalted.thread",
	"inst_retired.any",
	"br_inst_retired.all_branches",
	"br_misp_retired.all_branches",
	"longest_lat_cache.reference",
	"longest_lat_cache.miss",
	};

	int
	pmc_pmu_stat_mode(const char ***cntrs)
	{
	if (pmc_pmu_enabled()) {
	*cntrs = stat_mode_cntrs;
	return (0);
	}
	return (EOPNOTSUPP);
	}

	#else

	uint64_t
	pmc_pmu_sample_rate_get(const char *event_name __unused)
	{
	return (DEFAULT_SAMPLE_COUNT);
	}

	void
	pmc_pmu_print_counters(const char *event_name __unused)
	{
	}

	void
	pmc_pmu_print_counter_desc(const char *e __unused)
	{
	}

	void
	pmc_pmu_print_counter_desc_long(const char *e __unused)
	{
	}

	void
	pmc_pmu_print_counter_full(const char *e __unused)
	{

	}

	int
	pmc_pmu_enabled(void)
	{
	return (0);
	}

	int
	pmc_pmu_pmcallocate(const char e __unused, struct pmc_op_pmcallocate p __unused)
	{
	return (EOPNOTSUPP);
	}

	const char *
	pmc_pmu_event_get_by_idx(const char *c __unused, int idx __unused)
	{
	return (NULL);
	}

	int
	pmc_pmu_stat_mode(const char ***a __unused)
	{
	return (EOPNOTSUPP);
	}

	int
	pmc_pmu_idx_get_by_event(const char c __unused, const char e __unused)
	{
	return (-1);
	}

	#endif
	diff --git a/release/sparc64/mkisoimages.sh b/release/sparc64/mkisoimages.sh
	deleted file mode 100644
	index 80894096f01e..000000000000
	--- a/release/sparc64/mkisoimages.sh
	+++ /dev/null
	@@ -1,86 +0,0 @@
	-#!/bin/sh
	-#
	-# Module: mkisoimages.sh
	-# Author: Jordan K Hubbard
	-# Date: 22 June 2001
	-#
	-# $FreeBSD$
	-#
	-# This script is used by release/Makefile to build the (optional) ISO images
	-# for a FreeBSD release. It is considered architecture dependent since each
	-# platform has a slightly unique way of making bootable CDs. This script
	-# is also allowed to generate any number of images since that is more of
	-# publishing decision than anything else.
	-#
	-# Usage:
	-#
	-# mkisoimages.sh [-b] image-label image-name base-bits-dir [extra-bits-dir]
	-#
	-# Where -b is passed if the ISO image should be made "bootable" by
	-# whatever standards this architecture supports (may be unsupported),
	-# image-label is the ISO image label, image-name is the filename of the
	-# resulting ISO image, base-bits-dir contains the image contents and
	-# extra-bits-dir, if provided, contains additional files to be merged
	-# into base-bits-dir as part of making the image.
	-set -e
	-
	-if [ $# -lt 3 ]; then
	- echo "Usage: $0 [-b] image-label image-name base-bits-dir [extra-bits-dir]" > /dev/stderr
	- exit 1
	-fi
	-
	-case "$1" in
	--b) BOPT="$1"; shift ;;
	-esac
	-LABEL=`echo "$1" \| tr '[:lower:]' '[:upper:]'`; shift
	-NAME="$1"; shift
	-BASEBITSDIR="$1"
	-
	-# Create an ISO image
	-publisher="The FreeBSD Project. https://www.FreeBSD.org/"
	-echo "/dev/iso9660/$LABEL / cd9660 ro 0 0" > "$BASEBITSDIR/etc/fstab"
	-makefs -t cd9660 -o rockridge -o label="$LABEL" -o publisher="$publisher" "$NAME.tmp" "$@"
	-rm -f "$BASEBITSDIR/etc/fstab"
	-
	-if [ "$BOPT" != "-b" ]; then
	- mv "$NAME.tmp" "$NAME"
	- exit 0
	-fi
	-
	-TMPIMGDIR=`mktemp -d /tmp/bootfs.XXXXXXXX` \|\| exit 1
	-BOOTFSDIR="$TMPIMGDIR/bootfs"
	-BOOTFSIMG="$TMPIMGDIR/bootfs.img"
	-
	-# Create a boot filesystem
	-mkdir -p "$BOOTFSDIR/boot"
	-cp -p "$BASEBITSDIR/boot/loader" "$BOOTFSDIR/boot"
	-makefs -t ffs -B be -M 512k "$BOOTFSIMG" "$BOOTFSDIR"
	-dd if="$BASEBITSDIR/boot/boot1" of="$BOOTFSIMG" bs=512 conv=notrunc,sync
	-
	-# Create a boot ISO image
	-: ${CYLSIZE:=640}
	-ISOSIZE=$(stat -f %z "$NAME.tmp")
	-ISOBLKS=$((($ISOSIZE + 511) / 512))
	-ISOCYLS=$((($ISOBLKS + ($CYLSIZE - 1)) / $CYLSIZE))
	-
	-BOOTFSSIZE=$(stat -f %z "$BOOTFSIMG")
	-BOOTFSBLKS=$((($BOOTFSSIZE + 511) / 512))
	-BOOTFSCYLS=$((($BOOTFSBLKS + ($CYLSIZE - 1)) / $CYLSIZE))
	-
	-ENDCYL=$(($ISOCYLS + $BOOTFSCYLS))
	-NSECTS=$(($ENDCYL * 1 * $CYLSIZE))
	-
	-dd if="$NAME.tmp" of="$NAME" bs="${CYLSIZE}b" conv=notrunc,sync
	-dd if="$BOOTFSIMG" of="$NAME" bs="${CYLSIZE}b" seek=$ISOCYLS conv=notrunc,sync
	-# The number of alternative cylinders is always 2.
	-dd if=/dev/zero of="$NAME" bs="${CYLSIZE}b" seek=$ENDCYL count=2 conv=notrunc,sync
	-rm -rf "$NAME.tmp" "$TMPIMGDIR"
	-
	-# Write VTOC8 label to boot ISO image
	-MD=`mdconfig -a -t vnode -S 512 -y 1 -x "$CYLSIZE" -f "$NAME"`
	-gpart create -s VTOC8 $MD
	-# !4: usr, for ISO image part
	-gpart add -i 1 -s "$(($ISOCYLS * $CYLSIZE * 512))b" -t \!4 $MD
	-# !2: root, for bootfs part.
	-gpart add -i 6 -s "$(($BOOTFSCYLS * $CYLSIZE * 512))b" -t \!2 $MD
	-mdconfig -d -u ${MD#md}
	diff --git a/release/sparc64/sparc64.conf b/release/sparc64/sparc64.conf
	deleted file mode 100644
	index c0579eaadb4b..000000000000
	--- a/release/sparc64/sparc64.conf
	+++ /dev/null
	@@ -1,10 +0,0 @@
	-#!/bin/sh
	-#
	-# $FreeBSD$
	-#
	-
	-# Configuration file for release/release.sh to build sparc64/sparc64.
	-
	-TARGET="sparc64"
	-TARGET_ARCH="sparc64"
	-KERNEL="GENERIC"
	diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile
	index 9249e00a7250..9681ebaddfa3 100644
	--- a/rescue/rescue/Makefile
	+++ b/rescue/rescue/Makefile
	@@ -1,237 +1,233 @@
	#$FreeBSD$
	# @(#)Makefile 8.1 (Berkeley) 6/2/93

	.include <src.opts.mk>

	PACKAGE=rescue
	MAN=
	MK_SSP= no
	NO_SHARED= yes

	PROG= rescue
	BINDIR?=/rescue

	# Shell scripts need #! line to be edited from /bin/sh to /rescue/sh
	SCRIPTS= nextboot_FIXED
	SCRIPTSNAME_nextboot_FIXED= nextboot
	nextboot_FIXED: ../../sbin/reboot/nextboot.sh
	sed '1s/\/bin\//\/rescue\//' ${.ALLSRC} > ${.TARGET}
	CLEANFILES+= nextboot_FIXED

	SCRIPTS+= dhclient_FIXED
	SCRIPTSNAME_dhclient_FIXED= dhclient-script
	dhclient_FIXED: ../../sbin/dhclient/dhclient-script
	sed '1s/\/bin\//\/rescue\//' ${.ALLSRC} > ${.TARGET}
	CLEANFILES+= dhclient_FIXED

	# The help which used to be here is now in mk/bsd.crunchgen.mk

	# Define Makefile variable RESCUE
	CRUNCH_BUILDOPTS+= -DRESCUE
	# Define compile-time RESCUE symbol when compiling components
	CRUNCH_BUILDOPTS+= CRUNCH_CFLAGS=-DRESCUE

	# An experiment that failed: try overriding bsd.lib.mk and bsd.prog.mk
	# rather than incorporating rescue-specific logic into standard files.
	#MAKEFLAGS= -m ${.CURDIR} ${.MAKEFLAGS}

	# Hackery: 'librescue' exists merely as a tool for appropriately
	# recompiling specific library entries. We _know_ they're needed, and
	# regular archive searching creates ugly library ordering problems.
	# Easiest fix: tell the linker to include them into the executable
	# first, so they are guaranteed to override the regular lib entries.
	# Note that if 'librescue' hasn't been compiled, we'll just get the
	# regular lib entries from libc and friends.
	CRUNCH_LIBS+= ${.OBJDIR}/../librescue/*.o

	###################################################################
	# Programs from stock /bin
	#
	# WARNING: Changing this list may require adjusting
	# /usr/include/paths.h as well! You were warned!
	#
	CRUNCH_SRCDIRS+= bin
	CRUNCH_PROGS_bin= cat chflags chio chmod cp date dd df echo \
	ed expr getfacl hostname kenv kill ln ls mkdir mv \
	pkill ps pwd realpath rm rmdir setfacl sh sleep stty \
	sync test
	CRUNCH_LIBS+= -lcrypt -ledit -ljail -lkvm -lelf -ltermcapw -lutil -lxo
	CRUNCH_BUILDTOOLS+= bin/sh

	# Additional options for specific programs
	CRUNCH_ALIAS_test= [
	CRUNCH_ALIAS_sh= -sh
	# The -sh alias shouldn't appear in /rescue as a hard link
	CRUNCH_SUPPRESS_LINK_-sh= 1
	CRUNCH_ALIAS_ln= link
	CRUNCH_ALIAS_rm= unlink
	CRUNCH_ALIAS_ed= red
	CRUNCH_ALIAS_pkill= pgrep

	.if ${MK_TCSH} != "no"
	CRUNCH_PROGS_bin+= csh
	CRUNCH_ALIAS_csh= -csh tcsh -tcsh
	CRUNCH_BUILDTOOLS+= bin/csh
	CRUNCH_SUPPRESS_LINK_-csh= 1
	CRUNCH_SUPPRESS_LINK_-tcsh= 1
	.endif

	###################################################################
	# Programs from standard /sbin
	#
	# WARNING: Changing this list may require adjusting
	# /usr/include/paths.h as well! You were warned!
	#
	# Note that mdmfs have their own private 'pathnames.h'
	# headers in addition to the standard 'paths.h' header.
	#
	CRUNCH_SRCDIRS+= sbin
	CRUNCH_PROGS_sbin= \
	camcontrol clri devfs dmesg dump \
	dumpfs dumpon fsck fsck_ffs fsck_msdosfs fsdb \
	fsirand gbde geom ifconfig init \
	kldconfig kldload kldstat kldunload ldconfig \
	md5 mdconfig mdmfs mknod mount mount_cd9660 \
	mount_msdosfs mount_nfs mount_nullfs \
	mount_udf mount_unionfs newfs \
	newfs_msdos nos-tun ping reboot \
	restore rcorder route savecore \
	shutdown spppcontrol swapon sysctl tunefs umount

	.if ${MK_CCD} != "no"
	CRUNCH_PROGS_sbin+= ccdconfig
	.endif

	.if ${MK_INET6_SUPPORT} != "no"
	CRUNCH_PROGS_sbin+= ping6
	CRUNCH_PROGS_sbin+= rtsol
	.endif

	.if ${MK_IPFILTER} != "no"
	CRUNCH_PROGS_sbin+= ipf
	CRUNCH_LIBS_ipf+= ${LIBIPF}
	.endif

	.if ${MK_ROUTED} != "no"
	CRUNCH_PROGS_sbin+= routed rtquery
	.endif

	.if ${MK_ZFS} != "no"
	CRUNCH_PROGS_sbin+= bectl
	CRUNCH_PROGS_sbin+= zfs
	CRUNCH_PROGS_sbin+= zpool
	CRUNCH_PROGS_usr.sbin+= zdb
	.endif

	# crunchgen does not like C++ programs; this should be fixed someday
	# CRUNCH_PROGS+= devd

	CRUNCH_LIBS+= -l80211 -lalias -lcam -lncursesw -ldevstat -lipsec -llzma
	.if ${MK_ZFS} != "no"
	CRUNCH_LIBS+= -lavl -lzpool -lzfs_core -lzfs -lnvpair -lpthread -luutil -lumem
	CRUNCH_LIBS+= -lbe
	.else
	# liblzma needs pthread
	CRUNCH_LIBS+= -lpthread
	.endif
	CRUNCH_LIBS+= -lgeom -lbsdxml -lkiconv
	.if ${MK_OPENSSL} == "no"
	CRUNCH_LIBS+= -lmd
	.endif
	CRUNCH_LIBS+= -lmt -lsbuf -lufs -lz

	.if ${MACHINE_CPUARCH} == "i386"
	CRUNCH_PROGS_sbin+= bsdlabel sconfig fdisk
	CRUNCH_ALIAS_bsdlabel= disklabel
	#CRUNCH_PROGS+= mount_smbfs
	#CRUNCH_LIBS+= -lsmb
	.endif

	-.if ${MACHINE_CPUARCH} == "sparc64"
	-CRUNCH_PROGS_sbin+= bsdlabel sunlabel
	-.endif
	-
	.if ${MACHINE_CPUARCH} == "amd64"
	CRUNCH_PROGS_sbin+= bsdlabel fdisk
	CRUNCH_ALIAS_bsdlabel= disklabel
	.endif

	CRUNCH_SRCDIR_rtquery= ${SRCTOP}/sbin/routed/rtquery
	CRUNCH_SRCDIR_ipf= ${SRCTOP}/sbin/ipf/ipf
	.if ${MK_ZFS} != "no"
	CRUNCH_SRCDIR_zfs= ${SRCTOP}/cddl/sbin/zfs
	CRUNCH_SRCDIR_zpool= ${SRCTOP}/cddl/sbin/zpool
	CRUNCH_SRCDIR_zdb= ${SRCTOP}/cddl/usr.sbin/zdb
	.endif
	CRUNCH_ALIAS_reboot= fastboot halt fasthalt
	CRUNCH_ALIAS_restore= rrestore
	CRUNCH_ALIAS_dump= rdump
	CRUNCH_ALIAS_fsck_ffs= fsck_4.2bsd fsck_ufs
	CRUNCH_ALIAS_geom= glabel gpart
	CRUNCH_ALIAS_shutdown= poweroff

	# dhclient has historically been troublesome...
	CRUNCH_PROGS_sbin+= dhclient

	##################################################################
	# Programs from stock /usr/bin
	#
	CRUNCH_SRCDIRS+= usr.bin

	CRUNCH_PROGS_usr.bin= head mt sed tail tee

	CRUNCH_PROGS_usr.bin+= gzip
	CRUNCH_ALIAS_gzip= gunzip gzcat zcat

	CRUNCH_PROGS_usr.bin+= bzip2
	CRUNCH_ALIAS_bzip2= bunzip2 bzcat
	CRUNCH_LIBS+= -lbz2

	CRUNCH_PROGS_usr.bin+= less
	CRUNCH_ALIAS_less= more

	CRUNCH_PROGS_usr.bin+= xz
	CRUNCH_ALIAS_xz= unxz lzma unlzma xzcat lzcat

	CRUNCH_PROGS_usr.bin+= zstd
	CRUNCH_ALIAS_zstd= unzstd zstdcat zstdmt
	CRUNCH_LIBS+= ${LDADD_zstd}

	CRUNCH_PROGS_usr.bin+= tar
	CRUNCH_LIBS+= -larchive
	.if ${MK_OPENSSL} != "no"
	CRUNCH_LIBS+= -lcrypto
	.endif
	CRUNCH_LIBS+= -lmd

	.if ${MK_NETCAT} != "no"
	CRUNCH_PROGS_usr.bin+= nc
	.endif

	.if ${MK_VI} != "no"
	CRUNCH_PROGS_usr.bin+= vi
	CRUNCH_ALIAS_vi= ex
	.endif

	CRUNCH_PROGS_usr.bin+= id
	CRUNCH_ALIAS_id= groups whoami

	##################################################################
	# Programs from stock /usr/sbin
	#
	CRUNCH_SRCDIRS+= usr.sbin

	CRUNCH_PROGS_usr.sbin+= chroot

	CRUNCH_PROGS_usr.sbin+= chown
	CRUNCH_ALIAS_chown= chgrp
	##################################################################
	CRUNCH_LIBS+= -lm

	.if ${MK_ISCSI} != "no"
	CRUNCH_PROGS_usr.bin+= iscsictl
	CRUNCH_PROGS_usr.sbin+= iscsid
	.endif

	.include <bsd.crunchgen.mk>
	.include <bsd.prog.mk>
	diff --git a/sbin/init/ttys.sparc64 b/sbin/init/ttys.sparc64
	deleted file mode 100644
	index 5c7f3cff6a3f..000000000000
	--- a/sbin/init/ttys.sparc64
	+++ /dev/null
	@@ -1,54 +0,0 @@
	-#
	-# $FreeBSD$
	-# @(#)ttys 5.1 (Berkeley) 4/17/89
	-#
	-# This file specifies various information about terminals on the system.
	-# It is used by several different programs. Common entries for the
	-# various columns include:
	-#
	-# name The name of the terminal device.
	-#
	-# getty The program to start running on the terminal. Typically a
	-# getty program, as the name implies. Other common entries
	-# include none, when no getty is needed, and xdm, to start the
	-# X Window System.
	-#
	-# type The initial terminal type for this port. For hardwired
	-# terminal lines, this will contain the type of terminal used.
	-# For virtual consoles, the correct type is typically xterm.
	-# Other common values include dialup for incoming modem ports, and
	-# unknown when the terminal type cannot be predetermined.
	-#
	-# status Must be on or off. If on, init will run the getty program on
	-# the specified port. If the word "secure" appears, this tty
	-# allows root login.
	-#
	-# name getty type status comments
	-#
	-# If console is marked "insecure", then init will ask for the root password
	-# when going to single-user mode.
	-console none unknown off secure
	-# ofw_console(4)
	-screen "/usr/libexec/getty Pc" vt100 off secure
	-ttya "/usr/libexec/getty 3wire.9600" vt100 off secure
	-ttyb "/usr/libexec/getty 3wire.9600" vt100 off secure
	-# syscons(4)
	-ttyv0 "/usr/libexec/getty Pc" xterm onifexists secure
	-# Virtual terminals
	-ttyv1 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv2 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv3 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv4 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv5 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv6 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv7 "/usr/libexec/getty Pc" xterm onifexists secure
	-ttyv8 "/usr/local/bin/xdm -nodaemon" xterm off secure
	-# Serial terminals
	-# The 'dialup' keyword identifies dialin lines to login, fingerd etc.
	-# uart(4)
	-ttyu0 "/usr/libexec/getty 3wire" vt100 onifconsole secure
	-ttyu1 "/usr/libexec/getty 3wire" vt100 onifconsole secure
	-ttyu2 "/usr/libexec/getty 3wire" vt100 onifconsole secure
	-ttyu3 "/usr/libexec/getty 3wire" vt100 onifconsole secure
	-# Dumb console
	-dcons "/usr/libexec/getty std.9600" vt100 off secure
	diff --git a/share/man/man3/siginfo.3 b/share/man/man3/siginfo.3
	index f0cc9aa2487b..fc4ea2ba1df7 100644
	--- a/share/man/man3/siginfo.3
	+++ b/share/man/man3/siginfo.3
	@@ -1,344 +1,343 @@
	.\" Copyright (c) 2005 David Xu <davidxu@FreeBSD.org>
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice(s), this list of conditions and the following disclaimer as
	.\" the first lines of this file unmodified other than the possible
	.\" addition of one or more copyright notices.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice(s), this list of conditions and the following disclaimer in
	.\" the documentation and/or other materials provided with the
	.\" distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
	.\" EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
	.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
	.\" BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	.\" WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
	.\" OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
	.\" EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd May 8, 2020
	+.Dd July 28, 2020
	.Dt SIGINFO 3
	.Os
	.Sh NAME
	.Nm siginfo
	.Nd "signal generation information"
	.Sh SYNOPSIS
	.In signal.h
	.Sh DESCRIPTION
	A process may request signal information when it is catching a signal.
	The information specifies why the system generated that signal.
	To request signal information in a signal handler, the user can set
	.Dv SA_SIGINFO
	in
	.Va sa_flags
	before
	.Xr sigaction 2
	is called,
	otherwise the user can use
	.Xr sigwaitinfo 2
	and
	.Xr sigtimedwait 2
	to get signal information.
	In either case, the system returns the information in a structure of type
	.Vt siginfo_t ,
	which includes the following information:
	.Bl -column ".Vt union signal" ".Va si_overrun"
	.It Sy Type Ta Sy Member Ta Sy Description
	.It Vt int Ta Va si_signo Ta
	signal number
	.It Vt int Ta Va si_errno Ta
	error number
	.It Vt int Ta Va si_code Ta
	signal code
	.It Vt union sigval Ta Va si_value Ta
	signal value
	.It Vt pid_t Ta Va si_pid Ta
	sending process ID
	.It Vt uid_t Ta Va si_uid Ta
	sending process's real user ID
	.It Vt void Ta Va *si_addr Ta
	-address of faulting instruction
	+virtual address
	.It Vt int Ta Va si_status Ta
	exit value or signal
	.It Vt long Ta Va si_band Ta
	band event for
	.Dv SIGPOLL
	.It Vt int Ta Va si_trapno Ta
	machine trap code
	.It Vt int Ta Va si_timerid Ta
	.Tn POSIX
	timer ID
	.It Vt int Ta Va si_overrun Ta
	.Tn POSIX
	timer overrun count
	.It Vt int Ta Va si_mqd Ta
	.Tn POSIX
	message queue ID
	.El
	.Pp
	The
	.Va si_signo
	member contains the signal number.
	.Pp
	The
	.Va si_errno
	member contains an error number defined in the file
	.In errno.h .
	.Pp
	The
	.Va si_code
	member contains a code which describes the cause of the signal.
	The macros specified in the
	.Sy Code
	column of the following table are defined
	for use as values of
	.Va si_code
	that are signal-specific or non-signal-specific reasons why the signal was
	generated:
	.Bl -column ".Dv SIGPOLL" ".Dv CLD_CONTINUED"
	.It Sy Signal Ta Sy Code Ta Sy Reason
	.It Dv SIGILL Ta Dv ILL_ILLOPC Ta
	illegal opcode
	.It Ta Dv ILL_ILLOPN Ta
	illegal operand
	.It Ta Dv ILL_ILLADR Ta
	illegal addressing mode
	.It Ta Dv ILL_ILLTRP Ta
	illegal trap
	.It Ta Dv ILL_PRVOPC Ta
	illegal privileged opcode
	.It Ta Dv ILL_PRVREG Ta
	illegal privileged register
	.It Ta Dv ILL_COPROC Ta
	coprocessor error
	.It Ta Dv ILL_BADSTK Ta
	internal stack error
	.It Dv SIGFPE Ta Dv FPE_INTDIV Ta
	integer divide by zero
	.It Ta Dv FPE_INTOVF Ta
	integer overflow
	.It Ta Dv FPE_FLTDIV Ta
	floating-point divide by zero
	.It Ta Dv FPE_FLTOVF Ta
	floating-point overflow
	.It Ta Dv FPE_FLTUND Ta
	floating-point underflow
	.It Ta Dv FPE_FLTRES Ta
	floating-point inexact result
	.It Ta Dv FPE_FLTINV Ta
	invalid floating-point operation
	.It Ta Dv FPE_FLTSUB Ta
	subscript out of range
	.It Dv SIGSEGV Ta Dv SEGV_MAPERR Ta
	address not mapped to object
	.It Ta Dv SEGV_ACCERR Ta
	invalid permissions for mapped object
	.It Dv SIGBUS Ta Dv BUS_ADRALN Ta
	invalid address alignment
	.It Ta Dv BUS_ADRERR Ta
	nonexistent physical address
	.It Ta Dv BUS_OBJERR Ta
	object-specific hardware error
	.It Ta Dv BUS_OOMERR Ta
	cannot alloc a page to map at fault
	.It Dv SIGTRAP Ta Dv TRAP_BRKPT Ta
	process breakpoint
	.It Ta Dv TRAP_TRACE Ta
	process trace trap
	.It Ta Dv TRAP_DTRACE Ta
	DTrace induced trap
	.It Ta Dv TRAP_CAP Ta
	capabilities protective trap
	.It Dv SIGCHLD Ta Dv CLD_EXITED Ta
	child has exited
	.It Ta Dv CLD_KILLED Ta
	child has terminated abnormally and did not create a core file
	.It Ta Dv CLD_DUMPED Ta
	child has terminated abnormally and created a core file
	.It Ta Dv CLD_TRAPPED Ta
	traced child has trapped
	.It Ta Dv CLD_STOPPED Ta
	child has stopped
	.It Ta Dv CLD_CONTINUED Ta
	stopped child has continued
	.It Dv SIGPOLL Ta Dv POLL_IN Ta
	data input available
	.It Ta Dv POLL_OUT Ta
	output buffers available
	.It Ta Dv POLL_MSG Ta
	input message available
	.It Ta Dv POLL_ERR Ta
	I/O error
	.It Ta Dv POLL_PRI Ta
	high priority input available
	.It Ta Dv POLL_HUP Ta
	device disconnected
	.It Any Ta Dv SI_NOINFO Ta
	Only the
	.Va si_signo
	member is meaningful; the value of all other members is unspecified.
	.It Ta Dv SI_USER Ta
	signal sent by
	.Xr kill 2
	.It Ta Dv SI_QUEUE Ta
	signal sent by
	.Xr sigqueue 2
	.It Ta Dv SI_TIMER Ta
	signal generated by expiration of a timer set by
	.Xr timer_settime 2
	.It Ta Dv SI_ASYNCIO Ta
	signal generated by completion of an asynchronous I/O request
	.It Ta Dv SI_MESGQ Ta
	signal generated by arrival of a message on an empty message queue
	.It Ta Dv SI_KERNEL Ta
	signal generated by miscellaneous parts of the kernel
	.It Ta Dv SI_LWP Ta
	signal sent by
	.Xr pthread_kill 3
	.El
	.Pp
	+For synchronous signals,
	+.Va si_addr
	+is generally set to the address of the faulting instruction.
	+However, synchronous signals raised by a faulting memory access such as
	+.Dv SIGSEGV
	+and
	+.Dv SIGBUS
	+may report the address of the faulting memory access (if available) in
	+.Va si_addr
	+instead.
	+.Pp
	+Sychronous signals set
	+.Va si_trapno
	+to a machine-dependent trap number.
	+.Pp
	In addition, the following signal-specific information is available:
	.Bl -column ".Dv SIGPOLL" ".Dv CLD_CONTINUED"
	.It Sy Signal Ta Sy Member Ta Sy Value
	-.It Dv SIGILL Ta Va si_addr Ta
	-address of faulting instruction
	-.It Ta Va si_trapno Ta
	-machine dependent of trap code
	-.It Dv SIGFPE Ta Va si_addr Ta
	-address of faulting instruction
	-.It Ta Va si_trapno Ta
	-machine dependent of trap code
	-.It Dv SIGSEGV Ta Va si_addr Ta
	-address of faulting memory reference
	-.It Ta Va si_trapno Ta
	-machine dependent of trap code
	-.It Dv SIGBUS Ta Va si_addr Ta
	-address of faulting instruction
	-.It Ta Va si_trapno Ta
	-machine dependent of trap code
	.It Dv SIGCHLD Ta Va si_pid Ta
	child process ID
	.It Ta Va si_status Ta
	exit value or signal; if
	.Va si_code
	is equal to
	.Dv CLD_EXITED ,
	then it is equal to the exit value of the child process, otherwise,
	it is equal to a signal that caused the child process to change state.
	.It Ta Va si_uid Ta "real user ID of the process that sent the signal"
	.It Dv SIGPOLL Ta Va si_band Ta "band event for"
	.Dv POLL_IN , POLL_OUT ,
	or
	.Dv POLL_MSG
	.El
	.Pp
	Finally, the following code-specific information is available:
	.Bl -column ".Dv SI_ASYNCIO" ".Va si_overrun"
	.It Sy Code Ta Sy Member Ta Sy Value
	.It Dv SI_USER Ta Va si_pid Ta
	the process ID that sent the signal
	.It Ta Va si_uid Ta
	real user ID of the process that sent the signal
	.It Dv SI_QUEUE Ta Va si_value Ta
	the value passed to
	.Xr sigqueue 2
	system call
	.It Ta Va si_pid Ta
	the process ID that sent the signal
	.It Ta Va si_uid Ta
	real user ID of the process that sent the signal
	.It Dv SI_TIMER Ta Va si_value Ta
	the value passed to
	.Xr timer_create 2
	system call
	.It Ta Va si_timerid Ta
	the timer ID returned by
	.Xr timer_create 2
	system call
	.It Ta Va si_overrun Ta
	timer overrun count corresponding to the signal
	.It Ta Va si_errno Ta
	If timer overrun will be
	.Brq Dv DELAYTIMER_MAX ,
	an error code defined in
	.In errno.h
	is set
	.It Dv SI_ASYNCIO Ta Va si_value Ta
	the value passed to aio system calls
	.It Dv SI_MESGQ Ta Va si_value Ta
	the value passed to
	.Xr mq_notify 2
	system call
	.It Ta Va si_mqd Ta
	the ID of the message queue which generated the signal
	.It Dv SI_LWP Ta Va si_pid Ta
	the process ID that sent the signal
	.It Ta Va si_uid Ta
	real user ID of the process that sent the signal
	.El
	.Sh NOTES
	Currently, the kernel never generates the
	.Dv SIGPOLL
	signal.
	.Dv SIGCHLD
	signal is queued when a process changed its status or exited.
	.Tn POSIX
	Realtime Extensions like aio, timer, and message queue also queue
	signals.
	Signals with code
	.Dv SI_USER ,
	.Dv SI_KERNEL
	or
	.Dv SI_LWP
	are only queued if there are sufficient resources;
	otherwise,
	.Dv SI_NOINFO
	results.
	For some hardware architectures, the exact value of
	.Va si_addr
	might not be available.
	.Sh SEE ALSO
	.Xr aio_read 2 ,
	.Xr kill 2 ,
	.Xr mq_notify 2 ,
	.Xr sigaction 2 ,
	.Xr sigqueue 2 ,
	.Xr sigwaitinfo 2 ,
	.Xr timer_create 2 ,
	.Xr timer_settime 2 ,
	.Xr waitpid 2 ,
	.Xr pthread_kill 3
	.Sh STANDARDS
	The
	.Vt siginfo_t
	type conforms to
	.St -p1003.1-2004 .
	.Sh HISTORY
	Full support for
	.Tn POSIX
	signal information first appeared in
	.Fx 7.0 .
	The codes
	.Dv SI_USER
	and
	.Dv SI_KERNEL
	can be generated as of
	.Fx 8.1 .
	The code
	.Dv SI_LWP
	can be generated as of
	.Fx 9.0 .
	.Sh AUTHORS
	This manual page was written by
	.An David Xu Aq Mt davidxu@FreeBSD.org .
	diff --git a/share/man/man3/tree.3 b/share/man/man3/tree.3
	index e6ecc6e44237..a29f06d3c21e 100644
	--- a/share/man/man3/tree.3
	+++ b/share/man/man3/tree.3
	@@ -1,716 +1,717 @@
	.\" $OpenBSD: tree.3,v 1.7 2002/06/12 01:09:20 provos Exp $
	.\"
	.\" Copyright 2002 Niels Provos <provos@citi.umich.edu>
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\" 3. All advertising materials mentioning features or use of this software
	.\" must display the following acknowledgement:
	.\" This product includes software developed by Niels Provos.
	.\" 4. The name of the author may not be used to endorse or promote products
	.\" derived from this software without specific prior written permission.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	.\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd February 25, 2020
	+.Dd July 27, 2020
	.Dt TREE 3
	.Os
	.Sh NAME
	.Nm SPLAY_PROTOTYPE ,
	.Nm SPLAY_GENERATE ,
	.Nm SPLAY_ENTRY ,
	.Nm SPLAY_HEAD ,
	.Nm SPLAY_INITIALIZER ,
	.Nm SPLAY_ROOT ,
	.Nm SPLAY_EMPTY ,
	.Nm SPLAY_NEXT ,
	.Nm SPLAY_MIN ,
	.Nm SPLAY_MAX ,
	.Nm SPLAY_FIND ,
	.Nm SPLAY_LEFT ,
	.Nm SPLAY_RIGHT ,
	.Nm SPLAY_FOREACH ,
	.Nm SPLAY_INIT ,
	.Nm SPLAY_INSERT ,
	.Nm SPLAY_REMOVE ,
	.Nm RB_PROTOTYPE ,
	.Nm RB_PROTOTYPE_STATIC ,
	.Nm RB_PROTOTYPE_INSERT ,
	.Nm RB_PROTOTYPE_INSERT_COLOR ,
	.Nm RB_PROTOTYPE_REMOVE ,
	.Nm RB_PROTOTYPE_REMOVE_COLOR ,
	.Nm RB_PROTOTYPE_FIND ,
	.Nm RB_PROTOTYPE_NFIND ,
	.Nm RB_PROTOTYPE_NEXT ,
	.Nm RB_PROTOTYPE_PREV ,
	.Nm RB_PROTOTYPE_MINMAX ,
	.Nm RB_PROTOTYPE_REINSERT ,
	.Nm RB_GENERATE ,
	.Nm RB_GENERATE_STATIC ,
	.Nm RB_GENERATE_INSERT ,
	.Nm RB_GENERATE_INSERT_COLOR ,
	.Nm RB_GENERATE_REMOVE ,
	.Nm RB_GENERATE_REMOVE_COLOR ,
	.Nm RB_GENERATE_FIND ,
	.Nm RB_GENERATE_NFIND ,
	.Nm RB_GENERATE_NEXT ,
	.Nm RB_GENERATE_PREV ,
	.Nm RB_GENERATE_MINMAX ,
	.Nm RB_GENERATE_REINSERT ,
	.Nm RB_ENTRY ,
	.Nm RB_HEAD ,
	.Nm RB_INITIALIZER ,
	.Nm RB_ROOT ,
	.Nm RB_EMPTY ,
	.Nm RB_NEXT ,
	.Nm RB_PREV ,
	.Nm RB_MIN ,
	.Nm RB_MAX ,
	.Nm RB_FIND ,
	.Nm RB_NFIND ,
	.Nm RB_LEFT ,
	.Nm RB_RIGHT ,
	.Nm RB_PARENT ,
	.Nm RB_FOREACH ,
	.Nm RB_FOREACH_FROM ,
	.Nm RB_FOREACH_SAFE ,
	.Nm RB_FOREACH_REVERSE ,
	.Nm RB_FOREACH_REVERSE_FROM ,
	.Nm RB_FOREACH_REVERSE_SAFE ,
	.Nm RB_INIT ,
	.Nm RB_INSERT ,
	.Nm RB_REMOVE ,
	.Nm RB_REINSERT
	.Nd "implementations of splay and rank-balanced (wavl) trees"
	.Sh SYNOPSIS
	.In sys/tree.h
	.Fn SPLAY_PROTOTYPE NAME TYPE FIELD CMP
	.Fn SPLAY_GENERATE NAME TYPE FIELD CMP
	.Fn SPLAY_ENTRY TYPE
	.Fn SPLAY_HEAD HEADNAME TYPE
	.Ft "struct TYPE *"
	.Fn SPLAY_INITIALIZER "SPLAY_HEAD *head"
	.Fn SPLAY_ROOT "SPLAY_HEAD *head"
	.Ft bool
	.Fn SPLAY_EMPTY "SPLAY_HEAD *head"
	.Ft "struct TYPE *"
	.Fn SPLAY_NEXT NAME "SPLAY_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn SPLAY_MIN NAME "SPLAY_HEAD *head"
	.Ft "struct TYPE *"
	.Fn SPLAY_MAX NAME "SPLAY_HEAD *head"
	.Ft "struct TYPE *"
	.Fn SPLAY_FIND NAME "SPLAY_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn SPLAY_LEFT "struct TYPE *elm" "SPLAY_ENTRY NAME"
	.Ft "struct TYPE *"
	.Fn SPLAY_RIGHT "struct TYPE *elm" "SPLAY_ENTRY NAME"
	.Fn SPLAY_FOREACH VARNAME NAME "SPLAY_HEAD *head"
	.Ft void
	.Fn SPLAY_INIT "SPLAY_HEAD *head"
	.Ft "struct TYPE *"
	.Fn SPLAY_INSERT NAME "SPLAY_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn SPLAY_REMOVE NAME "SPLAY_HEAD head" "struct TYPE elm"
	.Fn RB_PROTOTYPE NAME TYPE FIELD CMP
	.Fn RB_PROTOTYPE_STATIC NAME TYPE FIELD CMP
	.Fn RB_PROTOTYPE_INSERT NAME TYPE ATTR
	.Fn RB_PROTOTYPE_INSERT_COLOR NAME TYPE ATTR
	.Fn RB_PROTOTYPE_REMOVE NAME TYPE ATTR
	.Fn RB_PROTOTYPE_REMOVE_COLOR NAME TYPE ATTR
	.Fn RB_PROTOTYPE_FIND NAME TYPE ATTR
	.Fn RB_PROTOTYPE_NFIND NAME TYPE ATTR
	.Fn RB_PROTOTYPE_NEXT NAME TYPE ATTR
	.Fn RB_PROTOTYPE_PREV NAME TYPE ATTR
	.Fn RB_PROTOTYPE_MINMAX NAME TYPE ATTR
	.Fn RB_PROTOTYPE_REINSERT NAME TYPE ATTR
	.Fn RB_GENERATE NAME TYPE FIELD CMP
	.Fn RB_GENERATE_STATIC NAME TYPE FIELD CMP
	.Fn RB_GENERATE_INSERT NAME TYPE FIELD CMP ATTR
	.Fn RB_GENERATE_INSERT_COLOR NAME TYPE FIELD ATTR
	.Fn RB_GENERATE_REMOVE NAME TYPE FIELD ATTR
	.Fn RB_GENERATE_REMOVE_COLOR NAME TYPE FIELD ATTR
	.Fn RB_GENERATE_FIND NAME TYPE FIELD CMP ATTR
	.Fn RB_GENERATE_NFIND NAME TYPE FIELD CMP ATTR
	.Fn RB_GENERATE_NEXT NAME TYPE FIELD ATTR
	.Fn RB_GENERATE_PREV NAME TYPE FIELD ATTR
	.Fn RB_GENERATE_MINMAX NAME TYPE FIELD ATTR
	.Fn RB_GENERATE_REINSERT NAME TYPE FIELD CMP ATTR
	.Fn RB_ENTRY TYPE
	.Fn RB_HEAD HEADNAME TYPE
	.Fn RB_INITIALIZER "RB_HEAD *head"
	.Ft "struct TYPE *"
	.Fn RB_ROOT "RB_HEAD *head"
	.Ft "bool"
	.Fn RB_EMPTY "RB_HEAD *head"
	.Ft "struct TYPE *"
	.Fn RB_NEXT NAME "RB_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn RB_PREV NAME "RB_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn RB_MIN NAME "RB_HEAD *head"
	.Ft "struct TYPE *"
	.Fn RB_MAX NAME "RB_HEAD *head"
	.Ft "struct TYPE *"
	.Fn RB_FIND NAME "RB_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn RB_NFIND NAME "RB_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn RB_LEFT "struct TYPE *elm" "RB_ENTRY NAME"
	.Ft "struct TYPE *"
	.Fn RB_RIGHT "struct TYPE *elm" "RB_ENTRY NAME"
	.Ft "struct TYPE *"
	.Fn RB_PARENT "struct TYPE *elm" "RB_ENTRY NAME"
	.Fn RB_FOREACH VARNAME NAME "RB_HEAD *head"
	.Fn RB_FOREACH_FROM "VARNAME" "NAME" "POS_VARNAME"
	.Fn RB_FOREACH_SAFE "VARNAME" "NAME" "RB_HEAD *head" "TEMP_VARNAME"
	.Fn RB_FOREACH_REVERSE VARNAME NAME "RB_HEAD *head"
	.Fn RB_FOREACH_REVERSE_FROM "VARNAME" "NAME" "POS_VARNAME"
	.Fn RB_FOREACH_REVERSE_SAFE "VARNAME" "NAME" "RB_HEAD *head" "TEMP_VARNAME"
	.Ft void
	.Fn RB_INIT "RB_HEAD *head"
	.Ft "struct TYPE *"
	.Fn RB_INSERT NAME "RB_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn RB_REMOVE NAME "RB_HEAD head" "struct TYPE elm"
	.Ft "struct TYPE *"
	.Fn RB_REINSERT NAME "RB_HEAD head" "struct TYPE elm"
	.Sh DESCRIPTION
	These macros define data structures for different types of trees:
	splay trees and rank-balanced (wavl) trees.
	.Pp
	In the macro definitions,
	.Fa TYPE
	is the name tag of a user defined structure that must contain a field of type
	.Vt SPLAY_ENTRY ,
	or
	.Vt RB_ENTRY ,
	named
	.Fa ENTRYNAME .
	The argument
	.Fa HEADNAME
	is the name tag of a user defined structure that must be declared
	using the macros
	.Fn SPLAY_HEAD ,
	or
	.Fn RB_HEAD .
	The argument
	.Fa NAME
	has to be a unique name prefix for every tree that is defined.
	.Pp
	The function prototypes are declared with
	.Fn SPLAY_PROTOTYPE ,
	.Fn RB_PROTOTYPE ,
	or
	.Fn RB_PROTOTYPE_STATIC .
	The function bodies are generated with
	.Fn SPLAY_GENERATE ,
	.Fn RB_GENERATE ,
	or
	.Fn RB_GENERATE_STATIC .
	See the examples below for further explanation of how these macros are used.
	.Sh SPLAY TREES
	A splay tree is a self-organizing data structure.
	Every operation on the tree causes a splay to happen.
	The splay moves the requested
	node to the root of the tree and partly rebalances it.
	.Pp
	This has the benefit that request locality causes faster lookups as
	the requested nodes move to the top of the tree.
	On the other hand, every lookup causes memory writes.
	.Pp
	The Balance Theorem bounds the total access time for
	.Ar m
	operations and
	.Ar n
	inserts on an initially empty tree as
	.Fn O "\[lp]m + n\[rp]lg n" .
	The
	amortized cost for a sequence of
	.Ar m
	accesses to a splay tree is
	.Fn O "lg n" .
	.Pp
	A splay tree is headed by a structure defined by the
	.Fn SPLAY_HEAD
	macro.
	A
	structure is declared as follows:
	.Bd -ragged -offset indent
	.Fn SPLAY_HEAD HEADNAME TYPE
	.Va head ;
	.Ed
	.Pp
	where
	.Fa HEADNAME
	is the name of the structure to be defined, and struct
	.Fa TYPE
	is the type of the elements to be inserted into the tree.
	.Pp
	The
	.Fn SPLAY_ENTRY
	macro declares a structure that allows elements to be connected in the tree.
	.Pp
	In order to use the functions that manipulate the tree structure,
	their prototypes need to be declared with the
	.Fn SPLAY_PROTOTYPE
	macro,
	where
	.Fa NAME
	is a unique identifier for this particular tree.
	The
	.Fa TYPE
	argument is the type of the structure that is being managed
	by the tree.
	The
	.Fa FIELD
	argument is the name of the element defined by
	.Fn SPLAY_ENTRY .
	.Pp
	The function bodies are generated with the
	.Fn SPLAY_GENERATE
	macro.
	It takes the same arguments as the
	.Fn SPLAY_PROTOTYPE
	macro, but should be used only once.
	.Pp
	Finally,
	the
	.Fa CMP
	argument is the name of a function used to compare tree nodes
	with each other.
	The function takes two arguments of type
	.Vt "struct TYPE *" .
	If the first argument is smaller than the second, the function returns a
	value smaller than zero.
	If they are equal, the function returns zero.
	Otherwise, it should return a value greater than zero.
	The compare
	function defines the order of the tree elements.
	.Pp
	The
	.Fn SPLAY_INIT
	macro initializes the tree referenced by
	.Fa head .
	.Pp
	The splay tree can also be initialized statically by using the
	.Fn SPLAY_INITIALIZER
	macro like this:
	.Bd -ragged -offset indent
	.Fn SPLAY_HEAD HEADNAME TYPE
	.Va head
	=
	.Fn SPLAY_INITIALIZER &head ;
	.Ed
	.Pp
	The
	.Fn SPLAY_INSERT
	macro inserts the new element
	.Fa elm
	into the tree.
	.Pp
	The
	.Fn SPLAY_REMOVE
	macro removes the element
	.Fa elm
	from the tree pointed by
	.Fa head .
	.Pp
	The
	.Fn SPLAY_FIND
	macro can be used to find a particular element in the tree.
	.Bd -literal -offset indent
	struct TYPE find, *res;
	find.key = 30;
	res = SPLAY_FIND(NAME, head, &find);
	.Ed
	.Pp
	The
	.Fn SPLAY_ROOT ,
	.Fn SPLAY_MIN ,
	.Fn SPLAY_MAX ,
	and
	.Fn SPLAY_NEXT
	macros can be used to traverse the tree:
	.Bd -literal -offset indent
	for (np = SPLAY_MIN(NAME, &head); np != NULL; np = SPLAY_NEXT(NAME, &head, np))
	.Ed
	.Pp
	Or, for simplicity, one can use the
	.Fn SPLAY_FOREACH
	macro:
	.Bd -ragged -offset indent
	.Fn SPLAY_FOREACH np NAME head
	.Ed
	.Pp
	The
	.Fn SPLAY_EMPTY
	macro should be used to check whether a splay tree is empty.
	.Sh RANK-BALANCED TREES
	Rank-balanced (RB) trees are a framework for defining height-balanced
	binary search trees, including AVL and red-black trees.
	Each tree node has an associated rank.
	Balance conditions are expressed by conditions on the differences in
	rank between any node and its children.
	Rank differences are stored in each tree node.
	-.Pp
	+.Pp
	The balance conditions implemented by the RB macros lead to weak AVL
	(wavl) trees, which combine the best aspects of AVL and red-black
	trees.
	Wavl trees rebalance after an insertion in the same way AVL trees do,
	with the same worst-case time as red-black trees offer, and with
	better balance in the resulting tree.
	Wavl trees rebalance after a removal in a way that requires less
	restructuring, in the worst case, than either AVL or red-black trees
	-do. Removals can lead to a tree almost as unbalanced as a red-black
	+do.
	+Removals can lead to a tree almost as unbalanced as a red-black
	tree; insertions lead to a tree becoming as balanced as an AVL tree.
	.Pp
	A rank-balanced tree is headed by a structure defined by the
	.Fn RB_HEAD
	macro.
	A
	structure is declared as follows:
	.Bd -ragged -offset indent
	.Fn RB_HEAD HEADNAME TYPE
	.Va head ;
	.Ed
	.Pp
	where
	.Fa HEADNAME
	is the name of the structure to be defined, and struct
	.Fa TYPE
	is the type of the elements to be inserted into the tree.
	.Pp
	The
	.Fn RB_ENTRY
	macro declares a structure that allows elements to be connected in the tree.
	.Pp
	In order to use the functions that manipulate the tree structure,
	their prototypes need to be declared with the
	.Fn RB_PROTOTYPE
	or
	.Fn RB_PROTOTYPE_STATIC
	macro,
	where
	.Fa NAME
	is a unique identifier for this particular tree.
	The
	.Fa TYPE
	argument is the type of the structure that is being managed
	by the tree.
	The
	.Fa FIELD
	argument is the name of the element defined by
	.Fn RB_ENTRY .
	Individual prototypes can be declared with
	.Fn RB_PROTOTYPE_INSERT ,
	.Fn RB_PROTOTYPE_INSERT_COLOR ,
	.Fn RB_PROTOTYPE_REMOVE ,
	.Fn RB_PROTOTYPE_REMOVE_COLOR ,
	.Fn RB_PROTOTYPE_FIND ,
	.Fn RB_PROTOTYPE_NFIND ,
	.Fn RB_PROTOTYPE_NEXT ,
	.Fn RB_PROTOTYPE_PREV ,
	.Fn RB_PROTOTYPE_MINMAX ,
	and
	.Fn RB_PROTOTYPE_REINSERT
	in case not all functions are required.
	The individual prototype macros expect
	.Fa NAME ,
	.Fa TYPE ,
	and
	.Fa ATTR
	arguments.
	The
	.Fa ATTR
	argument must be empty for global functions or
	.Fa static
	for static functions.
	.Pp
	The function bodies are generated with the
	.Fn RB_GENERATE
	or
	.Fn RB_GENERATE_STATIC
	macro.
	These macros take the same arguments as the
	.Fn RB_PROTOTYPE
	and
	.Fn RB_PROTOTYPE_STATIC
	macros, but should be used only once.
	As an alternative individual function bodies are generated with the
	.Fn RB_GENERATE_INSERT ,
	.Fn RB_GENERATE_INSERT_COLOR ,
	.Fn RB_GENERATE_REMOVE ,
	.Fn RB_GENERATE_REMOVE_COLOR ,
	.Fn RB_GENERATE_FIND ,
	.Fn RB_GENERATE_NFIND ,
	.Fn RB_GENERATE_NEXT ,
	.Fn RB_GENERATE_PREV ,
	.Fn RB_GENERATE_MINMAX ,
	and
	.Fn RB_GENERATE_REINSERT
	macros.
	.Pp
	Finally,
	the
	.Fa CMP
	argument is the name of a function used to compare tree nodes
	with each other.
	The function takes two arguments of type
	.Vt "struct TYPE *" .
	If the first argument is smaller than the second, the function returns a
	value smaller than zero.
	If they are equal, the function returns zero.
	Otherwise, it should return a value greater than zero.
	The compare
	function defines the order of the tree elements.
	.Pp
	The
	.Fn RB_INIT
	macro initializes the tree referenced by
	.Fa head .
	.Pp
	The rank-balanced tree can also be initialized statically by using the
	.Fn RB_INITIALIZER
	macro like this:
	.Bd -ragged -offset indent
	.Fn RB_HEAD HEADNAME TYPE
	.Va head
	=
	.Fn RB_INITIALIZER &head ;
	.Ed
	.Pp
	The
	.Fn RB_INSERT
	macro inserts the new element
	.Fa elm
	into the tree.
	.Pp
	The
	.Fn RB_REMOVE
	macro removes the element
	.Fa elm
	from the tree pointed by
	.Fa head .
	.Pp
	The
	.Fn RB_FIND
	and
	.Fn RB_NFIND
	macros can be used to find a particular element in the tree.
	.Bd -literal -offset indent
	struct TYPE find, *res;
	find.key = 30;
	res = RB_FIND(NAME, head, &find);
	.Ed
	.Pp
	The
	.Fn RB_ROOT ,
	.Fn RB_MIN ,
	.Fn RB_MAX ,
	.Fn RB_NEXT ,
	and
	.Fn RB_PREV
	macros can be used to traverse the tree:
	.Pp
	.Dl "for (np = RB_MIN(NAME, &head); np != NULL; np = RB_NEXT(NAME, &head, np))"
	.Pp
	Or, for simplicity, one can use the
	.Fn RB_FOREACH
	or
	.Fn RB_FOREACH_REVERSE
	macro:
	.Bd -ragged -offset indent
	.Fn RB_FOREACH np NAME head
	.Ed
	.Pp
	The macros
	.Fn RB_FOREACH_SAFE
	and
	.Fn RB_FOREACH_REVERSE_SAFE
	traverse the tree referenced by head
	in a forward or reverse direction respectively,
	assigning each element in turn to np.
	However, unlike their unsafe counterparts,
	they permit both the removal of np
	as well as freeing it from within the loop safely
	without interfering with the traversal.
	.Pp
	Both
	.Fn RB_FOREACH_FROM
	and
	.Fn RB_FOREACH_REVERSE_FROM
	may be used to continue an interrupted traversal
	in a forward or reverse direction respectively.
	The head pointer is not required.
	The pointer to the node from where to resume the traversal
	should be passed as their last argument,
	and will be overwritten to provide safe traversal.
	.Pp
	The
	.Fn RB_EMPTY
	macro should be used to check whether a rank-balanced tree is empty.
	.Pp
	The
	.Fn RB_REINSERT
	macro updates the position of the element
	.Fa elm
	in the tree.
	This must be called if a member of a
	.Nm tree
	is modified in a way that affects comparison, such as by modifying
	a node's key.
	This is a lower overhead alternative to removing the element
	and reinserting it again.
	.Sh EXAMPLES
	The following example demonstrates how to declare a rank-balanced tree
	holding integers.
	Values are inserted into it and the contents of the tree are printed
	in order.
	Lastly, the internal structure of the tree is printed.
	.Bd -literal -offset 3n
	#include <sys/tree.h>
	#include <err.h>
	#include <stdio.h>
	#include <stdlib.h>

	struct node {
	RB_ENTRY(node) entry;
	int i;
	};

	int
	intcmp(struct node e1, struct node e2)
	{
	return (e1->i < e2->i ? -1 : e1->i > e2->i);
	}

	RB_HEAD(inttree, node) head = RB_INITIALIZER(&head);
	RB_GENERATE(inttree, node, entry, intcmp)

	int testdata[] = {
	20, 16, 17, 13, 3, 6, 1, 8, 2, 4, 10, 19, 5, 9, 12, 15, 18,
	7, 11, 14
	};

	void
	print_tree(struct node *n)
	{
	struct node left, right;

	if (n == NULL) {
	printf("nil");
	return;
	}
	left = RB_LEFT(n, entry);
	right = RB_RIGHT(n, entry);
	if (left == NULL && right == NULL)
	printf("%d", n->i);
	else {
	printf("%d(", n->i);
	print_tree(left);
	printf(",");
	print_tree(right);
	printf(")");
	}
	}

	int
	main(void)
	{
	int i;
	struct node *n;

	for (i = 0; i < sizeof(testdata) / sizeof(testdata[0]); i++) {
	if ((n = malloc(sizeof(struct node))) == NULL)
	err(1, NULL);
	n->i = testdata[i];
	RB_INSERT(inttree, &head, n);
	}

	RB_FOREACH(n, inttree, &head) {
	printf("%d\en", n->i);
	}
	print_tree(RB_ROOT(&head));
	printf("\en");
	return (0);
	}
	.Ed
	.Sh NOTES
	Trying to free a tree in the following way is a common error:
	.Bd -literal -offset indent
	SPLAY_FOREACH(var, NAME, head) {
	SPLAY_REMOVE(NAME, head, var);
	free(var);
	}
	free(head);
	.Ed
	.Pp
	Since
	.Va var
	is freed, the
	.Fn FOREACH
	macro refers to a pointer that may have been reallocated already.
	Proper code needs a second variable.
	.Bd -literal -offset indent
	for (var = SPLAY_MIN(NAME, head); var != NULL; var = nxt) {
	nxt = SPLAY_NEXT(NAME, head, var);
	SPLAY_REMOVE(NAME, head, var);
	free(var);
	}
	.Ed
	.Pp
	Both
	.Fn RB_INSERT
	and
	.Fn SPLAY_INSERT
	return
	.Dv NULL
	if the element was inserted in the tree successfully, otherwise they
	return a pointer to the element with the colliding key.
	.Pp
	Accordingly,
	.Fn RB_REMOVE
	and
	.Fn SPLAY_REMOVE
	return the pointer to the removed element otherwise they return
	.Dv NULL
	to indicate an error.
	.Sh SEE ALSO
	.Xr arb 3 ,
	.Xr queue 3
	.Rs
	.%A "Bernhard Haeupler"
	.%A "Siddhartha Sen"
	.%A "Robert E. Tarjan"
	.%T "Rank-Balanced Trees"
	.%U "http://sidsen.azurewebsites.net/papers/rb-trees-talg.pdf"
	.%J "ACM Transactions on Algorithms"
	.%V "11"
	.%N "4"
	.%D "June 2015"
	.Re
	.Sh HISTORY
	The tree macros first appeared in
	.Fx 4.6 .
	.Sh AUTHORS
	The author of the tree macros is
	.An Niels Provos .
	diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
	index fe715f48288d..9049f3feb5e3 100644
	--- a/share/man/man4/Makefile
	+++ b/share/man/man4/Makefile
	@@ -1,1028 +1,1025 @@
	# @(#)Makefile 8.1 (Berkeley) 6/18/93
	# $FreeBSD$

	.include <src.opts.mk>

	MAN= aac.4 \
	aacraid.4 \
	acpi.4 \
	${_acpi_asus.4} \
	${_acpi_asus_wmi.4} \
	${_acpi_dock.4} \
	${_acpi_fujitsu.4} \
	${_acpi_hp.4} \
	${_acpi_ibm.4} \
	${_acpi_panasonic.4} \
	${_acpi_rapidstart.4} \
	${_acpi_sony.4} \
	acpi_thermal.4 \
	acpi_battery.4 \
	${_acpi_toshiba.4} \
	acpi_video.4 \
	${_acpi_wmi.4} \
	ada.4 \
	adm6996fc.4 \
	ads111x.4 \
	ae.4 \
	${_aesni.4} \
	age.4 \
	agp.4 \
	ahc.4 \
	ahci.4 \
	ahd.4 \
	${_aibs.4} \
	aio.4 \
	alc.4 \
	ale.4 \
	alpm.4 \
	altera_atse.4 \
	altera_avgen.4 \
	altera_jtag_uart.4 \
	altera_sdcard.4 \
	altq.4 \
	amdpm.4 \
	${_amdsbwd.4} \
	${_amdsmb.4} \
	${_amdsmn.4} \
	${_amdtemp.4} \
	${_bxe.4} \
	amr.4 \
	an.4 \
	${_aout.4} \
	${_apic.4} \
	arcmsr.4 \
	${_asmc.4} \
	at45d.4 \
	ata.4 \
	ath.4 \
	ath_ahb.4 \
	ath_hal.4 \
	ath_pci.4 \
	atkbd.4 \
	atkbdc.4 \
	atp.4 \
	${_atf_test_case.4} \
	${_atrtc.4} \
	${_attimer.4} \
	audit.4 \
	auditpipe.4 \
	aue.4 \
	axe.4 \
	axge.4 \
	bce.4 \
	bcma.4 \
	bfe.4 \
	bge.4 \
	${_bhyve.4} \
	bhnd.4 \
	bhnd_chipc.4 \
	bhnd_pmu.4 \
	bhndb.4 \
	bhndb_pci.4 \
	blackhole.4 \
	bnxt.4 \
	bpf.4 \
	bridge.4 \
	bt.4 \
	bwi.4 \
	bwn.4 \
	${_bytgpio.4} \
	capsicum.4 \
	cardbus.4 \
	carp.4 \
	cas.4 \
	cc_cdg.4 \
	cc_chd.4 \
	cc_cubic.4 \
	cc_dctcp.4 \
	cc_hd.4 \
	cc_htcp.4 \
	cc_newreno.4 \
	cc_vegas.4 \
	${_ccd.4} \
	ccr.4 \
	cd.4 \
	cdce.4 \
	cdceem.4 \
	cfi.4 \
	cfumass.4 \
	ch.4 \
	chromebook_platform.4 \
	${_chvgpio.4} \
	ciss.4 \
	cloudabi.4 \
	cmx.4 \
	${_coretemp.4} \
	${_cpuctl.4} \
	cpufreq.4 \
	crypto.4 \
	ctl.4 \
	cue.4 \
	cxgb.4 \
	cxgbe.4 \
	cxgbev.4 \
	- cy.4 \
	cyapa.4 \
	da.4 \
	dc.4 \
	dcons.4 \
	dcons_crom.4 \
	ddb.4 \
	devctl.4 \
	disc.4 \
	divert.4 \
	${_dpms.4} \
	ds1307.4 \
	ds3231.4 \
	${_dtrace_provs} \
	dummynet.4 \
	edsc.4 \
	ehci.4 \
	em.4 \
	ena.4 \
	enc.4 \
	epair.4 \
	esp.4 \
	est.4 \
	et.4 \
	etherswitch.4 \
	eventtimers.4 \
	exca.4 \
	e6060sw.4 \
	fd.4 \
	fdc.4 \
	fdt.4 \
	fdt_pinctrl.4 \
	fdtbus.4 \
	ffclock.4 \
	filemon.4 \
	firewire.4 \
	full.4 \
	fwe.4 \
	fwip.4 \
	fwohci.4 \
	fxp.4 \
	gbde.4 \
	gdb.4 \
	gem.4 \
	geom.4 \
	geom_linux_lvm.4 \
	geom_map.4 \
	geom_uzip.4 \
	gif.4 \
	gpio.4 \
	gpioiic.4 \
	gpioled.4 \
	gpioths.4 \
	gre.4 \
	h_ertt.4 \
	hifn.4 \
	hme.4 \
	hpet.4 \
	${_hpt27xx.4} \
	${_hptiop.4} \
	${_hptmv.4} \
	${_hptnr.4} \
	${_hptrr.4} \
	${_hv_kvp.4} \
	${_hv_netvsc.4} \
	${_hv_storvsc.4} \
	${_hv_utils.4} \
	${_hv_vmbus.4} \
	${_hv_vss.4} \
	hwpmc.4 \
	${_hwpstate_intel.4} \
	iavf.4 \
	ichsmb.4 \
	${_ichwd.4} \
	icmp.4 \
	icmp6.4 \
	ida.4 \
	if_ipsec.4 \
	iflib.4 \
	ifmib.4 \
	ig4.4 \
	igmp.4 \
	iic.4 \
	iic_gpiomux.4 \
	iicbb.4 \
	iicbus.4 \
	iicmux.4 \
	iicsmb.4 \
	iir.4 \
	${_imcsmb.4} \
	inet.4 \
	inet6.4 \
	intpm.4 \
	intro.4 \
	${_io.4} \
	${_ioat.4} \
	ip.4 \
	ip6.4 \
	ipfirewall.4 \
	ipheth.4 \
	${_ipmi.4} \
	ips.4 \
	ipsec.4 \
	ipw.4 \
	ipwfw.4 \
	isci.4 \
	isl.4 \
	ismt.4 \
	isp.4 \
	ispfw.4 \
	${_itwd.4} \
	iwi.4 \
	iwifw.4 \
	iwm.4 \
	iwmfw.4 \
	iwn.4 \
	iwnfw.4 \
	ixgbe.4 \
	ixl.4 \
	jedec_dimm.4 \
	jme.4 \
	kbdmux.4 \
	keyboard.4 \
	kld.4 \
	ksyms.4 \
	ksz8995ma.4 \
	ktr.4 \
	kue.4 \
	lagg.4 \
	le.4 \
	led.4 \
	lge.4 \
	${_linux.4} \
	liquidio.4 \
	lm75.4 \
	lo.4 \
	lp.4 \
	lpbb.4 \
	lpt.4 \
	ltc430x.4 \
	mac.4 \
	mac_biba.4 \
	mac_bsdextended.4 \
	mac_ifoff.4 \
	mac_lomac.4 \
	mac_mls.4 \
	mac_none.4 \
	mac_ntpd.4 \
	mac_partition.4 \
	mac_portacl.4 \
	mac_seeotheruids.4 \
	mac_stub.4 \
	mac_test.4 \
	malo.4 \
	md.4 \
	mdio.4 \
	me.4 \
	mem.4 \
	meteor.4 \
	mfi.4 \
	miibus.4 \
	mk48txx.4 \
	mld.4 \
	mlx.4 \
	mlx4en.4 \
	mlx5en.4 \
	mly.4 \
	mmc.4 \
	mmcsd.4 \
	mn.4 \
	mod_cc.4 \
	mos.4 \
	mouse.4 \
	mpr.4 \
	mps.4 \
	mpt.4 \
	mrsas.4 \
	msk.4 \
	mtio.4 \
	multicast.4 \
	muge.4 \
	mvs.4 \
	mwl.4 \
	mwlfw.4 \
	mx25l.4 \
	mxge.4 \
	my.4 \
	${_ndis.4} \
	net80211.4 \
	netdump.4 \
	netfpga10g_nf10bmac.4 \
	netgdb.4 \
	netgraph.4 \
	netintro.4 \
	netmap.4 \
	${_nfe.4} \
	${_nfsmb.4} \
	ng_async.4 \
	ngatmbase.4 \
	ng_atmllc.4 \
	ng_bpf.4 \
	ng_bridge.4 \
	ng_bt3c.4 \
	ng_btsocket.4 \
	ng_car.4 \
	ng_ccatm.4 \
	ng_checksum.4 \
	ng_cisco.4 \
	ng_deflate.4 \
	ng_device.4 \
	nge.4 \
	ng_echo.4 \
	ng_eiface.4 \
	ng_etf.4 \
	ng_ether.4 \
	ng_ether_echo.4 \
	ng_frame_relay.4 \
	ng_gif.4 \
	ng_gif_demux.4 \
	ng_h4.4 \
	ng_hci.4 \
	ng_hole.4 \
	ng_hub.4 \
	ng_iface.4 \
	ng_ipfw.4 \
	ng_ip_input.4 \
	ng_ksocket.4 \
	ng_l2cap.4 \
	ng_l2tp.4 \
	ng_lmi.4 \
	ng_mppc.4 \
	ng_nat.4 \
	ng_netflow.4 \
	ng_one2many.4 \
	ng_patch.4 \
	ng_pipe.4 \
	ng_ppp.4 \
	ng_pppoe.4 \
	ng_pptpgre.4 \
	ng_pred1.4 \
	ng_rfc1490.4 \
	ng_socket.4 \
	ng_source.4 \
	ng_split.4 \
	ng_sppp.4 \
	ng_sscfu.4 \
	ng_sscop.4 \
	ng_tag.4 \
	ng_tcpmss.4 \
	ng_tee.4 \
	ng_tty.4 \
	ng_ubt.4 \
	ng_UI.4 \
	ng_uni.4 \
	ng_vjc.4 \
	ng_vlan.4 \
	nmdm.4 \
	${_ntb.4} \
	${_ntb_hw_amd.4} \
	${_ntb_hw_intel.4} \
	${_ntb_hw_plx.4} \
	${_ntb_transport.4} \
	${_nda.4} \
	${_if_ntb.4} \
	null.4 \
	numa.4 \
	${_nvd.4} \
	${_nvdimm.4} \
	${_nvme.4} \
	${_nvram.4} \
	${_nvram2env.4} \
	oce.4 \
	ocs_fc.4\
	ohci.4 \
	orm.4 \
	ow.4 \
	ow_temp.4 \
	owc.4 \
	${_padlock.4} \
	pass.4 \
	pccard.4 \
	pccbb.4 \
	pcf.4 \
	${_pchtherm.4} \
	pci.4 \
	pcib.4 \
	pcic.4 \
	pcm.4 \
	${_pf.4} \
	${_pflog.4} \
	${_pfsync.4} \
	pim.4 \
	pms.4 \
	polling.4 \
	ppbus.4 \
	ppc.4 \
	ppi.4 \
	procdesc.4 \
	proto.4 \
	psm.4 \
	pst.4 \
	pt.4 \
	ptnet.4 \
	pts.4 \
	pty.4 \
	puc.4 \
	pwmc.4 \
	${_qlxge.4} \
	${_qlxgb.4} \
	${_qlxgbe.4} \
	${_qlnxe.4} \
	ral.4 \
	random.4 \
	- rc.4 \
	rctl.4 \
	re.4 \
	rgephy.4 \
	rights.4 \
	rl.4 \
	rndtest.4 \
	route.4 \
	- rp.4 \
	rtwn.4 \
	rtwnfw.4 \
	rtwn_pci.4 \
	rue.4 \
	sa.4 \
	safe.4 \
	safexcel.4 \
	sbp.4 \
	sbp_targ.4 \
	scc.4 \
	sched_4bsd.4 \
	sched_ule.4 \
	screen.4 \
	scsi.4 \
	sctp.4 \
	sdhci.4 \
	sem.4 \
	send.4 \
	ses.4 \
	${_sfxge.4} \
	sge.4 \
	siba.4 \
	siftr.4 \
	siis.4 \
	simplebus.4 \
	sis.4 \
	sk.4 \
	${_smartpqi.4} \
	smb.4 \
	smbios.4 \
	smbus.4 \
	smp.4 \
	smsc.4 \
	snd_ad1816.4 \
	snd_als4000.4 \
	snd_atiixp.4 \
	snd_cmi.4 \
	snd_cs4281.4 \
	snd_csa.4 \
	snd_ds1.4 \
	snd_emu10k1.4 \
	snd_emu10kx.4 \
	snd_envy24.4 \
	snd_envy24ht.4 \
	snd_es137x.4 \
	snd_ess.4 \
	snd_fm801.4 \
	snd_gusc.4 \
	snd_hda.4 \
	snd_hdspe.4 \
	snd_ich.4 \
	snd_maestro3.4 \
	snd_maestro.4 \
	snd_mss.4 \
	snd_neomagic.4 \
	snd_sbc.4 \
	snd_solo.4 \
	snd_spicds.4 \
	snd_t4dwave.4 \
	snd_uaudio.4 \
	snd_via8233.4 \
	snd_via82c686.4 \
	snd_vibes.4 \
	snp.4 \
	spigen.4 \
	${_spkr.4} \
	splash.4 \
	sppp.4 \
	ste.4 \
	stf.4 \
	stge.4 \
	${_superio.4} \
	sym.4 \
	syncache.4 \
	syncer.4 \
	syscons.4 \
	sysmouse.4 \
	tap.4 \
	targ.4 \
	tcp.4 \
	tdfx.4 \
	terasic_mtl.4 \
	termios.4 \
	textdump.4 \
	ti.4 \
	timecounters.4 \
	${_tpm.4} \
	tty.4 \
	tun.4 \
	twa.4 \
	twe.4 \
	tws.4 \
	udp.4 \
	udplite.4 \
	ure.4 \
	vale.4 \
	vga.4 \
	vge.4 \
	viapm.4 \
	${_viawd.4} \
	${_virtio.4} \
	${_virtio_balloon.4} \
	${_virtio_blk.4} \
	${_virtio_console.4} \
	${_virtio_random.4} \
	${_virtio_scsi.4} \
	${_vmci.4} \
	vkbd.4 \
	vlan.4 \
	vxlan.4 \
	${_vmd.4} \
	${_vmm.4} \
	${_vmx.4} \
	vr.4 \
	vt.4 \
	vte.4 \
	${_vtnet.4} \
	watchdog.4 \
	${_wbwd.4} \
	wi.4 \
	witness.4 \
	wlan.4 \
	wlan_acl.4 \
	wlan_amrr.4 \
	wlan_ccmp.4 \
	wlan_tkip.4 \
	wlan_wep.4 \
	wlan_xauth.4 \
	wmt.4 \
	${_wpi.4} \
	wsp.4 \
	${_xen.4} \
	xhci.4 \
	xl.4 \
	${_xnb.4} \
	xpt.4 \
	zero.4

	MLINKS= ads111x.4 ads1013.4 \
	ads111x.4 ads1014.4 \
	ads111x.4 ads1015.4 \
	ads111x.4 ads1113.4 \
	ads111x.4 ads1114.4 \
	ads111x.4 ads1115.4
	MLINKS+=ae.4 if_ae.4
	MLINKS+=age.4 if_age.4
	MLINKS+=agp.4 agpgart.4
	MLINKS+=alc.4 if_alc.4
	MLINKS+=ale.4 if_ale.4
	MLINKS+=altera_atse.4 atse.4
	MLINKS+=altera_sdcard.4 altera_sdcardc.4
	MLINKS+=altq.4 ALTQ.4
	MLINKS+=ath.4 if_ath.4
	MLINKS+=ath_pci.4 if_ath_pci.4
	MLINKS+=an.4 if_an.4
	MLINKS+=aue.4 if_aue.4
	MLINKS+=axe.4 if_axe.4
	MLINKS+=bce.4 if_bce.4
	MLINKS+=bfe.4 if_bfe.4
	MLINKS+=bge.4 if_bge.4
	MLINKS+=bnxt.4 if_bnxt.4
	MLINKS+=bridge.4 if_bridge.4
	MLINKS+=bwi.4 if_bwi.4
	MLINKS+=bwn.4 if_bwn.4
	MLINKS+=${_bxe.4} ${_if_bxe.4}
	MLINKS+=cas.4 if_cas.4
	MLINKS+=cdce.4 if_cdce.4
	MLINKS+=cfi.4 cfid.4
	MLINKS+=cloudabi.4 cloudabi32.4 \
	cloudabi.4 cloudabi64.4
	MLINKS+=crypto.4 cryptodev.4
	MLINKS+=cue.4 if_cue.4
	MLINKS+=cxgb.4 if_cxgb.4
	MLINKS+=cxgbe.4 if_cxgbe.4 \
	cxgbe.4 vcxgbe.4 \
	cxgbe.4 if_vcxgbe.4 \
	cxgbe.4 cxl.4 \
	cxgbe.4 if_cxl.4 \
	cxgbe.4 vcxl.4 \
	cxgbe.4 if_vcxl.4 \
	cxgbe.4 cc.4 \
	cxgbe.4 if_cc.4 \
	cxgbe.4 vcc.4 \
	cxgbe.4 if_vcc.4
	MLINKS+=cxgbev.4 if_cxgbev.4 \
	cxgbev.4 cxlv.4 \
	cxgbev.4 if_cxlv.4 \
	cxgbev.4 ccv.4 \
	cxgbev.4 if_ccv.4
	MLINKS+=dc.4 if_dc.4
	MLINKS+=disc.4 if_disc.4
	MLINKS+=edsc.4 if_edsc.4
	MLINKS+=em.4 if_em.4 \
	em.4 igb.4 \
	em.4 if_igb.4
	MLINKS+=enc.4 if_enc.4
	MLINKS+=epair.4 if_epair.4
	MLINKS+=et.4 if_et.4
	MLINKS+=fd.4 stderr.4 \
	fd.4 stdin.4 \
	fd.4 stdout.4
	MLINKS+=fdt.4 FDT.4
	MLINKS+=firewire.4 ieee1394.4
	MLINKS+=fwe.4 if_fwe.4
	MLINKS+=fwip.4 if_fwip.4
	MLINKS+=fxp.4 if_fxp.4
	MLINKS+=gem.4 if_gem.4
	MLINKS+=geom.4 GEOM.4
	MLINKS+=gif.4 if_gif.4
	MLINKS+=gpio.4 gpiobus.4
	MLINKS+=gpioths.4 dht11.4
	MLINKS+=gpioths.4 dht22.4
	MLINKS+=gre.4 if_gre.4
	MLINKS+=hme.4 if_hme.4
	MLINKS+=hpet.4 acpi_hpet.4
	MLINKS+=${_hptrr.4} ${_rr232x.4}
	MLINKS+=${_attimer.4} ${_i8254.4}
	MLINKS+=ip.4 rawip.4
	MLINKS+=ipfirewall.4 ipaccounting.4 \
	ipfirewall.4 ipacct.4 \
	ipfirewall.4 ipfw.4
	MLINKS+=ipheth.4 if_ipheth.4
	MLINKS+=ipw.4 if_ipw.4
	MLINKS+=iwi.4 if_iwi.4
	MLINKS+=iwm.4 if_iwm.4
	MLINKS+=iwn.4 if_iwn.4
	MLINKS+=ixgbe.4 ix.4
	MLINKS+=ixgbe.4 if_ix.4
	MLINKS+=ixgbe.4 if_ixgbe.4
	MLINKS+=ixl.4 if_ixl.4
	MLINKS+=iavf.4 if_iavf.4
	MLINKS+=jme.4 if_jme.4
	MLINKS+=kue.4 if_kue.4
	MLINKS+=lagg.4 trunk.4
	MLINKS+=lagg.4 if_lagg.4
	MLINKS+=le.4 if_le.4
	MLINKS+=lge.4 if_lge.4
	MLINKS+=lo.4 loop.4
	MLINKS+=lp.4 plip.4
	MLINKS+=malo.4 if_malo.4
	MLINKS+=md.4 vn.4
	MLINKS+=mem.4 kmem.4
	MLINKS+=mfi.4 mfi_linux.4 \
	mfi.4 mfip.4
	MLINKS+=mlx5en.4 mce.4
	MLINKS+=mn.4 if_mn.4
	MLINKS+=mos.4 if_mos.4
	MLINKS+=msk.4 if_msk.4
	MLINKS+=mwl.4 if_mwl.4
	MLINKS+=mxge.4 if_mxge.4
	MLINKS+=my.4 if_my.4
	MLINKS+=${_ndis.4} ${_if_ndis.4}
	MLINKS+=netfpga10g_nf10bmac.4 if_nf10bmac.4
	MLINKS+=netintro.4 net.4 \
	netintro.4 networking.4
	MLINKS+=${_nfe.4} ${_if_nfe.4}
	MLINKS+=nge.4 if_nge.4
	MLINKS+=ow.4 onewire.4
	MLINKS+=pccbb.4 cbb.4
	MLINKS+=pcm.4 snd.4 \
	pcm.4 sound.4
	MLINKS+=pms.4 pmspcv.4
	MLINKS+=ptnet.4 if_ptnet.4
	MLINKS+=ral.4 if_ral.4
	MLINKS+=re.4 if_re.4
	MLINKS+=rl.4 if_rl.4
	MLINKS+=rtwn_pci.4 if_rtwn_pci.4
	MLINKS+=rue.4 if_rue.4
	MLINKS+=scsi.4 CAM.4 \
	scsi.4 cam.4 \
	scsi.4 scbus.4 \
	scsi.4 SCSI.4
	MLINKS+=sge.4 if_sge.4
	MLINKS+=sis.4 if_sis.4
	MLINKS+=sk.4 if_sk.4
	MLINKS+=smp.4 SMP.4
	MLINKS+=smsc.4 if_smsc.4
	MLINKS+=snd_envy24.4 snd_ak452x.4
	MLINKS+=snd_sbc.4 snd_sb16.4 \
	snd_sbc.4 snd_sb8.4
	MLINKS+=${_spkr.4} ${_speaker.4}
	MLINKS+=splash.4 screensaver.4
	MLINKS+=ste.4 if_ste.4
	MLINKS+=stf.4 if_stf.4
	MLINKS+=stge.4 if_stge.4
	MLINKS+=syncache.4 syncookies.4
	MLINKS+=syscons.4 sc.4
	MLINKS+=tap.4 if_tap.4 \
	tap.4 vmnet.4 \
	tap.4 if_vmnet.4
	MLINKS+=tdfx.4 tdfx_linux.4
	MLINKS+=ti.4 if_ti.4
	MLINKS+=tun.4 if_tun.4
	MLINKS+=ure.4 if_ure.4
	MLINKS+=vge.4 if_vge.4
	MLINKS+=vlan.4 if_vlan.4
	MLINKS+=vxlan.4 if_vxlan.4
	MLINKS+=${_vmx.4} ${_if_vmx.4}
	MLINKS+=vr.4 if_vr.4
	MLINKS+=vte.4 if_vte.4
	MLINKS+=${_vtnet.4} ${_if_vtnet.4}
	MLINKS+=watchdog.4 SW_WATCHDOG.4
	MLINKS+=wi.4 if_wi.4
	MLINKS+=${_wpi.4} ${_if_wpi.4}
	MLINKS+=xl.4 if_xl.4

	.if ${MACHINE_CPUARCH} == "amd64" \|\| ${MACHINE_CPUARCH} == "i386"
	_acpi_asus.4= acpi_asus.4
	_acpi_asus_wmi.4= acpi_asus_wmi.4
	_acpi_dock.4= acpi_dock.4
	_acpi_fujitsu.4=acpi_fujitsu.4
	_acpi_hp.4= acpi_hp.4
	_acpi_ibm.4= acpi_ibm.4
	_acpi_panasonic.4=acpi_panasonic.4
	_acpi_rapidstart.4=acpi_rapidstart.4
	_acpi_sony.4= acpi_sony.4
	_acpi_toshiba.4=acpi_toshiba.4
	_acpi_wmi.4= acpi_wmi.4
	_aesni.4= aesni.4
	_aout.4= aout.4
	_apic.4= apic.4
	_atrtc.4= atrtc.4
	_attimer.4= attimer.4
	_aibs.4= aibs.4
	_amdsbwd.4= amdsbwd.4
	_amdsmb.4= amdsmb.4
	_amdsmn.4= amdsmn.4
	_amdtemp.4= amdtemp.4
	_asmc.4= asmc.4
	_bxe.4= bxe.4
	_bytgpio.4= bytgpio.4
	_chvgpio.4= chvgpio.4
	_coretemp.4= coretemp.4
	_cpuctl.4= cpuctl.4
	_dpms.4= dpms.4
	_hpt27xx.4= hpt27xx.4
	_hptiop.4= hptiop.4
	_hptmv.4= hptmv.4
	_hptnr.4= hptnr.4
	_hptrr.4= hptrr.4
	_hv_kvp.4= hv_kvp.4
	_hv_netvsc.4= hv_netvsc.4
	_hv_storvsc.4= hv_storvsc.4
	_hv_utils.4= hv_utils.4
	_hv_vmbus.4= hv_vmbus.4
	_hv_vss.4= hv_vss.4
	_hwpstate_intel.4= hwpstate_intel.4
	_i8254.4= i8254.4
	_ichwd.4= ichwd.4
	_if_bxe.4= if_bxe.4
	_if_ndis.4= if_ndis.4
	_if_nfe.4= if_nfe.4
	_if_urtw.4= if_urtw.4
	_if_vmx.4= if_vmx.4
	_if_vtnet.4= if_vtnet.4
	_if_wpi.4= if_wpi.4
	_imcsmb.4= imcsmb.4
	_ipmi.4= ipmi.4
	_io.4= io.4
	_itwd.4= itwd.4
	_linux.4= linux.4
	_nda.4= nda.4
	_ndis.4= ndis.4
	_nfe.4= nfe.4
	_nfsmb.4= nfsmb.4
	_if_ntb.4= if_ntb.4
	_ntb.4= ntb.4
	_ntb_hw_amd.4= ntb_hw_amd.4
	_ntb_hw_intel.4= ntb_hw_intel.4
	_ntb_hw_plx.4= ntb_hw_plx.4
	_ntb_transport.4=ntb_transport.4
	_nvd.4= nvd.4
	_nvme.4= nvme.4
	_nvram.4= nvram.4
	_padlock.4= padlock.4
	_pchtherm.4= pchtherm.4
	_rr232x.4= rr232x.4
	_speaker.4= speaker.4
	_spkr.4= spkr.4
	_superio.4= superio.4
	_tpm.4= tpm.4
	_urtw.4= urtw.4
	_viawd.4= viawd.4
	_virtio.4= virtio.4
	_virtio_balloon.4=virtio_balloon.4
	_virtio_blk.4= virtio_blk.4
	_virtio_console.4=virtio_console.4
	_virtio_random.4= virtio_random.4
	_virtio_scsi.4= virtio_scsi.4
	_vmci.4= vmci.4
	_vmx.4= vmx.4
	_vtnet.4= vtnet.4
	_wbwd.4= wbwd.4
	_wpi.4= wpi.4
	_xen.4= xen.4
	_xnb.4= xnb.4

	.endif

	.if ${MACHINE_CPUARCH} == "amd64"
	_ioat.4= ioat.4
	_nvdimm.4= nvdimm.4
	_qlxge.4= qlxge.4
	_qlxgb.4= qlxgb.4
	_qlxgbe.4= qlxgbe.4
	_qlnxe.4= qlnxe.4
	_sfxge.4= sfxge.4
	_smartpqi.4= smartpqi.4
	_vmd.4= vmd.4

	MLINKS+=qlxge.4 if_qlxge.4
	MLINKS+=qlxgb.4 if_qlxgb.4
	MLINKS+=qlxgbe.4 if_qlxgbe.4
	MLINKS+=qlnxe.4 if_qlnxe.4
	MLINKS+=sfxge.4 if_sfxge.4

	.if ${MK_BHYVE} != "no"
	_bhyve.4= bhyve.4
	_vmm.4= vmm.4
	.endif
	.endif

	.if ${MACHINE_CPUARCH} == "mips"
	_nvram2env.4= nvram2env.4
	.endif

	.if ${MACHINE_CPUARCH} == "powerpc"
	_if_vtnet.4= if_vtnet.4
	_nvd.4= nvd.4
	_nvme.4= nvme.4
	_virtio.4= virtio.4
	_virtio_balloon.4=virtio_balloon.4
	_virtio_blk.4= virtio_blk.4
	_virtio_console.4=virtio_console.4
	_virtio_random.4= virtio_random.4
	_virtio_scsi.4= virtio_scsi.4
	_vtnet.4= vtnet.4
	.endif

	.if empty(MAN_ARCH)
	__arches= ${MACHINE} ${MACHINE_ARCH} ${MACHINE_CPUARCH}
	.elif ${MAN_ARCH} == "all"
	__arches= ${:!/bin/sh -c "/bin/ls -d ${.CURDIR}/man4.*"!:E}
	.else
	__arches= ${MAN_ARCH}
	.endif
	.for __arch in ${__arches:O:u}
	.if exists(${.CURDIR}/man4.${__arch})
	SUBDIR+= man4.${__arch}
	.endif
	.endfor

	.if ${MK_BLUETOOTH} != "no"
	MAN+= ng_bluetooth.4
	.endif

	.if ${MK_CCD} != "no"
	_ccd.4= ccd.4
	.endif

	.if ${MK_CDDL} != "no"
	_dtrace_provs= dtrace_audit.4 \
	dtrace_io.4 \
	dtrace_ip.4 \
	dtrace_lockstat.4 \
	dtrace_proc.4 \
	dtrace_sched.4 \
	dtrace_sctp.4 \
	dtrace_tcp.4 \
	dtrace_udp.4 \
	dtrace_udplite.4

	MLINKS+= dtrace_audit.4 dtaudit.4
	.endif

	.if ${MK_EFI} != "no"
	MAN+= efidev.4

	MLINKS+= efidev.4 efirtc.4
	.endif

	.if ${MK_ISCSI} != "no"
	MAN+= cfiscsi.4
	MAN+= iscsi.4
	MAN+= iscsi_initiator.4
	MAN+= iser.4
	.endif

	.if ${MK_OFED} != "no"
	MAN+= mlx4ib.4
	MAN+= mlx5ib.4
	.endif

	.if ${MK_MLX5TOOL} != "no"
	MAN+= mlx5io.4
	.endif

	.if ${MK_TESTS} != "no"
	ATF= ${SRCTOP}/contrib/atf
	.PATH: ${ATF}/doc
	_atf_test_case.4= atf-test-case.4
	.endif

	.if ${MK_PF} != "no"
	_pf.4= pf.4
	_pflog.4= pflog.4
	_pfsync.4= pfsync.4
	.endif

	.if ${MK_USB} != "no"
	MAN+= \
	otus.4 \
	otusfw.4 \
	rsu.4 \
	rsufw.4 \
	rtwn_usb.4 \
	rum.4 \
	run.4 \
	runfw.4 \
	u3g.4 \
	uark.4 \
	uart.4 \
	uath.4 \
	ubsa.4 \
	ubser.4 \
	ubtbcmfw.4 \
	uchcom.4 \
	ucom.4 \
	ucycom.4 \
	udav.4 \
	udbp.4 \
	udl.4 \
	uep.4 \
	ufm.4 \
	ufoma.4 \
	uftdi.4 \
	ugen.4 \
	ugold.4 \
	uhci.4 \
	uhid.4 \
	uhso.4 \
	uipaq.4 \
	ukbd.4 \
	uled.4 \
	ulpt.4 \
	umass.4 \
	umcs.4 \
	umct.4 \
	umodem.4 \
	umoscom.4 \
	ums.4 \
	unix.4 \
	upgt.4 \
	uplcom.4 \
	ural.4 \
	urio.4 \
	urndis.4 \
	${_urtw.4} \
	usb.4 \
	usb_quirk.4 \
	usb_template.4 \
	usfs.4 \
	uslcom.4 \
	uvisor.4 \
	uvscom.4 \
	zyd.4

	MLINKS+=otus.4 if_otus.4
	MLINKS+=rsu.4 if_rsu.4
	MLINKS+=rtwn_usb.4 if_rtwn_usb.4
	MLINKS+=rum.4 if_rum.4
	MLINKS+=run.4 if_run.4
	MLINKS+=u3g.4 u3gstub.4
	MLINKS+=uath.4 if_uath.4
	MLINKS+=udav.4 if_udav.4
	MLINKS+=upgt.4 if_upgt.4
	MLINKS+=ural.4 if_ural.4
	MLINKS+=urndis.4 if_urndis.4
	MLINKS+=${_urtw.4} ${_if_urtw.4}
	MLINKS+=zyd.4 if_zyd.4
	.endif

	.include <bsd.prog.mk>
	diff --git a/share/man/man4/aesni.4 b/share/man/man4/aesni.4
	index aacbe79cc337..08465bdf5e88 100644
	--- a/share/man/man4/aesni.4
	+++ b/share/man/man4/aesni.4
	@@ -1,111 +1,112 @@
	.\" Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd September 26, 2017
	+.Dd July 29, 2020
	.Dt AESNI 4
	.Os
	.Sh NAME
	.Nm aesni
	.Nd "driver for the AES and SHA accelerator on x86 CPUs"
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device cryptodev"
	.Cd "device aesni"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	aesni_load="YES"
	.Ed
	.Sh DESCRIPTION
	Starting with Intel Westmere and AMD Bulldozer, some x86 processors implement a
	new set of instructions called AESNI.
	The set of six instructions accelerates the calculation of the key
	schedule for key lengths of 128, 192, and 256 of the Advanced
	Encryption Standard (AES) symmetric cipher, and provides a hardware
	implementation of the regular and the last encryption and decryption
	rounds.
	.Pp
	The processor capability is reported as AESNI in the Features2 line at boot.
	.Pp
	Starting with the Intel Goldmont and AMD Ryzen microarchitectures, some x86
	processors implement a new set of SHA instructions.
	The set of seven instructions accelerates the calculation of SHA1 and SHA256
	hashes.
	.Pp
	The processor capability is reported as SHA in the Structured Extended Features
	line at boot.
	.Pp
	The
	.Nm
	driver does not attach on systems that lack both CPU capabilities.
	On systems that support only one of AESNI or SHA extensions, the driver will
	attach and support that one function.
	.Pp
	The
	.Nm
	driver registers itself to accelerate AES and SHA operations for
	.Xr crypto 4 .
	Besides speed, the advantage of using the
	.Nm
	driver is that the AESNI operation
	is data-independent, thus eliminating some attack vectors based on
	measuring cache use and timings typically present in table-driven
	implementations.
	.Sh SEE ALSO
	.Xr crypt 3 ,
	.Xr crypto 4 ,
	.Xr intro 4 ,
	.Xr ipsec 4 ,
	.Xr padlock 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr crypto 9
	.Sh HISTORY
	The
	.Nm
	driver first appeared in
	.Fx 9.0 .
	SHA support was added in
	.Fx 12.0 .
	.Sh AUTHORS
	.An -nosplit
	The
	.Nm
	driver was written by
	.An Konstantin Belousov Aq Mt kib@FreeBSD.org
	and
	.An Conrad Meyer Aq Mt cem@FreeBSD.org .
	The key schedule calculation code was adopted from the sample provided
	by Intel and used in the analogous
	.Ox
	driver.
	The hash step intrinsics implementations were supplied by Intel.
	diff --git a/share/man/man4/cy.4 b/share/man/man4/cy.4
	deleted file mode 100644
	index c0c807286f1e..000000000000
	--- a/share/man/man4/cy.4
	+++ /dev/null
	@@ -1,257 +0,0 @@
	-.\" Copyright (c) 1990, 1991 The Regents of the University of California.
	-.\" All rights reserved.
	-.\"
	-.\" This code is derived from software contributed to Berkeley by
	-.\" the Systems Programming Group of the University of Utah Computer
	-.\" Science Department.
	-.\" Redistribution and use in source and binary forms, with or without
	-.\" modification, are permitted provided that the following conditions
	-.\" are met:
	-.\" 1. Redistributions of source code must retain the above copyright
	-.\" notice, this list of conditions and the following disclaimer.
	-.\" 2. Redistributions in binary form must reproduce the above copyright
	-.\" notice, this list of conditions and the following disclaimer in the
	-.\" documentation and/or other materials provided with the distribution.
	-.\" 3. Neither the name of the University nor the names of its contributors
	-.\" may be used to endorse or promote products derived from this software
	-.\" without specific prior written permission.
	-.\"
	-.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	-.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	-.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	-.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	-.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	-.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	-.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	-.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	-.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	-.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	-.\" SUCH DAMAGE.
	-.\"
	-.\" from: @(#)dca.4 5.2 (Berkeley) 3/27/91
	-.\" from: com.4,v 1.1 1993/08/06 11:19:07 cgd Exp
	-.\" from: sio.4,v 1.16 1995/06/26 06:05:30 bde Exp $
	-.\" $FreeBSD$
	-.\"
	-.Dd May 24, 2004
	-.Dt CY 4
	-.Os
	-.Sh NAME
	-.Nm cy
	-.Nd Cyclades Cyclom-Y serial driver
	-.Sh SYNOPSIS
	-For one ISA card:
	-.Bd -ragged -offset indent -compact
	-.Cd "device cy"
	-.Pp
	-In
	-.Pa /boot/device.hints :
	-.Cd hint.cy.0.at="isa"
	-.Cd hint.cy.0.irq="10"
	-.Cd hint.cy.0.maddr="0xd4000"
	-.Cd hint.cy.0.msize="0x2000"
	-.Ed
	-.Pp
	-For two ISA cards:
	-.Bd -ragged -offset indent -compact
	-.Cd "device cy"
	-.Pp
	-In
	-.Pa /boot/device.hints :
	-.Cd hint.cy.0.at="isa"
	-.Cd hint.cy.0.irq="10"
	-.Cd hint.cy.0.maddr="0xd4000"
	-.Cd hint.cy.0.msize="0x2000"
	-.Cd hint.cy.1.at="isa"
	-.Cd hint.cy.1.irq="11"
	-.Cd hint.cy.1.maddr="0xd6000"
	-.Cd hint.cy.1.msize="0x2000"
	-.Ed
	-.Pp
	-For PCI cards:
	-.Bd -ragged -offset indent -compact
	-.Cd "device cy"
	-.Cd "options CY_PCI_FASTINTR"
	-.Pp
	-No lines are required in
	-.Pa /boot/device.hints
	-for PCI cards.
	-.Ed
	-.Pp
	-Minor numbering:
	-.Bd -literal -offset indent -compact
	-0b\fIMMMMMMMMMMMMMMMMxxxxxxxxOLIMMMMM\fR
	- call\fBO\fRut
	- \fBL\fRock
	- \fBI\fRnitial
	- \fBMMMMMMMMMMMMMMMM MMMMMM\fRinor
	-.Ed
	-.Sh DESCRIPTION
	-The
	-.Nm
	-driver provides support for Cirrus Logic CD1400-based
	-.Tn EIA
	-.Tn RS-232C
	-.Pf ( Tn CCITT
	-.Tn V.24 )
	-communications interfaces (ports) on Cyclades Cyclom-Y boards.
	-Each CD1400 provides 4 ports.
	-Cyclom-Y boards with various numbers of CD1400's are available.
	-This driver supports up to 8 CD1400's (32 ports) per board.
	-.Pp
	-Input and output for each line may set independently
	-to the following speeds:
	-50, 75, 110, 134.5, 150, 300, 600, 1200, 1800, 2400, 4800, 9600,
	-19200, 38400, 57600, or 115200 bps.
	-Other speeds of up to 150000 are supported by the termios interface
	-but not by the sgttyb compatibility interface.
	-The CD1400 is not fast enough to handle speeds above 115200 bps
	-effectively.
	-It can transmit on a single line at slightly more than 115200 bps,
	-but when 4 lines are active in both directions its limit is about
	-90000 bps on each line.
	-.\" XXX the following should be true for all serial drivers and
	-.\" should not be repeated in the man pages for all serial drivers.
	-.\" It was copied from sio.4. The only change was s/sio/cy/g.
	-.Pp
	-Serial ports controlled by the
	-.Nm
	-driver can be used for both `callin' and `callout'.
	-For each port there is a callin device and a callout device.
	-The minor number of the callout device is 128 higher
	-than that of the corresponding callin port.
	-The callin device is general purpose.
	-Processes opening it normally wait for carrier
	-and for the callout device to become inactive.
	-The callout device is used to steal the port from
	-processes waiting for carrier on the callin device.
	-Processes opening it do not wait for carrier
	-and put any processes waiting for carrier on the callin device into
	-a deeper sleep so that they do not conflict with the callout session.
	-The callout device is abused for handling programs that are supposed
	-to work on general ports and need to open the port without waiting
	-but are too stupid to do so.
	-.Pp
	-The
	-.Nm
	-driver also supports an initial-state and a lock-state control
	-device for each of the callin and the callout "data" devices.
	-The minor number of the initial-state device is 32 higher
	-than that of the corresponding data device.
	-The minor number of the lock-state device is 64 higher
	-than that of the corresponding data device.
	-The termios settings of a data device are copied
	-from those of the corresponding initial-state device
	-on first opens and are not inherited from previous opens.
	-Use
	-.Xr stty 1
	-in the normal way on the initial-state devices to program
	-initial termios states suitable for your setup.
	-.Pp
	-The lock termios state acts as flags to disable changing
	-the termios state.
	-E.g., to lock a flag variable such as
	-CRTSCTS, use
	-.Em "stty crtscts"
	-on the lock-state device.
	-Speeds and special characters
	-may be locked by setting the corresponding value in the lock-state
	-device to any nonzero value.
	-.Pp
	-Correct programs talking to correctly wired external devices
	-work with almost arbitrary initial states and almost no locking,
	-but other setups may benefit from changing some of the default
	-initial state and locking the state.
	-In particular, the initial states for non (POSIX) standard flags
	-should be set to suit the devices attached and may need to be
	-locked to prevent buggy programs from changing them.
	-E.g., CRTSCTS should be locked on for devices that support
	-RTS/CTS handshaking at all times and off for devices that do not
	-support it at all.
	-CLOCAL should be locked on for devices
	-that do not support carrier.
	-HUPCL may be locked off if you do not
	-want to hang up for some reason.
	-In general, very bad things happen
	-if something is locked to the wrong state, and things should not
	-be locked for devices that support more than one setting.
	-The
	-CLOCAL flag on callin ports should be locked off for logins
	-to avoid certain security holes, but this needs to be done by
	-getty if the callin port is used for anything else.
	-.Ss Kernel Configuration Options
	-The
	-.Em CY_PCI_FASTINTR
	-option should be used to avoid suboptimal interrupt handling for
	-PCI Cyclades boards.
	-The PCI BIOS must be configured with the
	-.Nm
	-interrupt not shared with any other active device
	-for this option to work.
	-This option is not the default because it is currently harmful in
	-certain cases where it does not work.
	-.Sh FILES
	-.\" XXX more cloning: s/d/c/g and add a ? for the card number.
	-.Bl -tag -width /dev/ttyic?? -compact
	-.It Pa /dev/ttyc??
	-for callin ports
	-.It Pa /dev/ttyic??
	-.It Pa /dev/ttylc??
	-corresponding callin initial-state and lock-state devices
	-.Pp
	-.\" XXX more cloning: s/a/c/g. No consistency :-(.
	-.It Pa /dev/cuac??
	-for callout ports
	-.It Pa /dev/cuaic??
	-.It Pa /dev/cualc??
	-corresponding callout initial-state and lock-state devices
	-.El
	-.Pp
	-.Bl -tag -width /etc/rc.serial -compact
	-.It Pa /etc/rc.serial
	-examples of setting the initial-state and lock-state devices
	-.El
	-.Pp
	-The first question mark in these device names is short for the
	-card number
	-(a decimal number between 0 and 65535 inclusive).
	-The second question mark is short for the port number
	-(a letter in the range [0-9a-v]).
	-.Sh DIAGNOSTICS
	-.Bl -diag
	-.\" XXX back to s/sio/cy/g.
	-.It cy%d: silo overflow.
	-Problem in the interrupt handler.
	-.El
	-.Bl -diag
	-.It cy%d: interrupt-level buffer overflow.
	-Problem in the bottom half of the driver.
	-.El
	-.Bl -diag
	-.It cy%d: tty-level buffer overflow.
	-Problem in the application.
	-Input has arrived faster than the given module could process it
	-and some has been lost.
	-.El
	-.\" .Bl -diag
	-.\" .It sio%d: reduced fifo trigger level to %d.
	-.\" Attempting to avoid further silo overflows.
	-.\" .El
	-.Sh SEE ALSO
	-.Xr stty 1 ,
	-.Xr termios 4 ,
	-.Xr tty 4 ,
	-.Xr comcontrol 8 ,
	-.Xr pstat 8
	-.Sh HISTORY
	-The
	-.Nm
	-driver is derived from the
	-.Nm sio
	-driver and the
	-.Nx
	-.Nm
	-driver and is
	-.Ud
	-.Sh BUGS
	-Serial consoles are not implemented.
	diff --git a/share/man/man4/hifn.4 b/share/man/man4/hifn.4
	index f9952ac153ad..a9a06c1fbd91 100644
	--- a/share/man/man4/hifn.4
	+++ b/share/man/man4/hifn.4
	@@ -1,134 +1,134 @@
	.\" $OpenBSD: hifn.4,v 1.32 2002/09/26 07:55:40 miod Exp $
	.\"
	.\" Copyright (c) 2000 Theo de Raadt
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	.\" DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
	.\" INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	.\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	.\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	.\" ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	.\" POSSIBILITY OF SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd May 11, 2020
	+.Dd July 29, 2020
	.Dt HIFN 4
	.Os
	.Sh NAME
	.Nm hifn
	.Nd Hifn 7751/7951/7811/7955/7956 crypto accelerator
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device cryptodev"
	.Cd "device hifn"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	hifn_load="YES"
	.Ed
	.Sh DESCRIPTION
	The
	.Nm
	driver supports various cards containing the Hifn 7751, 7951,
	7811, 7955, and 7956 chipsets.
	.Pp
	The
	.Nm
	driver registers itself to accelerate
	AES (7955 and 7956 only),
	SHA1, and SHA1-HMAC operations for
	.Xr ipsec 4
	and
	.Xr crypto 4 .
	.Pp
	The Hifn
	.Tn 7951 ,
	.Tn 7811 ,
	.Tn 7955 ,
	and
	.Tn 7956
	will also supply data to the kernel
	.Xr random 4
	subsystem.
	.Sh HARDWARE
	The
	.Nm
	driver supports various cards containing the Hifn 7751, 7951,
	7811, 7955, and 7956
	chipsets, such as:
	.Bl -tag -width namenamenamena -offset indent
	.It Invertex AEON
	No longer being made.
	Came as 128KB SRAM model, or 2MB DRAM model.
	.It Hifn 7751
	Reference board with 512KB SRAM.
	.It PowerCrypt
	Comes with 512KB SRAM.
	.It XL-Crypt
	Only board based on 7811 (which is faster than 7751 and has
	a random number generator).
	.It NetSec 7751
	Supports the most IPsec sessions, with 1MB SRAM.
	.It Soekris Engineering vpn1201 and vpn1211
	See
	.Pa http://www.soekris.com/ .
	Contains a 7951 and supports symmetric and random number operations.
	.It Soekris Engineering vpn1401 and vpn1411
	See
	.Pa http://www.soekris.com/ .
	Contains a 7955 and supports symmetric and random number operations.
	.El
	.Sh SEE ALSO
	-.Xr crypt 3 ,
	.Xr crypto 4 ,
	.Xr intro 4 ,
	.Xr ipsec 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr crypto 9
	.Sh HISTORY
	The
	.Nm
	device driver appeared in
	.Ox 2.7 .
	The
	.Nm
	device driver was imported to
	.Fx 5.0 .
	.Sh CAVEATS
	The Hifn 9751 shares the same PCI ID.
	This chip is basically a 7751, but with the cryptographic functions missing.
	Instead, the 9751 is only capable of doing compression.
	Since we do not currently attempt to use any of these chips to do
	compression, the 9751-based cards are not useful.
	.Pp
	Support for the 7955 and 7956 is incomplete; the asymmetric crypto
	facilities are to be added and the performance is suboptimal.
	.Sh BUGS
	The 7751 chip starts out at initialization by only supporting compression.
	A proprietary algorithm, which has been reverse engineered, is required to
	unlock the cryptographic functionality of the chip.
	It is possible for vendors to make boards which have a lock ID not known
	to the driver, but all vendors currently just use the obvious ID which is
	13 bytes of 0.
	diff --git a/share/man/man4/man4.aarch64/armv8crypto.4 b/share/man/man4/man4.aarch64/armv8crypto.4
	index 5b91049d404e..a80b0801d722 100644
	--- a/share/man/man4/man4.aarch64/armv8crypto.4
	+++ b/share/man/man4/man4.aarch64/armv8crypto.4
	@@ -1,83 +1,84 @@
	.\" Copyright (c) 2016 The FreeBSD Foundation
	.\" All rights reserved.
	.\"
	.\" This software was developed by Andrew Turner under
	.\" sponsorship from the FreeBSD Foundation.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd October 20, 2016
	+.Dd July 29, 2020
	.Dt ARMV8CRYPTO 4
	.Os
	.Sh NAME
	.Nm armv8crypto
	.Nd "driver for the AES accelerator on ARM CPUs"
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device armv8crypto"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	armv8crypto_load="YES"
	.Ed
	.Sh DESCRIPTION
	Starting with the ARMv8 architecture ARM Limited has added optional
	cryptography instructions to accelerate AES, SHA-1, SHA-2, and
	finite field arithmetic.
	.Pp
	The processor capability is reported as AES in the Instruction Set
	Attributes 0 line at boot.
	The
	.Nm
	driver does not attach on systems that lack the required CPU capability.
	.Pp
	The
	.Nm
	driver registers itself to accelerate AES operations for
	.Xr crypto 4 .
	.Sh SEE ALSO
	.Xr crypt 3 ,
	.Xr crypto 4 ,
	.Xr intro 4 ,
	.Xr ipsec 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr crypto 9
	.Sh HISTORY
	The
	.Nm
	driver first appeared in
	.Fx 11.0 .
	.Sh AUTHORS
	.An -nosplit
	The
	.Nm
	driver was written by
	.An Andrew Turner Aq Mt andrew@FreeBSD.org .
	diff --git a/share/man/man4/man4.i386/glxsb.4 b/share/man/man4/man4.i386/glxsb.4
	index d2a30352fe7f..c08d57906136 100644
	--- a/share/man/man4/man4.i386/glxsb.4
	+++ b/share/man/man4/man4.i386/glxsb.4
	@@ -1,97 +1,98 @@
	.\" $OpenBSD: glxsb.4,v 1.5 2007/05/31 19:19:54 jmc Exp $
	.\"
	.\"Copyright (c) 2006 Tom Cosgrove <tom@openbsd.org>
	.\"
	.\"Permission to use, copy, modify, and distribute this software for any
	.\"purpose with or without fee is hereby granted, provided that the above
	.\"copyright notice and this permission notice appear in all copies.
	.\"
	.\"THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
	.\"WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
	.\"MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
	.\"ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	.\"WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
	.\"ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
	.\"OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd June 8, 2008
	+.Dd July 29, 2020
	.Dt GLXSB 4 i386
	.Os
	.Sh NAME
	.Nm glxsb
	.Nd Geode LX Security Block crypto accelerator
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device glxsb"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	glxsb_load="YES"
	.Ed
	.Sh DESCRIPTION
	The
	.Nm
	driver supports the security block of the Geode LX series processors.
	The Geode LX is a member of the AMD Geode family
	of integrated x86 system chips.
	.Pp
	Driven by periodic checks for available data from the generator,
	.Nm
	supplies entropy to the
	.Xr random 4
	driver for common usage.
	.Pp
	.Nm
	also supports acceleration of AES-128-CBC operations for
	.Xr crypto 4 .
	It also registers itself to accelerate other HMAC algorithms, although
	there is no hardware acceleration for those algorithms.
	This is only needed so
	.Nm
	can work with
	.Xr ipsec 4 .
	.Sh CAVEAT
	The
	.Xr crypto 9
	framework will fail to open the crypto session on the device if the AES
	key's length is != 128 bits.
	This prevents the use of the
	.Nm
	device driver with AES keys of length != 128 bits.
	.Sh SEE ALSO
	.Xr crypto 4 ,
	.Xr intro 4 ,
	.Xr ipsec 4 ,
	.Xr pci 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr crypto 9
	.Sh HISTORY
	The
	.Nm
	device driver first appeared in
	.Ox 4.1 .
	The
	.Nm
	device driver was imported into
	.Fx 7.1 .
	.Sh AUTHORS
	.An -nosplit
	The
	.Nm
	device driver was written for
	.Ox
	by
	.An Tom Cosgrove .
	It was ported to
	.Fx
	by
	.An Patrick Lamaiziere Aq Mt patfbsd@davenulle.org .
	diff --git a/share/man/man4/ng_iface.4 b/share/man/man4/ng_iface.4
	index 0b406cdd298f..2259ea1076d3 100644
	--- a/share/man/man4/ng_iface.4
	+++ b/share/man/man4/ng_iface.4
	@@ -1,172 +1,166 @@
	.\" Copyright (c) 1996-1999 Whistle Communications, Inc.
	.\" All rights reserved.
	.\"
	.\" Subject to the following obligations and disclaimer of warranty, use and
	.\" redistribution of this software, in source or object code forms, with or
	.\" without modifications are expressly permitted by Whistle Communications;
	.\" provided, however, that:
	.\" 1. Any and all reproductions of the source or object code must include the
	.\" copyright notice above and the following disclaimer of warranties; and
	.\" 2. No rights are granted, in any manner or form, to use Whistle
	.\" Communications, Inc. trademarks, including the mark "WHISTLE
	.\" COMMUNICATIONS" on advertising, endorsements, or otherwise except as
	.\" such appears in the above copyright notice or in the software.
	.\"
	.\" THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
	.\" TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
	.\" REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
	.\" INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
	.\" MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
	.\" WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
	.\" REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
	.\" SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
	.\" IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
	.\" RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
	.\" WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
	.\" PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
	.\" SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
	.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	.\" THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
	.\" OF SUCH DAMAGE.
	.\"
	.\" Author: Archie Cobbs <archie@FreeBSD.org>
	.\"
	.\" $FreeBSD$
	.\" $Whistle: ng_iface.8,v 1.5 1999/01/25 23:46:26 archie Exp $
	.\"
	-.Dd February 6, 2019
	+.Dd July 31, 2020
	.Dt NG_IFACE 4
	.Os
	.Sh NAME
	.Nm ng_iface
	.Nd interface netgraph node type
	.Sh SYNOPSIS
	.In netgraph/ng_iface.h
	.Sh DESCRIPTION
	An
	.Nm iface
	node is both a netgraph node and a system networking interface.
	When an
	.Nm iface
	node is created, a new interface appears which is accessible via
	.Xr ifconfig 8 .
	.Nm Iface
	node interfaces are named
	.Dv ng0 ,
	.Dv ng1 ,
	etc.
	When a node is shutdown, the corresponding interface is removed
	and the interface name becomes available for reuse by future
	.Nm iface
	nodes; new nodes always take the first unused interface.
	The node itself is assigned the same name as its interface, unless the name
	already exists, in which case the node remains unnamed.
	.Pp
	An
	.Nm iface
	node has a single hook corresponding to each supported protocol.
	Packets transmitted via the interface flow out the corresponding
	protocol-specific hook.
	Similarly, packets received on a hook appear on the interface as
	packets received into the corresponding protocol stack.
	-The currently supported protocols are IP, IPv6, ATM, NATM, and NS.
	+The currently supported protocols are IP and IPv6.
	.Pp
	An
	.Nm iface
	node can be configured as a point-to-point interface or a broadcast interface.
	The configuration can only be changed when the interface is down.
	The default mode is point-to-point.
	.Pp
	.Nm Iface
	nodes support the Berkeley Packet Filter (BPF).
	.Sh HOOKS
	This node type supports the following hooks:
	.Bl -tag -width ".Va inet6"
	.It Va inet
	Transmission and reception of IP packets.
	.It Va inet6
	Transmission and reception of IPv6 packets.
	-.It Va atm
	-Transmission and reception of ATM packets.
	-.It Va natm
	-Transmission and reception of NATM packets.
	-.It Va ns
	-Transmission and reception of NS packets.
	.El
	.Sh CONTROL MESSAGES
	This node type supports the generic control messages, plus the following:
	.Bl -tag -width foo
	.It Dv NGM_IFACE_GET_IFNAME Pq Ic getifname
	Returns the name of the associated interface as a
	.Dv NUL Ns -terminated
	.Tn ASCII
	string.
	Normally this is the same as the name of the node.
	.It Dv NGM_IFACE_GET_IFINDEX Pq Ic getifindex
	Returns the global index of the associated interface as a 32 bit integer.
	.It Dv NGM_IFACE_POINT2POINT Pq Ic point2point
	Set the interface to point-to-point mode.
	The interface must not currently be up.
	.It Dv NGM_IFACE_BROADCAST Pq Ic broadcast
	Set the interface to broadcast mode.
	The interface must not currently be up.
	.El
	.Sh SHUTDOWN
	This node shuts down upon receipt of a
	.Dv NGM_SHUTDOWN
	control message.
	The associated interface is removed and becomes available
	for use by future
	.Nm iface
	nodes.
	.Pp
	Unlike most other node types, an
	.Nm iface
	node does
	.Em not
	go away when all hooks have been disconnected; rather, and explicit
	.Dv NGM_SHUTDOWN
	control message is required.
	.Sh ALTQ Support
	The
	.Nm
	interface supports ALTQ bandwidth management feature.
	However,
	.Nm
	is a special case, since it is not a physical interface with limited bandwidth.
	One should not turn ALTQ on
	.Nm
	if the latter corresponds to some tunneled connection, e.g.\& PPPoE or PPTP.
	In this case, ALTQ should be configured on the interface that is used to
	transmit the encapsulated packets.
	In case when your graph ends up with some kind of serial line, either
	synchronous or modem, the
	.Nm
	is the right place to turn ALTQ on.
	.Sh Nesting
	.Nm
	supports nesting, a configuration when traffic of one
	.Nm
	interface flows through the other.
	The default maximum allowed nesting level is 2.
	It can be changed at runtime setting
	.Xr sysctl 8
	variable
	.Va net.graph.iface.max_nesting
	to the desired level of nesting.
	.Sh SEE ALSO
	.Xr altq 4 ,
	.Xr bpf 4 ,
	.Xr netgraph 4 ,
	.Xr ng_cisco 4 ,
	.Xr ifconfig 8 ,
	.Xr ngctl 8
	.Xr sysctl
	.Sh HISTORY
	The
	.Nm iface
	node type was implemented in
	.Fx 4.0 .
	.Sh AUTHORS
	.An Archie Cobbs Aq Mt archie@FreeBSD.org
	diff --git a/share/man/man4/padlock.4 b/share/man/man4/padlock.4
	index ad648af9d946..44963d0fd140 100644
	--- a/share/man/man4/padlock.4
	+++ b/share/man/man4/padlock.4
	@@ -1,97 +1,98 @@
	.\" Copyright (c) 2005 Christian Brueffer
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd February 8, 2010
	+.Dd July 29, 2020
	.Dt PADLOCK 4
	.Os
	.Sh NAME
	.Nm padlock
	.Nd "driver for the cryptographic functions and RNG in VIA C3, C7 and Eden processors"
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device padlock"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	padlock_load="YES"
	.Ed
	.Sh DESCRIPTION
	The C3 and Eden processor series from VIA include hardware acceleration for
	AES.
	The C7 series includes hardware acceleration for AES, SHA1, SHA256 and RSA.
	All of the above processor series include a hardware random number generator.
	.Pp
	The
	.Nm
	driver registers itself to accelerate AES operations and, if available, HMAC/SHA1
	and HMAC/SHA256 for
	.Xr crypto 4 .
	It also registers itself to accelerate other HMAC algorithms, although
	there is no hardware acceleration for those algorithms.
	This is only needed so
	.Nm
	can work with
	.Xr ipsec 4 .
	.Pp
	The hardware random number generator supplies data for the kernel
	.Xr random 4
	subsystem.
	.Sh SEE ALSO
	.Xr crypt 3 ,
	.Xr crypto 4 ,
	.Xr intro 4 ,
	.Xr ipsec 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr crypto 9
	.Sh HISTORY
	The
	.Nm
	driver first appeared in
	.Ox .
	The first
	.Fx
	release to include it was
	.Fx 6.0 .
	.Sh AUTHORS
	.An -nosplit
	The
	.Nm
	driver with AES encryption support was written by
	.An Jason Wright Aq Mt jason@OpenBSD.org .
	It was ported to
	.Fx
	and then extended to support SHA1 and SHA256
	by
	.An Pawel Jakub Dawidek Aq Mt pjd@FreeBSD.org .
	This manual page was written by
	.An Christian Brueffer Aq Mt brueffer@FreeBSD.org .
	diff --git a/share/man/man4/rc.4 b/share/man/man4/rc.4
	deleted file mode 100644
	index 95bcae6c3981..000000000000
	--- a/share/man/man4/rc.4
	+++ /dev/null
	@@ -1,124 +0,0 @@
	-.\"
	-.\" Copyright (c) 2004 Tom Rhodes
	-.\" All rights reserved.
	-.\"
	-.\" Redistribution and use in source and binary forms, with or without
	-.\" modification, are permitted provided that the following conditions
	-.\" are met:
	-.\" 1. Redistributions of source code must retain the above copyright
	-.\" notice, this list of conditions and the following disclaimer.
	-.\" 2. Redistributions in binary form must reproduce the above copyright
	-.\" notice, this list of conditions and the following disclaimer in the
	-.\" documentation and/or other materials provided with the distribution.
	-.\"
	-.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	-.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	-.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	-.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	-.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	-.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	-.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	-.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	-.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	-.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	-.\" SUCH DAMAGE.
	-.\"
	-.\" $FreeBSD$
	-.\"
	-.Dd March 18, 2005
	-.Dt RC 4
	-.Os
	-.Sh NAME
	-.Nm rc
	-.Nd RISCom/8 multiport card
	-.Sh SYNOPSIS
	-.Cd device isa
	-.Cd device rc
	-.Sh DESCRIPTION
	-The
	-.Tn RISCom/8
	-is an eight port
	-.Tn ISA
	-.Tn RS-232C
	-communications multiplexer with a built in
	-.Tn RISC
	-processor.
	-It uses a block of sixteen
	-.Tn I/O
	-ports in the range 0x200 to 0x3f0 selectable by on-board
	-switches or jumpers.
	-The block must be aligned on a sixteen port boundary.
	-The jumper-selectable hardware interrupt level may be set to
	-be detected during system
	-initialization using settings found in the
	-.Pa /boot/device.hints
	-file.
	-.Pp
	-This driver is mostly based on the Cirrus Logic CL-CD180 driver.
	-.Sh HARDWARE
	-The
	-.Nm
	-driver provides support for the
	-.Tn SDL
	-Communications
	-.Tn RISCom/8
	-boards.
	-.Sh DIAGNOSTICS
	-The following driver specific error messages
	-may be reported:
	-.Bl -diag
	-.It "rc%d channel%d: interrupt-level buffer overflow"
	-An internal buffer overflow error has occurred on
	-the listed channel.
	-The
	-.Nm
	-driver will need to be reloaded to correct this.
	-.It "rc%d: Bad char chan %d"
	-The channel has obtained a bad set of characters.
	-.It "rc%d: Got extra chars chan %d"
	-The
	-.Nm
	-driver got more characters than expected on the channel shown.
	-.It "rc%d: data mismatch chan %d ptr %d (%d != %d)"
	-Data sent from channel
	-.Ar %d
	-to the rx buffer was different then expected.
	-.It "rc%d: channel %d command timeout, rc.c line: %d"
	-A command timeout has occurred on the channel, the
	-.Pa src/sys/dev/rc/rc.c
	-file can be consulted for more information.
	-.El
	-.Sh SEE ALSO
	-.Xr tty 1 ,
	-.Xr ttyname 3 ,
	-.Xr tty 4 ,
	-.Xr device.hints 5 ,
	-.Xr comcontrol 8 ,
	-.Xr getty 8 ,
	-.Xr mutex 9 ,
	-.Xr splx 9
	-.Pp
	-.Pa http://www.sdlcomm.com
	-.Sh HISTORY
	-The
	-.Nm
	-driver first appeared in
	-.Fx 2.0.5 .
	-This manual page first appeared in
	-.Fx 5.3 .
	-.Sh AUTHORS
	-This manual page was written by
	-.An Tom Rhodes Aq Mt trhodes@FreeBSD.org .
	-.Sh BUGS
	-The
	-.Nm
	-driver code still uses the
	-.Xr spl 9
	-functions.
	-These should be replaced by
	-.Xr mutex 9
	-functions.
	-.Pp
	-The various
	-.Fn ttyld_*
	-functions should be documented.
	diff --git a/share/man/man4/rp.4 b/share/man/man4/rp.4
	deleted file mode 100644
	index d8a130c810b3..000000000000
	--- a/share/man/man4/rp.4
	+++ /dev/null
	@@ -1,195 +0,0 @@
	-.\" Copyright (c) 1995 Comtrol, Inc.
	-.\" All rights reserved.
	-.\"
	-.\" $FreeBSD$
	-.Dd November 15, 1995
	-.Dt RP 4
	-.Os
	-.Sh NAME
	-.Nm rp
	-.Nd "driver for Comtrol RocketPort Intelligent Serial Port Cards"
	-.Sh SYNOPSIS
	-.Cd "device rp"
	-.Pp
	-For ISA cards, you must specify the port address in
	-.Pa /boot/device.hints :
	-.Cd hint.rp.0.at="isa"
	-.Cd hint.rp.0.port="0x100"
	-.Sh DESCRIPTION
	-This driver provides a kernel device driver for the
	-.Tn RocketPort
	-and
	-.Tn RocketPort RA
	-serial boards.
	-These boards provide 8, 16, or 32 high-speed serial ports
	-while requiring only 68 bytes of I/O space for all 8, 16,
	-or 32 ports, and do not require an interrupt channel.
	-This driver supports up to four
	-.Tn RocketPort
	-or
	-.Tn RocketPort RA
	-boards in one machine simultaneously.
	-If you are using four 32 port
	-.Tn RocketPort
	-boards, you can put as many as 128 intelligent serial ports
	-on your system.
	-.Pp
	-The
	-.Nm
	-driver supports the following speeds: 50, 75, 110, 134, 150,
	-200, 300, 600, 1200, 1800, 2400, 4800, 9600, 19200, 38400, 7200,
	-14400, 57600, 76800, 115200, and 230400.
	-(You must use
	-.Xr termios 4 ,
	-rather than the old style ioctl interface to use non-traditional
	-speeds.)
	-.Pp
	-An open on the
	-.Nm
	-driver will block until carrier is present, unless
	-.Dv O_NONBLOCK
	-or
	-.Dv CLOCAL
	-is set.
	-.Sh HARDWARE CONFIGURATION
	-The first
	-.Tn RocketPort
	-or
	-.Tn RocketPort RA
	-card requires a 68-byte contiguous block of I/O addresses,
	-starting at one of the following:
	-0x100h, 0x140h, 0x180h, 0x200h, 0x240h, 0x280h, 0x300h, 0x340h,
	-0x380h.
	-The second, third, and fourth
	-.Tn RocketPort
	-cards require only a
	-64-byte contiguous block of I/O addresses, starting at one of the
	-above address ranges.
	-The I/O address range used by any of the
	-.Tn RocketPort
	-cards must not conflict with any other cards in the system,
	-including other
	-.Tn RocketPort
	-cards.
	-The starting range of the I/O ports used by each card
	-must match with the I/O address specified in
	-.Pa /boot/device.hints .
	-.Pp
	-Since the first
	-.Tn RocketPort
	-uses 68 I/O addresses, if the first card is
	-set to use an I/O block starting at 0x100,
	-it will occupy the I/O ports between 0x100 and 0x143.
	-This means that the second, third, or fourth
	-.Tn RocketPort
	-board may not use the block of addresses starting at 0x140,
	-since the first three I/O addresses of that range
	-are used by the first board.
	-This is an important point to keep in mind.
	-.Pp
	-If you have two ISA cards, one installed at 0x100 and the
	-second installed at 0x180, then you should add the following to
	-.Pa /boot/device.hints :
	-.Pp
	-.Dl hint.rp.0.at="isa"
	-.Dl hint.rp.0.port="0x100"
	-.Dl hint.rp.1.at="isa"
	-.Dl hint.rp.1.port="0x180"
	-.Pp
	-The configuration of the
	-.Tn RocketPort
	-cards is done via the set of 8 DIP switches,
	-labeled SW1 on the
	-.Tn RocketPort
	-card:
	-.Bd -literal -offset indent
	-+-------------------------------+
	-\| 8 \| 7 \| 6 \| 5 \| 4 \| 3 \| 2 \| 1 \|
	-+-------+-------+---------------+
	-\| Unused\| Card \| I/O Port Block\|
	-+-------------------------------+
	-.Ed
	-.Pp
	-DIP switches 7 and 8 are unused, and must be left on.
	-.Pp
	-DIP switches 6 and 5 identify the card number of each
	-.Tn RocketPort
	-card.
	-The first card installed in the system must have its DIP switches set
	-as card number one; the second card installed in the system must have
	-its DIP switches set as card number two; and so on.
	-As shipped from
	-the factory, DIP switches 6 and 5 are both on by default, indicating
	-that this is the first card installed on the system:
	-.Bd -literal -offset indent
	-DIP Switches
	-6 5
	-===================
	-On On First Card
	-On Off Second Card
	-Off On Third Card
	-Off Off Fourth Card
	-.Ed
	-.Pp
	-DIP switches 4, 3, 2, and 1 indicate the I/O address range used by the
	-first
	-.Tn RocketPort
	-card.
	-If there are more than one
	-.Tn RocketPort
	-cards installed in a system,
	-the second, third and fourth
	-.Tn RocketPort
	-cards must
	-also be set to the I/O address range used by the first
	-.Tn RocketPort
	-card;
	-all cards must have these DIP switches set identically
	-for proper operation.
	-As shipped from the factory, DIP switch 4 is on,
	-and switches 3, 2, and 1 are off by default,
	-indicating an I/O address range used by the first
	-card which starts at 0x180 and extends to 0x1C3.
	-.Bd -literal -offset indent
	-DIP Switches I/O Address Range
	-4 3 2 1 Used by the First Card
	-=====================================
	-On Off On Off 100-143
	-On Off Off On 140-183
	-On Off Off Off 180-1C3
	-Off On On Off 200-243
	-Off On Off On 240-283
	-Off On Off Off 280-2C3
	-Off Off On Off 300-343
	-Off Off Off On 340-383
	-Off Off Off Off 380-3C3
	-.Ed
	-.Sh FILES
	-.Bl -tag -width ".Pa /dev/ttyR[0-4][0-9a-f]"
	-.It Pa /dev/ttyR[0-4][0-9a-f]
	-.El
	-.Sh AUTHORS
	-.An Theodore Ts'o Aq Mt tytso@mit.edu
	-.Pp
	-This driver was written under contract for Comtrol Corporation.
	-For dealer, distributor and other information regarding Comtrol
	-.Tn RocketPort ,
	-contact Comtrol Corporation at (800) 926-6876 or send email to
	-.Aq Mt info@comtrol.com .
	-To report bugs for this driver, please send email to
	-.Aq Mt bug-bsdi-rocketport@comtrol.com .
	-.Sh BUGS
	-If incoming software flow control is enabled on a 486 or Pentium
	-machine, and the flow control is very heavily exercised, on rare occasions
	-a character will get dropped.
	-This problem does not occur on a 386, and
	-it is not currently known whether the bug is in the
	-.Nm
	-driver
	-or in the
	-.Bsx
	-tty layer.
	-.\" (Although my bet is that it's in the higher-level tty layer;
	-.\" given the bugs I found while writing this driver, it's clear
	-.\" the BSD software flow control code has not been tested very much
	-.\" at all! -- TYT)
	diff --git a/share/man/man4/safe.4 b/share/man/man4/safe.4
	index 4680a9b3710f..3d5cbec03784 100644
	--- a/share/man/man4/safe.4
	+++ b/share/man/man4/safe.4
	@@ -1,129 +1,130 @@
	.\"-
	.\" Copyright (c) 2003 Sam Leffler, Errno Consulting
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"/
	-.Dd May 11, 2020
	+.Dd July 29, 2020
	.Dt SAFE 4
	.Os
	.Sh NAME
	.Nm safe
	.Nd SafeNet crypto accelerator
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device cryptodev"
	.Cd "device safe"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	safe_load="YES"
	.Ed
	.Pp
	.Nm sysctl Va hw.safe.debug
	.Nm sysctl Va hw.safe.dump
	.Nm sysctl Va hw.safe.rnginterval
	.Nm sysctl Va hw.safe.rngbufsize
	.Nm sysctl Va hw.safe.rngmaxalarm
	.Sh DESCRIPTION
	The
	.Nm
	driver supports cards containing SafeNet crypto accelerator chips.
	.Pp
	The
	.Nm
	driver registers itself to accelerate AES,
	SHA1-HMAC, and NULL operations for
	.Xr ipsec 4
	and
	.Xr crypto 4 .
	.Pp
	On all models, the driver registers itself to provide random data to the
	.Xr random 4
	subsystem.
	Periodically the driver will poll the hardware RNG and retrieve
	data for use by the system.
	If the driver detects that the hardware RNG is resonating with any local
	signal, it will reset the oscillators that generate random data.
	Three
	.Xr sysctl 8
	settings control this procedure:
	.Va hw.safe.rnginterval
	specifies the time, in seconds, between polling operations,
	.Va hw.safe.rngbufsize
	specifies the number of 32-bit words to retrieve on each poll,
	and
	.Va hw.safe.rngmaxalarm
	specifies the threshold for resetting the oscillators.
	.Pp
	When the driver is compiled with
	.Dv SAFE_DEBUG
	defined, two
	.Xr sysctl 8
	variables are provided for debugging purposes:
	.Va hw.safe.debug
	can be set to a non-zero value to enable debugging messages to be sent
	to the console for each cryptographic operation,
	.Va hw.safe.dump
	is a write-only variable that can be used to force driver state to be sent
	to the console.
	Set this variable to
	.Dq Li ring
	to dump the current state of the descriptor ring,
	to
	.Dq Li dma
	to dump the hardware DMA registers,
	or
	to
	.Dq Li int
	to dump the hardware interrupt registers.
	.Sh HARDWARE
	The
	.Nm
	driver supports cards containing any of the following chips:
	.Bl -tag -width "SafeNet 1141" -offset indent
	.It SafeNet 1141
	The original chipset.
	Supports DES, Triple-DES, AES, MD5, and SHA-1
	symmetric crypto operations, RNG, public key operations, and full IPsec
	packet processing.
	.It SafeNet 1741
	A faster version of the 1141.
	.El
	.Sh SEE ALSO
	.Xr crypt 3 ,
	.Xr crypto 4 ,
	.Xr intro 4 ,
	.Xr ipsec 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr crypto 9
	.Sh BUGS
	Public key support is not implemented.
	diff --git a/share/man/man4/safexcel.4 b/share/man/man4/safexcel.4
	index 6751570713f9..774dfddfb053 100644
	--- a/share/man/man4/safexcel.4
	+++ b/share/man/man4/safexcel.4
	@@ -1,84 +1,85 @@
	.\"-
	.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate)
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd June 23, 2020
	+.Dd July 29, 2020
	.Dt SAFEXCEL 4
	.Os
	.Sh NAME
	.Nm safexcel
	.Nd Inside Secure SafeXcel-IP-97 security packet engine
	.Sh SYNOPSIS
	To compile this driver into the kernel,
	place the following lines in your
	kernel configuration file:
	.Bd -ragged -offset indent
	.Cd "device crypto"
	.Cd "device cryptodev"
	.Cd "device safexcel"
	.Ed
	.Pp
	Alternatively, to load the driver as a
	module at boot time, place the following line in
	.Xr loader.conf 5 :
	.Bd -literal -offset indent
	safexcel_load="YES"
	.Ed
	.Sh DESCRIPTION
	The
	.Nm
	driver implements
	.Xr crypto 4
	support for the cryptographic acceleration functions of the EIP-97 device
	found on some Marvell systems-on-chip.
	The driver can accelerate the following AES modes:
	.Pp
	.Bl -bullet -compact
	.It
	AES-CBC
	.It
	AES-CTR
	.It
	AES-XTS
	.It
	AES-GCM
	.It
	AES-CCM
	.El
	.Pp
	.Nm
	also implements SHA1 and SHA2 transforms, and can combine AES-CBC and AES-CTR
	with SHA1-HMAC and SHA2-HMAC for encrypt-then-authenticate operations.
	.Sh SEE ALSO
	.Xr crypto 4 ,
	.Xr ipsec 4 ,
	.Xr random 4 ,
	+.Xr crypto 7 ,
	.Xr geli 8 ,
	.Xr crypto 9
	.Sh HISTORY
	The
	.Nm
	driver first appeared in
	.Fx 13.0 .
	diff --git a/share/man/man8/rc.subr.8 b/share/man/man8/rc.subr.8
	index fd23ca20c36c..f522a5d162c3 100644
	--- a/share/man/man8/rc.subr.8
	+++ b/share/man/man8/rc.subr.8
	@@ -1,939 +1,943 @@
	.\" $NetBSD: rc.subr.8,v 1.12 2004/01/06 00:52:24 lukem Exp $
	.\"
	.\" Copyright (c) 2002-2004 The NetBSD Foundation, Inc.
	.\" All rights reserved.
	.\"
	.\" This code is derived from software contributed to The NetBSD Foundation
	.\" by Luke Mewburn.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
	.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
	.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
	.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	.\" POSSIBILITY OF SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd July 24, 2020
	+.Dd July 31, 2020
	.Dt RC.SUBR 8
	.Os
	.Sh NAME
	.Nm rc.subr
	.Nd functions used by system shell scripts
	.Sh SYNOPSIS
	.Bl -item -compact
	.It
	.Ic .\& Pa /etc/rc.subr
	.Pp
	.It
	.Ic backup_file Ar action Ar file Ar current Ar backup
	.It
	.Ic checkyesno Ar var
	.It
	.Ic check_pidfile Ar pidfile Ar procname Op Ar interpreter
	.It
	.Ic check_process Ar procname Op Ar interpreter
	.It
	.Ic debug Ar message
	.It
	.Ic err Ar exitval Ar message
	.It
	.Ic force_depend Ar name
	.It
	.Ic info Ar message
	.It
	.Ic load_kld Oo Fl e Ar regex Oc Oo Fl m Ar module Oc Ar file
	.It
	.Ic load_rc_config Ar name
	.It
	.Ic load_rc_config_var Ar name Ar var
	.It
	.Ic mount_critical_filesystems Ar type
	.It
	.Ic rc_usage Ar command ...
	.It
	.Ic reverse_list Ar item ...
	.It
	.Ic run_rc_command Ar argument
	.It
	.Ic run_rc_script Ar file Ar argument
	.It
	.Ic wait_for_pids Op Ar pid ...
	.It
	.Ic warn Ar message
	.El
	.Sh DESCRIPTION
	The
	.Nm
	script
	contains commonly used shell script functions and variable
	definitions which are used by various scripts such as
	.Xr rc 8 .
	Scripts required by ports in
	.Pa /usr/local/etc/rc.d
	will also eventually
	be rewritten to make use of it.
	.Pp
	The
	.Nm
	functions were mostly imported from
	.Nx .
	.Pp
	They are accessed by sourcing
	.Pa /etc/rc.subr
	into the current shell.
	.Pp
	The following shell functions are available:
	.Bl -tag -width 4n
	.It Ic backup_file Ar action file current backup
	Make a backup copy of
	.Ar file
	into
	.Ar current .
	Save the previous version of
	.Ar current
	as
	.Ar backup .
	.Pp
	The
	.Ar action
	argument
	may be one of the following:
	.Bl -tag -width ".Cm remove"
	.It Cm add
	.Ar file
	is now being backed up by or possibly re-entered into this backup mechanism.
	.Ar current
	is created.
	.It Cm update
	.Ar file
	has changed and needs to be backed up.
	If
	.Ar current
	exists, it is copied to
	.Ar backup
	and then
	.Ar file
	is copied to
	.Ar current .
	.It Cm remove
	.Ar file
	is no longer being tracked by this backup mechanism.
	.Ar current
	is moved to
	.Ar backup .
	.El
	.It Ic checkyesno Ar var
	Return 0 if
	.Ar var
	is defined to
	.Dq Li YES ,
	.Dq Li TRUE ,
	.Dq Li ON ,
	or
	.Ql 1 .
	Return 1 if
	.Ar var
	is defined to
	.Dq Li NO ,
	.Dq Li FALSE ,
	.Dq Li OFF ,
	or
	.Ql 0 .
	Otherwise, warn that
	.Ar var
	is not set correctly.
	The values are case insensitive.
	.Sy Note :
	.Ar var
	should be a variable name, not its value;
	.Ic checkyesno
	will expand the variable by itself.
	.It Ic check_pidfile Ar pidfile procname Op Ar interpreter
	Parses the first word of the first line of
	.Ar pidfile
	for a PID, and ensures that the process with that PID
	is running and its first argument matches
	.Ar procname .
	Prints the matching PID if successful, otherwise nothing.
	If
	.Ar interpreter
	is provided, parse the first line of
	.Ar procname ,
	ensure that the line is of the form:
	.Pp
	.Dl "#! interpreter [...]"
	.Pp
	and use
	.Ar interpreter
	with its optional arguments and
	.Ar procname
	appended as the process string to search for.
	.It Ic check_process Ar procname Op Ar interpreter
	Prints the PIDs of any processes that are running with a first
	argument that matches
	.Ar procname .
	.Ar interpreter
	is handled as per
	.Ic check_pidfile .
	.It Ic debug Ar message
	Display a debugging message to
	.Va stderr ,
	log it to the system log using
	.Xr logger 1 ,
	and
	return to the caller.
	The error message consists of the script name
	(from
	.Va $0 ) ,
	followed by
	.Dq Li ": DEBUG: " ,
	and then
	.Ar message .
	This function is intended to be used by developers
	as an aid to debugging scripts.
	It can be turned on or off
	by the
	.Xr rc.conf 5
	variable
	.Va rc_debug .
	.It Ic err Ar exitval message
	Display an error message to
	.Va stderr ,
	log it to the system log
	using
	.Xr logger 1 ,
	and
	.Ic exit
	with an exit value of
	.Ar exitval .
	The error message consists of the script name
	(from
	.Va $0 ) ,
	followed by
	.Dq Li ": ERROR: " ,
	and then
	.Ar message .
	.It Ic force_depend Ar name
	Output an advisory message and force the
	.Ar name
	service to start.
	The
	.Ar name
	argument is the
	.Xr basename 1
	component of the path to the script located at
	.Pa /etc/rc.d
	(scripts stored in other locations such as
	.Pa /usr/local/etc/rc.d
	cannot be controlled with
	.Ic force_depend
	currently).
	If the script fails for any reason it will output a warning
	and return with a return value of 1.
	If it was successful
	it will return 0.
	.It Ic info Ar message
	Display an informational message to
	.Va stdout ,
	and log it to the system log using
	.Xr logger 1 .
	The message consists of the script name
	(from
	.Va $0 ) ,
	followed by
	.Dq Li ": INFO: " ,
	and then
	.Ar message .
	The display of this informational output can be
	turned on or off by the
	.Xr rc.conf 5
	variable
	.Va rc_info .
	.It Ic load_kld Oo Fl e Ar regex Oc Oo Fl m Ar module Oc Ar file
	Load
	.Ar file
	as a kernel module unless it is already loaded.
	For the purpose of checking the module status,
	either the exact module name can be specified using
	.Fl m ,
	or an
	.Xr egrep 1
	regular expression matching the module name can be supplied via
	.Fl e .
	By default, the module is assumed to have the same name as
	.Ar file ,
	which is not always the case.
	.It Ic load_rc_config Ar name
	Source in the configuration files for
	.Ar name .
	First,
	.Pa /etc/rc.conf
	is sourced if it has not yet been read in.
	Then,
	.Pa /etc/rc.conf.d/ Ns Ar name
	is sourced if it is an existing file.
	The latter may also contain other variable assignments to override
	.Ic run_rc_command
	arguments defined by the calling script, to provide an easy
	mechanism for an administrator to override the behaviour of a given
	.Xr rc.d 8
	script without requiring the editing of that script.
	.It Ic load_rc_config_var Ar name Ar var
	Read the
	.Xr rc.conf 5
	variable
	.Ar var
	for
	.Ar name
	and set in the current shell, using
	.Ic load_rc_config
	in a sub-shell to prevent unwanted side effects from other variable
	assignments.
	.It Ic mount_critical_filesystems Ar type
	Go through a list of critical file systems,
	as found in the
	.Xr rc.conf 5
	variable
	.Va critical_filesystems_ Ns Ar type ,
	mounting each one that
	is not currently mounted.
	.It Ic rc_usage Ar command ...
	Print a usage message for
	.Va $0 ,
	with
	.Ar commands
	being the list of valid arguments
	prefixed by
	.Sm off
	.Dq Bq Li fast \| force \| one \| quiet .
	.Sm on
	.It Ic reverse_list Ar item ...
	Print the list of
	.Ar items
	in reverse order.
	.It Ic run_rc_command Ar argument
	Run the
	.Ar argument
	method for the current
	.Xr rc.d 8
	script, based on the settings of various shell variables.
	.Ic run_rc_command
	is extremely flexible, and allows fully functional
	.Xr rc.d 8
	scripts to be implemented in a small amount of shell code.
	.Pp
	.Ar argument
	is searched for in the list of supported commands, which may be one
	of:
	.Bl -tag -width ".Cm restart" -offset indent
	.It Cm start
	Start the service.
	This should check that the service is to be started as specified by
	.Xr rc.conf 5 .
	Also checks if the service is already running and refuses to start if
	it is.
	This latter check is not performed by standard
	.Fx
	scripts if the system is starting directly to multi-user mode, to
	speed up the boot process.
	.It Cm stop
	If the service is to be started as specified by
	.Xr rc.conf 5 ,
	stop the service.
	This should check that the service is running and complain if it is not.
	.It Cm restart
	Perform a
	.Cm stop
	then a
	.Cm start .
	Defaults to displaying the process ID of the program (if running).
	.It Cm enabled
	Return 0 if the service is enabled and 1 if it is not.
	This command does not print anything.
	.It Cm rcvar
	Display which
	.Xr rc.conf 5
	variables are used to control the startup of the service (if any).
	.El
	.Pp
	If
	.Va pidfile
	or
	.Va procname
	is set, also support:
	.Bl -tag -width ".Cm restart" -offset indent
	.It Cm poll
	Wait for the command to exit.
	.It Cm status
	Show the status of the process.
	.El
	.Pp
	Other supported commands are listed in the optional variable
	.Va extra_commands .
	.Pp
	.Ar argument
	may have one of the following prefixes which alters its operation:
	.Bl -tag -width ".Li force" -offset indent
	.It Li fast
	Skip the check for an existing running process,
	and sets
	.Va rc_fast Ns = Ns Li YES .
	.It Li force
	Skip the checks for
	.Va rcvar
	being set to
	.Dq Li YES ,
	and sets
	.Va rc_force Ns = Ns Li YES .
	This ignores
	.Ar argument Ns Va _precmd
	returning non-zero, and ignores any of the
	.Va required_*
	tests failing, and always returns a zero exit status.
	.It Li one
	Skip the checks for
	.Va rcvar
	being set to
	.Dq Li YES ,
	but performs all the other prerequisite tests.
	.It Li quiet
	Inhibits some verbose diagnostics.
	Currently, this includes messages
	.Qq Starting ${name}
	(as checked by
	.Ic check_startmsgs
	inside
	.Nm )
	and errors about usage of services that are not enabled in
	.Xr rc.conf 5 .
	This prefix also sets
	.Va rc_quiet Ns = Ns Li YES .
	.Em Please, note\&:
	.Va rc_quiet
	is not intended to completely mask all debug and warning messages,
	but only certain small classes of them.
	.El
	.Pp
	.Ic run_rc_command
	uses the following shell variables to control its behaviour.
	Unless otherwise stated, these are optional.
	.Bl -tag -width ".Va procname" -offset indent
	.It Va name
	The name of this script.
	This is not optional.
	.It Va rcvar
	The value of
	.Va rcvar
	is checked with
	.Ic checkyesno
	to determine if this method should be run.
	.It Va command
	Full path to the command.
	Not required if
	.Ar argument Ns Va _cmd
	is defined for each supported keyword.
	Can be overridden by
	.Va ${name}_program .
	.It Va command_args
	Optional arguments and/or shell directives for
	.Va command .
	.It Va command_interpreter
	.Va command
	is started with:
	.Pp
	.Dl "#! command_interpreter [...]"
	.Pp
	which results in its
	.Xr ps 1
	command being:
	.Pp
	.Dl "command_interpreter [...] command"
	.Pp
	so use that string to find the PID(s) of the running command
	rather than
	.Va command .
	.It Va extra_commands
	Extra commands/keywords/arguments supported.
	.It Va pidfile
	Path to PID file.
	Used to determine the PID(s) of the running command.
	If
	.Va pidfile
	is set, use:
	.Pp
	.Dl "check_pidfile $pidfile $procname"
	.Pp
	to find the PID.
	Otherwise, if
	.Va command
	is set, use:
	.Pp
	.Dl "check_process $procname"
	.Pp
	to find the PID.
	.It Va procname
	Process name to check for.
	Defaults to the value of
	.Va command .
	.It Va required_dirs
	Check for the existence of the listed directories
	before running the
	.Cm start
	method.
	The list is checked before running
	.Va start_precmd .
	.It Va required_files
	Check for the readability of the listed files
	before running the
	.Cm start
	method.
	The list is checked before running
	.Va start_precmd .
	.It Va required_modules
	Ensure that the listed kernel modules are loaded
	before running the
	.Cm start
	method.
	The list is checked after running
	.Va start_precmd .
	This is done after invoking the commands from
	.Va start_precmd
	so that the missing modules are not loaded in vain
	if the preliminary commands indicate a error condition.
	A word in the list can have an optional
	.Dq Li \&: Ns Ar modname
	or
	.Dq Li ~ Ns Ar pattern
	suffix.
	The
	.Ar modname
	or
	.Ar pattern
	parameter is passed to
	.Ic load_kld
	through a
	.Fl m
	or
	.Fl e
	option, respectively.
	See the description of
	.Ic load_kld
	in this document for details.
	.It Va required_vars
	Perform
	.Ic checkyesno
	on each of the list variables
	before running the
	.Cm start
	method.
	The list is checked after running
	.Va start_precmd .
	.It Va ${name}_chdir
	Directory to
	.Ic cd
	to before running
	.Va command ,
	if
	.Va ${name}_chroot
	is not provided.
	.It Va ${name}_chroot
	Directory to
	.Xr chroot 8
	to before running
	.Va command .
	Only supported after
	.Pa /usr
	is mounted.
	.It Va ${name}_env
	A list of environment variables to run
	.Va command
	with.
	Those variables will be passed as arguments to the
	.Xr env 1
	utility unless
	.Ar argument Ns Va _cmd
	is defined.
	In that case the contents of
	.Va ${name}_env
	will be exported via the
	.Xr export 1
	builtin of
	.Xr sh 1 ,
	which puts some limitations on the names of variables
	(e.g., a variable name may not start with a digit).
	.It Va ${name}_env_file
	A file to source for environmental variables to run
	.Va command
	with.
	Note that all the variables which are being assigned in this file are going
	to be exported into the environment of
	.Va command .
	.It Va ${name}_fib
	FIB
	.Pa Routing Table
	number to run
	.Va command
	with.
	See
	.Xr setfib 1
	for more details.
	.It Va ${name}_flags
	Arguments to call
	.Va command
	with.
	This is usually set in
	.Xr rc.conf 5 ,
	and not in the
	.Xr rc.d 8
	script.
	The environment variable
	.Sq Ev flags
	can be used to override this.
	.It Va ${name}_nice
	.Xr nice 1
	level to run
	.Va command
	as.
	Only supported after
	.Pa /usr
	is mounted.
	.It Va ${name}_limits
	Resource limits to apply to
	.Va command .
	This will be passed as arguments to the
	.Xr limits 1
	utility.
	By default, the resource limits are based on the login class defined in
	.Va ${name}_login_class .
	.It Va ${name}_login_class
	Login class to use with
	.Va ${name}_limits .
	Defaults to
	.Dq Li daemon .
	.It Va ${name}_oomprotect
	.Xr protect 1
	.Va command
	from being killed when swap space is exhausted.
	If
	.Dq Li YES
	is used, no child processes are protected.
	If
	.Dq Li ALL ,
	protect all child processes.
	.It Va ${name}_program
	Full path to the command.
	Overrides
	.Va command
	if both are set, but has no effect if
	.Va command
	is unset.
	As a rule,
	.Va command
	should be set in the script while
	.Va ${name}_program
	should be set in
	.Xr rc.conf 5 .
	.It Va ${name}_user
	User to run
	.Va command
	as, using
	.Xr chroot 8
	if
	.Va ${name}_chroot
	is set, otherwise
	uses
	.Xr su 1 .
	Only supported after
	.Pa /usr
	is mounted.
	.It Va ${name}_group
	Group to run the chrooted
	.Va command
	as.
	.It Va ${name}_groups
	Comma separated list of supplementary groups to run the chrooted
	.Va command
	with.
	.It Va ${name}_prepend
	Commands to be prepended to
	.Va command .
	This is a generic version of
	.Va ${name}_env ,
	.Va ${name}_fib ,
	or
	.Va ${name}_nice .
	.It Ar argument Ns Va _cmd
	Shell commands which override the default method for
	.Ar argument .
	.It Ar argument Ns Va _precmd
	Shell commands to run just before running
	.Ar argument Ns Va _cmd
	or the default method for
	.Ar argument .
	If this returns a non-zero exit code, the main method is not performed.
	If the default method is being executed, this check is performed after
	the
	.Va required_*
	checks and process (non-)existence checks.
	.It Ar argument Ns Va _postcmd
	Shell commands to run if running
	.Ar argument Ns Va _cmd
	or the default method for
	.Ar argument
	returned a zero exit code.
	.It Va sig_stop
	Signal to send the processes to stop in the default
	.Cm stop
	method.
	Defaults to
	.Dv SIGTERM .
	.It Va sig_reload
	Signal to send the processes to reload in the default
	.Cm reload
	method.
	Defaults to
	.Dv SIGHUP .
	.El
	.Pp
	For a given method
	.Ar argument ,
	if
	.Ar argument Ns Va _cmd
	is not defined, then a default method is provided by
	.Ic run_rc_command :
	.Bl -tag -width ".Sy Argument" -offset indent
	.It Sy Argument
	.Sy Default method
	.It Cm start
	If
	.Va command
	is not running and
	.Ic checkyesno Va rcvar
	succeeds, start
	.Va command .
	.It Cm stop
	Determine the PIDs of
	.Va command
	with
	.Ic check_pidfile
	or
	.Ic check_process
	(as appropriate),
	.Ic kill Va sig_stop
	those PIDs, and run
	.Ic wait_for_pids
	on those PIDs.
	.It Cm reload
	Similar to
	.Cm stop ,
	except that it uses
	.Va sig_reload
	instead, and does not run
	.Ic wait_for_pids .
	Another difference from
	.Cm stop
	is that
	.Cm reload
	is not provided by default.
	It can be enabled via
	.Va extra_commands
	if appropriate:
	.Pp
	.Dl "extra_commands=reload"
	.It Cm restart
	Runs the
	.Cm stop
	method, then the
	.Cm start
	method.
	.It Cm status
	Show the PID of
	.Va command ,
	or some other script specific status operation.
	.It Cm poll
	Wait for
	.Va command
	to exit.
	.It Cm rcvar
	Display which
	.Xr rc.conf 5
	variable is used (if any).
	This method always works, even if the appropriate
	.Xr rc.conf 5
	variable is set to
	.Dq Li NO .
	.El
	.Pp
	The following variables are available to the methods
	(such as
	.Ar argument Ns Va _cmd )
	as well as after
	.Ic run_rc_command
	has completed:
	.Bl -tag -width ".Va rc_service" -offset indent
	.It Va rc_arg
	Argument provided to
	.Ic run_rc_command ,
	after fast and force processing has been performed.
	.It Va rc_flags
	Flags to start the default command with.
	Defaults to
	.Va ${name}_flags ,
	unless overridden by the environment variable
	.Sq Ev flags .
	This variable may be changed by the
	.Ar argument Ns Va _precmd
	method.
	.It Va rc_service
	Path to the service script being executed, in case it needs to re-invoke itself.
	.It Va rc_pid
	PID of
	.Va command
	(if appropriate).
	.It Va rc_fast
	Not empty if
	.Dq Li fast
	prefix was used.
	.It Va rc_force
	Not empty if
	.Dq Li force
	prefix was used.
	.El
	.It Ic run_rc_script Ar file argument
	Start the script
	.Ar file
	with an argument of
	.Ar argument ,
	and handle the return value from the script.
	.Pp
	Various shell variables are unset before
	.Ar file
	is started:
	.Bd -ragged -offset indent
	.Va name ,
	.Va command ,
	.Va command_args ,
	.Va command_interpreter ,
	.Va extra_commands ,
	.Va pidfile ,
	.Va rcvar ,
	.Va required_dirs ,
	.Va required_files ,
	.Va required_vars ,
	.Ar argument Ns Va _cmd ,
	.Ar argument Ns Va _precmd .
	.Ar argument Ns Va _postcmd .
	.Ed
	.Pp
	The startup behaviour of
	.Ar file
	depends upon the following checks:
	.Bl -enum
	.It
	If
	.Ar file
	ends in
	.Pa .sh ,
	it is sourced into the current shell.
	.It
	If
	.Ar file
	appears to be a backup or scratch file
	(e.g., with a suffix of
	.Pa ~ , # , .OLD ,
	or
	.Pa .orig ) ,
	ignore it.
	.It
	If
	.Ar file
	is not executable, ignore it.
	.It
	If the
	.Xr rc.conf 5
	variable
	.Va rc_fast_and_loose
	is empty,
	source
	.Ar file
	in a sub shell,
	otherwise source
	.Ar file
	into the current shell.
	.El
	.It Ic stop_boot Op Ar always
	Prevent booting to multiuser mode.
	If the
	.Va autoboot
	variable is set to
	-.Ql yes ,
	+.Ql yes
	+(see
	+.Xr rc 8
	+to learn more about
	+.Va autoboot ) ,
	or
	.Ic checkyesno Ar always
	indicates a truth value, then a
	.Dv SIGTERM
	signal is sent to the parent
	process, which is assumed to be
	.Xr rc 8 .
	Otherwise, the shell exits with a non-zero status.
	.It Ic wait_for_pids Op Ar pid ...
	Wait until all of the provided
	.Ar pids
	do not exist any more, printing the list of outstanding
	.Ar pids
	every two seconds.
	.It Ic warn Ar message
	Display a warning message to
	.Va stderr
	and log it to the system log
	using
	.Xr logger 1 .
	The warning message consists of the script name
	(from
	.Va $0 ) ,
	followed by
	.Dq Li ": WARNING: " ,
	and then
	.Ar message .
	.El
	.Sh FILES
	.Bl -tag -width ".Pa /etc/rc.subr" -compact
	.It Pa /etc/rc.subr
	The
	.Nm
	file resides in
	.Pa /etc .
	.El
	.Sh SEE ALSO
	.Xr rc.conf 5 ,
	.Xr rc 8
	.Sh HISTORY
	The
	.Nm
	script
	appeared in
	.Nx 1.3 .
	The
	.Xr rc.d 8
	support functions appeared in
	.Nx 1.5 .
	The
	.Nm
	script
	first appeared in
	.Fx 5.0 .
	diff --git a/share/misc/committers-doc.dot b/share/misc/committers-doc.dot
	index 691248727521..4edffffeed1b 100644
	--- a/share/misc/committers-doc.dot
	+++ b/share/misc/committers-doc.dot
	@@ -1,200 +1,206 @@
	# $FreeBSD$

	# This file is meant to list all FreeBSD doc+www committers and describe the
	# mentor-mentee relationships between them.
	# The graphical output can be generated from this file with the following
	# command:
	# $ dot -T png -o file.png committers-doc.dot
	#
	# The dot binary is part of the graphics/graphviz port.

	digraph doc {

	# Node definitions follow this example:
	#
	# foo [label="Foo Bar\nfoo@FreeBSD.org\n????/??/??"]
	#
	# ????/??/?? is the date when the commit bit was obtained, usually the one you
	# can find looking at svn logs for the svnadmin/conf/access file.
	# Use YYYY/MM/DD format.
	#
	# For returned commit bits, the node definition will follow this example:
	#
	# foo [label="Foo Bar\nfoo@FreeBSD.org\n????/??/??\n????/??/??"]
	#
	# The first date is the same as for an active committer, the second date is
	# the date when the commit bit has been returned. Again, check svn logs.

	node [color=grey62, style=filled, bgcolor=black];

	# Alumni go here. Try to keep things sorted.

	ache [label="Andrey Chernov\nache@FreeBSD.org\n1997/06/13\n2010/12/11"]
	bmah [label="Bruce A. Mah\nbmah@FreeBSD.org\n2000/08/22\n2009/09/13"]
	bvs [label="Vitaly Bogdanov\nbvs@FreeBSD.org\n2005/10/03\n2010/12/11"]
	ceri [label="Ceri Davies\nceri@FreeBSD.org\n2002/03/17\n2012/02/29"]
	den [label="Denis Peplin\nden@FreeBSD.org\n2003/09/13\n2009/07/09"]
	garys [label="Gary W. Swearingen\ngarys@FreeBSD.org\n2005/08/21\n2008/03/02"]
	jcamou [label="Jesus R. Camou\njcamou@FreeBSD.org\n2005/03/02\n2008/12/20"]
	jesusr [label="Jesus Rodriguez Cuesta\njesusr@FreeBSD.org\n1998/12/10\n2010/12/11"]
	jim [label="Jim Mock\njim@FreeBSD.org\n1999/08/11\n2003/12/15"]
	josef [label="Josef El-Rayes\njosef@FreeBSD.org\n2004/01/15\n2008/03/29"]
	marcel [label="Marcel Moolenaar\nmarcel@FreeBSD.org\n1999/07/03\n2012/04/25"]
	mheinen [label="Martin Heinen\nmheinen@FreeBSD.org\n2002/10/04\n2006/04/26"]
	murray [label="Murray Stokely\nmurray@FreeBSD.org\n2000/04/05\n2012/04/25"]
	nik [label="Nik Clayton\nnik@FreeBSD.org\n1998/02/26\n2008/12/20"]
	pgj [label="Gabor Pali\npgj@FreeBSD.org\n2008/04/21\n2010/12/01"]
	roam [label="Peter Pentchev\nroam@FreeBSD.org\n2003/02/14\n2012/02/29"]

	node [color=lightblue2, style=filled, bgcolor=black];

	# Current doc committers go here. Try to keep things sorted.

	"0mp" [label="Mateusz Piotrowski\n0mp@FreeBSD.org\n2019/09/29"]
	ale [label="Alex Dupre\nale@FreeBSD.org\n2003/12/22"]
	allanjude [label="Allan Jude\nallanjude@FreeBSD.org\n2014/05/17"]
	bcr [label="Benedict Reuschling\nbcr@FreeBSD.org\n2009/12/24"]
	bhd [label="Björn Heidotting\nbhd@FreeBSD.org\n2014/10/14"]
	blackend [label="Marc Fonvieille\nblackend@FreeBSD.org\n2002/06/16"]
	brd [label="Brad Davis\nbrd@FreeBSD.org\n2005/06/01"]
	brueffer [label="Christian Brueffer\nbrueffer@FreeBSD.org\n2003/01/13"]
	carlavilla [label="Sergio Carlavilla\ncarlavilla@FreeBSD.org\n2019/05/16"]
	chinsan [label="Chinsan Huang\nchinsan@FreeBSD.org\n2006/09/20"]
	crees [label="Chris Rees\ncrees@FreeBSD.org\n2013/05/27"]
	danger [label="Daniel Gerzo\ndanger@FreeBSD.org\n2006/08/20"]
	+debdrup [label="Daniel Ebdrup Jensen\ndebdrup@FreeBSD.org\n2020/07/26"]
	delphij [label="Xin Li\ndelphij@FreeBSD.org\n2004/09/14"]
	dexter [label="Michael Dexter\ndexter@FreeBSD.org\n2016/11/15"]
	dru [label="Dru Lavigne\ndru@FreeBSD.org\n2013/01/22"]
	eadler [label="Eitan Adler\neadler@FreeBSD.org\n2012/10/15"]
	ebrandi [label="Edson Brandi\nebrandi@FreeBSD.org\n2012/09/13"]
	gabor [label="Gabor Kovesdan\ngabor@FreeBSD.org\n2007/02/02"]
	ganbold [label="Ganbold Tsagaankhuu\nganbold@FreeBSD.org\n2008/02/26"]
	gavin [label="Gavin Atkinson\ngavin@FreeBSD.org\n2011/07/18"]
	gbe [label="Gordon Bergling\ngbe@FreeBSD.org\n2020/06/08"]
	gjb [label="Glen Barber\ngjb@FreeBSD.org\n2010/09/01"]
	hrs [label="Hiroki Sato\nhrs@FreeBSD.org\n2000/07/06"]
	issyl0 [label="Isabell Long\nissyl0@FreeBSD.org\n2012/04/25"]
	jgh [label="Jason Helfman\njgh@FreeBSD.org\n2014/01/20"]
	jkois [label="Johann Kois\njkois@FreeBSD.org\n2004/11/11"]
	joel [label="Joel Dahl\njoel@FreeBSD.org\n2005/04/05"]
	keramida [label="Giorgos Keramidas\nkeramida@FreeBSD.org\n2001/10/12"]
	linimon [label="Mark Linimon\nlinimon@FreeBSD.org\n2004/03/31"]
	loader [label="Fukang Chen\nloader@FreeBSD.org\n2007/07/30"]
	manolis [label="Manolis Kiagias\nmanolis@FreeBSD.org\n2008/05/24"]
	marck [label="Dmitry Morozovsky\nmarck@FreeBSD.org\n2004/08/10"]
	maxim [label="Maxim Konovalov\nmaxim@FreeBSD.org\n2002/02/07"]
	miwi [label="Martin Wilke\nmiwi@FreeBSD.org\n2007/10/26"]
	pav [label="Pav Lucistnik\npav@FreeBSD.org\n2005/08/12"]
	pluknet [label="Sergey Kandaurov\npluknet@FreeBSD.org\n2012/02/14"]
	remko [label="Remko Lodder\nremko@FreeBSD.org\n2004/10/16"]
	rene [label="Rene Ladan\nrene@FreeBSD.org\n2008/11/03"]
	ryusuke [label="Ryusuke Suzuki\nryusuke@FreeBSD.org\n2009/12/21"]
	sevan [label="Sevan Janiyan\nsevan@FreeBSD.org\n2016/09/16"]
	sg [label="Stephen Gregoratto\nsg@FreeBSD.org\n2019/09/10"]
	simon [label="Simon L. Nielsen\nsimon@FreeBSD.org\n2003/07/20"]
	skreuzer [label="Steven Kreuzer\nskreuzer@FreeBSD.org\n2014/01/15"]
	taras [label="Taras Korenko\ntaras@FreeBSD.org\n2010/06/25"]
	trhodes [label="Tom Rhodes\ntrhodes@FreeBSD.org\n2002/03/25"]
	wblock [label="Warren Block\nwblock@FreeBSD.org\n2011/09/12"]
	ygy [label="Guangyuan Yang\nygy@FreeBSD.org\n2017/09/18"]
	zeising [label="Niclas Zeising\nzeising@FreeBSD.org\n2012/07/03"]

	# Here are the mentor/mentee relationships.
	# Group together all the mentees for a particular mentor.
	# Keep the list sorted by mentor login.

	+"0mp" -> debdrup
	+
	+allanjude -> debdrup
	+
	bcr -> gavin
	bcr -> wblock
	bcr -> eadler
	bcr -> dru
	bcr -> crees
	bcr -> jgh
	bcr -> allanjude
	bcr -> bhd
	bcr -> sevan
	bcr -> dexter
	bcr -> sg
	bcr -> carlavilla
	bcr -> "0mp"
	bcr -> gbe
	+bcr -> debdrup

	blackend -> ale

	brueffer -> joel

	ceri -> brd
	ceri -> brueffer
	ceri -> linimon
	ceri -> roam
	ceri -> simon

	den -> marck

	delphij -> chinsan
	delphij -> loader

	eadler -> allanjude

	gabor -> pgj
	gabor -> manolis
	gabor -> taras
	gabor -> issyl0
	gabor -> ebrandi
	gabor -> carlavilla

	gjb -> wblock
	gjb -> rene
	gjb -> dru
	gjb -> crees

	hrs -> ryusuke
	hrs -> dru
	hrs -> skreuzer

	jesusr -> jcamou

	jim -> trhodes

	jkois -> miwi
	jkois -> bcr
	jkois -> gavin
	jkois -> gjb
	jkois -> eadler

	joel -> zeising

	keramida -> blackend
	keramida -> danger
	keramida -> gabor
	keramida -> ganbold
	keramida -> garys
	keramida -> gjb
	keramida -> pav

	marck -> bvs
	marck -> pluknet
	marck -> taras

	maxim -> taras

	mheinen -> jkois

	murray -> ceri
	murray -> delphij

	nik -> bmah
	nik -> keramida

	remko -> jkois
	remko -> rene
	remko -> jgh

	simon -> josef
	simon -> remko

	trhodes -> danger
	trhodes -> jcamou

	wblock -> jgh
	wblock -> allanjude
	}
	diff --git a/share/misc/committers-src.dot b/share/misc/committers-src.dot
	index a7550b9a6ce2..212c1b613029 100644
	--- a/share/misc/committers-src.dot
	+++ b/share/misc/committers-src.dot
	@@ -1,906 +1,910 @@
	# $FreeBSD$

	# This file is meant to list all FreeBSD src committers and describe the
	# mentor-mentee relationships between them.
	# The graphical output can be generated from this file with the following
	# command:
	# $ dot -T png -o file.png committers-src.dot
	#
	# The dot binary is part of the graphics/graphviz port.

	digraph src {

	# Node definitions follow this example:
	#
	# foo [label="Foo Bar\nfoo@FreeBSD.org\n????/??/??"]
	#
	# ????/??/?? is the date when the commit bit was obtained, usually the one you
	# can find looking at svn logs for the svnadmin/conf/access file.
	# Use YYYY/MM/DD format.
	#
	# For returned commit bits, the node definition will follow this example:
	#
	# foo [label="Foo Bar\nfoo@FreeBSD.org\n????/??/??\n????/??/??"]
	#
	# The first date is the same as for an active committer, the second date is
	# the date when the commit bit has been returned. Again, check svn logs.

	node [color=grey62, style=filled, bgcolor=black];

	# Alumni go here.. Try to keep things sorted.

	alm [label="Andrew Moore\nalm@FreeBSD.org\n1993/06/12\n????/??/??"]
	anholt [label="Eric Anholt\nanholt@FreeBSD.org\n2002/04/22\n2008/08/07"]
	archie [label="Archie Cobbs\narchie@FreeBSD.org\n1998/11/06\n2006/06/09"]
	arr [label="Andrew R. Reiter\narr@FreeBSD.org\n2001/11/02\n2005/05/25"]
	arun [label="Arun Sharma\narun@FreeBSD.org\n2003/03/06\n2006/12/16"]
	asmodai [label="Jeroen Ruigrok\nasmodai@FreeBSD.org\n1999/12/16\n2001/11/16"]
	benjsc [label="Benjamin Close\nbenjsc@FreeBSD.org\n2007/02/09\n2010/09/15"]
	billf [label="Bill Fumerola\nbillf@FreeBSD.org\n1998/11/11\n2008/11/10"]
	bmah [label="Bruce A. Mah\nbmah@FreeBSD.org\n2002/01/29\n2009/09/13"]
	bmilekic [label="Bosko Milekic\nbmilekic@FreeBSD.org\n2000/09/21\n2008/11/10"]
	bushman [label="Michael Bushkov\nbushman@FreeBSD.org\n2007/03/10\n2010/04/29"]
	carl [label="Carl Delsey\ncarl@FreeBSD.org\n2013/01/14\n2014/03/06"]
	ceri [label="Ceri Davies\nceri@FreeBSD.org\n2006/11/07\n2012/03/07"]
	cjc [label="Crist J. Clark\ncjc@FreeBSD.org\n2001/06/01\n2006/12/29"]
	davidxu [label="David Xu\ndavidxu@FreeBSD.org\n2002/09/02\n2014/04/14"]
	dds [label="Diomidis Spinellis\ndds@FreeBSD.org\n2003/06/20\n2010/09/22"]
	dhartmei [label="Daniel Hartmeier\ndhartmei@FreeBSD.org\n2004/04/06\n2008/12/08"]
	dmlb [label="Duncan Barclay\ndmlb@FreeBSD.org\n2001/12/14\n2008/11/10"]
	dougb [label="Doug Barton\ndougb@FreeBSD.org\n2000/10/26\n2012/10/08"]
	eik [label="Oliver Eikemeier\neik@FreeBSD.org\n2004/05/20\n2008/11/10"]
	furuta [label="Atsushi Furuta\nfuruta@FreeBSD.org\n2000/06/21\n2003/03/08"]
	gj [label="Gary L. Jennejohn\ngj@FreeBSD.org\n1994/??/??\n2006/04/28"]
	groudier [label="Gerard Roudier\ngroudier@FreeBSD.org\n1999/12/30\n2006/04/06"]
	jake [label="Jake Burkholder\njake@FreeBSD.org\n2000/05/16\n2008/11/10"]
	jayanth [label="Jayanth Vijayaraghavan\njayanth@FreeBSD.org\n2000/05/08\n2008/11/10"]
	jb [label="John Birrell\njb@FreeBSD.org\n1997/03/27\n2009/12/15"]
	jdp [label="John Polstra\njdp@FreeBSD.org\n1995/12/07\n2008/02/26"]
	jedgar [label="Chris D. Faulhaber\njedgar@FreeBSD.org\n1999/12/15\n2006/04/07"]
	jkh [label="Jordan K. Hubbard\njkh@FreeBSD.org\n1993/06/12\n2008/06/13"]
	jlemon [label="Jonathan Lemon\njlemon@FreeBSD.org\n1997/08/14\n2008/11/10"]
	joe [label="Josef Karthauser\njoe@FreeBSD.org\n1999/10/22\n2008/08/10"]
	jtc [label="J.T. Conklin\njtc@FreeBSD.org\n1993/06/12\n????/??/??"]
	kargl [label="Steven G. Kargl\nkargl@FreeBSD.org\n2011/01/17\n2015/06/28"]
	kbyanc [label="Kelly Yancey\nkbyanc@FreeBSD.org\n2000/07/11\n2006/07/25"]
	keichii [label="Michael Wu\nkeichii@FreeBSD.org\n2001/03/07\n2006/04/28"]
	linimon [label="Mark Linimon\nlinimon@FreeBSD.org\n2006/09/30\n2008/05/04"]
	lulf [label="Ulf Lilleengen\nlulf@FreeBSD.org\n2007/10/24\n2012/01/19"]
	mb [label="Maxim Bolotin\nmb@FreeBSD.org\n2000/04/06\n2003/03/08"]
	marks [label="Mark Santcroos\nmarks@FreeBSD.org\n2004/03/18\n2008/09/29"]
	mike [label="Mike Barcroft\nmike@FreeBSD.org\n2001/07/17\n2006/04/28"]
	msmith [label="Mike Smith\nmsmith@FreeBSD.org\n1996/10/22\n2003/12/15"]
	murray [label="Murray Stokely\nmurray@FreeBSD.org\n2000/04/05\n2010/07/25"]
	mux [label="Maxime Henrion\nmux@FreeBSD.org\n2002/03/03\n2011/06/22"]
	nate [label="Nate Willams\nnate@FreeBSD.org\n1993/06/12\n2003/12/15"]
	njl [label="Nate Lawson\nnjl@FreeBSD.org\n2002/08/07\n2008/02/16"]
	non [label="Noriaki Mitsnaga\nnon@FreeBSD.org\n2000/06/19\n2007/03/06"]
	onoe [label="Atsushi Onoe\nonoe@FreeBSD.org\n2000/07/21\n2008/11/10"]
	rafan [label="Rong-En Fan\nrafan@FreeBSD.org\n2007/01/31\n2012/07/23"]
	randi [label="Randi Harper\nrandi@FreeBSD.org\n2010/04/20\n2012/05/10"]
	rink [label="Rink Springer\nrink@FreeBSD.org\n2006/01/16\n2010/11/04"]
	robert [label="Robert Drehmel\nrobert@FreeBSD.org\n2001/08/23\n2006/05/13"]
	sah [label="Sam Hopkins\nsah@FreeBSD.org\n2004/12/15\n2008/11/10"]
	shafeeq [label="Shafeeq Sinnamohideen\nshafeeq@FreeBSD.org\n2000/06/19\n2006/04/06"]
	sheldonh [label="Sheldon Hearn\nsheldonh@FreeBSD.org\n1999/06/14\n2006/05/13"]
	shiba [label="Takeshi Shibagaki\nshiba@FreeBSD.org\n2000/06/19\n2008/11/10"]
	shin [label="Yoshinobu Inoue\nshin@FreeBSD.org\n1999/07/29\n2003/03/08"]
	snb [label="Nick Barkas\nsnb@FreeBSD.org\n2009/05/05\n2010/11/04"]
	tmm [label="Thomas Moestl\ntmm@FreeBSD.org\n2001/03/07\n2006/07/12"]
	toshi [label="Toshihiko Arai\ntoshi@FreeBSD.org\n2000/07/06\n2003/03/08"]
	tshiozak [label="Takuya SHIOZAKI\ntshiozak@FreeBSD.org\n2001/04/25\n2003/03/08"]
	uch [label="UCHIYAMA Yasushi\nuch@FreeBSD.org\n2000/06/21\n2002/04/24"]
	wilko [label="Wilko Bulte\nwilko@FreeBSD.org\n2000/01/13\n2013/01/17"]
	yar [label="Yar Tikhiy\nyar@FreeBSD.org\n2001/03/25\n2012/05/23"]
	zack [label="Zack Kirsch\nzack@FreeBSD.org\n2010/11/05\n2012/09/08"]


	node [color=lightblue2, style=filled, bgcolor=black];

	# Current src committers go here. Try to keep things sorted.

	ache [label="Andrey Chernov\nache@FreeBSD.org\n1993/10/31"]
	achim [label="Achim Leubner\nachim@FreeBSD.org\n2013/01/23"]
	adrian [label="Adrian Chadd\nadrian@FreeBSD.org\n2000/07/03"]
	ae [label="Andrey V. Elsukov\nae@FreeBSD.org\n2010/06/03"]
	akiyama [label="Shunsuke Akiyama\nakiyama@FreeBSD.org\n2000/06/19"]
	alc [label="Alan Cox\nalc@FreeBSD.org\n1999/02/23"]
	alfredo [label="Alfredo Dal'Ava Junior\nalfredo@FreeBSD.org\n2020/01/27"]
	allanjude [label="Allan Jude\nallanjude@FreeBSD.org\n2015/07/30"]
	ambrisko [label="Doug Ambrisko\nambrisko@FreeBSD.org\n2001/12/19"]
	anchie [label="Ana Kukec\nanchie@FreeBSD.org\n2010/04/14"]
	andre [label="Andre Oppermann\nandre@FreeBSD.org\n2003/11/12"]
	andreast [label="Andreas Tobler\nandreast@FreeBSD.org\n2010/09/05"]
	andrew [label="Andrew Turner\nandrew@FreeBSD.org\n2010/07/19"]
	antoine [label="Antoine Brodin\nantoine@FreeBSD.org\n2008/02/03"]
	araujo [label="Marcelo Araujo\naraujo@FreeBSD.org\n2015/08/04"]
	arichardson [label="Alex Richardson\narichardson@FreeBSD.org\n2017/10/30"]
	ariff [label="Ariff Abdullah\nariff@FreeBSD.org\n2005/11/14"]
	art [label="Artem Belevich\nart@FreeBSD.org\n2011/03/29"]
	arybchik [label="Andrew Rybchenko\narybchik@FreeBSD.org\n2014/10/12"]
	asomers [label="Alan Somers\nasomers@FreeBSD.org\n2013/04/24"]
	avg [label="Andriy Gapon\navg@FreeBSD.org\n2009/02/18"]
	avos [label="Andriy Voskoboinyk\navos@FreeBSD.org\n2015/09/24"]
	badger [label="Eric Badger\nbadger@FreeBSD.org\n2016/07/01"]
	bapt [label="Baptiste Daroussin\nbapt@FreeBSD.org\n2011/12/23"]
	bcran [label="Rebecca Cran\nbcran@FreeBSD.org\n2010/01/29"]
	bde [label="Bruce Evans\nbde@FreeBSD.org\n1994/08/20"]
	bdragon [label="Brandon Bergren\nbdragon@FreeBSD.org\n2019/05/31"]
	bdrewery [label="Bryan Drewery\nbdrewery@FreeBSD.org\n2013/12/14"]
	benl [label="Ben Laurie\nbenl@FreeBSD.org\n2011/05/18"]
	benno [label="Benno Rice\nbenno@FreeBSD.org\n2000/11/02"]
	bms [label="Bruce M Simpson\nbms@FreeBSD.org\n2003/08/06"]
	br [label="Ruslan Bukin\nbr@FreeBSD.org\n2013/09/02"]
	brian [label="Brian Somers\nbrian@FreeBSD.org\n1996/12/16"]
	brooks [label="Brooks Davis\nbrooks@FreeBSD.org\n2001/06/21"]
	brueffer [label="Christian Brueffer\nbrueffer@FreeBSD.org\n2006/02/28"]
	bruno [label="Bruno Ducrot\nbruno@FreeBSD.org\n2005/07/18"]
	bryanv [label="Bryan Venteicher\nbryanv@FreeBSD.org\n2012/11/03"]
	bschmidt [label="Bernhard Schmidt\nbschmidt@FreeBSD.org\n2010/02/06"]
	bwidawsk [label="Ben Widawsky\nbwidawsk@FreeBSD.org\n2018/07/05"]
	bz [label="Bjoern A. Zeeb\nbz@FreeBSD.org\n2004/07/27"]
	cem [label="Conrad Meyer\ncem@FreeBSD.org\n2015/07/05"]
	chuck [label="Chuck Tuffli\nchuck@FreeBSD.org\n2017/09/06"]
	cognet [label="Olivier Houchard\ncognet@FreeBSD.org\n2002/10/09"]
	cokane [label="Coleman Kane\ncokane@FreeBSD.org\n2000/06/19"]
	cperciva [label="Colin Percival\ncperciva@FreeBSD.org\n2004/01/20"]
	csjp [label="Christian S.J. Peron\ncsjp@FreeBSD.org\n2004/05/04"]
	cy [label="Cy Schubert\ncy@FreeBSD.org\n2013/04/23"]
	dab [label="David Bright\ndab@FreeBSD.org\n2016/10/24"]
	das [label="David Schultz\ndas@FreeBSD.org\n2003/02/21"]
	davide [label="Davide Italiano\ndavide@FreeBSD.org\n2012/01/27"]
	dchagin [label="Dmitry Chagin\ndchagin@FreeBSD.org\n2009/02/28"]
	def [label="Konrad Witaszczyk\ndef@FreeBSD.org\n2016/11/02"]
	delphij [label="Xin Li\ndelphij@FreeBSD.org\n2004/09/14"]
	des [label="Dag-Erling Smorgrav\ndes@FreeBSD.org\n1998/04/03"]
	dexuan [label="Dexuan Cui\ndexuan@FreeBSD.org\n2016/10/24"]
	dfr [label="Doug Rabson\ndfr@FreeBSD.org\n????/??/??"]
	dg [label="David Greenman\ndg@FreeBSD.org\n1993/06/14"]
	dim [label="Dimitry Andric\ndim@FreeBSD.org\n2010/08/30"]
	dougm [label="Doug Moore\ndougm@FreeBSD.org\n2019/04/30"]
	dteske [label="Devin Teske\ndteske@FreeBSD.org\n2012/04/10"]
	dumbbell [label="Jean-Sebastien Pedron\ndumbbell@FreeBSD.org\n2004/11/29"]
	dwmalone [label="David Malone\ndwmalone@FreeBSD.org\n2000/07/11"]
	eadler [label="Eitan Adler\neadler@FreeBSD.org\n2012/01/18"]
	ed [label="Ed Schouten\ned@FreeBSD.org\n2008/05/22"]
	edavis [label="Eric Davis\nedavis@FreeBSD.org\n2013/10/09"]
	edwin [label="Edwin Groothuis\nedwin@FreeBSD.org\n2007/06/25"]
	eivind [label="Eivind Eklund\neivind@FreeBSD.org\n1997/02/02"]
	emaste [label="Ed Maste\nemaste@FreeBSD.org\n2005/10/04"]
	emax [label="Maksim Yevmenkin\nemax@FreeBSD.org\n2003/10/12"]
	eri [label="Ermal Luci\neri@FreeBSD.org\n2008/06/11"]
	erj [label="Eric Joyner\nerj@FreeBSD.org\n2014/12/14"]
	eugen [label="Eugene Grosbein\neugen@FreeBSD.org\n2017/09/19"]
	fabient [label="Fabien Thomas\nfabient@FreeBSD.org\n2009/03/16"]
	fanf [label="Tony Finch\nfanf@FreeBSD.org\n2002/05/05"]
	fjoe [label="Max Khon\nfjoe@FreeBSD.org\n2001/08/06"]
	flz [label="Florent Thoumie\nflz@FreeBSD.org\n2006/03/30"]
	freqlabs [label="Ryan Moeller\nfreqlabs@FreeBSD.org\n2020/02/10"]
	fsu [label="Fedor Uporov\nfsu@FreeBSD.org\n2017/08/28"]
	gabor [label="Gabor Kovesdan\ngabor@FreeBSD.org\n2010/02/02"]
	gad [label="Garance A. Drosehn\ngad@FreeBSD.org\n2000/10/27"]
	gallatin [label="Andrew Gallatin\ngallatin@FreeBSD.org\n1999/01/15"]
	ganbold [label="Ganbold Tsagaankhuu\nganbold@FreeBSD.org\n2013/12/18"]
	gavin [label="Gavin Atkinson\ngavin@FreeBSD.org\n2009/12/07"]
	gibbs [label="Justin T. Gibbs\ngibbs@FreeBSD.org\n????/??/??"]
	gjb [label="Glen Barber\ngjb@FreeBSD.org\n2013/06/04"]
	gleb [label="Gleb Kurtsou\ngleb@FreeBSD.org\n2011/09/19"]
	glebius [label="Gleb Smirnoff\nglebius@FreeBSD.org\n2004/07/14"]
	gnn [label="George V. Neville-Neil\ngnn@FreeBSD.org\n2004/10/11"]
	gordon [label="Gordon Tetlow\ngordon@FreeBSD.org\n2002/05/17"]
	grehan [label="Peter Grehan\ngrehan@FreeBSD.org\n2002/08/08"]
	grog [label="Greg Lehey\ngrog@FreeBSD.org\n1998/08/30"]
	gshapiro [label="Gregory Shapiro\ngshapiro@FreeBSD.org\n2000/07/12"]
	harti [label="Hartmut Brandt\nharti@FreeBSD.org\n2003/01/29"]
	hiren [label="Hiren Panchasara\nhiren@FreeBSD.org\n2013/04/12"]
	hmp [label="Hiten Pandya\nhmp@FreeBSD.org\n2004/03/23"]
	hselasky [label="Hans Petter Selasky\nhselasky@FreeBSD.org\n"]
	ian [label="Ian Lepore\nian@FreeBSD.org\n2013/01/07"]
	iedowse [label="Ian Dowse\niedowse@FreeBSD.org\n2000/12/01"]
	imp [label="Warner Losh\nimp@FreeBSD.org\n1996/09/20"]
	ivoras [label="Ivan Voras\nivoras@FreeBSD.org\n2008/06/10"]
	jah [label="Jason A. Harmening\njah@FreeBSD.org\n2015/03/08"]
	jamie [label="Jamie Gritton\njamie@FreeBSD.org\n2009/01/28"]
	jasone [label="Jason Evans\njasone@FreeBSD.org\n1999/03/03"]
	jceel [label="Jakub Klama\njceel@FreeBSD.org\n2011/09/25"]
	jch [label="Julien Charbon\njch@FreeBSD.org\n2014/09/24"]
	jchandra [label="Jayachandran C.\njchandra@FreeBSD.org\n2010/05/19"]
	jeb [label="Jeb Cramer\njeb@FreeBSD.org\n2018/01/25"]
	jeff [label="Jeff Roberson\njeff@FreeBSD.org\n2002/02/21"]
	jh [label="Jaakko Heinonen\njh@FreeBSD.org\n2009/10/02"]
	jhb [label="John Baldwin\njhb@FreeBSD.org\n1999/08/23"]
	jhibbits [label="Justin Hibbits\njhibbits@FreeBSD.org\n2011/11/30"]
	jilles [label="Jilles Tjoelker\njilles@FreeBSD.org\n2009/05/22"]
	jimharris [label="Jim Harris\njimharris@FreeBSD.org\n2011/12/09"]
	jinmei [label="JINMEI Tatuya\njinmei@FreeBSD.org\n2007/03/17"]
	jkim [label="Jung-uk Kim\njkim@FreeBSD.org\n2005/07/06"]
	jkoshy [label="A. Joseph Koshy\njkoshy@FreeBSD.org\n1998/05/13"]
	jlh [label="Jeremie Le Hen\njlh@FreeBSD.org\n2012/04/22"]
	jls [label="Jordan Sissel\njls@FreeBSD.org\n2006/12/06"]
	jmcneill [label="Jared McNeill\njmcneill@FreeBSD.org\n2016/02/24"]
	jmg [label="John-Mark Gurney\njmg@FreeBSD.org\n1997/02/13"]
	jmmv [label="Julio Merino\njmmv@FreeBSD.org\n2013/11/02"]
	joerg [label="Joerg Wunsch\njoerg@FreeBSD.org\n1993/11/14"]
	johalun [label="Johannes Lundberg\njohalun@FreeBSD.org\n2019/01/19"]
	jon [label="Jonathan Chen\njon@FreeBSD.org\n2000/10/17"]
	jonathan [label="Jonathan Anderson\njonathan@FreeBSD.org\n2010/10/07"]
	jpaetzel [label="Josh Paetzel\njpaetzel@FreeBSD.org\n2011/01/21"]
	jtl [label="Jonathan T. Looney\njtl@FreeBSD.org\n2015/10/26"]
	julian [label="Julian Elischer\njulian@FreeBSD.org\n1993/04/19"]
	jwd [label="John De Boskey\njwd@FreeBSD.org\n2000/05/19"]
	kaiw [label="Kai Wang\nkaiw@FreeBSD.org\n2007/09/26"]
	kaktus [label="Pawel Biernacki\nkaktus@FreeBSD.org\n2019/09/26"]
	kan [label="Alexander Kabaev\nkan@FreeBSD.org\n2002/07/21"]
	karels [label="Mike Karels\nkarels@FreeBSD.org\n2016/06/09"]
	ken [label="Ken Merry\nken@FreeBSD.org\n1998/09/08"]
	kensmith [label="Ken Smith\nkensmith@FreeBSD.org\n2004/01/23"]
	kevans [label="Kyle Evans\nkevans@FreeBSD.org\n2017/06/20"]
	kevlo [label="Kevin Lo\nkevlo@FreeBSD.org\n2006/07/23"]
	kib [label="Konstantin Belousov\nkib@FreeBSD.org\n2006/06/03"]
	kibab [label="Ilya Bakulin\nkibab@FreeBSD.org\n2017/09/02"]
	kmacy [label="Kip Macy\nkmacy@FreeBSD.org\n2005/06/01"]
	kp [label="Kristof Provost\nkp@FreeBSD.org\n2015/03/22"]
	landonf [label="Landon Fuller\nlandonf@FreeBSD.org\n2016/05/31"]
	le [label="Lukas Ertl\nle@FreeBSD.org\n2004/02/02"]
	leitao [label="Breno Leitao\nleitao@FreeBSD.org\n2018/05/22"]
	lidl [label="Kurt Lidl\nlidl@FreeBSD.org\n2015/10/21"]
	loos [label="Luiz Otavio O Souza\nloos@FreeBSD.org\n2013/07/03"]
	lstewart [label="Lawrence Stewart\nlstewart@FreeBSD.org\n2008/10/06"]
	luporl [label="Leandro Lupori\nluporl@FreeBSD.org\n2018/05/21"]
	lwhsu [label="Li-Wen Hsu\nlwhsu@FreeBSD.org\n2018/08/09"]
	manu [label="Emmanuel Vadot\nmanu@FreeBSD.org\n2016/04/24"]
	marcel [label="Marcel Moolenaar\nmarcel@FreeBSD.org\n1999/07/03"]
	marius [label="Marius Strobl\nmarius@FreeBSD.org\n2004/04/17"]
	markj [label="Mark Johnston\nmarkj@FreeBSD.org\n2012/12/18"]
	markm [label="Mark Murray\nmarkm@FreeBSD.org\n1995/04/24"]
	markus [label="Markus Brueffer\nmarkus@FreeBSD.org\n2006/06/01"]
	matteo [label="Matteo Riondato\nmatteo@FreeBSD.org\n2006/01/18"]
	mav [label="Alexander Motin\nmav@FreeBSD.org\n2007/04/12"]
	maxim [label="Maxim Konovalov\nmaxim@FreeBSD.org\n2002/02/07"]
	mdf [label="Matthew Fleming\nmdf@FreeBSD.org\n2010/06/04"]
	mdodd [label="Matthew N. Dodd\nmdodd@FreeBSD.org\n1999/07/27"]
	melifaro [label="Alexander V. Chernikov\nmelifaro@FreeBSD.org\n2011/10/04"]
	mhorne [label="Mitchell Horne\nmhorne@FreeBSD.org\n2019/03/20"]
	miwi [label="Martin Wilke\nmiwi@FreeBSD.org\n2011/02/18\n2018/06/14"]
	mizhka [label="Michael Zhilin\nmizhka@FreeBSD.org\n2016/07/19"]
	mjacob [label="Matt Jacob\nmjacob@FreeBSD.org\n1997/08/13"]
	mjg [label="Mateusz Guzik\nmjg@FreeBSD.org\n2012/06/04"]
	mjoras [label="Matt Joras\nmjoras@FreeBSD.org\n2017/07/12"]
	mlaier [label="Max Laier\nmlaier@FreeBSD.org\n2004/02/10"]
	mmel [label="Michal Meloun\nmmel@FreeBSD.org\n2015/11/01"]
	monthadar [label="Monthadar Al Jaberi\nmonthadar@FreeBSD.org\n2012/04/02"]
	mp [label="Mark Peek\nmp@FreeBSD.org\n2001/07/27"]
	mr [label="Michael Reifenberger\nmr@FreeBSD.org\n2001/09/30"]
	mw [label="Marcin Wojtas\nmw@FreeBSD.org\n2017/07/18"]
	neel [label="Neel Natu\nneel@FreeBSD.org\n2009/09/20"]
	netchild [label="Alexander Leidinger\nnetchild@FreeBSD.org\n2005/03/31"]
	ngie [label="Enji Cooper\nngie@FreeBSD.org\n2014/07/27"]
	nick [label="Nick O'Brien\nnick@FreeBSD.org\n2020/03/09"]
	nork [label="Norikatsu Shigemura\nnork@FreeBSD.org\n2009/06/09"]
	np [label="Navdeep Parhar\nnp@FreeBSD.org\n2009/06/05"]
	nwhitehorn [label="Nathan Whitehorn\nnwhitehorn@FreeBSD.org\n2008/07/03"]
	n_hibma [label="Nick Hibma\nn_hibma@FreeBSD.org\n1998/11/26"]
	obrien [label="David E. O'Brien\nobrien@FreeBSD.org\n1996/10/29"]
	olli [label="Oliver Fromme\nolli@FreeBSD.org\n2008/02/14"]
	oshogbo [label="Mariusz Zaborski\noshogbo@FreeBSD.org\n2015/04/15"]
	peadar [label="Peter Edwards\npeadar@FreeBSD.org\n2004/03/08"]
	peter [label="Peter Wemm\npeter@FreeBSD.org\n1995/07/04"]
	peterj [label="Peter Jeremy\npeterj@FreeBSD.org\n2012/09/14"]
	pfg [label="Pedro Giffuni\npfg@FreeBSD.org\n2011/12/01"]
	phil [label="Phil Shafer\nphil@FreeBSD.ogr\n2015/12/30"]
	philip [label="Philip Paeps\nphilip@FreeBSD.org\n2004/01/21"]
	phk [label="Poul-Henning Kamp\nphk@FreeBSD.org\n1994/02/21"]
	pho [label="Peter Holm\npho@FreeBSD.org\n2008/11/16"]
	pjd [label="Pawel Jakub Dawidek\npjd@FreeBSD.org\n2004/02/02"]
	pkelsey [label="Patrick Kelsey\pkelsey@FreeBSD.org\n2014/05/29"]
	pluknet [label="Sergey Kandaurov\npluknet@FreeBSD.org\n2010/10/05"]
	ps [label="Paul Saab\nps@FreeBSD.org\n2000/02/23"]
	qingli [label="Qing Li\nqingli@FreeBSD.org\n2005/04/13"]
	ram [label="Ram Kishore Vegesna\nram@FreeBSD.org\n2018/04/04"]
	ray [label="Aleksandr Rybalko\nray@FreeBSD.org\n2011/05/25"]
	rdivacky [label="Roman Divacky\nrdivacky@FreeBSD.org\n2008/03/13"]
	remko [label="Remko Lodder\nremko@FreeBSD.org\n2007/02/23"]
	+rew [label="Robert Wing\nrew@FreeBSD.org\n2020/07/23"]
	rgrimes [label="Rodney W. Grimes\nrgrimes@FreeBSD.org\n1993/06/12\n2017/03/03"]
	rik [label="Roman Kurakin\nrik@FreeBSD.org\n2003/12/18"]
	rlibby [label="Ryan Libby\nrlibby@FreeBSD.org\n2017/06/07"]
	rmacklem [label="Rick Macklem\nrmacklem@FreeBSD.org\n2009/03/27"]
	rmh [label="Robert Millan\nrmh@FreeBSD.org\n2011/09/18"]
	rnoland [label="Robert Noland\nrnoland@FreeBSD.org\n2008/09/15"]
	roberto [label="Ollivier Robert\nroberto@FreeBSD.org\n1995/02/22"]
	rodrigc [label="Craig Rodrigues\nrodrigc@FreeBSD.org\n2005/05/14"]
	royger [label="Roger Pau Monne\nroyger@FreeBSD.org\n2013/11/26"]
	rpaulo [label="Rui Paulo\nrpaulo@FreeBSD.org\n2007/09/25"]
	rpokala [label="Ravi Pokala\nrpokala@FreeBSD.org\n2015/11/19"]
	rrs [label="Randall R Stewart\nrrs@FreeBSD.org\n2007/02/08"]
	rscheff [label="Richard Scheffenegger\nrscheff@FreeBSD.org\n2020/04/06"]
	rse [label="Ralf S. Engelschall\nrse@FreeBSD.org\n1997/07/31"]
	rstone [label="Ryan Stone\nrstone@FreeBSD.org\n2010/04/19"]
	ru [label="Ruslan Ermilov\nru@FreeBSD.org\n1999/05/27"]
	rwatson [label="Robert N. M. Watson\nrwatson@FreeBSD.org\n1999/12/16"]
	sam [label="Sam Leffler\nsam@FreeBSD.org\n2002/07/02"]
	sanpei [label="MIHIRA Sanpei Yoshiro\nsanpei@FreeBSD.org\n2000/06/19"]
	sbruno [label="Sean Bruno\nsbruno@FreeBSD.org\n2008/08/02"]
	scf [label="Sean C. Farley\nscf@FreeBSD.org\n2007/06/24"]
	schweikh [label="Jens Schweikhardt\nschweikh@FreeBSD.org\n2001/04/06"]
	scottl [label="Scott Long\nscottl@FreeBSD.org\n2000/09/28"]
	scottph [label="D Scott Phillips\nscottph@FreeBSD.org\n2019/05/28"]
	se [label="Stefan Esser\nse@FreeBSD.org\n1994/08/26"]
	sephe [label="Sepherosa Ziehau\nsephe@FreeBSD.org\n2007/03/28"]
	sepotvin [label="Stephane E. Potvin\nsepotvin@FreeBSD.org\n2007/02/15"]
	sgalabov [label="Stanislav Galabov\nsgalabov@FreeBSD.org\n2016/02/24"]
	shurd [label="Stephen Hurd\nshurd@FreeBSD.org\n2017/09/02"]
	simon [label="Simon L. Nielsen\nsimon@FreeBSD.org\n2006/03/07"]
	sjg [label="Simon J. Gerraty\nsjg@FreeBSD.org\n2012/10/23"]
	skra [label="Svatopluk Kraus\nskra@FreeBSD.org\n2015/10/28"]
	slavash [label="Slava Shwartsman\nslavash@FreeBSD.org\n2018/02/08"]
	slm [label="Stephen McConnell\nslm@FreeBSD.org\n2014/05/07"]
	smh [label="Steven Hartland\nsmh@FreeBSD.org\n2012/11/12"]
	sobomax [label="Maxim Sobolev\nsobomax@FreeBSD.org\n2001/07/25"]
	sos [label="Soren Schmidt\nsos@FreeBSD.org\n????/??/??"]
	sson [label="Stacey Son\nsson@FreeBSD.org\n2008/07/08"]
	stas [label="Stanislav Sedov\nstas@FreeBSD.org\n2008/08/22"]
	stevek [label="Stephen J. Kiernan\nstevek@FreeBSD.org\n2016/07/18"]
	suz [label="SUZUKI Shinsuke\nsuz@FreeBSD.org\n2002/03/26"]
	syrinx [label="Shteryana Shopova\nsyrinx@FreeBSD.org\n2006/10/07"]
	takawata [label="Takanori Watanabe\ntakawata@FreeBSD.org\n2000/07/06"]
	theraven [label="David Chisnall\ntheraven@FreeBSD.org\n2011/11/11"]
	thj [label="Tom Jones\nthj@FreeBSD.org\n2018/04/07"]
	thompsa [label="Andrew Thompson\nthompsa@FreeBSD.org\n2005/05/25"]
	ticso [label="Bernd Walter\nticso@FreeBSD.org\n2002/01/31"]
	tijl [label="Tijl Coosemans\ntijl@FreeBSD.org\n2010/07/16"]
	tsoome [label="Toomas Soome\ntsoome@FreeBSD.org\n2016/08/10"]
	trasz [label="Edward Tomasz Napierala\ntrasz@FreeBSD.org\n2008/08/22"]
	trhodes [label="Tom Rhodes\ntrhodes@FreeBSD.org\n2002/05/28"]
	trociny [label="Mikolaj Golub\ntrociny@FreeBSD.org\n2011/03/10"]
	tuexen [label="Michael Tuexen\ntuexen@FreeBSD.org\n2009/06/06"]
	tychon [label="Tycho Nightingale\ntychon@FreeBSD.org\n2014/01/21"]
	ume [label="Hajimu UMEMOTO\nume@FreeBSD.org\n2000/02/26"]
	uqs [label="Ulrich Spoerlein\nuqs@FreeBSD.org\n2010/01/28"]
	vangyzen [label="Eric van Gyzen\nvangyzen@FreeBSD.org\n2015/03/08"]
	vanhu [label="Yvan Vanhullebus\nvanhu@FreeBSD.org\n2008/07/21"]
	versus [label="Konrad Jankowski\nversus@FreeBSD.org\n2008/10/27"]
	vmaffione [label="Vincenzo Maffione\nvmaffione@FreeBSD.org\n2018/03/19"]
	weongyo [label="Weongyo Jeong\nweongyo@FreeBSD.org\n2007/12/21"]
	wes [label="Wes Peters\nwes@FreeBSD.org\n1998/11/25"]
	whu [label="Wei Hu\nwhu@FreeBSD.org\n2015/02/11"]
	will [label="Will Andrews\nwill@FreeBSD.org\n2000/03/20"]
	wkoszek [label="Wojciech A. Koszek\nwkoszek@FreeBSD.org\n2006/02/21"]
	wma [label="Wojciech Macek\nwma@FreeBSD.org\n2016/01/18"]
	wollman [label="Garrett Wollman\nwollman@FreeBSD.org\n????/??/??"]
	wsalamon [label="Wayne Salamon\nwsalamon@FreeBSD.org\n2005/06/25"]
	wulf [label="Vladimir Kondratyev\nwulf@FreeBSD.org\n2017/04/27"]
	yongari [label="Pyun YongHyeon\nyongari@FreeBSD.org\n2004/08/01"]
	yuripv [label="Yuri Pankov\nyuripv@FreeBSD.org\n2018/10/09"]
	zbb [label="Zbigniew Bodek\nzbb@FreeBSD.org\n2013/09/02"]
	zec [label="Marko Zec\nzec@FreeBSD.org\n2008/06/22"]
	zml [label="Zachary Loafman\nzml@FreeBSD.org\n2009/05/27"]
	zont [label="Andrey Zonov\nzont@FreeBSD.org\n2012/08/21"]

	# Pseudo target representing rev 1.1 of commit.allow
	day1 [label="Birth of FreeBSD"]

	# Here are the mentor/mentee relationships.
	# Group together all the mentees for a particular mentor.
	# Keep the list sorted by mentor login.

	day1 -> jtc
	day1 -> jkh
	day1 -> nate
	day1 -> rgrimes
	day1 -> alm
	day1 -> dg

	adrian -> avos
	adrian -> jmcneill
	adrian -> landonf
	adrian -> lidl
	adrian -> loos
	adrian -> mizhka
	adrian -> monthadar
	adrian -> ray
	adrian -> rmh
	adrian -> sephe
	adrian -> sgalabov

	ae -> melifaro

	+allanjude -> rew
	allanjude -> tsoome

	alc -> davide

	andre -> qingli

	andrew -> manu

	anholt -> jkim

	araujo -> miwi

	avg -> art
	avg -> eugen
	avg -> pluknet
	avg -> smh

	bapt -> allanjude
	bapt -> araujo
	bapt -> bdrewery
	bapt -> wulf

	bde -> rgrimes

	benno -> grehan

	billf -> dougb
	billf -> gad
	billf -> jedgar
	billf -> jhb
	billf -> shafeeq
	billf -> will

	bmilekic -> csjp

	bms -> dhartmei
	bms -> mlaier
	bms -> thompsa

	brian -> joe

	brooks -> bushman
	brooks -> jamie
	brooks -> theraven
	brooks -> arichardson

	bz -> anchie
	bz -> jamie
	bz -> syrinx

	cognet -> br
	cognet -> jceel
	cognet -> kevlo
	cognet -> ian
	cognet -> manu
	cognet -> mw
	cognet -> wkoszek
	cognet -> wma
	cognet -> zbb

	cperciva -> eadler
	cperciva -> flz
	cperciva -> randi
	cperciva -> simon

	csjp -> bushman

	das -> kargl
	das -> rodrigc

	delphij -> gabor
	delphij -> rafan
	delphij -> sephe

	des -> anholt
	des -> hmp
	des -> mike
	des -> olli
	des -> ru
	des -> bapt

	dds -> versus

	dfr -> gallatin
	dfr -> zml

	dg -> peter

	dim -> theraven

	dwmalone -> fanf
	dwmalone -> peadar
	dwmalone -> snb

	eadler -> bcran

	ed -> dim
	ed -> gavin
	ed -> jilles
	ed -> rdivacky
	ed -> uqs

	eivind -> des
	eivind -> rwatson

	emaste -> achim
	emaste -> bwidawsk
	emaste -> dteske
	emaste -> kevans
	emaste -> lwhsu
	emaste -> markj
	emaste -> ngie
	emaste -> rstone
	emaste -> scottph

	emax -> markus

	erj -> jeb

	fjoe -> versus

	gallatin -> ticso

	gavin -> versus

	gibbs -> mjacob
	gibbs -> njl
	gibbs -> royger
	gibbs -> whu

	glebius -> mav

	gnn -> jinmei
	gnn -> rrs
	gnn -> ivoras
	gnn -> vanhu
	gnn -> lstewart
	gnn -> np
	gnn -> davide
	gnn -> arybchik
	gnn -> erj
	gnn -> kp
	gnn -> jtl
	gnn -> karels

	gonzo -> jmcneill
	gonzo -> wulf

	grehan -> bryanv
	grehan -> rgrimes

	grog -> edwin
	grog -> le
	grog -> peterj

	hselasky -> slavash

	imp -> akiyama
	imp -> ambrisko
	imp -> andrew
	imp -> bmah
	imp -> bruno
	imp -> chuck
	imp -> dmlb
	imp -> emax
	imp -> furuta
	imp -> joe
	imp -> johalun
	imp -> jon
	imp -> keichii
	imp -> kibab
	imp -> mb
	imp -> mr
	imp -> neel
	imp -> non
	imp -> nork
	imp -> onoe
	imp -> remko
	imp -> rik
	imp -> rink
	imp -> sanpei
	imp -> shiba
	imp -> takawata
	imp -> toshi
	imp -> tsoome
	imp -> uch

	jake -> bms
	jake -> gordon
	jake -> harti
	jake -> jeff
	jake -> kmacy
	jake -> robert
	jake -> yongari

	jb -> sson

	jdp -> fjoe

	jfv -> erj

	jhb -> arr
	jhb -> avg
	jhb -> jch
	jhb -> jeff
	jhb -> kbyanc
	jhb -> peterj
	jhb -> pfg
	jhb -> rnoland
	jhb -> rpokala
	jhb -> arichardson
	jhb -> scottph

	jhibbits -> alfredo
	jhibbits -> bdragon
	jhibbits -> leitao
	jhibbits -> luporl

	jimharris -> carl

	jkh -> dfr
	jkh -> gj
	jkh -> grog
	jkh -> imp
	jkh -> jlemon
	jkh -> joerg
	jkh -> jwd
	jkh -> msmith
	jkh -> murray
	jkh -> phk
	jkh -> wes
	jkh -> yar

	jkoshy -> kaiw
	jkoshy -> fabient
	jkoshy -> rstone

	jlemon -> bmilekic
	jlemon -> brooks

	jmallett -> pkelsey

	jmmv -> ngie

	joerg -> brian
	joerg -> eik
	joerg -> jmg
	joerg -> le
	joerg -> netchild
	joerg -> schweikh

	jtl -> ngie
	jtl -> thj

	julian -> glebius
	julian -> davidxu
	julian -> archie
	julian -> adrian
	julian -> zec
	julian -> mp

	kan -> kib

	ken -> asomers
	ken -> chuck
	ken -> ram
	ken -> slm
	ken -> will

	+kevans -> rew
	+
	kib -> ae
	kib -> badger
	kib -> dchagin
	kib -> dougm
	kib -> gjb
	kib -> jah
	kib -> jlh
	kib -> jpaetzel
	kib -> kaktus
	kib -> lulf
	kib -> melifaro
	kib -> mmel
	kib -> pho
	kib -> pluknet
	kib -> rdivacky
	kib -> rmacklem
	kib -> rmh
	kib -> skra
	kib -> slavash
	kib -> stas
	kib -> tijl
	kib -> trociny
	kib -> vangyzen
	kib -> yuripv
	kib -> zont

	kmacy -> lstewart

	kp -> nick

	marcel -> allanjude
	marcel -> art
	marcel -> arun
	marcel -> marius
	marcel -> nwhitehorn
	marcel -> sjg

	markj -> cem
	markj -> dougm
	markj -> lwhsu
	markj -> mhorne
	markj -> rlibby

	markm -> jasone
	markm -> sheldonh

	mav -> ae
	mav -> eugen
	mav -> freqlabs
	mav -> ram

	mdf -> gleb

	mdodd -> jake

	mike -> das

	mjg -> kaktus

	mlaier -> benjsc
	mlaier -> dhartmei
	mlaier -> thompsa
	mlaier -> eri

	mmacy -> freqlabs

	msmith -> cokane
	msmith -> jasone
	msmith -> scottl

	murray -> delphij

	mux -> cognet
	mux -> dumbbell

	netchild -> ariff

	njl -> marks
	njl -> philip
	njl -> rpaulo
	njl -> sepotvin

	nwhitehorn -> andreast
	nwhitehorn -> jhibbits
	nwhitehorn -> leitao
	nwhitehorn -> luporl

	obrien -> benno
	obrien -> groudier
	obrien -> gshapiro
	obrien -> kan
	obrien -> sam

	pfg -> pstef
	pfg -> fsu

	peter -> asmodai
	peter -> jayanth
	peter -> ps

	philip -> benl
	philip -> ed
	philip -> jls
	philip -> matteo
	philip -> uqs
	philip -> kp
	philip -> nick

	phk -> jkoshy
	phk -> mux
	phk -> rgrimes

	pjd -> def
	pjd -> kib
	pjd -> lulf
	pjd -> oshogbo
	pjd -> smh
	pjd -> trociny

	rgrimes -> markm
	rgrimes -> rscheff

	rmacklem -> jwd

	royger -> whu

	rpaulo -> avg
	rpaulo -> bschmidt
	rpaulo -> dim
	rpaulo -> jmmv
	rpaulo -> lidl
	rpaulo -> ngie

	rrs -> bcran
	rrs -> jchandra
	rrs -> tuexen

	rstone -> markj
	rstone -> mjoras

	ru -> ceri
	ru -> cjc
	ru -> eik
	ru -> maxim
	ru -> sobomax

	rwatson -> adrian
	rwatson -> antoine
	rwatson -> bmah
	rwatson -> brueffer
	rwatson -> bz
	rwatson -> cperciva
	rwatson -> emaste
	rwatson -> gnn
	rwatson -> jh
	rwatson -> jonathan
	rwatson -> kensmith
	rwatson -> kmacy
	rwatson -> linimon
	rwatson -> rmacklem
	rwatson -> shafeeq
	rwatson -> tmm
	rwatson -> trasz
	rwatson -> trhodes
	rwatson -> wsalamon

	rodrigc -> araujo

	sam -> andre
	sam -> benjsc
	sam -> sephe

	sbruno -> hiren
	sbruno -> jeb
	sbruno -> jimharris
	sbruno -> shurd

	schweikh -> dds

	scottl -> achim
	scottl -> jimharris
	scottl -> pjd
	scottl -> sah
	scottl -> sbruno
	scottl -> scottph
	scottl -> slm
	scottl -> yongari

	sephe -> dexuan

	sheldonh -> dwmalone
	sheldonh -> iedowse

	shin -> ume

	simon -> benl

	sjg -> phil
	sjg -> stevek

	sos -> marcel

	stas -> ganbold

	theraven -> phil

	thompsa -> weongyo
	thompsa -> eri

	trasz -> jh
	trasz -> mjg

	tuexen -> rscheff

	ume -> jinmei
	ume -> suz
	ume -> tshiozak

	vangyzen -> badger
	vangyzen -> dab

	wes -> scf

	wkoszek -> jceel

	wollman -> gad

	zml -> mdf
	zml -> zack

	}
	diff --git a/stand/common/install.c b/stand/common/install.c
	index de0fd88fc4e3..ea6eaaa7cebc 100644
	--- a/stand/common/install.c
	+++ b/stand/common/install.c
	@@ -1,409 +1,405 @@
	/*-
	* Copyright (c) 2008-2014, Juniper Networks, Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/socket.h>
	#include <net/if.h>
	#include <netinet/in.h>
	#include <netinet/in_systm.h>

	#include <stand.h>
	#include <net.h>
	#include <string.h>

	#include "bootstrap.h"

	extern struct in_addr servip;

	extern int pkgfs_init(const char , struct fs_ops );
	extern void pkgfs_cleanup(void);

	COMMAND_SET(install, "install", "install software package", command_install);

	static char *inst_kernel;
	static char **inst_modules;
	static char *inst_rootfs;
	static char *inst_loader_rc;

	static int
	setpath(char *what, char val)
	{
	char *path;
	size_t len;
	int rel;

	len = strlen(val) + 1;
	rel = (val[0] != '/') ? 1 : 0;
	path = malloc(len + rel);
	if (path == NULL)
	return (ENOMEM);
	path[0] = '/';
	strcpy(path + rel, val);

	*what = path;
	return (0);
	}

	static int
	setmultipath(char **what, char val)
	{
	char s, v;
	int count, error, idx;

	count = 0;
	v = val;
	do {
	count++;
	s = strchr(v, ',');
	v = (s == NULL) ? NULL : s + 1;
	} while (v != NULL);

	what = calloc(count + 1, sizeof(char ));
	if (*what == NULL)
	return (ENOMEM);

	for (idx = 0; idx < count; idx++) {
	s = strchr(val, ',');
	if (s != NULL)
	*s++ = '\0';
	error = setpath(*what + idx, val);
	if (error)
	return (error);
	val = s;
	}

	return (0);
	}

	static int
	read_metatags(int fd)
	{
	char buf[1024];
	char p, tag, *val;
	ssize_t fsize;
	int error;

	fsize = read(fd, buf, sizeof(buf));
	if (fsize == -1)
	return (errno);

	/*
	* Assume that if we read a whole buffer worth of data, we
	* haven't read the entire file. In other words, the buffer
	* size must always be larger than the file size. That way
	* we can append a '\0' and use standard string operations.
	* Return an error if this is not possible.
	*/
	if (fsize == sizeof(buf))
	return (ENOMEM);

	buf[fsize] = '\0';
	error = 0;
	tag = buf;
	while (!error && *tag != '\0') {
	val = strchr(tag, '=');
	if (val == NULL) {
	error = EINVAL;
	break;
	}
	*val++ = '\0';
	p = strchr(val, '\n');
	if (p == NULL) {
	error = EINVAL;
	break;
	}
	*p++ = '\0';

	if (strcmp(tag, "KERNEL") == 0)
	error = setpath(&inst_kernel, val);
	else if (strcmp(tag, "MODULES") == 0)
	error = setmultipath(&inst_modules, val);
	else if (strcmp(tag, "ROOTFS") == 0)
	error = setpath(&inst_rootfs, val);
	else if (strcmp(tag, "LOADER_RC") == 0)
	error = setpath(&inst_loader_rc, val);

	tag = p;
	}

	return (error);
	}

	static void
	cleanup(void)
	{
	u_int i;

	if (inst_kernel != NULL) {
	free(inst_kernel);
	inst_kernel = NULL;
	}
	if (inst_modules != NULL) {
	i = 0;
	while (inst_modules[i] != NULL)
	free(inst_modules[i++]);
	free(inst_modules);
	inst_modules = NULL;
	}
	if (inst_rootfs != NULL) {
	free(inst_rootfs);
	inst_rootfs = NULL;
	}
	if (inst_loader_rc != NULL) {
	free(inst_loader_rc);
	inst_loader_rc = NULL;
	}
	pkgfs_cleanup();
	}

	/*
	* usage: install URL
	* where: URL = tftp://[host]/<package>
	* or file://[devname[:fstype]]/<package>
	*/
	static int
	install(char *pkgname)
	{
	static char buf[256];
	struct fs_ops *proto;
	struct preloaded_file *fp;
	char e, s, *currdev;
	char *devname;
	size_t devnamelen;
	int error, fd, i, local;

	s = strstr(pkgname, "://");
	if (s == NULL)
	goto invalid_url;

	i = s - pkgname;
	s += 3;
	if (*s == '\0')
	goto invalid_url;

	devname = NULL;
	devnamelen = 0;
	proto = NULL;
	local = 0;

	if (i == 4 && !strncasecmp(pkgname, "tftp", i)) {
	devname = "net0";
	devnamelen = 4;
	proto = &tftp_fsops;
	} else if (i == 4 && !strncasecmp(pkgname, "file", i)) {
	currdev = getenv("currdev");
	local = 1;

	if (s == '/') { / file:/// */
	if (devname == NULL)
	devname = currdev;
	if (devname == NULL)
	devname = "disk1";
	} else { /* file://devname[:fstype]/ */
	devname = s;
	e = strchr(devname, '/');
	if (!e)
	goto invalid_url;
	devnamelen = e - devname;
	s = e; /* consume devname */
	}
	if ((e = strchr(devname, ':')) != NULL) {
	/* could be :fstype */
	devnamelen = e - devname;
	switch (e[1]) {
	case '\0': /* just currdev */
	break;
	case 'd':
	proto = &dosfs_fsops;
	break;
	#ifdef HOSTPROG
	case 'h':
	{
	extern struct fs_ops host_fsops;

	proto = &host_fsops;
	}
	break;
	#endif
	case 'u':
	proto = &ufs_fsops;
	break;
	}
	}
	if (proto == NULL && strncmp(devname, "disk", 4) == 0) {
	proto = &dosfs_fsops;
	}
	}

	if (devname == NULL)
	goto invalid_url;

	if (devnamelen == 0) {
	/* default is currdev which ends with ':' */
	devnamelen = strlen(devname);
	if (devname[devnamelen - 1] == ':')
	devnamelen--;
	}

	if (*s != '/' ) {
	if (local)
	goto invalid_url;

	pkgname = strchr(s, '/');
	if (pkgname == NULL)
	goto invalid_url;

	*pkgname = '\0';
	servip.s_addr = inet_addr(s);
	if (servip.s_addr == htonl(INADDR_NONE))
	goto invalid_url;

	setenv("serverip", inet_ntoa(servip), 1);

	- if (proto == &tftp_fsops) {
	- tftpip.s_addr = servip.s_addr;
	- }
	-
	*pkgname = '/';
	} else
	pkgname = s;

	i = snprintf(buf, sizeof(buf), "%.*s:%s",
	(int) devnamelen, devname, pkgname);
	if (i >= (int) sizeof(buf)) {
	command_errmsg = "package name too long";
	return (CMD_ERROR);
	}
	setenv("install_package", buf, 1);

	error = pkgfs_init(buf, proto);
	if (error) {
	command_errmsg = "cannot open package";
	goto fail;
	}

	/*
	* Point of no return: unload anything that may have been
	* loaded and prune the environment from harmful variables.
	*/
	unload();
	unsetenv("vfs.root.mountfrom");

	/*
	* read the metatags file.
	*/
	fd = open("/metatags", O_RDONLY);
	if (fd != -1) {
	error = read_metatags(fd);
	close(fd);
	if (error) {
	command_errmsg = "cannot load metatags";
	goto fail;
	}
	}

	s = (inst_kernel == NULL) ? "/kernel" : inst_kernel;
	error = mod_loadkld(s, 0, NULL);
	if (error) {
	command_errmsg = "cannot load kernel from package";
	goto fail;
	}

	/* If there is a loader.rc in the package, execute it */
	s = (inst_loader_rc == NULL) ? "/loader.rc" : inst_loader_rc;
	fd = open(s, O_RDONLY);
	if (fd != -1) {
	close(fd);
	- error = inter_include(s);
	+ error = interp_include(s);
	if (error == CMD_ERROR)
	goto fail;
	}

	i = 0;
	while (inst_modules != NULL && inst_modules[i] != NULL) {
	error = mod_loadkld(inst_modules[i], 0, NULL);
	if (error) {
	command_errmsg = "cannot load module(s) from package";
	goto fail;
	}
	i++;
	}

	s = (inst_rootfs == NULL) ? "/install.iso" : inst_rootfs;
	if (file_loadraw(s, "mfs_root", 1) == NULL) {
	error = errno;
	command_errmsg = "cannot load root file system";
	goto fail;
	}

	cleanup();

	fp = file_findfile(NULL, NULL);
	if (fp != NULL)
	file_formats[fp->f_loader]->l_exec(fp);
	error = CMD_ERROR;
	command_errmsg = "unable to start installation";

	fail:
	sprintf(buf, "%s (error %d)", command_errmsg, error);
	cleanup();
	unload();
	exclusive_file_system = NULL;
	command_errmsg = buf; /* buf is static. */
	return (CMD_ERROR);

	invalid_url:
	command_errmsg = "invalid URL";
	return (CMD_ERROR);
	}

	static int
	command_install(int argc, char *argv[])
	{
	int argidx;

	unsetenv("install_format");

	argidx = 1;
	while (1) {
	if (argc == argidx) {
	command_errmsg =
	"usage: install [--format] <URL>";
	return (CMD_ERROR);
	}
	if (!strcmp(argv[argidx], "--format")) {
	setenv("install_format", "yes", 1);
	argidx++;
	continue;
	}
	break;
	}

	return (install(argv[argidx]));
	}
	diff --git a/sys/arm/allwinner/clkng/ccu_sun8i_r.c b/sys/arm/allwinner/clkng/ccu_sun8i_r.c
	index 74f822ca761d..a0b7093ea8fd 100644
	--- a/sys/arm/allwinner/clkng/ccu_sun8i_r.c
	+++ b/sys/arm/allwinner/clkng/ccu_sun8i_r.c
	@@ -1,265 +1,265 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2017,2018 Emmanuel Vadot <manu@freebsd.org>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/rman.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <machine/bus.h>

	#include <dev/fdt/simplebus.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#if defined(__aarch64__)
	#include "opt_soc.h"
	#endif

	#include <dev/extres/clk/clk_div.h>
	#include <dev/extres/clk/clk_fixed.h>
	#include <dev/extres/clk/clk_mux.h>

	#include <arm/allwinner/clkng/aw_ccung.h>

	#include <gnu/dts/include/dt-bindings/clock/sun8i-r-ccu.h>
	#include <gnu/dts/include/dt-bindings/reset/sun8i-r-ccu.h>

	/* Non-exported clocks */
	#define CLK_AHB0 1
	#define CLK_APB0 2

	static struct aw_ccung_reset ccu_sun8i_r_resets[] = {
	CCU_RESET(RST_APB0_IR, 0xb0, 1)
	CCU_RESET(RST_APB0_TIMER, 0xb0, 2)
	CCU_RESET(RST_APB0_RSB, 0xb0, 3)
	CCU_RESET(RST_APB0_UART, 0xb0, 4)
	CCU_RESET(RST_APB0_I2C, 0xb0, 6)
	};

	static struct aw_ccung_gate ccu_sun8i_r_gates[] = {
	CCU_GATE(CLK_APB0_PIO, "apb0-pio", "apb0", 0x28, 0)
	CCU_GATE(CLK_APB0_IR, "apb0-ir", "apb0", 0x28, 1)
	CCU_GATE(CLK_APB0_TIMER, "apb0-timer", "apb0", 0x28, 2)
	CCU_GATE(CLK_APB0_RSB, "apb0-rsb", "apb0", 0x28, 3)
	CCU_GATE(CLK_APB0_UART, "apb0-uart", "apb0", 0x28, 4)
	CCU_GATE(CLK_APB0_I2C, "apb0-i2c", "apb0", 0x28, 6)
	CCU_GATE(CLK_APB0_TWD, "apb0-twd", "apb0", 0x28, 7)
	};

	static const char *ar100_parents[] = {"osc32k", "osc24M", "pll_periph0", "iosc"};
	static const char *a83t_ar100_parents[] = {"osc16M-d512", "osc24M", "pll_periph", "osc16M"};
	PREDIV_CLK(ar100_clk, CLK_AR100, /* id */
	"ar100", ar100_parents, /* name, parents */
	0x00, /* offset */
	16, 2, /* mux */
	4, 2, 0, AW_CLK_FACTOR_POWER_OF_TWO, /* div */
	8, 5, 0, AW_CLK_FACTOR_HAS_COND, /* prediv */
	16, 2, 2); /* prediv condition */
	PREDIV_CLK(a83t_ar100_clk, CLK_AR100, /* id */
	"ar100", a83t_ar100_parents, /* name, parents */
	0x00, /* offset */
	16, 2, /* mux */
	4, 2, 0, AW_CLK_FACTOR_POWER_OF_TWO, /* div */
	8, 5, 0, AW_CLK_FACTOR_HAS_COND, /* prediv */
	16, 2, 2); /* prediv condition */

	static const char *ahb0_parents[] = {"ar100"};
	FIXED_CLK(ahb0_clk,
	CLK_AHB0, /* id */
	"ahb0", /* name */
	ahb0_parents, /* parent */
	0, /* freq */
	1, /* mult */
	1, /* div */
	0); /* flags */

	static const char *apb0_parents[] = {"ahb0"};
	DIV_CLK(apb0_clk,
	CLK_APB0, /* id */
	"apb0", apb0_parents, /* name, parents */
	0x0c, /* offset */
	0, 2, /* shift, width */
	0, NULL); /* flags, div table */

	static const char *r_ccu_ir_parents[] = {"osc32k", "osc24M"};
	NM_CLK(r_ccu_ir_clk,
	CLK_IR, /* id */
	"ir", r_ccu_ir_parents, /* names, parents */
	0x54, /* offset */
	0, 4, 0, 0, /* N factor */
	16, 2, 0, 0, /* M flags */
	24, 2, /* mux */
	31, /* gate */
	- AW_CLK_HAS_MUX \| AW_CLK_REPARENT); /* flags */
	+ AW_CLK_HAS_MUX \| AW_CLK_REPARENT \| AW_CLK_HAS_GATE);/* flags */

	static const char *a83t_ir_parents[] = {"osc16M", "osc24M"};
	static struct aw_clk_nm_def a83t_ir_clk = {
	.clkdef = {
	.id = CLK_IR,
	.name = "ir",
	.parent_names = a83t_ir_parents,
	.parent_cnt = nitems(a83t_ir_parents),
	},
	.offset = 0x54,
	.n = {.shift = 0, .width = 4, .flags = AW_CLK_FACTOR_POWER_OF_TWO, },
	.m = {.shift = 16, .width = 2},
	.prediv = {
	.cond_shift = 24,
	.cond_width = 2,
	.cond_value = 0,
	.value = 16
	},
	.mux_shift = 24,
	.mux_width = 2,
	.flags = AW_CLK_HAS_MUX \| AW_CLK_HAS_PREDIV,
	};

	static struct aw_ccung_clk clks[] = {
	{ .type = AW_CLK_PREDIV_MUX, .clk.prediv_mux = &ar100_clk},
	{ .type = AW_CLK_DIV, .clk.div = &apb0_clk},
	{ .type = AW_CLK_FIXED, .clk.fixed = &ahb0_clk},
	{ .type = AW_CLK_NM, .clk.nm = &r_ccu_ir_clk},
	};

	static struct aw_ccung_clk a83t_clks[] = {
	{ .type = AW_CLK_PREDIV_MUX, .clk.prediv_mux = &a83t_ar100_clk},
	{ .type = AW_CLK_DIV, .clk.div = &apb0_clk},
	{ .type = AW_CLK_FIXED, .clk.fixed = &ahb0_clk},
	{ .type = AW_CLK_NM, .clk.nm = &a83t_ir_clk},
	};

	static struct ofw_compat_data compat_data[] = {
	#if defined(SOC_ALLWINNER_H3) \|\| defined(SOC_ALLWINNER_H5)
	{ "allwinner,sun8i-h3-r-ccu", 1 },
	#endif
	#if defined(SOC_ALLWINNER_A64)
	{ "allwinner,sun50i-a64-r-ccu", 1 },
	#endif
	{ NULL, 0},
	};

	static int
	ccu_sun8i_r_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	device_set_desc(dev, "Allwinner SUN8I_R Clock Control Unit NG");
	return (BUS_PROBE_DEFAULT);
	}

	static int
	ccu_sun8i_r_attach(device_t dev)
	{
	struct aw_ccung_softc *sc;

	sc = device_get_softc(dev);

	sc->resets = ccu_sun8i_r_resets;
	sc->nresets = nitems(ccu_sun8i_r_resets);
	sc->gates = ccu_sun8i_r_gates;
	sc->ngates = nitems(ccu_sun8i_r_gates);
	sc->clks = clks;
	sc->nclks = nitems(clks);

	return (aw_ccung_attach(dev));
	}

	static device_method_t ccu_sun8i_r_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ccu_sun8i_r_probe),
	DEVMETHOD(device_attach, ccu_sun8i_r_attach),

	DEVMETHOD_END
	};

	static devclass_t ccu_sun8i_r_devclass;

	DEFINE_CLASS_1(ccu_sun8i_r, ccu_sun8i_r_driver, ccu_sun8i_r_methods,
	sizeof(struct aw_ccung_softc), aw_ccung_driver);

	EARLY_DRIVER_MODULE(ccu_sun8i_r, simplebus, ccu_sun8i_r_driver,
	ccu_sun8i_r_devclass, 0, 0, BUS_PASS_RESOURCE + BUS_PASS_ORDER_MIDDLE);

	static int
	ccu_a83t_r_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "allwinner,sun8i-a83t-r-ccu"))
	return (ENXIO);

	device_set_desc(dev, "Allwinner A83T_R Clock Control Unit NG");
	return (BUS_PROBE_DEFAULT);
	}

	static int
	ccu_a83t_r_attach(device_t dev)
	{
	struct aw_ccung_softc *sc;

	sc = device_get_softc(dev);

	sc->resets = ccu_sun8i_r_resets;
	sc->nresets = nitems(ccu_sun8i_r_resets);
	sc->gates = ccu_sun8i_r_gates;
	sc->ngates = nitems(ccu_sun8i_r_gates);
	sc->clks = a83t_clks;
	sc->nclks = nitems(a83t_clks);

	return (aw_ccung_attach(dev));
	}

	static device_method_t ccu_a83t_r_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ccu_a83t_r_probe),
	DEVMETHOD(device_attach, ccu_a83t_r_attach),

	DEVMETHOD_END
	};

	static devclass_t ccu_a83t_r_devclass;

	DEFINE_CLASS_1(ccu_a83t_r, ccu_a83t_r_driver, ccu_a83t_r_methods,
	sizeof(struct aw_ccung_softc), aw_ccung_driver);

	EARLY_DRIVER_MODULE(ccu_a83t_r, simplebus, ccu_a83t_r_driver,
	ccu_a83t_r_devclass, 0, 0, BUS_PASS_RESOURCE + BUS_PASS_ORDER_MIDDLE);
	diff --git a/sys/arm/arm/trap-v6.c b/sys/arm/arm/trap-v6.c
	index 84acf479bfe1..7ade224f949e 100644
	--- a/sys/arm/arm/trap-v6.c
	+++ b/sys/arm/arm/trap-v6.c
	@@ -1,665 +1,667 @@
	/*-
	* Copyright 2014 Olivier Houchard <cognet@FreeBSD.org>
	* Copyright 2014 Svatopluk Kraus <onwahe@gmail.com>
	* Copyright 2014 Michal Meloun <meloun@miracle.cz>
	* Copyright 2014 Andrew Turner <andrew@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include "opt_ktrace.h"

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/systm.h>
	#include <sys/proc.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/signalvar.h>
	#include <sys/ktr.h>
	#include <sys/vmmeter.h>
	#ifdef KTRACE
	#include <sys/uio.h>
	#include <sys/ktrace.h>
	#endif

	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_map.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_param.h>

	#include <machine/cpu.h>
	#include <machine/frame.h>
	#include <machine/machdep.h>
	#include <machine/pcb.h>

	#ifdef KDB
	#include <sys/kdb.h>
	#include <machine/db_machdep.h>
	#endif

	#ifdef KDTRACE_HOOKS
	#include <sys/dtrace_bsd.h>
	#endif

	extern char cachebailout[];

	#ifdef DEBUG
	int last_fault_code; /* For the benefit of pmap_fault_fixup() */
	#endif

	struct ksig {
	int sig;
	u_long code;
	vm_offset_t addr;
	};

	typedef int abort_func_t(struct trapframe *, u_int, u_int, u_int, u_int,
	struct thread , struct ksig );

	static abort_func_t abort_fatal;
	static abort_func_t abort_align;
	static abort_func_t abort_icache;

	struct abort {
	abort_func_t *func;
	const char *desc;
	};

	/*
	* How are the aborts handled?
	*
	* Undefined Code:
	* - Always fatal as we do not know what does it mean.
	* Imprecise External Abort:
	* - Always fatal, but can be handled somehow in the future.
	* Now, due to PCIe buggy hardware, ignored.
	* Precise External Abort:
	* - Always fatal, but who knows in the future???
	* Debug Event:
	* - Special handling.
	* External Translation Abort (L1 & L2)
	* - Always fatal as something is screwed up in page tables or hardware.
	* Domain Fault (L1 & L2):
	* - Always fatal as we do not play game with domains.
	* Alignment Fault:
	* - Everything should be aligned in kernel with exception of user to kernel
	* and vice versa data copying, so if pcb_onfault is not set, it's fatal.
	* We generate signal in case of abort from user mode.
	* Instruction cache maintenance:
	* - According to manual, this is translation fault during cache maintenance
	* operation. So, it could be really complex in SMP case and fuzzy too
	* for cache operations working on virtual addresses. For now, we will
	* consider this abort as fatal. In fact, no cache maintenance on
	* not mapped virtual addresses should be called. As cache maintenance
	* operation (except DMB, DSB, and Flush Prefetch Buffer) are priviledged,
	* the abort is fatal for user mode as well for now. (This is good place to
	* note that cache maintenance on virtual address fill TLB.)
	* Acces Bit (L1 & L2):
	* - Fast hardware emulation for kernel and user mode.
	* Translation Fault (L1 & L2):
	* - Standard fault mechanism is held including vm_fault().
	* Permission Fault (L1 & L2):
	* - Fast hardware emulation of modify bits and in other cases, standard
	* fault mechanism is held including vm_fault().
	*/

	static const struct abort aborts[] = {
	{abort_fatal, "Undefined Code (0x000)"},
	{abort_align, "Alignment Fault"},
	{abort_fatal, "Debug Event"},
	{NULL, "Access Bit (L1)"},
	{NULL, "Instruction cache maintenance"},
	{NULL, "Translation Fault (L1)"},
	{NULL, "Access Bit (L2)"},
	{NULL, "Translation Fault (L2)"},

	{abort_fatal, "External Abort"},
	{abort_fatal, "Domain Fault (L1)"},
	{abort_fatal, "Undefined Code (0x00A)"},
	{abort_fatal, "Domain Fault (L2)"},
	{abort_fatal, "External Translation Abort (L1)"},
	{NULL, "Permission Fault (L1)"},
	{abort_fatal, "External Translation Abort (L2)"},
	{NULL, "Permission Fault (L2)"},

	{abort_fatal, "TLB Conflict Abort"},
	{abort_fatal, "Undefined Code (0x401)"},
	{abort_fatal, "Undefined Code (0x402)"},
	{abort_fatal, "Undefined Code (0x403)"},
	{abort_fatal, "Undefined Code (0x404)"},
	{abort_fatal, "Undefined Code (0x405)"},
	{abort_fatal, "Asynchronous External Abort"},
	{abort_fatal, "Undefined Code (0x407)"},

	{abort_fatal, "Asynchronous Parity Error on Memory Access"},
	{abort_fatal, "Parity Error on Memory Access"},
	{abort_fatal, "Undefined Code (0x40A)"},
	{abort_fatal, "Undefined Code (0x40B)"},
	{abort_fatal, "Parity Error on Translation (L1)"},
	{abort_fatal, "Undefined Code (0x40D)"},
	{abort_fatal, "Parity Error on Translation (L2)"},
	{abort_fatal, "Undefined Code (0x40F)"}
	};

	static __inline void
	-call_trapsignal(struct thread *td, int sig, int code, vm_offset_t addr)
	+call_trapsignal(struct thread *td, int sig, int code, vm_offset_t addr,
	+ int trapno)
	{
	ksiginfo_t ksi;

	CTR4(KTR_TRAP, "%s: addr: %#x, sig: %d, code: %d",
	__func__, addr, sig, code);

	/*
	* TODO: some info would be nice to know
	* if we are serving data or prefetch abort.
	*/

	ksiginfo_init_trap(&ksi);
	ksi.ksi_signo = sig;
	ksi.ksi_code = code;
	ksi.ksi_addr = (void *)addr;
	+ ksi.ksi_trapno = trapno;
	trapsignal(td, &ksi);
	}

	/*
	* abort_imprecise() handles the following abort:
	*
	* FAULT_EA_IMPREC - Imprecise External Abort
	*
	* The imprecise means that we don't know where the abort happened,
	* thus FAR is undefined. The abort should not never fire, but hot
	* plugging or accidental hardware failure can be the cause of it.
	* If the abort happens, it can even be on different (thread) context.
	* Without any additional support, the abort is fatal, as we do not
	* know what really happened.
	*
	* QQQ: Some additional functionality, like pcb_onfault but global,
	* can be implemented. Imprecise handlers could be registered
	* which tell us if the abort is caused by something they know
	* about. They should return one of three codes like:
	* FAULT_IS_MINE,
	* FAULT_CAN_BE_MINE,
	* FAULT_IS_NOT_MINE.
	* The handlers should be called until some of them returns
	* FAULT_IS_MINE value or all was called. If all handlers return
	* FAULT_IS_NOT_MINE value, then the abort is fatal.
	*/
	static __inline void
	abort_imprecise(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode)
	{

	/*
	* XXX - We can got imprecise abort as result of access
	* to not-present PCI/PCIe configuration space.
	*/
	#if 0
	goto out;
	#endif
	abort_fatal(tf, FAULT_EA_IMPREC, fsr, 0, prefetch, curthread, NULL);

	/*
	* Returning from this function means that we ignore
	* the abort for good reason. Note that imprecise abort
	* could fire any time even in user mode.
	*/

	#if 0
	out:
	if (usermode)
	userret(curthread, tf);
	#endif
	}

	/*
	* abort_debug() handles the following abort:
	*
	* FAULT_DEBUG - Debug Event
	*
	*/
	static __inline void
	abort_debug(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode,
	u_int far)
	{

	if (usermode) {
	struct thread *td;

	td = curthread;
	- call_trapsignal(td, SIGTRAP, TRAP_BRKPT, far);
	+ call_trapsignal(td, SIGTRAP, TRAP_BRKPT, far, FAULT_DEBUG);
	userret(td, tf);
	} else {
	#ifdef KDB
	kdb_trap((prefetch) ? T_BREAKPOINT : T_WATCHPOINT, 0, tf);
	#else
	printf("No debugger in kernel.\n");
	#endif
	}
	}

	/*
	* Abort handler.
	*
	* FAR, FSR, and everything what can be lost after enabling
	* interrupts must be grabbed before the interrupts will be
	* enabled. Note that when interrupts will be enabled, we
	* could even migrate to another CPU ...
	*
	* TODO: move quick cases to ASM
	*/
	void
	abort_handler(struct trapframe *tf, int prefetch)
	{
	struct thread *td;
	vm_offset_t far, va;
	int idx, rv;
	uint32_t fsr;
	struct ksig ksig;
	struct proc *p;
	struct pcb *pcb;
	struct vm_map *map;
	struct vmspace *vm;
	vm_prot_t ftype;
	bool usermode;
	int bp_harden, ucode;
	#ifdef INVARIANTS
	void *onfault;
	#endif

	VM_CNT_INC(v_trap);
	td = curthread;

	fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get();
	#if __ARM_ARCH >= 7
	far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get();
	#else
	far = (prefetch) ? TRAPF_PC(tf) : cp15_dfar_get();
	#endif

	idx = FSR_TO_FAULT(fsr);
	usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */

	/*
	* Apply BP hardening by flushing the branch prediction cache
	* for prefaults on kernel addresses.
	*/
	if (__predict_false(prefetch && far > VM_MAXUSER_ADDRESS &&
	(idx == FAULT_TRAN_L2 \|\| idx == FAULT_PERM_L2))) {
	bp_harden = PCPU_GET(bp_harden_kind);
	if (bp_harden == PCPU_BP_HARDEN_KIND_BPIALL)
	_CP15_BPIALL();
	else if (bp_harden == PCPU_BP_HARDEN_KIND_ICIALLU)
	_CP15_ICIALLU();
	}

	if (usermode)
	td->td_frame = tf;

	CTR6(KTR_TRAP, "%s: fsr %#x (idx %u) far %#x prefetch %u usermode %d",
	__func__, fsr, idx, far, prefetch, usermode);

	/*
	* Firstly, handle aborts that are not directly related to mapping.
	*/
	if (__predict_false(idx == FAULT_EA_IMPREC)) {
	abort_imprecise(tf, fsr, prefetch, usermode);
	return;
	}

	if (__predict_false(idx == FAULT_DEBUG)) {
	abort_debug(tf, fsr, prefetch, usermode, far);
	return;
	}

	/*
	* ARM has a set of unprivileged load and store instructions
	* (LDRT/LDRBT/STRT/STRBT ...) which are supposed to be used in other
	* than user mode and OS should recognize their aborts and behave
	* appropriately. However, there is no way how to do that reasonably
	* in general unless we restrict the handling somehow.
	*
	* For now, these instructions are used only in copyin()/copyout()
	* like functions where usermode buffers are checked in advance that
	* they are not from KVA space. Thus, no action is needed here.
	*/

	/*
	* (1) Handle access and R/W hardware emulation aborts.
	* (2) Check that abort is not on pmap essential address ranges.
	* There is no way how to fix it, so we don't even try.
	*/
	rv = pmap_fault(PCPU_GET(curpmap), far, fsr, idx, usermode);
	if (rv == KERN_SUCCESS)
	return;
	#ifdef KDB
	if (kdb_active) {
	kdb_reenter();
	goto out;
	}
	#endif
	if (rv == KERN_INVALID_ADDRESS)
	goto nogo;

	if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
	/*
	* Due to both processor errata and lazy TLB invalidation when
	* access restrictions are removed from virtual pages, memory
	* accesses that are allowed by the physical mapping layer may
	* nonetheless cause one spurious page fault per virtual page.
	* When the thread is executing a "no faulting" section that
	* is bracketed by vm_fault_{disable,enable}_pagefaults(),
	* every page fault is treated as a spurious page fault,
	* unless it accesses the same virtual address as the most
	* recent page fault within the same "no faulting" section.
	*/
	if (td->td_md.md_spurflt_addr != far \|\|
	(td->td_pflags & TDP_RESETSPUR) != 0) {
	td->td_md.md_spurflt_addr = far;
	td->td_pflags &= ~TDP_RESETSPUR;

	tlb_flush_local(far & ~PAGE_MASK);
	return;
	}
	} else {
	/*
	* If we get a page fault while in a critical section, then
	* it is most likely a fatal kernel page fault. The kernel
	* is already going to panic trying to get a sleep lock to
	* do the VM lookup, so just consider it a fatal trap so the
	* kernel can print out a useful trap message and even get
	* to the debugger.
	*
	* If we get a page fault while holding a non-sleepable
	* lock, then it is most likely a fatal kernel page fault.
	* If WITNESS is enabled, then it's going to whine about
	* bogus LORs with various VM locks, so just skip to the
	* fatal trap handling directly.
	*/
	if (td->td_critnest != 0 \|\|
	WITNESS_CHECK(WARN_SLEEPOK \| WARN_GIANTOK, NULL,
	"Kernel page fault") != 0) {
	abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
	return;
	}
	}

	/* Re-enable interrupts if they were enabled previously. */
	if (td->td_md.md_spinlock_count == 0) {
	if (__predict_true(tf->tf_spsr & PSR_I) == 0)
	enable_interrupts(PSR_I);
	if (__predict_true(tf->tf_spsr & PSR_F) == 0)
	enable_interrupts(PSR_F);
	}

	p = td->td_proc;
	if (usermode) {
	td->td_pticks = 0;
	if (td->td_cowgen != p->p_cowgen)
	thread_cow_update(td);
	}

	/* Invoke the appropriate handler, if necessary. */
	if (__predict_false(aborts[idx].func != NULL)) {
	if ((aborts[idx].func)(tf, idx, fsr, far, prefetch, td, &ksig))
	goto do_trapsignal;
	goto out;
	}

	/*
	* At this point, we're dealing with one of the following aborts:
	*
	* FAULT_ICACHE - I-cache maintenance
	* FAULT_TRAN_xx - Translation
	* FAULT_PERM_xx - Permission
	*/

	/*
	* Don't pass faulting cache operation to vm_fault(). We don't want
	* to handle all vm stuff at this moment.
	*/
	pcb = td->td_pcb;
	if (__predict_false(pcb->pcb_onfault == cachebailout)) {
	tf->tf_r0 = far; /* return failing address */
	tf->tf_pc = (register_t)pcb->pcb_onfault;
	return;
	}

	/* Handle remaining I-cache aborts. */
	if (idx == FAULT_ICACHE) {
	if (abort_icache(tf, idx, fsr, far, prefetch, td, &ksig))
	goto do_trapsignal;
	goto out;
	}

	va = trunc_page(far);
	if (va >= KERNBASE) {
	/*
	* Don't allow user-mode faults in kernel address space.
	*/
	if (usermode)
	goto nogo;

	map = kernel_map;
	} else {
	/*
	* This is a fault on non-kernel virtual memory. If curproc
	* is NULL or curproc->p_vmspace is NULL the fault is fatal.
	*/
	vm = (p != NULL) ? p->p_vmspace : NULL;
	if (vm == NULL)
	goto nogo;

	map = &vm->vm_map;
	if (!usermode && (td->td_intr_nesting_level != 0 \|\|
	pcb->pcb_onfault == NULL)) {
	abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
	return;
	}
	}

	ftype = (fsr & FSR_WNR) ? VM_PROT_WRITE : VM_PROT_READ;
	if (prefetch)
	ftype \|= VM_PROT_EXECUTE;

	#ifdef DEBUG
	last_fault_code = fsr;
	#endif

	#ifdef INVARIANTS
	onfault = pcb->pcb_onfault;
	pcb->pcb_onfault = NULL;
	#endif

	/* Fault in the page. */
	rv = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &ksig.sig,
	&ucode);
	ksig.code = ucode;

	#ifdef INVARIANTS
	pcb->pcb_onfault = onfault;
	#endif

	if (__predict_true(rv == KERN_SUCCESS))
	goto out;
	nogo:
	if (!usermode) {
	if (td->td_intr_nesting_level == 0 &&
	pcb->pcb_onfault != NULL) {
	tf->tf_r0 = rv;
	tf->tf_pc = (int)pcb->pcb_onfault;
	return;
	}
	CTR2(KTR_TRAP, "%s: vm_fault() failed with %d", __func__, rv);
	abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
	return;
	}

	ksig.addr = far;

	do_trapsignal:
	- call_trapsignal(td, ksig.sig, ksig.code, ksig.addr);
	+ call_trapsignal(td, ksig.sig, ksig.code, ksig.addr, idx);
	out:
	if (usermode)
	userret(td, tf);
	}

	/*
	* abort_fatal() handles the following data aborts:
	*
	* FAULT_DEBUG - Debug Event
	* FAULT_ACCESS_xx - Acces Bit
	* FAULT_EA_PREC - Precise External Abort
	* FAULT_DOMAIN_xx - Domain Fault
	* FAULT_EA_TRAN_xx - External Translation Abort
	* FAULT_EA_IMPREC - Imprecise External Abort
	* + all undefined codes for ABORT
	*
	* We should never see these on a properly functioning system.
	*
	* This function is also called by the other handlers if they
	* detect a fatal problem.
	*
	* Note: If 'l' is NULL, we assume we're dealing with a prefetch abort.
	*/
	static int
	abort_fatal(struct trapframe *tf, u_int idx, u_int fsr, u_int far,
	u_int prefetch, struct thread td, struct ksig ksig)
	{
	bool usermode;
	const char *mode;
	const char *rw_mode;

	usermode = TRAPF_USERMODE(tf);
	#ifdef KDTRACE_HOOKS
	if (!usermode) {
	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(tf, far))
	return (0);
	}
	#endif

	mode = usermode ? "user" : "kernel";
	rw_mode = fsr & FSR_WNR ? "write" : "read";
	disable_interrupts(PSR_I\|PSR_F);

	if (td != NULL) {
	printf("Fatal %s mode data abort: '%s' on %s\n", mode,
	aborts[idx].desc, rw_mode);
	printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr);
	if (idx != FAULT_EA_IMPREC)
	printf("%08x, ", far);
	else
	printf("Invalid, ");
	printf("spsr=%08x\n", tf->tf_spsr);
	} else {
	printf("Fatal %s mode prefetch abort at 0x%08x\n",
	mode, tf->tf_pc);
	printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr);
	}

	printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n",
	tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
	printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n",
	tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
	printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n",
	tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
	printf("r12=%08x, ", tf->tf_r12);

	if (usermode)
	printf("usp=%08x, ulr=%08x",
	tf->tf_usr_sp, tf->tf_usr_lr);
	else
	printf("ssp=%08x, slr=%08x",
	tf->tf_svc_sp, tf->tf_svc_lr);
	printf(", pc =%08x\n\n", tf->tf_pc);

	#ifdef KDB
	if (debugger_on_trap) {
	kdb_why = KDB_WHY_TRAP;
	kdb_trap(fsr, 0, tf);
	kdb_why = KDB_WHY_UNSET;
	}
	#endif
	panic("Fatal abort");
	/NOTREACHED/
	}

	/*
	* abort_align() handles the following data abort:
	*
	* FAULT_ALIGN - Alignment fault
	*
	* Everything should be aligned in kernel with exception of user to kernel
	* and vice versa data copying, so if pcb_onfault is not set, it's fatal.
	* We generate signal in case of abort from user mode.
	*/
	static int
	abort_align(struct trapframe *tf, u_int idx, u_int fsr, u_int far,
	u_int prefetch, struct thread td, struct ksig ksig)
	{
	bool usermode;

	usermode = TRAPF_USERMODE(tf);
	if (!usermode) {
	if (td->td_intr_nesting_level == 0 && td != NULL &&
	td->td_pcb->pcb_onfault != NULL) {
	tf->tf_r0 = EFAULT;
	tf->tf_pc = (int)td->td_pcb->pcb_onfault;
	return (0);
	}
	abort_fatal(tf, idx, fsr, far, prefetch, td, ksig);
	}
	/* Deliver a bus error signal to the process */
	ksig->code = BUS_ADRALN;
	ksig->sig = SIGBUS;
	ksig->addr = far;
	return (1);
	}

	/*
	* abort_icache() handles the following data abort:
	*
	* FAULT_ICACHE - Instruction cache maintenance
	*
	* According to manual, FAULT_ICACHE is translation fault during cache
	* maintenance operation. In fact, no cache maintenance operation on
	* not mapped virtual addresses should be called. As cache maintenance
	* operation (except DMB, DSB, and Flush Prefetch Buffer) are priviledged,
	* the abort is concider as fatal for now. However, all the matter with
	* cache maintenance operation on virtual addresses could be really complex
	* and fuzzy in SMP case, so maybe in future standard fault mechanism
	* should be held here including vm_fault() calling.
	*/
	static int
	abort_icache(struct trapframe *tf, u_int idx, u_int fsr, u_int far,
	u_int prefetch, struct thread td, struct ksig ksig)
	{

	abort_fatal(tf, idx, fsr, far, prefetch, td, ksig);
	return(0);
	}
	diff --git a/sys/arm/broadcom/bcm2835/bcm2835_cpufreq.c b/sys/arm/broadcom/bcm2835/bcm2835_cpufreq.c
	index 1b5e3a63e09e..15c1106cf36f 100644
	--- a/sys/arm/broadcom/bcm2835/bcm2835_cpufreq.c
	+++ b/sys/arm/broadcom/bcm2835/bcm2835_cpufreq.c
	@@ -1,1656 +1,1609 @@
	/*-
	* Copyright (C) 2013-2015 Daisuke Aoyama <aoyama@peach.ne.jp>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/cpu.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/sema.h>
	#include <sys/sysctl.h>

	#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <machine/intr.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	-#include <arm/broadcom/bcm2835/bcm2835_mbox.h>
	-#include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h>
	+#include <arm/broadcom/bcm2835/bcm2835_firmware.h>
	#include <arm/broadcom/bcm2835/bcm2835_vcbus.h>

	#include "cpufreq_if.h"
	-#include "mbox_if.h"

	#ifdef DEBUG
	#define DPRINTF(fmt, ...) do { \
	printf("%s:%u: ", __func__, __LINE__); \
	printf(fmt, ##__VA_ARGS__); \
	} while (0)
	#else
	#define DPRINTF(fmt, ...)
	#endif

	#define HZ2MHZ(freq) ((freq) / (1000 * 1000))
	#define MHZ2HZ(freq) ((freq) * (1000 * 1000))

	#ifdef SOC_BCM2835
	#define OFFSET2MVOLT(val) (1200 + ((val) * 25))
	#define MVOLT2OFFSET(val) (((val) - 1200) / 25)
	#define DEFAULT_ARM_FREQUENCY 700
	#define DEFAULT_LOWEST_FREQ 300
	#else
	#define OFFSET2MVOLT(val) (((val) / 1000))
	#define MVOLT2OFFSET(val) (((val) * 1000))
	#define DEFAULT_ARM_FREQUENCY 600
	#define DEFAULT_LOWEST_FREQ 600
	#endif
	#define DEFAULT_CORE_FREQUENCY 250
	#define DEFAULT_SDRAM_FREQUENCY 400
	#define TRANSITION_LATENCY 1000
	#define MIN_OVER_VOLTAGE -16
	#define MAX_OVER_VOLTAGE 6
	#define MSG_ERROR -999999999
	#define MHZSTEP 100
	#define HZSTEP (MHZ2HZ(MHZSTEP))
	#define TZ_ZEROC 2731

	#define VC_LOCK(sc) do { \
	sema_wait(&vc_sema); \
	} while (0)
	#define VC_UNLOCK(sc) do { \
	sema_post(&vc_sema); \
	} while (0)

	/* ARM->VC mailbox property semaphore */
	static struct sema vc_sema;

	static struct sysctl_ctx_list bcm2835_sysctl_ctx;

	struct bcm2835_cpufreq_softc {
	device_t dev;
	+ device_t firmware;
	int arm_max_freq;
	int arm_min_freq;
	int core_max_freq;
	int core_min_freq;
	int sdram_max_freq;
	int sdram_min_freq;
	int max_voltage_core;
	int min_voltage_core;

	/* the values written in mbox */
	int voltage_core;
	int voltage_sdram;
	int voltage_sdram_c;
	int voltage_sdram_i;
	int voltage_sdram_p;
	int turbo_mode;

	/* initial hook for waiting mbox intr */
	struct intr_config_hook init_hook;
	};

	static struct ofw_compat_data compat_data[] = {
	{ "broadcom,bcm2835-vc", 1 },
	{ "broadcom,bcm2708-vc", 1 },
	{ "brcm,bcm2709", 1 },
	{ "brcm,bcm2835", 1 },
	{ "brcm,bcm2836", 1 },
	{ "brcm,bcm2837", 1 },
	{ "brcm,bcm2711", 1 },
	{ NULL, 0 }
	};

	static int cpufreq_verbose = 0;
	TUNABLE_INT("hw.bcm2835.cpufreq.verbose", &cpufreq_verbose);
	static int cpufreq_lowest_freq = DEFAULT_LOWEST_FREQ;
	TUNABLE_INT("hw.bcm2835.cpufreq.lowest_freq", &cpufreq_lowest_freq);

	#ifdef PROP_DEBUG
	static void
	bcm2835_dump(const void *data, int len)
	{
	const uint8_t p = (const uint8_t)data;
	int i;

	printf("dump @ %p:\n", data);
	for (i = 0; i < len; i++) {
	printf("%2.2x ", p[i]);
	if ((i % 4) == 3)
	printf(" ");
	if ((i % 16) == 15)
	printf("\n");
	}
	printf("\n");
	}
	#endif

	static int
	bcm2835_cpufreq_get_clock_rate(struct bcm2835_cpufreq_softc *sc,
	uint32_t clock_id)
	{
	- struct msg_get_clock_rate msg;
	+ union msg_get_clock_rate_body msg;
	int rate;
	int err;

	/*
	* Get clock rate
	* Tag: 0x00030002
	* Request:
	* Length: 4
	* Value:
	* u32: clock id
	* Response:
	* Length: 8
	* Value:
	* u32: clock id
	* u32: rate (in Hz)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_CLOCK_RATE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.clock_id = clock_id;
	- msg.end_tag = 0;
	+ msg.req.clock_id = clock_id;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_CLOCK_RATE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get clock rate (id=%u)\n",
	clock_id);
	return (MSG_ERROR);
	}

	/* result (Hz) */
	- rate = (int)msg.body.resp.rate_hz;
	+ rate = (int)msg.resp.rate_hz;
	DPRINTF("clock = %d(Hz)\n", rate);
	return (rate);
	}

	static int
	bcm2835_cpufreq_get_max_clock_rate(struct bcm2835_cpufreq_softc *sc,
	uint32_t clock_id)
	{
	- struct msg_get_max_clock_rate msg;
	+ union msg_get_clock_rate_body msg;
	int rate;
	int err;

	/*
	* Get max clock rate
	* Tag: 0x00030004
	* Request:
	* Length: 4
	* Value:
	* u32: clock id
	* Response:
	* Length: 8
	* Value:
	* u32: clock id
	* u32: rate (in Hz)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_MAX_CLOCK_RATE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.clock_id = clock_id;
	- msg.end_tag = 0;
	+ msg.req.clock_id = clock_id;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_MAX_CLOCK_RATE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get max clock rate (id=%u)\n",
	clock_id);
	return (MSG_ERROR);
	}

	/* result (Hz) */
	- rate = (int)msg.body.resp.rate_hz;
	+ rate = (int)msg.resp.rate_hz;
	DPRINTF("clock = %d(Hz)\n", rate);
	return (rate);
	}

	static int
	bcm2835_cpufreq_get_min_clock_rate(struct bcm2835_cpufreq_softc *sc,
	uint32_t clock_id)
	{
	- struct msg_get_min_clock_rate msg;
	+ union msg_get_clock_rate_body msg;
	int rate;
	int err;

	/*
	* Get min clock rate
	* Tag: 0x00030007
	* Request:
	* Length: 4
	* Value:
	* u32: clock id
	* Response:
	* Length: 8
	* Value:
	* u32: clock id
	* u32: rate (in Hz)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_MIN_CLOCK_RATE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.clock_id = clock_id;
	- msg.end_tag = 0;
	+ msg.req.clock_id = clock_id;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_MIN_CLOCK_RATE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get min clock rate (id=%u)\n",
	clock_id);
	return (MSG_ERROR);
	}

	/* result (Hz) */
	- rate = (int)msg.body.resp.rate_hz;
	+ rate = (int)msg.resp.rate_hz;
	DPRINTF("clock = %d(Hz)\n", rate);
	return (rate);
	}

	static int
	bcm2835_cpufreq_set_clock_rate(struct bcm2835_cpufreq_softc *sc,
	uint32_t clock_id, uint32_t rate_hz)
	{
	- struct msg_set_clock_rate msg;
	+ union msg_set_clock_rate_body msg;
	int rate;
	int err;

	/*
	* Set clock rate
	* Tag: 0x00038002
	* Request:
	* Length: 8
	* Value:
	* u32: clock id
	* u32: rate (in Hz)
	* Response:
	* Length: 8
	* Value:
	* u32: clock id
	* u32: rate (in Hz)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_SET_CLOCK_RATE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.clock_id = clock_id;
	- msg.body.req.rate_hz = rate_hz;
	- msg.end_tag = 0;
	+ msg.req.clock_id = clock_id;
	+ msg.req.rate_hz = rate_hz;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_SET_CLOCK_RATE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't set clock rate (id=%u)\n",
	clock_id);
	return (MSG_ERROR);
	}

	/* workaround for core clock */
	- if (clock_id == BCM2835_MBOX_CLOCK_ID_CORE) {
	+ if (clock_id == BCM2835_FIRMWARE_CLOCK_ID_CORE) {
	/* for safety (may change voltage without changing clock) */
	DELAY(TRANSITION_LATENCY);

	/*
	* XXX: the core clock is unable to change at once,
	* to change certainly, write it twice now.
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_SET_CLOCK_RATE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.clock_id = clock_id;
	- msg.body.req.rate_hz = rate_hz;
	- msg.end_tag = 0;
	+ msg.req.clock_id = clock_id;
	+ msg.req.rate_hz = rate_hz;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_SET_CLOCK_RATE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev,
	"can't set clock rate (id=%u)\n", clock_id);
	return (MSG_ERROR);
	}
	}

	/* result (Hz) */
	- rate = (int)msg.body.resp.rate_hz;
	+ rate = (int)msg.resp.rate_hz;
	DPRINTF("clock = %d(Hz)\n", rate);
	return (rate);
	}

	static int
	bcm2835_cpufreq_get_turbo(struct bcm2835_cpufreq_softc *sc)
	{
	- struct msg_get_turbo msg;
	+ union msg_get_turbo_body msg;
	int level;
	int err;

	/*
	* Get turbo
	* Tag: 0x00030009
	* Request:
	* Length: 4
	* Value:
	* u32: id
	* Response:
	* Length: 8
	* Value:
	* u32: id
	* u32: level
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_TURBO;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.id = 0;
	- msg.end_tag = 0;
	+ msg.req.id = 0;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_TURBO, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get turbo\n");
	return (MSG_ERROR);
	}

	/* result 0=non-turbo, 1=turbo */
	- level = (int)msg.body.resp.level;
	+ level = (int)msg.resp.level;
	DPRINTF("level = %d\n", level);
	return (level);
	}

	static int
	bcm2835_cpufreq_set_turbo(struct bcm2835_cpufreq_softc *sc, uint32_t level)
	{
	- struct msg_set_turbo msg;
	+ union msg_set_turbo_body msg;
	int value;
	int err;

	/*
	* Set turbo
	* Tag: 0x00038009
	* Request:
	* Length: 8
	* Value:
	* u32: id
	* u32: level
	* Response:
	* Length: 8
	* Value:
	* u32: id
	* u32: level
	*/

	/* replace unknown value to OFF */
	- if (level != BCM2835_MBOX_TURBO_ON && level != BCM2835_MBOX_TURBO_OFF)
	- level = BCM2835_MBOX_TURBO_OFF;
	+ if (level != BCM2835_FIRMWARE_TURBO_ON &&
	+ level != BCM2835_FIRMWARE_TURBO_OFF)
	+ level = BCM2835_FIRMWARE_TURBO_OFF;

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_SET_TURBO;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.id = 0;
	- msg.body.req.level = level;
	- msg.end_tag = 0;
	+ msg.req.id = 0;
	+ msg.req.level = level;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_SET_TURBO, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't set turbo\n");
	return (MSG_ERROR);
	}

	/* result 0=non-turbo, 1=turbo */
	- value = (int)msg.body.resp.level;
	+ value = (int)msg.resp.level;
	DPRINTF("level = %d\n", value);
	return (value);
	}

	static int
	bcm2835_cpufreq_get_voltage(struct bcm2835_cpufreq_softc *sc,
	uint32_t voltage_id)
	{
	- struct msg_get_voltage msg;
	+ union msg_get_voltage_body msg;
	int value;
	int err;

	/*
	* Get voltage
	* Tag: 0x00030003
	* Request:
	* Length: 4
	* Value:
	* u32: voltage id
	* Response:
	* Length: 8
	* Value:
	* u32: voltage id
	* u32: value (offset from 1.2V in units of 0.025V)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_VOLTAGE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.voltage_id = voltage_id;
	- msg.end_tag = 0;
	+ msg.req.voltage_id = voltage_id;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_VOLTAGE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get voltage\n");
	return (MSG_ERROR);
	}

	/* result (offset from 1.2V) */
	- value = (int)msg.body.resp.value;
	+ value = (int)msg.resp.value;
	DPRINTF("value = %d\n", value);
	return (value);
	}

	static int
	bcm2835_cpufreq_get_max_voltage(struct bcm2835_cpufreq_softc *sc,
	uint32_t voltage_id)
	{
	- struct msg_get_max_voltage msg;
	+ union msg_get_voltage_body msg;
	int value;
	int err;

	/*
	* Get voltage
	* Tag: 0x00030005
	* Request:
	* Length: 4
	* Value:
	* u32: voltage id
	* Response:
	* Length: 8
	* Value:
	* u32: voltage id
	* u32: value (offset from 1.2V in units of 0.025V)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_MAX_VOLTAGE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.voltage_id = voltage_id;
	- msg.end_tag = 0;
	+ msg.req.voltage_id = voltage_id;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_MAX_VOLTAGE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get max voltage\n");
	return (MSG_ERROR);
	}

	/* result (offset from 1.2V) */
	- value = (int)msg.body.resp.value;
	+ value = (int)msg.resp.value;
	DPRINTF("value = %d\n", value);
	return (value);
	}
	static int
	bcm2835_cpufreq_get_min_voltage(struct bcm2835_cpufreq_softc *sc,
	uint32_t voltage_id)
	{
	- struct msg_get_min_voltage msg;
	+ union msg_get_voltage_body msg;
	int value;
	int err;

	/*
	* Get voltage
	* Tag: 0x00030008
	* Request:
	* Length: 4
	* Value:
	* u32: voltage id
	* Response:
	* Length: 8
	* Value:
	* u32: voltage id
	* u32: value (offset from 1.2V in units of 0.025V)
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_MIN_VOLTAGE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.voltage_id = voltage_id;
	- msg.end_tag = 0;
	+ msg.req.voltage_id = voltage_id;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_MIN_VOLTAGE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get min voltage\n");
	return (MSG_ERROR);
	}

	/* result (offset from 1.2V) */
	- value = (int)msg.body.resp.value;
	+ value = (int)msg.resp.value;
	DPRINTF("value = %d\n", value);
	return (value);
	}

	static int
	bcm2835_cpufreq_set_voltage(struct bcm2835_cpufreq_softc *sc,
	uint32_t voltage_id, int32_t value)
	{
	- struct msg_set_voltage msg;
	+ union msg_set_voltage_body msg;
	int err;

	/*
	* Set voltage
	* Tag: 0x00038003
	* Request:
	* Length: 4
	* Value:
	* u32: voltage id
	* u32: value (offset from 1.2V in units of 0.025V)
	* Response:
	* Length: 8
	* Value:
	* u32: voltage id
	* u32: value (offset from 1.2V in units of 0.025V)
	*/

	/*
	* over_voltage:
	* 0 (1.2 V). Values above 6 are only allowed when force_turbo or
	* current_limit_override are specified (which set the warranty bit).
	*/
	if (value > MAX_OVER_VOLTAGE \|\| value < MIN_OVER_VOLTAGE) {
	/* currently not supported */
	device_printf(sc->dev, "not supported voltage: %d\n", value);
	return (MSG_ERROR);
	}

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_SET_VOLTAGE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.voltage_id = voltage_id;
	- msg.body.req.value = (uint32_t)value;
	- msg.end_tag = 0;
	+ msg.req.voltage_id = voltage_id;
	+ msg.req.value = (uint32_t)value;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_SET_VOLTAGE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't set voltage\n");
	return (MSG_ERROR);
	}

	/* result (offset from 1.2V) */
	- value = (int)msg.body.resp.value;
	+ value = (int)msg.resp.value;
	DPRINTF("value = %d\n", value);
	return (value);
	}

	static int
	bcm2835_cpufreq_get_temperature(struct bcm2835_cpufreq_softc *sc)
	{
	- struct msg_get_temperature msg;
	+ union msg_get_temperature_body msg;
	int value;
	int err;

	/*
	* Get temperature
	* Tag: 0x00030006
	* Request:
	* Length: 4
	* Value:
	* u32: temperature id
	* Response:
	* Length: 8
	* Value:
	* u32: temperature id
	* u32: value
	*/

	/* setup single tag buffer */
	memset(&msg, 0, sizeof(msg));
	- msg.hdr.buf_size = sizeof(msg);
	- msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	- msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_TEMPERATURE;
	- msg.tag_hdr.val_buf_size = sizeof(msg.body);
	- msg.tag_hdr.val_len = sizeof(msg.body.req);
	- msg.body.req.temperature_id = 0;
	- msg.end_tag = 0;
	+ msg.req.temperature_id = 0;

	/* call mailbox property */
	- err = bcm2835_mbox_property(&msg, sizeof(msg));
	+ err = bcm2835_firmware_property(sc->firmware,
	+ BCM2835_FIRMWARE_TAG_GET_TEMPERATURE, &msg, sizeof(msg));
	if (err) {
	device_printf(sc->dev, "can't get temperature\n");
	return (MSG_ERROR);
	}

	/* result (temperature of degree C) */
	- value = (int)msg.body.resp.value;
	+ value = (int)msg.resp.value;
	DPRINTF("value = %d\n", value);
	return (value);
	}



	static int
	sysctl_bcm2835_cpufreq_arm_freq(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_ARM);
	+ val = bcm2835_cpufreq_get_clock_rate(sc, BCM2835_FIRMWARE_CLOCK_ID_ARM);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_ARM,
	+ err = bcm2835_cpufreq_set_clock_rate(sc, BCM2835_FIRMWARE_CLOCK_ID_ARM,
	val);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set clock arm_freq error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_core_freq(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_CORE);
	+ val = bcm2835_cpufreq_get_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_CORE,
	+ err = bcm2835_cpufreq_set_clock_rate(sc, BCM2835_FIRMWARE_CLOCK_ID_CORE,
	val);
	if (err == MSG_ERROR) {
	VC_UNLOCK(sc);
	device_printf(sc->dev, "set clock core_freq error\n");
	return (EIO);
	}
	VC_UNLOCK(sc);
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_sdram_freq(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_SDRAM);
	+ val = bcm2835_cpufreq_get_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_SDRAM,
	- val);
	+ err = bcm2835_cpufreq_set_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM, val);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set clock sdram_freq error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_turbo(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	val = bcm2835_cpufreq_get_turbo(sc);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	if (val > 0)
	- sc->turbo_mode = BCM2835_MBOX_TURBO_ON;
	+ sc->turbo_mode = BCM2835_FIRMWARE_TURBO_ON;
	else
	- sc->turbo_mode = BCM2835_MBOX_TURBO_OFF;
	+ sc->turbo_mode = BCM2835_FIRMWARE_TURBO_OFF;

	VC_LOCK(sc);
	err = bcm2835_cpufreq_set_turbo(sc, sc->turbo_mode);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set turbo error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_voltage_core(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_CORE);
	+ val = bcm2835_cpufreq_get_voltage(sc, BCM2835_FIRMWARE_VOLTAGE_ID_CORE);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	if (val > MAX_OVER_VOLTAGE \|\| val < MIN_OVER_VOLTAGE)
	return (EINVAL);
	sc->voltage_core = val;

	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_CORE,
	+ err = bcm2835_cpufreq_set_voltage(sc, BCM2835_FIRMWARE_VOLTAGE_ID_CORE,
	sc->voltage_core);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set voltage core error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_voltage_sdram_c(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_C);
	+ val = bcm2835_cpufreq_get_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	if (val > MAX_OVER_VOLTAGE \|\| val < MIN_OVER_VOLTAGE)
	return (EINVAL);
	sc->voltage_sdram_c = val;

	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_C,
	+ err = bcm2835_cpufreq_set_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C,
	sc->voltage_sdram_c);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set voltage sdram_c error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_voltage_sdram_i(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_I);
	+ val = bcm2835_cpufreq_get_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	if (val > MAX_OVER_VOLTAGE \|\| val < MIN_OVER_VOLTAGE)
	return (EINVAL);
	sc->voltage_sdram_i = val;

	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_I,
	- sc->voltage_sdram_i);
	+ err = bcm2835_cpufreq_set_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I, sc->voltage_sdram_i);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set voltage sdram_i error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_voltage_sdram_p(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	- val = bcm2835_cpufreq_get_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_P);
	+ val = bcm2835_cpufreq_get_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	if (val > MAX_OVER_VOLTAGE \|\| val < MIN_OVER_VOLTAGE)
	return (EINVAL);
	sc->voltage_sdram_p = val;

	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_P,
	- sc->voltage_sdram_p);
	+ err = bcm2835_cpufreq_set_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P, sc->voltage_sdram_p);
	VC_UNLOCK(sc);
	if (err == MSG_ERROR) {
	device_printf(sc->dev, "set voltage sdram_p error\n");
	return (EIO);
	}
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_voltage_sdram(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* multiple write only */
	if (!req->newptr)
	return (EINVAL);
	val = 0;
	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err)
	return (err);

	/* write request */
	if (val > MAX_OVER_VOLTAGE \|\| val < MIN_OVER_VOLTAGE)
	return (EINVAL);
	sc->voltage_sdram = val;

	VC_LOCK(sc);
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_C,
	- val);
	+ err = bcm2835_cpufreq_set_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C, val);
	if (err == MSG_ERROR) {
	VC_UNLOCK(sc);
	device_printf(sc->dev, "set voltage sdram_c error\n");
	return (EIO);
	}
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_I,
	- val);
	+ err = bcm2835_cpufreq_set_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I, val);
	if (err == MSG_ERROR) {
	VC_UNLOCK(sc);
	device_printf(sc->dev, "set voltage sdram_i error\n");
	return (EIO);
	}
	- err = bcm2835_cpufreq_set_voltage(sc, BCM2835_MBOX_VOLTAGE_ID_SDRAM_P,
	- val);
	+ err = bcm2835_cpufreq_set_voltage(sc,
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P, val);
	if (err == MSG_ERROR) {
	VC_UNLOCK(sc);
	device_printf(sc->dev, "set voltage sdram_p error\n");
	return (EIO);
	}
	VC_UNLOCK(sc);
	DELAY(TRANSITION_LATENCY);

	return (0);
	}

	static int
	sysctl_bcm2835_cpufreq_temperature(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	val = bcm2835_cpufreq_get_temperature(sc);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	return (EINVAL);
	}

	static int
	sysctl_bcm2835_devcpu_temperature(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_cpufreq_softc *sc = arg1;
	int val;
	int err;

	/* get realtime value */
	VC_LOCK(sc);
	val = bcm2835_cpufreq_get_temperature(sc);
	VC_UNLOCK(sc);
	if (val == MSG_ERROR)
	return (EIO);

	/* 1/1000 celsius (raw) to 1/10 kelvin */
	val = val / 100 + TZ_ZEROC;

	err = sysctl_handle_int(oidp, &val, 0, req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	return (EINVAL);
	}


	static void
	bcm2835_cpufreq_init(void *arg)
	{
	struct bcm2835_cpufreq_softc *sc = arg;
	struct sysctl_ctx_list *ctx;
	device_t cpu;
	int arm_freq, core_freq, sdram_freq;
	int arm_max_freq, arm_min_freq, core_max_freq, core_min_freq;
	int sdram_max_freq, sdram_min_freq;
	int voltage_core, voltage_sdram_c, voltage_sdram_i, voltage_sdram_p;
	int max_voltage_core, min_voltage_core;
	int max_voltage_sdram_c, min_voltage_sdram_c;
	int max_voltage_sdram_i, min_voltage_sdram_i;
	int max_voltage_sdram_p, min_voltage_sdram_p;
	int turbo, temperature;

	VC_LOCK(sc);

	/* current clock */
	arm_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM);
	core_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE);
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE);
	sdram_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM);
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM);

	/* max/min clock */
	arm_max_freq = bcm2835_cpufreq_get_max_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM);
	arm_min_freq = bcm2835_cpufreq_get_min_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM);
	core_max_freq = bcm2835_cpufreq_get_max_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE);
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE);
	core_min_freq = bcm2835_cpufreq_get_min_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE);
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE);
	sdram_max_freq = bcm2835_cpufreq_get_max_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM);
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM);
	sdram_min_freq = bcm2835_cpufreq_get_min_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM);
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM);

	/* turbo mode */
	turbo = bcm2835_cpufreq_get_turbo(sc);
	if (turbo > 0)
	- sc->turbo_mode = BCM2835_MBOX_TURBO_ON;
	+ sc->turbo_mode = BCM2835_FIRMWARE_TURBO_ON;
	else
	- sc->turbo_mode = BCM2835_MBOX_TURBO_OFF;
	+ sc->turbo_mode = BCM2835_FIRMWARE_TURBO_OFF;

	/* voltage */
	voltage_core = bcm2835_cpufreq_get_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_CORE);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_CORE);
	voltage_sdram_c = bcm2835_cpufreq_get_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_C);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C);
	voltage_sdram_i = bcm2835_cpufreq_get_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_I);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I);
	voltage_sdram_p = bcm2835_cpufreq_get_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_P);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P);

	/* current values (offset from 1.2V) */
	sc->voltage_core = voltage_core;
	sc->voltage_sdram = voltage_sdram_c;
	sc->voltage_sdram_c = voltage_sdram_c;
	sc->voltage_sdram_i = voltage_sdram_i;
	sc->voltage_sdram_p = voltage_sdram_p;

	/* max/min voltage */
	max_voltage_core = bcm2835_cpufreq_get_max_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_CORE);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_CORE);
	min_voltage_core = bcm2835_cpufreq_get_min_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_CORE);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_CORE);
	max_voltage_sdram_c = bcm2835_cpufreq_get_max_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_C);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C);
	max_voltage_sdram_i = bcm2835_cpufreq_get_max_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_I);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I);
	max_voltage_sdram_p = bcm2835_cpufreq_get_max_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_P);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P);
	min_voltage_sdram_c = bcm2835_cpufreq_get_min_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_C);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C);
	min_voltage_sdram_i = bcm2835_cpufreq_get_min_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_I);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I);
	min_voltage_sdram_p = bcm2835_cpufreq_get_min_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_SDRAM_P);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P);

	/* temperature */
	temperature = bcm2835_cpufreq_get_temperature(sc);

	/* show result */
	if (cpufreq_verbose \|\| bootverbose) {
	device_printf(sc->dev, "Boot settings:\n");
	device_printf(sc->dev,
	"current ARM %dMHz, Core %dMHz, SDRAM %dMHz, Turbo %s\n",
	HZ2MHZ(arm_freq), HZ2MHZ(core_freq), HZ2MHZ(sdram_freq),
	- (sc->turbo_mode == BCM2835_MBOX_TURBO_ON) ? "ON" : "OFF");
	+ (sc->turbo_mode == BCM2835_FIRMWARE_TURBO_ON) ? "ON":"OFF");

	device_printf(sc->dev,
	"max/min ARM %d/%dMHz, Core %d/%dMHz, SDRAM %d/%dMHz\n",
	HZ2MHZ(arm_max_freq), HZ2MHZ(arm_min_freq),
	HZ2MHZ(core_max_freq), HZ2MHZ(core_min_freq),
	HZ2MHZ(sdram_max_freq), HZ2MHZ(sdram_min_freq));

	device_printf(sc->dev,
	"current Core %dmV, SDRAM_C %dmV, SDRAM_I %dmV, "
	"SDRAM_P %dmV\n",
	OFFSET2MVOLT(voltage_core), OFFSET2MVOLT(voltage_sdram_c),
	OFFSET2MVOLT(voltage_sdram_i),
	OFFSET2MVOLT(voltage_sdram_p));

	device_printf(sc->dev,
	"max/min Core %d/%dmV, SDRAM_C %d/%dmV, SDRAM_I %d/%dmV, "
	"SDRAM_P %d/%dmV\n",
	OFFSET2MVOLT(max_voltage_core),
	OFFSET2MVOLT(min_voltage_core),
	OFFSET2MVOLT(max_voltage_sdram_c),
	OFFSET2MVOLT(min_voltage_sdram_c),
	OFFSET2MVOLT(max_voltage_sdram_i),
	OFFSET2MVOLT(min_voltage_sdram_i),
	OFFSET2MVOLT(max_voltage_sdram_p),
	OFFSET2MVOLT(min_voltage_sdram_p));

	device_printf(sc->dev,
	"Temperature %d.%dC\n", (temperature / 1000),
	(temperature % 1000) / 100);
	} else { /* !cpufreq_verbose && !bootverbose */
	device_printf(sc->dev,
	"ARM %dMHz, Core %dMHz, SDRAM %dMHz, Turbo %s\n",
	HZ2MHZ(arm_freq), HZ2MHZ(core_freq), HZ2MHZ(sdram_freq),
	- (sc->turbo_mode == BCM2835_MBOX_TURBO_ON) ? "ON" : "OFF");
	+ (sc->turbo_mode == BCM2835_FIRMWARE_TURBO_ON) ? "ON":"OFF");
	}

	/* keep in softc (MHz/mV) */
	sc->arm_max_freq = HZ2MHZ(arm_max_freq);
	sc->arm_min_freq = HZ2MHZ(arm_min_freq);
	sc->core_max_freq = HZ2MHZ(core_max_freq);
	sc->core_min_freq = HZ2MHZ(core_min_freq);
	sc->sdram_max_freq = HZ2MHZ(sdram_max_freq);
	sc->sdram_min_freq = HZ2MHZ(sdram_min_freq);
	sc->max_voltage_core = OFFSET2MVOLT(max_voltage_core);
	sc->min_voltage_core = OFFSET2MVOLT(min_voltage_core);

	/* if turbo is on, set to max values */
	- if (sc->turbo_mode == BCM2835_MBOX_TURBO_ON) {
	- bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_ARM,
	- arm_max_freq);
	+ if (sc->turbo_mode == BCM2835_FIRMWARE_TURBO_ON) {
	+ bcm2835_cpufreq_set_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM, arm_max_freq);
	DELAY(TRANSITION_LATENCY);
	- bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_CORE,
	- core_max_freq);
	+ bcm2835_cpufreq_set_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE, core_max_freq);
	DELAY(TRANSITION_LATENCY);
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM, sdram_max_freq);
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM, sdram_max_freq);
	DELAY(TRANSITION_LATENCY);
	} else {
	- bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_ARM,
	- arm_min_freq);
	+ bcm2835_cpufreq_set_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM, arm_min_freq);
	DELAY(TRANSITION_LATENCY);
	- bcm2835_cpufreq_set_clock_rate(sc, BCM2835_MBOX_CLOCK_ID_CORE,
	- core_min_freq);
	+ bcm2835_cpufreq_set_clock_rate(sc,
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE, core_min_freq);
	DELAY(TRANSITION_LATENCY);
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM, sdram_min_freq);
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM, sdram_min_freq);
	DELAY(TRANSITION_LATENCY);
	}

	VC_UNLOCK(sc);

	/* add human readable temperature to dev.cpu node */
	cpu = device_get_parent(sc->dev);
	if (cpu != NULL) {
	ctx = device_get_sysctl_ctx(cpu);
	SYSCTL_ADD_PROC(ctx,
	SYSCTL_CHILDREN(device_get_sysctl_tree(cpu)), OID_AUTO,
	"temperature",
	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_devcpu_temperature, "IK",
	"Current SoC temperature");
	}

	/* release this hook (continue boot) */
	config_intrhook_disestablish(&sc->init_hook);
	}

	static void
	bcm2835_cpufreq_identify(driver_t *driver, device_t parent)
	{
	const struct ofw_compat_data *compat;
	phandle_t root;

	root = OF_finddevice("/");
	for (compat = compat_data; compat->ocd_str != NULL; compat++)
	if (ofw_bus_node_is_compatible(root, compat->ocd_str))
	break;

	if (compat->ocd_data == 0)
	return;

	DPRINTF("driver=%p, parent=%p\n", driver, parent);
	if (device_find_child(parent, "bcm2835_cpufreq", -1) != NULL)
	return;
	if (BUS_ADD_CHILD(parent, 0, "bcm2835_cpufreq", -1) == NULL)
	device_printf(parent, "add child failed\n");
	}

	static int
	bcm2835_cpufreq_probe(device_t dev)
	{

	if (device_get_unit(dev) != 0)
	return (ENXIO);
	device_set_desc(dev, "CPU Frequency Control");

	return (0);
	}

	static int
	bcm2835_cpufreq_attach(device_t dev)
	{
	struct bcm2835_cpufreq_softc *sc;
	struct sysctl_oid *oid;

	/* set self dev */
	sc = device_get_softc(dev);
	sc->dev = dev;
	+ sc->firmware = devclass_get_device(
	+ devclass_find("bcm2835_firmware"), 0);
	+ if (sc->firmware == NULL) {
	+ device_printf(dev, "Unable to find firmware device\n");
	+ return (ENXIO);
	+ }

	/* initial values */
	sc->arm_max_freq = -1;
	sc->arm_min_freq = -1;
	sc->core_max_freq = -1;
	sc->core_min_freq = -1;
	sc->sdram_max_freq = -1;
	sc->sdram_min_freq = -1;
	sc->max_voltage_core = 0;
	sc->min_voltage_core = 0;

	/* setup sysctl at first device */
	if (device_get_unit(dev) == 0) {
	sysctl_ctx_init(&bcm2835_sysctl_ctx);
	/* create node for hw.cpufreq */
	oid = SYSCTL_ADD_NODE(&bcm2835_sysctl_ctx,
	SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, "cpufreq",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "");

	/* Frequency (Hz) */
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "arm_freq",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_cpufreq_arm_freq, "IU",
	"ARM frequency (Hz)");
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "core_freq",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_cpufreq_core_freq, "IU",
	"Core frequency (Hz)");
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "sdram_freq",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_cpufreq_sdram_freq, "IU",
	"SDRAM frequency (Hz)");

	/* Turbo state */
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "turbo",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_cpufreq_turbo, "IU",
	"Disables dynamic clocking");

	/* Voltage (offset from 1.2V in units of 0.025V) */
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "voltage_core",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_cpufreq_voltage_core, "I",
	"ARM/GPU core voltage"
	"(offset from 1.2V in units of 0.025V)");
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "voltage_sdram",
	CTLTYPE_INT \| CTLFLAG_WR \| CTLFLAG_NEEDGIANT, sc,
	0, sysctl_bcm2835_cpufreq_voltage_sdram, "I",
	"SDRAM voltage (offset from 1.2V in units of 0.025V)");

	/* Voltage individual SDRAM */
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "voltage_sdram_c",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc,
	0, sysctl_bcm2835_cpufreq_voltage_sdram_c, "I",
	"SDRAM controller voltage"
	"(offset from 1.2V in units of 0.025V)");
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "voltage_sdram_i",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc,
	0, sysctl_bcm2835_cpufreq_voltage_sdram_i, "I",
	"SDRAM I/O voltage (offset from 1.2V in units of 0.025V)");
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "voltage_sdram_p",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc,
	0, sysctl_bcm2835_cpufreq_voltage_sdram_p, "I",
	"SDRAM phy voltage (offset from 1.2V in units of 0.025V)");

	/* Temperature */
	SYSCTL_ADD_PROC(&bcm2835_sysctl_ctx, SYSCTL_CHILDREN(oid),
	OID_AUTO, "temperature",
	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_bcm2835_cpufreq_temperature, "I",
	"SoC temperature (thousandths of a degree C)");
	}

	/* ARM->VC lock */
	sema_init(&vc_sema, 1, "vcsema");

	/* register callback for using mbox when interrupts are enabled */
	sc->init_hook.ich_func = bcm2835_cpufreq_init;
	sc->init_hook.ich_arg = sc;

	if (config_intrhook_establish(&sc->init_hook) != 0) {
	device_printf(dev, "config_intrhook_establish failed\n");
	return (ENOMEM);
	}

	/* this device is controlled by cpufreq(4) */
	cpufreq_register(dev);

	return (0);
	}

	static int
	bcm2835_cpufreq_detach(device_t dev)
	{

	sema_destroy(&vc_sema);

	return (cpufreq_unregister(dev));
	}

	static int
	bcm2835_cpufreq_set(device_t dev, const struct cf_setting *cf)
	{
	struct bcm2835_cpufreq_softc *sc;
	uint32_t rate_hz, rem;
	int resp_freq, arm_freq, min_freq, core_freq;
	#ifdef DEBUG
	int cur_freq;
	#endif

	if (cf == NULL \|\| cf->freq < 0)
	return (EINVAL);

	sc = device_get_softc(dev);

	/* setting clock (Hz) */
	rate_hz = (uint32_t)MHZ2HZ(cf->freq);
	rem = rate_hz % HZSTEP;
	rate_hz -= rem;
	if (rate_hz == 0)
	return (EINVAL);

	/* adjust min freq */
	min_freq = sc->arm_min_freq;
	- if (sc->turbo_mode != BCM2835_MBOX_TURBO_ON)
	+ if (sc->turbo_mode != BCM2835_FIRMWARE_TURBO_ON)
	if (min_freq > cpufreq_lowest_freq)
	min_freq = cpufreq_lowest_freq;

	if (rate_hz < MHZ2HZ(min_freq) \|\| rate_hz > MHZ2HZ(sc->arm_max_freq))
	return (EINVAL);

	/* set new value and verify it */
	VC_LOCK(sc);
	#ifdef DEBUG
	cur_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM);
	#endif
	resp_freq = bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM, rate_hz);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM, rate_hz);
	DELAY(TRANSITION_LATENCY);
	arm_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM);

	/*
	* if non-turbo and lower than or equal min_freq,
	* clock down core and sdram to default first.
	*/
	- if (sc->turbo_mode != BCM2835_MBOX_TURBO_ON) {
	+ if (sc->turbo_mode != BCM2835_FIRMWARE_TURBO_ON) {
	core_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE);
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE);
	if (rate_hz > MHZ2HZ(sc->arm_min_freq)) {
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE,
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE,
	MHZ2HZ(sc->core_max_freq));
	DELAY(TRANSITION_LATENCY);
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM,
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM,
	MHZ2HZ(sc->sdram_max_freq));
	DELAY(TRANSITION_LATENCY);
	} else {
	if (sc->core_min_freq < DEFAULT_CORE_FREQUENCY &&
	core_freq > DEFAULT_CORE_FREQUENCY) {
	/* first, down to 250, then down to min */
	DELAY(TRANSITION_LATENCY);
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE,
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE,
	MHZ2HZ(DEFAULT_CORE_FREQUENCY));
	DELAY(TRANSITION_LATENCY);
	/* reset core voltage */
	bcm2835_cpufreq_set_voltage(sc,
	- BCM2835_MBOX_VOLTAGE_ID_CORE, 0);
	+ BCM2835_FIRMWARE_VOLTAGE_ID_CORE, 0);
	DELAY(TRANSITION_LATENCY);
	}
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_CORE,
	+ BCM2835_FIRMWARE_CLOCK_ID_CORE,
	MHZ2HZ(sc->core_min_freq));
	DELAY(TRANSITION_LATENCY);
	bcm2835_cpufreq_set_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_SDRAM,
	+ BCM2835_FIRMWARE_CLOCK_ID_SDRAM,
	MHZ2HZ(sc->sdram_min_freq));
	DELAY(TRANSITION_LATENCY);
	}
	}

	VC_UNLOCK(sc);

	if (resp_freq < 0 \|\| arm_freq < 0 \|\| resp_freq != arm_freq) {
	device_printf(dev, "wrong freq\n");
	return (EIO);
	}
	DPRINTF("cpufreq: %d -> %d\n", cur_freq, arm_freq);

	return (0);
	}

	static int
	bcm2835_cpufreq_get(device_t dev, struct cf_setting *cf)
	{
	struct bcm2835_cpufreq_softc *sc;
	int arm_freq;

	if (cf == NULL)
	return (EINVAL);

	sc = device_get_softc(dev);
	memset(cf, CPUFREQ_VAL_UNKNOWN, sizeof(*cf));
	cf->dev = NULL;

	/* get cuurent value */
	VC_LOCK(sc);
	arm_freq = bcm2835_cpufreq_get_clock_rate(sc,
	- BCM2835_MBOX_CLOCK_ID_ARM);
	+ BCM2835_FIRMWARE_CLOCK_ID_ARM);
	VC_UNLOCK(sc);
	if (arm_freq < 0) {
	device_printf(dev, "can't get clock\n");
	return (EINVAL);
	}

	/* CPU clock in MHz or 100ths of a percent. */
	cf->freq = HZ2MHZ(arm_freq);
	/* Voltage in mV. */
	cf->volts = CPUFREQ_VAL_UNKNOWN;
	/* Power consumed in mW. */
	cf->power = CPUFREQ_VAL_UNKNOWN;
	/* Transition latency in us. */
	cf->lat = TRANSITION_LATENCY;
	/* Driver providing this setting. */
	cf->dev = dev;

	return (0);
	}

	static int
	bcm2835_cpufreq_make_freq_list(device_t dev, struct cf_setting *sets,
	int *count)
	{
	struct bcm2835_cpufreq_softc *sc;
	int freq, min_freq, volts, rem;
	int idx;

	sc = device_get_softc(dev);
	freq = sc->arm_max_freq;
	min_freq = sc->arm_min_freq;

	/* adjust head freq to STEP */
	rem = freq % MHZSTEP;
	freq -= rem;
	if (freq < min_freq)
	freq = min_freq;

	/* if non-turbo, add extra low freq */
	- if (sc->turbo_mode != BCM2835_MBOX_TURBO_ON)
	+ if (sc->turbo_mode != BCM2835_FIRMWARE_TURBO_ON)
	if (min_freq > cpufreq_lowest_freq)
	min_freq = cpufreq_lowest_freq;

	#ifdef SOC_BCM2835
	/* from freq to min_freq */
	for (idx = 0; idx < *count && freq >= min_freq; idx++) {
	if (freq > sc->arm_min_freq)
	volts = sc->max_voltage_core;
	else
	volts = sc->min_voltage_core;
	sets[idx].freq = freq;
	sets[idx].volts = volts;
	sets[idx].lat = TRANSITION_LATENCY;
	sets[idx].dev = dev;
	freq -= MHZSTEP;
	}
	#else
	/* XXX RPi2 have only 900/600MHz */
	idx = 0;
	volts = sc->min_voltage_core;
	sets[idx].freq = freq;
	sets[idx].volts = volts;
	sets[idx].lat = TRANSITION_LATENCY;
	sets[idx].dev = dev;
	idx++;
	if (freq != min_freq) {
	sets[idx].freq = min_freq;
	sets[idx].volts = volts;
	sets[idx].lat = TRANSITION_LATENCY;
	sets[idx].dev = dev;
	idx++;
	}
	#endif
	*count = idx;

	return (0);
	}

	static int
	bcm2835_cpufreq_settings(device_t dev, struct cf_setting sets, int count)
	{
	struct bcm2835_cpufreq_softc *sc;

	if (sets == NULL \|\| count == NULL)
	return (EINVAL);

	sc = device_get_softc(dev);
	if (sc->arm_min_freq < 0 \|\| sc->arm_max_freq < 0) {
	printf("device is not configured\n");
	return (EINVAL);
	}

	/* fill data with unknown value */
	memset(sets, CPUFREQ_VAL_UNKNOWN, sizeof(sets) (*count));
	/* create new array up to count */
	bcm2835_cpufreq_make_freq_list(dev, sets, count);

	return (0);
	}

	static int
	bcm2835_cpufreq_type(device_t dev, int *type)
	{

	if (type == NULL)
	return (EINVAL);
	*type = CPUFREQ_TYPE_ABSOLUTE;

	return (0);
	}

	static device_method_t bcm2835_cpufreq_methods[] = {
	/* Device interface */
	DEVMETHOD(device_identify, bcm2835_cpufreq_identify),
	DEVMETHOD(device_probe, bcm2835_cpufreq_probe),
	DEVMETHOD(device_attach, bcm2835_cpufreq_attach),
	DEVMETHOD(device_detach, bcm2835_cpufreq_detach),

	/* cpufreq interface */
	DEVMETHOD(cpufreq_drv_set, bcm2835_cpufreq_set),
	DEVMETHOD(cpufreq_drv_get, bcm2835_cpufreq_get),
	DEVMETHOD(cpufreq_drv_settings, bcm2835_cpufreq_settings),
	DEVMETHOD(cpufreq_drv_type, bcm2835_cpufreq_type),

	DEVMETHOD_END
	};

	static devclass_t bcm2835_cpufreq_devclass;
	static driver_t bcm2835_cpufreq_driver = {
	"bcm2835_cpufreq",
	bcm2835_cpufreq_methods,
	sizeof(struct bcm2835_cpufreq_softc),
	};

	DRIVER_MODULE(bcm2835_cpufreq, cpu, bcm2835_cpufreq_driver,
	bcm2835_cpufreq_devclass, 0, 0);
	+MODULE_DEPEND(bcm2835_cpufreq, bcm2835_firmware, 1, 1, 1);
	diff --git a/sys/arm/broadcom/bcm2835/bcm2835_firmware.c b/sys/arm/broadcom/bcm2835/bcm2835_firmware.c
	index dcc23fe1cd61..44201ec80f7d 100644
	--- a/sys/arm/broadcom/bcm2835/bcm2835_firmware.c
	+++ b/sys/arm/broadcom/bcm2835/bcm2835_firmware.c
	@@ -1,181 +1,182 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause
	*
	* Copyright (c) 2020 Andrew Turner
	*
	* This work was supported by Innovate UK project 105694, "Digital Security
	* by Design (DSbD) Technology Platform Prototype".
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/sysctl.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <arm/broadcom/bcm2835/bcm2835_firmware.h>
	#include <arm/broadcom/bcm2835/bcm2835_mbox.h>
	#include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h>
	#include <arm/broadcom/bcm2835/bcm2835_vcbus.h>

	struct bcm2835_firmware_softc {
	device_t sc_dev;
	phandle_t sc_mbox;
	};

	static struct ofw_compat_data compat_data[] = {
	{"raspberrypi,bcm2835-firmware", 1},
	{NULL, 0}
	};

	static int sysctl_bcm2835_firmware_get_revision(SYSCTL_HANDLER_ARGS);

	static int
	bcm2835_firmware_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	device_set_desc(dev, "BCM2835 Firmware");
	return (BUS_PROBE_DEFAULT);
	}

	static int
	bcm2835_firmware_attach(device_t dev)
	{
	struct bcm2835_firmware_softc *sc;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *tree_node;
	struct sysctl_oid_list *tree;
	phandle_t node, mbox;
	int rv;

	sc = device_get_softc(dev);
	sc->sc_dev = dev;

	node = ofw_bus_get_node(dev);
	rv = OF_getencprop(node, "mboxes", &mbox, sizeof(mbox));
	if (rv <= 0) {
	device_printf(dev, "can't read mboxes property\n");
	return (ENXIO);
	}
	sc->sc_mbox = mbox;

	OF_device_register_xref(OF_xref_from_node(node), dev);

	ctx = device_get_sysctl_ctx(sc->sc_dev);
	tree_node = device_get_sysctl_tree(sc->sc_dev);
	tree = SYSCTL_CHILDREN(tree_node);
	SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "revision",
	CTLTYPE_UINT \| CTLFLAG_RD, sc, sizeof(*sc),
	sysctl_bcm2835_firmware_get_revision, "IU",
	"Firmware revision");
	return (0);
	}

	int
	bcm2835_firmware_property(device_t dev, uint32_t prop, void *data, size_t len)
	{
	struct {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	uint32_t data[];
	} *msg_hdr;
	size_t msg_len;
	int err;

	/*
	* The message is processed in 32-bit chunks so must be a multiple
	* of 32-bits.
	*/
	if ((len & (sizeof(uint32_t) - 1)) != 0)
	return (EINVAL);

	msg_len = sizeof(*msg_hdr) + len + sizeof(uint32_t);
	msg_hdr = malloc(sizeof(*msg_hdr) + msg_len + sizeof(uint32_t),
	M_DEVBUF, M_WAITOK \| M_ZERO);

	msg_hdr->hdr.buf_size = msg_len;
	msg_hdr->hdr.code = BCM2835_MBOX_CODE_REQ;
	msg_hdr->tag_hdr.tag = prop;
	msg_hdr->tag_hdr.val_buf_size = len;
	memcpy(msg_hdr->data, data, len);
	msg_hdr->data[len / sizeof(uint32_t)] = 0;

	err = bcm2835_mbox_property(msg_hdr, msg_len);
	if (err == 0) {
	memcpy(data, msg_hdr->data, len);
	}

	free(msg_hdr, M_DEVBUF);

	return (err);
	}

	static int
	sysctl_bcm2835_firmware_get_revision(SYSCTL_HANDLER_ARGS)
	{
	struct bcm2835_firmware_softc *sc = arg1;
	uint32_t rev;
	int err;

	if (bcm2835_firmware_property(sc->sc_dev,
	BCM2835_MBOX_TAG_FIRMWARE_REVISION, &rev, sizeof(rev)) != 0)
	return (ENXIO);

	err = sysctl_handle_int(oidp, &rev, sizeof(rev), req);
	if (err \|\| !req->newptr) /* error \|\| read request */
	return (err);

	/* write request */
	return (EINVAL);
	}

	static device_method_t bcm2835_firmware_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, bcm2835_firmware_probe),
	DEVMETHOD(device_attach, bcm2835_firmware_attach),

	DEVMETHOD_END
	};

	static devclass_t bcm2835_firmware_devclass;
	static driver_t bcm2835_firmware_driver = {
	"bcm2835_firmware",
	bcm2835_firmware_methods,
	sizeof(struct bcm2835_firmware_softc),
	};

	-DRIVER_MODULE(bcm2835_firmware, simplebus, bcm2835_firmware_driver,
	- bcm2835_firmware_devclass, 0, 0);
	+EARLY_DRIVER_MODULE(bcm2835_firmware, simplebus, bcm2835_firmware_driver,
	+ bcm2835_firmware_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST);
	+MODULE_DEPEND(bcm2835_firmware, mbox, 1, 1, 1);
	diff --git a/sys/arm/broadcom/bcm2835/bcm2835_firmware.h b/sys/arm/broadcom/bcm2835/bcm2835_firmware.h
	index da6a137083ed..666e450914f8 100644
	--- a/sys/arm/broadcom/bcm2835/bcm2835_firmware.h
	+++ b/sys/arm/broadcom/bcm2835/bcm2835_firmware.h
	@@ -1,38 +1,202 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause
	*
	* Copyright (c) 2020 Andrew Turner
	*
	* This work was supported by Innovate UK project 105694, "Digital Security
	* by Design (DSbD) Technology Platform Prototype".
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef _BCM2835_FIRMWARE_H_
	#define _BCM2835_FIRMWARE_H_

	+#define BCM2835_FIRMWARE_TAG_GET_CLOCK_RATE 0x00030002
	+#define BCM2835_FIRMWARE_TAG_SET_CLOCK_RATE 0x00038002
	+#define BCM2835_FIRMWARE_TAG_GET_MAX_CLOCK_RATE 0x00030004
	+#define BCM2835_FIRMWARE_TAG_GET_MIN_CLOCK_RATE 0x00030007
	+
	+#define BCM2835_FIRMWARE_CLOCK_ID_EMMC 0x00000001
	+#define BCM2835_FIRMWARE_CLOCK_ID_UART 0x00000002
	+#define BCM2835_FIRMWARE_CLOCK_ID_ARM 0x00000003
	+#define BCM2835_FIRMWARE_CLOCK_ID_CORE 0x00000004
	+#define BCM2835_FIRMWARE_CLOCK_ID_V3D 0x00000005
	+#define BCM2835_FIRMWARE_CLOCK_ID_H264 0x00000006
	+#define BCM2835_FIRMWARE_CLOCK_ID_ISP 0x00000007
	+#define BCM2835_FIRMWARE_CLOCK_ID_SDRAM 0x00000008
	+#define BCM2835_FIRMWARE_CLOCK_ID_PIXEL 0x00000009
	+#define BCM2835_FIRMWARE_CLOCK_ID_PWM 0x0000000a
	+#define BCM2838_FIRMWARE_CLOCK_ID_EMMC2 0x0000000c
	+
	+union msg_get_clock_rate_body {
	+ struct {
	+ uint32_t clock_id;
	+ } req;
	+ struct {
	+ uint32_t clock_id;
	+ uint32_t rate_hz;
	+ } resp;
	+};
	+
	+union msg_set_clock_rate_body {
	+ struct {
	+ uint32_t clock_id;
	+ uint32_t rate_hz;
	+ } req;
	+ struct {
	+ uint32_t clock_id;
	+ uint32_t rate_hz;
	+ } resp;
	+};
	+
	+#define BCM2835_FIRMWARE_TAG_GET_VOLTAGE 0x00030003
	+#define BCM2835_FIRMWARE_TAG_SET_VOLTAGE 0x00038003
	+#define BCM2835_FIRMWARE_TAG_GET_MAX_VOLTAGE 0x00030005
	+#define BCM2835_FIRMWARE_TAG_GET_MIN_VOLTAGE 0x00030008
	+
	+#define BCM2835_FIRMWARE_VOLTAGE_ID_CORE 0x00000001
	+#define BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_C 0x00000002
	+#define BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_P 0x00000003
	+#define BCM2835_FIRMWARE_VOLTAGE_ID_SDRAM_I 0x00000004
	+
	+union msg_get_voltage_body {
	+ struct {
	+ uint32_t voltage_id;
	+ } req;
	+ struct {
	+ uint32_t voltage_id;
	+ uint32_t value;
	+ } resp;
	+};
	+
	+union msg_set_voltage_body {
	+ struct {
	+ uint32_t voltage_id;
	+ uint32_t value;
	+ } req;
	+ struct {
	+ uint32_t voltage_id;
	+ uint32_t value;
	+ } resp;
	+};
	+
	+#define BCM2835_FIRMWARE_TAG_GET_TEMPERATURE 0x00030006
	+#define BCM2835_FIRMWARE_TAG_GET_MAX_TEMPERATURE 0x0003000a
	+
	+union msg_get_temperature_body {
	+ struct {
	+ uint32_t temperature_id;
	+ } req;
	+ struct {
	+ uint32_t temperature_id;
	+ uint32_t value;
	+ } resp;
	+};
	+
	+#define BCM2835_FIRMWARE_TAG_GET_TURBO 0x00030009
	+#define BCM2835_FIRMWARE_TAG_SET_TURBO 0x00038009
	+
	+#define BCM2835_FIRMWARE_TURBO_ON 1
	+#define BCM2835_FIRMWARE_TURBO_OFF 0
	+
	+union msg_get_turbo_body {
	+ struct {
	+ uint32_t id;
	+ } req;
	+ struct {
	+ uint32_t id;
	+ uint32_t level;
	+ } resp;
	+};
	+
	+union msg_set_turbo_body {
	+ struct {
	+ uint32_t id;
	+ uint32_t level;
	+ } req;
	+ struct {
	+ uint32_t id;
	+ uint32_t level;
	+ } resp;
	+};
	+
	+#define BCM2835_FIRMWARE_TAG_GET_GPIO_STATE 0x00030041
	+#define BCM2835_FIRMWARE_TAG_SET_GPIO_STATE 0x00038041
	+#define BCM2835_FIRMWARE_TAG_GET_GPIO_CONFIG 0x00030043
	+#define BCM2835_FIRMWARE_TAG_SET_GPIO_CONFIG 0x00038043
	+
	+#define BCM2835_FIRMWARE_GPIO_IN 0
	+#define BCM2835_FIRMWARE_GPIO_OUT 1
	+
	+union msg_get_gpio_state {
	+ struct {
	+ uint32_t gpio;
	+ } req;
	+ struct {
	+ uint32_t gpio;
	+ uint32_t state;
	+ } resp;
	+};
	+
	+union msg_set_gpio_state {
	+ struct {
	+ uint32_t gpio;
	+ uint32_t state;
	+ } req;
	+ struct {
	+ uint32_t gpio;
	+ } resp;
	+};
	+
	+union msg_get_gpio_config {
	+ struct {
	+ uint32_t gpio;
	+ } req;
	+ struct {
	+ uint32_t gpio;
	+ uint32_t dir;
	+ uint32_t pol;
	+ uint32_t term_en;
	+ uint32_t term_pull_up;
	+ } resp;
	+};
	+
	+union msg_set_gpio_config {
	+ struct {
	+ uint32_t gpio;
	+ uint32_t dir;
	+ uint32_t pol;
	+ uint32_t term_en;
	+ uint32_t term_pull_up;
	+ uint32_t state;
	+ } req;
	+ struct {
	+ uint32_t gpio;
	+ } resp;
	+};
	+
	int bcm2835_firmware_property(device_t, uint32_t, void *, size_t);

	#endif
	diff --git a/sys/arm/broadcom/bcm2835/bcm2835_mbox.c b/sys/arm/broadcom/bcm2835/bcm2835_mbox.c
	index 4d6ab7e9a6c3..d722c7aae7a7 100644
	--- a/sys/arm/broadcom/bcm2835/bcm2835_mbox.c
	+++ b/sys/arm/broadcom/bcm2835/bcm2835_mbox.c
	@@ -1,562 +1,574 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2012 Oleksandr Tymoshenko <gonzo@freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/sx.h>
	#include <sys/rman.h>
	#include <machine/bus.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	+#include <arm/broadcom/bcm2835/bcm2835_firmware.h>
	#include <arm/broadcom/bcm2835/bcm2835_mbox.h>
	#include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h>
	#include <arm/broadcom/bcm2835/bcm2835_vcbus.h>

	#include "mbox_if.h"

	#define REG_READ 0x00
	#define REG_POL 0x10
	#define REG_SENDER 0x14
	#define REG_STATUS 0x18
	#define STATUS_FULL 0x80000000
	#define STATUS_EMPTY 0x40000000
	#define REG_CONFIG 0x1C
	#define CONFIG_DATA_IRQ 0x00000001
	#define REG_WRITE 0x20 /* This is Mailbox 1 address */

	#define MBOX_MSG(chan, data) (((data) & ~0xf) \| ((chan) & 0xf))
	#define MBOX_CHAN(msg) ((msg) & 0xf)
	#define MBOX_DATA(msg) ((msg) & ~0xf)

	#define MBOX_LOCK(sc) do { \
	mtx_lock(&(sc)->lock); \
	} while(0)

	#define MBOX_UNLOCK(sc) do { \
	mtx_unlock(&(sc)->lock); \
	} while(0)

	#ifdef DEBUG
	#define dprintf(fmt, args...) printf(fmt, ##args)
	#else
	#define dprintf(fmt, args...)
	#endif

	struct bcm_mbox_softc {
	struct mtx lock;
	struct resource * mem_res;
	struct resource * irq_res;
	void* intr_hl;
	bus_space_tag_t bst;
	bus_space_handle_t bsh;
	int msg[BCM2835_MBOX_CHANS];
	int have_message[BCM2835_MBOX_CHANS];
	struct sx property_chan_lock;
	};

	#define mbox_read_4(sc, reg) \
	bus_space_read_4((sc)->bst, (sc)->bsh, reg)
	#define mbox_write_4(sc, reg, val) \
	bus_space_write_4((sc)->bst, (sc)->bsh, reg, val)

	static struct ofw_compat_data compat_data[] = {
	{"broadcom,bcm2835-mbox", 1},
	{"brcm,bcm2835-mbox", 1},
	{NULL, 0}
	};

	static int
	bcm_mbox_read_msg(struct bcm_mbox_softc sc, int ochan)
	{
	#ifdef DEBUG
	uint32_t data;
	#endif
	uint32_t msg;
	int chan;

	msg = mbox_read_4(sc, REG_READ);
	dprintf("bcm_mbox_intr: raw data %08x\n", msg);
	chan = MBOX_CHAN(msg);
	#ifdef DEBUG
	data = MBOX_DATA(msg);
	#endif
	if (sc->msg[chan]) {
	printf("bcm_mbox_intr: channel %d oveflow\n", chan);
	return (1);
	}
	dprintf("bcm_mbox_intr: chan %d, data %08x\n", chan, data);
	sc->msg[chan] = msg;

	if (ochan != NULL)
	*ochan = chan;

	return (0);
	}

	static void
	bcm_mbox_intr(void *arg)
	{
	struct bcm_mbox_softc *sc = arg;
	int chan;

	MBOX_LOCK(sc);
	while (!(mbox_read_4(sc, REG_STATUS) & STATUS_EMPTY))
	if (bcm_mbox_read_msg(sc, &chan) == 0) {
	sc->have_message[chan] = 1;
	wakeup(&sc->have_message[chan]);
	}
	MBOX_UNLOCK(sc);
	}

	static int
	bcm_mbox_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	device_set_desc(dev, "BCM2835 VideoCore Mailbox");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	bcm_mbox_attach(device_t dev)
	{
	struct bcm_mbox_softc *sc = device_get_softc(dev);
	int i;
	int rid = 0;

	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (sc->mem_res == NULL) {
	device_printf(dev, "could not allocate memory resource\n");
	return (ENXIO);
	}

	sc->bst = rman_get_bustag(sc->mem_res);
	sc->bsh = rman_get_bushandle(sc->mem_res);

	rid = 0;
	sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE);
	if (sc->irq_res == NULL) {
	device_printf(dev, "could not allocate interrupt resource\n");
	return (ENXIO);
	}

	/* Setup and enable the timer */
	if (bus_setup_intr(dev, sc->irq_res, INTR_MPSAFE \| INTR_TYPE_MISC,
	NULL, bcm_mbox_intr, sc, &sc->intr_hl) != 0) {
	bus_release_resource(dev, SYS_RES_IRQ, rid, sc->irq_res);
	device_printf(dev, "Unable to setup the clock irq handler.\n");
	return (ENXIO);
	}

	mtx_init(&sc->lock, "vcio mbox", NULL, MTX_DEF);
	for (i = 0; i < BCM2835_MBOX_CHANS; i++) {
	sc->msg[i] = 0;
	sc->have_message[i] = 0;
	}

	sx_init(&sc->property_chan_lock, "mboxprop");

	/* Read all pending messages */
	while ((mbox_read_4(sc, REG_STATUS) & STATUS_EMPTY) == 0)
	(void)mbox_read_4(sc, REG_READ);

	mbox_write_4(sc, REG_CONFIG, CONFIG_DATA_IRQ);

	return (0);
	}

	/*
	* Mailbox API
	*/
	static int
	bcm_mbox_write(device_t dev, int chan, uint32_t data)
	{
	int limit = 1000;
	struct bcm_mbox_softc *sc = device_get_softc(dev);

	dprintf("bcm_mbox_write: chan %d, data %08x\n", chan, data);
	MBOX_LOCK(sc);
	sc->have_message[chan] = 0;
	while ((mbox_read_4(sc, REG_STATUS) & STATUS_FULL) && --limit)
	DELAY(5);
	if (limit == 0) {
	printf("bcm_mbox_write: STATUS_FULL stuck");
	MBOX_UNLOCK(sc);
	return (EAGAIN);
	}
	mbox_write_4(sc, REG_WRITE, MBOX_MSG(chan, data));
	MBOX_UNLOCK(sc);

	return (0);
	}

	static int
	bcm_mbox_read(device_t dev, int chan, uint32_t *data)
	{
	struct bcm_mbox_softc *sc = device_get_softc(dev);
	int err, read_chan;

	dprintf("bcm_mbox_read: chan %d\n", chan);

	err = 0;
	MBOX_LOCK(sc);
	if (!cold) {
	if (sc->have_message[chan] == 0) {
	if (mtx_sleep(&sc->have_message[chan], &sc->lock, 0,
	"mbox", 10*hz) != 0) {
	device_printf(dev, "timeout waiting for message on chan %d\n", chan);
	err = ETIMEDOUT;
	}
	}
	} else {
	do {
	/* Wait for a message */
	while ((mbox_read_4(sc, REG_STATUS) & STATUS_EMPTY))
	;
	/* Read the message */
	if (bcm_mbox_read_msg(sc, &read_chan) != 0) {
	err = EINVAL;
	goto out;
	}
	} while (read_chan != chan);
	}
	/*
	* get data from intr handler, the same channel is never coming
	* because of holding sc lock.
	*/
	*data = MBOX_DATA(sc->msg[chan]);
	sc->msg[chan] = 0;
	sc->have_message[chan] = 0;
	out:
	MBOX_UNLOCK(sc);
	dprintf("bcm_mbox_read: chan %d, data %08x\n", chan, *data);

	return (err);
	}

	static device_method_t bcm_mbox_methods[] = {
	DEVMETHOD(device_probe, bcm_mbox_probe),
	DEVMETHOD(device_attach, bcm_mbox_attach),

	DEVMETHOD(mbox_read, bcm_mbox_read),
	DEVMETHOD(mbox_write, bcm_mbox_write),

	DEVMETHOD_END
	};

	static driver_t bcm_mbox_driver = {
	"mbox",
	bcm_mbox_methods,
	sizeof(struct bcm_mbox_softc),
	};

	static devclass_t bcm_mbox_devclass;

	-DRIVER_MODULE(mbox, simplebus, bcm_mbox_driver, bcm_mbox_devclass, 0, 0);
	+EARLY_DRIVER_MODULE(mbox, simplebus, bcm_mbox_driver, bcm_mbox_devclass, 0, 0,
	+ BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST);

	static void
	bcm2835_mbox_dma_cb(void arg, bus_dma_segment_t segs, int nseg, int err)
	{
	bus_addr_t *addr;

	if (err)
	return;
	addr = (bus_addr_t *)arg;
	*addr = ARMC_TO_VCBUS(segs[0].ds_addr);
	}

	static void *
	bcm2835_mbox_init_dma(device_t dev, size_t len, bus_dma_tag_t *tag,
	bus_dmamap_t map, bus_addr_t phys)
	{
	void *buf;
	int err;

	err = bus_dma_tag_create(bus_get_dma_tag(dev), 16, 0,
	bcm283x_dmabus_peripheral_lowaddr(), BUS_SPACE_MAXADDR, NULL, NULL,
	len, 1, len, 0, NULL, NULL, tag);
	if (err != 0) {
	device_printf(dev, "can't create DMA tag\n");
	return (NULL);
	}

	err = bus_dmamem_alloc(*tag, &buf, 0, map);
	if (err != 0) {
	bus_dma_tag_destroy(*tag);
	device_printf(dev, "can't allocate dmamem\n");
	return (NULL);
	}

	err = bus_dmamap_load(tag, map, buf, len, bcm2835_mbox_dma_cb,
	phys, 0);
	if (err != 0) {
	bus_dmamem_free(tag, buf, map);
	bus_dma_tag_destroy(*tag);
	device_printf(dev, "can't load DMA map\n");
	return (NULL);
	}

	return (buf);
	}

	static int
	bcm2835_mbox_err(device_t dev, bus_addr_t msg_phys, uint32_t resp_phys,
	struct bcm2835_mbox_hdr *msg, size_t len)
	{
	int idx;
	struct bcm2835_mbox_tag_hdr *tag;
	uint8_t *last;

	if ((uint32_t)msg_phys != resp_phys) {
	device_printf(dev, "response channel mismatch\n");
	return (EIO);
	}
	if (msg->code != BCM2835_MBOX_CODE_RESP_SUCCESS) {
	device_printf(dev, "mbox response error\n");
	return (EIO);
	}

	/* Loop until the end tag. */
	tag = (struct bcm2835_mbox_tag_hdr *)(msg + 1);
	last = (uint8_t *)msg + len;
	for (idx = 0; tag->tag != 0; idx++) {
	+ /*
	+ * When setting the GPIO config or state the firmware doesn't
	+ * set tag->val_len correctly.
	+ */
	+ if ((tag->tag == BCM2835_FIRMWARE_TAG_SET_GPIO_CONFIG \|\|
	+ tag->tag == BCM2835_FIRMWARE_TAG_SET_GPIO_STATE) &&
	+ tag->val_len == 0) {
	+ tag->val_len = BCM2835_MBOX_TAG_VAL_LEN_RESPONSE \|
	+ tag->val_buf_size;
	+ }
	if ((tag->val_len & BCM2835_MBOX_TAG_VAL_LEN_RESPONSE) == 0) {
	device_printf(dev, "tag %d response error\n", idx);
	return (EIO);
	}
	/* Clear the response bit. */
	tag->val_len &= ~BCM2835_MBOX_TAG_VAL_LEN_RESPONSE;

	/* Next tag. */
	tag = (struct bcm2835_mbox_tag_hdr )((uint8_t )tag +
	sizeof(*tag) + tag->val_buf_size);

	if ((uint8_t *)tag > last) {
	device_printf(dev, "mbox buffer size error\n");
	return (EIO);
	}
	}

	return (0);
	}

	int
	bcm2835_mbox_property(void *msg, size_t msg_size)
	{
	struct bcm_mbox_softc *sc;
	struct msg_set_power_state *buf;
	bus_dma_tag_t msg_tag;
	bus_dmamap_t msg_map;
	bus_addr_t msg_phys;
	uint32_t reg;
	device_t mbox;
	int err;

	/* get mbox device */
	mbox = devclass_get_device(devclass_find("mbox"), 0);
	if (mbox == NULL)
	return (ENXIO);

	sc = device_get_softc(mbox);
	sx_xlock(&sc->property_chan_lock);

	/* Allocate memory for the message */
	buf = bcm2835_mbox_init_dma(mbox, msg_size, &msg_tag, &msg_map,
	&msg_phys);
	if (buf == NULL) {
	err = ENOMEM;
	goto out;
	}

	memcpy(buf, msg, msg_size);

	bus_dmamap_sync(msg_tag, msg_map,
	BUS_DMASYNC_PREWRITE);

	MBOX_WRITE(mbox, BCM2835_MBOX_CHAN_PROP, (uint32_t)msg_phys);
	MBOX_READ(mbox, BCM2835_MBOX_CHAN_PROP, &reg);

	bus_dmamap_sync(msg_tag, msg_map,
	BUS_DMASYNC_PREREAD);

	memcpy(msg, buf, msg_size);

	err = bcm2835_mbox_err(mbox, msg_phys, reg,
	(struct bcm2835_mbox_hdr *)msg, msg_size);

	bus_dmamap_unload(msg_tag, msg_map);
	bus_dmamem_free(msg_tag, buf, msg_map);
	bus_dma_tag_destroy(msg_tag);
	out:
	sx_xunlock(&sc->property_chan_lock);
	return (err);
	}

	int
	bcm2835_mbox_set_power_state(uint32_t device_id, boolean_t on)
	{
	struct msg_set_power_state msg;
	int err;

	memset(&msg, 0, sizeof(msg));
	msg.hdr.buf_size = sizeof(msg);
	msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	msg.tag_hdr.tag = BCM2835_MBOX_TAG_SET_POWER_STATE;
	msg.tag_hdr.val_buf_size = sizeof(msg.body);
	msg.tag_hdr.val_len = sizeof(msg.body.req);
	msg.body.req.device_id = device_id;
	msg.body.req.state = (on ? BCM2835_MBOX_POWER_ON : 0) \|
	BCM2835_MBOX_POWER_WAIT;
	msg.end_tag = 0;

	err = bcm2835_mbox_property(&msg, sizeof(msg));

	return (err);
	}

	int
	bcm2835_mbox_get_clock_rate(uint32_t clock_id, uint32_t *hz)
	{
	struct msg_get_clock_rate msg;
	int err;

	memset(&msg, 0, sizeof(msg));
	msg.hdr.buf_size = sizeof(msg);
	msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	msg.tag_hdr.tag = BCM2835_MBOX_TAG_GET_CLOCK_RATE;
	msg.tag_hdr.val_buf_size = sizeof(msg.body);
	msg.tag_hdr.val_len = sizeof(msg.body.req);
	msg.body.req.clock_id = clock_id;
	msg.end_tag = 0;

	err = bcm2835_mbox_property(&msg, sizeof(msg));
	*hz = msg.body.resp.rate_hz;

	return (err);
	}

	int
	bcm2835_mbox_fb_get_w_h(struct bcm2835_fb_config *fb)
	{
	int err;
	struct msg_fb_get_w_h msg;

	memset(&msg, 0, sizeof(msg));
	msg.hdr.buf_size = sizeof(msg);
	msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	BCM2835_MBOX_INIT_TAG(&msg.physical_w_h, GET_PHYSICAL_W_H);
	msg.physical_w_h.tag_hdr.val_len = 0;
	msg.end_tag = 0;

	err = bcm2835_mbox_property(&msg, sizeof(msg));
	if (err == 0) {
	fb->xres = msg.physical_w_h.body.resp.width;
	fb->yres = msg.physical_w_h.body.resp.height;
	}

	return (err);
	}

	int
	bcm2835_mbox_fb_get_bpp(struct bcm2835_fb_config *fb)
	{
	int err;
	struct msg_fb_get_bpp msg;

	memset(&msg, 0, sizeof(msg));
	msg.hdr.buf_size = sizeof(msg);
	msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	BCM2835_MBOX_INIT_TAG(&msg.bpp, GET_DEPTH);
	msg.bpp.tag_hdr.val_len = 0;
	msg.end_tag = 0;

	err = bcm2835_mbox_property(&msg, sizeof(msg));
	if (err == 0)
	fb->bpp = msg.bpp.body.resp.bpp;

	return (err);
	}

	int
	bcm2835_mbox_fb_init(struct bcm2835_fb_config *fb)
	{
	int err;
	struct msg_fb_setup msg;

	memset(&msg, 0, sizeof(msg));
	msg.hdr.buf_size = sizeof(msg);
	msg.hdr.code = BCM2835_MBOX_CODE_REQ;
	BCM2835_MBOX_INIT_TAG(&msg.physical_w_h, SET_PHYSICAL_W_H);
	msg.physical_w_h.body.req.width = fb->xres;
	msg.physical_w_h.body.req.height = fb->yres;
	BCM2835_MBOX_INIT_TAG(&msg.virtual_w_h, SET_VIRTUAL_W_H);
	msg.virtual_w_h.body.req.width = fb->vxres;
	msg.virtual_w_h.body.req.height = fb->vyres;
	BCM2835_MBOX_INIT_TAG(&msg.offset, SET_VIRTUAL_OFFSET);
	msg.offset.body.req.x = fb->xoffset;
	msg.offset.body.req.y = fb->yoffset;
	BCM2835_MBOX_INIT_TAG(&msg.depth, SET_DEPTH);
	msg.depth.body.req.bpp = fb->bpp;
	BCM2835_MBOX_INIT_TAG(&msg.alpha, SET_ALPHA_MODE);
	msg.alpha.body.req.alpha = BCM2835_MBOX_ALPHA_MODE_IGNORED;
	BCM2835_MBOX_INIT_TAG(&msg.buffer, ALLOCATE_BUFFER);
	msg.buffer.body.req.alignment = PAGE_SIZE;
	BCM2835_MBOX_INIT_TAG(&msg.pitch, GET_PITCH);
	msg.end_tag = 0;

	err = bcm2835_mbox_property(&msg, sizeof(msg));
	if (err == 0) {
	fb->xres = msg.physical_w_h.body.resp.width;
	fb->yres = msg.physical_w_h.body.resp.height;
	fb->vxres = msg.virtual_w_h.body.resp.width;
	fb->vyres = msg.virtual_w_h.body.resp.height;
	fb->xoffset = msg.offset.body.resp.x;
	fb->yoffset = msg.offset.body.resp.y;
	fb->pitch = msg.pitch.body.resp.pitch;
	fb->base = VCBUS_TO_ARMC(msg.buffer.body.resp.fb_address);
	fb->size = msg.buffer.body.resp.fb_size;
	}

	return (err);
	}
	diff --git a/sys/arm/broadcom/bcm2835/bcm2835_mbox_prop.h b/sys/arm/broadcom/bcm2835/bcm2835_mbox_prop.h
	index 211820c09c31..15f48aefa536 100644
	--- a/sys/arm/broadcom/bcm2835/bcm2835_mbox_prop.h
	+++ b/sys/arm/broadcom/bcm2835/bcm2835_mbox_prop.h
	@@ -1,506 +1,448 @@
	/*-
	* Copyright (C) 2013-2014 Daisuke Aoyama <aoyama@peach.ne.jp>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef _BCM2835_MBOX_PROP_H_
	#define _BCM2835_MBOX_PROP_H_

	#include <sys/cdefs.h>
	#include <sys/types.h>

	/*
	* Mailbox property interface:
	* https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface
	*/
	#define BCM2835_MBOX_CODE_REQ 0
	#define BCM2835_MBOX_CODE_RESP_SUCCESS 0x80000000
	#define BCM2835_MBOX_CODE_RESP_ERROR 0x80000001
	#define BCM2835_MBOX_TAG_VAL_LEN_RESPONSE 0x80000000

	struct bcm2835_mbox_hdr {
	uint32_t buf_size;
	uint32_t code;
	};

	struct bcm2835_mbox_tag_hdr {
	uint32_t tag;
	uint32_t val_buf_size;
	uint32_t val_len;
	};

	#define BCM2835_MBOX_INIT_TAG(tag_, tagid_) do { \
	(tag_)->tag_hdr.tag = BCM2835_MBOX_TAG_##tagid_; \
	(tag_)->tag_hdr.val_buf_size = sizeof((tag_)->body); \
	(tag_)->tag_hdr.val_len = sizeof((tag_)->body.req); \
	} while (0)

	#define BCM2835_MBOX_TAG_FIRMWARE_REVISION 0x00000001



	#define BCM2835_MBOX_POWER_ID_EMMC 0x00000000
	#define BCM2835_MBOX_POWER_ID_UART0 0x00000001
	#define BCM2835_MBOX_POWER_ID_UART1 0x00000002
	#define BCM2835_MBOX_POWER_ID_USB_HCD 0x00000003
	#define BCM2835_MBOX_POWER_ID_I2C0 0x00000004
	#define BCM2835_MBOX_POWER_ID_I2C1 0x00000005
	#define BCM2835_MBOX_POWER_ID_I2C2 0x00000006
	#define BCM2835_MBOX_POWER_ID_SPI 0x00000007
	#define BCM2835_MBOX_POWER_ID_CCP2TX 0x00000008

	#define BCM2835_MBOX_POWER_ON (1 << 0)
	#define BCM2835_MBOX_POWER_WAIT (1 << 1)

	#define BCM2835_MBOX_TAG_GET_POWER_STATE 0x00020001
	#define BCM2835_MBOX_TAG_SET_POWER_STATE 0x00028001

	struct msg_get_power_state {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t device_id;
	} req;
	struct {
	uint32_t device_id;
	uint32_t state;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct msg_set_power_state {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t device_id;
	uint32_t state;
	} req;
	struct {
	uint32_t device_id;
	uint32_t state;
	} resp;
	} body;
	uint32_t end_tag;
	};

	/* Sets the power state for a given device */
	int bcm2835_mbox_set_power_state(uint32_t, boolean_t);

	#define BCM2835_MBOX_CLOCK_ID_EMMC 0x00000001
	-#define BCM2835_MBOX_CLOCK_ID_UART 0x00000002
	-#define BCM2835_MBOX_CLOCK_ID_ARM 0x00000003
	-#define BCM2835_MBOX_CLOCK_ID_CORE 0x00000004
	-#define BCM2835_MBOX_CLOCK_ID_V3D 0x00000005
	-#define BCM2835_MBOX_CLOCK_ID_H264 0x00000006
	-#define BCM2835_MBOX_CLOCK_ID_ISP 0x00000007
	-#define BCM2835_MBOX_CLOCK_ID_SDRAM 0x00000008
	-#define BCM2835_MBOX_CLOCK_ID_PIXEL 0x00000009
	-#define BCM2835_MBOX_CLOCK_ID_PWM 0x0000000a
	#define BCM2838_MBOX_CLOCK_ID_EMMC2 0x0000000c

	#define BCM2835_MBOX_TAG_GET_CLOCK_RATE 0x00030002
	-#define BCM2835_MBOX_TAG_SET_CLOCK_RATE 0x00038002
	-#define BCM2835_MBOX_TAG_GET_MAX_CLOCK_RATE 0x00030004
	-#define BCM2835_MBOX_TAG_GET_MIN_CLOCK_RATE 0x00030007

	struct msg_get_clock_rate {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t clock_id;
	} req;
	struct {
	uint32_t clock_id;
	uint32_t rate_hz;
	} resp;
	} body;
	uint32_t end_tag;
	};

	-struct msg_set_clock_rate {
	- struct bcm2835_mbox_hdr hdr;
	- struct bcm2835_mbox_tag_hdr tag_hdr;
	- union {
	- struct {
	- uint32_t clock_id;
	- uint32_t rate_hz;
	- } req;
	- struct {
	- uint32_t clock_id;
	- uint32_t rate_hz;
	- } resp;
	- } body;
	- uint32_t end_tag;
	-};
	-
	-struct msg_get_max_clock_rate {
	- struct bcm2835_mbox_hdr hdr;
	- struct bcm2835_mbox_tag_hdr tag_hdr;
	- union {
	- struct {
	- uint32_t clock_id;
	- } req;
	- struct {
	- uint32_t clock_id;
	- uint32_t rate_hz;
	- } resp;
	- } body;
	- uint32_t end_tag;
	-};
	-
	-struct msg_get_min_clock_rate {
	- struct bcm2835_mbox_hdr hdr;
	- struct bcm2835_mbox_tag_hdr tag_hdr;
	- union {
	- struct {
	- uint32_t clock_id;
	- } req;
	- struct {
	- uint32_t clock_id;
	- uint32_t rate_hz;
	- } resp;
	- } body;
	- uint32_t end_tag;
	-};
	-
	int bcm2835_mbox_get_clock_rate(uint32_t, uint32_t *);

	#define BCM2835_MBOX_TURBO_ON 1
	#define BCM2835_MBOX_TURBO_OFF 0

	#define BCM2835_MBOX_TAG_GET_TURBO 0x00030009
	#define BCM2835_MBOX_TAG_SET_TURBO 0x00038009

	struct msg_get_turbo {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t id;
	} req;
	struct {
	uint32_t id;
	uint32_t level;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct msg_set_turbo {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t id;
	uint32_t level;
	} req;
	struct {
	uint32_t id;
	uint32_t level;
	} resp;
	} body;
	uint32_t end_tag;
	};

	#define BCM2835_MBOX_VOLTAGE_ID_CORE 0x00000001
	#define BCM2835_MBOX_VOLTAGE_ID_SDRAM_C 0x00000002
	#define BCM2835_MBOX_VOLTAGE_ID_SDRAM_P 0x00000003
	#define BCM2835_MBOX_VOLTAGE_ID_SDRAM_I 0x00000004

	#define BCM2835_MBOX_TAG_GET_VOLTAGE 0x00030003
	#define BCM2835_MBOX_TAG_SET_VOLTAGE 0x00038003
	#define BCM2835_MBOX_TAG_GET_MAX_VOLTAGE 0x00030005
	#define BCM2835_MBOX_TAG_GET_MIN_VOLTAGE 0x00030008

	struct msg_get_voltage {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t voltage_id;
	} req;
	struct {
	uint32_t voltage_id;
	uint32_t value;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct msg_set_voltage {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t voltage_id;
	uint32_t value;
	} req;
	struct {
	uint32_t voltage_id;
	uint32_t value;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct msg_get_max_voltage {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t voltage_id;
	} req;
	struct {
	uint32_t voltage_id;
	uint32_t value;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct msg_get_min_voltage {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t voltage_id;
	} req;
	struct {
	uint32_t voltage_id;
	uint32_t value;
	} resp;
	} body;
	uint32_t end_tag;
	};

	#define BCM2835_MBOX_TAG_GET_TEMPERATURE 0x00030006
	#define BCM2835_MBOX_TAG_GET_MAX_TEMPERATURE 0x0003000a

	struct msg_get_temperature {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t temperature_id;
	} req;
	struct {
	uint32_t temperature_id;
	uint32_t value;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct msg_get_max_temperature {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t temperature_id;
	} req;
	struct {
	uint32_t temperature_id;
	uint32_t value;
	} resp;
	} body;
	uint32_t end_tag;
	};

	#define BCM2835_MBOX_TAG_GET_PHYSICAL_W_H 0x00040003
	#define BCM2835_MBOX_TAG_SET_PHYSICAL_W_H 0x00048003
	#define BCM2835_MBOX_TAG_GET_VIRTUAL_W_H 0x00040004
	#define BCM2835_MBOX_TAG_SET_VIRTUAL_W_H 0x00048004

	struct bcm2835_mbox_tag_fb_w_h {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t width;
	uint32_t height;
	} req;
	struct {
	uint32_t width;
	uint32_t height;
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_GET_DEPTH 0x00040005
	#define BCM2835_MBOX_TAG_SET_DEPTH 0x00048005

	struct bcm2835_mbox_tag_depth {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t bpp;
	} req;
	struct {
	uint32_t bpp;
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_GET_ALPHA_MODE 0x00040007
	#define BCM2835_MBOX_TAG_SET_ALPHA_MODE 0x00048007

	#define BCM2835_MBOX_ALPHA_MODE_0_OPAQUE 0
	#define BCM2835_MBOX_ALPHA_MODE_0_TRANSPARENT 1
	#define BCM2835_MBOX_ALPHA_MODE_IGNORED 2

	struct bcm2835_mbox_tag_alpha_mode {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t alpha;
	} req;
	struct {
	uint32_t alpha;
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_GET_VIRTUAL_OFFSET 0x00040009
	#define BCM2835_MBOX_TAG_SET_VIRTUAL_OFFSET 0x00048009

	struct bcm2835_mbox_tag_virtual_offset {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t x;
	uint32_t y;
	} req;
	struct {
	uint32_t x;
	uint32_t y;
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_GET_PITCH 0x00040008

	struct bcm2835_mbox_tag_pitch {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	} req;
	struct {
	uint32_t pitch;
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_ALLOCATE_BUFFER 0x00040001

	struct bcm2835_mbox_tag_allocate_buffer {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	uint32_t alignment;
	} req;
	struct {
	uint32_t fb_address;
	uint32_t fb_size;
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_RELEASE_BUFFER 0x00048001

	struct bcm2835_mbox_tag_release_buffer {
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	} req;
	struct {
	} resp;
	} body;
	};

	#define BCM2835_MBOX_TAG_GET_TOUCHBUF 0x0004000f

	struct bcm2835_mbox_tag_touchbuf {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_hdr tag_hdr;
	union {
	struct {
	} req;
	struct {
	uint32_t address;
	} resp;
	} body;
	uint32_t end_tag;
	};

	struct bcm2835_fb_config {
	uint32_t xres;
	uint32_t yres;
	uint32_t vxres;
	uint32_t vyres;
	uint32_t xoffset;
	uint32_t yoffset;
	uint32_t bpp;
	uint32_t pitch;
	uint32_t base;
	uint32_t size;
	};

	struct msg_fb_get_w_h {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_fb_w_h physical_w_h;
	uint32_t end_tag;
	};

	int bcm2835_mbox_fb_get_w_h(struct bcm2835_fb_config *);

	struct msg_fb_get_bpp {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_depth bpp;
	uint32_t end_tag;
	};

	int bcm2835_mbox_fb_get_bpp(struct bcm2835_fb_config *);

	struct msg_fb_setup {
	struct bcm2835_mbox_hdr hdr;
	struct bcm2835_mbox_tag_fb_w_h physical_w_h;
	struct bcm2835_mbox_tag_fb_w_h virtual_w_h;
	struct bcm2835_mbox_tag_virtual_offset offset;
	struct bcm2835_mbox_tag_depth depth;
	struct bcm2835_mbox_tag_alpha_mode alpha;
	struct bcm2835_mbox_tag_allocate_buffer buffer;
	struct bcm2835_mbox_tag_pitch pitch;
	uint32_t end_tag;
	};

	int bcm2835_mbox_fb_init(struct bcm2835_fb_config *);

	int bcm2835_mbox_property(void *, size_t);

	#endif /* _BCM2835_MBOX_PROP_H_ */
	diff --git a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c
	index 0140afcc4a97..cd9b60743be3 100644
	--- a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c
	+++ b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c
	@@ -1,829 +1,869 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2012 Oleksandr Tymoshenko <gonzo@freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	#include <sys/taskqueue.h>

	#include <machine/bus.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/mmc/bridge.h>
	#include <dev/mmc/mmcreg.h>
	+#include <dev/mmc/mmc_fdt_helpers.h>

	#include <dev/sdhci/sdhci.h>

	#include "mmcbr_if.h"
	#include "sdhci_if.h"

	#include "opt_mmccam.h"

	#include "bcm2835_dma.h"
	#include <arm/broadcom/bcm2835/bcm2835_mbox_prop.h>
	#ifdef NOTYET
	#include <arm/broadcom/bcm2835/bcm2835_clkman.h>
	#endif
	#include <arm/broadcom/bcm2835/bcm2835_vcbus.h>

	#define BCM2835_DEFAULT_SDHCI_FREQ 50
	#define BCM2838_DEFAULT_SDHCI_FREQ 100

	#define BCM_SDHCI_BUFFER_SIZE 512
	/*
	* NUM_DMA_SEGS is the number of DMA segments we want to accommodate on average.
	* We add in a number of segments based on how much we may need to spill into
	* another segment due to crossing page boundaries. e.g. up to PAGE_SIZE, an
	* extra page is needed as we can cross a page boundary exactly once.
	*/
	#define NUM_DMA_SEGS 1
	#define NUM_DMA_SPILL_SEGS \
	((((NUM_DMA_SEGS * BCM_SDHCI_BUFFER_SIZE) - 1) / PAGE_SIZE) + 1)
	#define ALLOCATED_DMA_SEGS (NUM_DMA_SEGS + NUM_DMA_SPILL_SEGS)
	#define BCM_DMA_MAXSIZE (NUM_DMA_SEGS * BCM_SDHCI_BUFFER_SIZE)

	#define BCM_SDHCI_SLOT_LEFT(slot) \
	((slot)->curcmd->data->len - (slot)->offset)

	#define BCM_SDHCI_SEGSZ_LEFT(slot) \
	min(BCM_DMA_MAXSIZE, \
	rounddown(BCM_SDHCI_SLOT_LEFT(slot), BCM_SDHCI_BUFFER_SIZE))

	#define DATA_PENDING_MASK (SDHCI_INT_DATA_AVAIL \| SDHCI_INT_SPACE_AVAIL)
	#define DATA_XFER_MASK (DATA_PENDING_MASK \| SDHCI_INT_DATA_END)

	#ifdef DEBUG
	static int bcm2835_sdhci_debug = 0;

	TUNABLE_INT("hw.bcm2835.sdhci.debug", &bcm2835_sdhci_debug);
	SYSCTL_INT(_hw_sdhci, OID_AUTO, bcm2835_sdhci_debug, CTLFLAG_RWTUN,
	&bcm2835_sdhci_debug, 0, "bcm2835 SDHCI debug level");

	#define dprintf(fmt, args...) \
	do { \
	if (bcm2835_sdhci_debug) \
	printf("%s: " fmt, __func__, ##args); \
	} while (0)
	#else
	#define dprintf(fmt, args...)
	#endif

	static int bcm2835_sdhci_hs = 1;
	static int bcm2835_sdhci_pio_mode = 0;

	struct bcm_mmc_conf {
	int clock_id;
	int clock_src;
	int default_freq;
	int quirks;
	int emmc_dreq;
	};

	struct bcm_mmc_conf bcm2835_sdhci_conf = {
	.clock_id = BCM2835_MBOX_CLOCK_ID_EMMC,
	.clock_src = -1,
	.default_freq = BCM2835_DEFAULT_SDHCI_FREQ,
	.quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK \|
	SDHCI_QUIRK_BROKEN_TIMEOUT_VAL \| SDHCI_QUIRK_DONT_SET_HISPD_BIT \|
	SDHCI_QUIRK_MISSING_CAPS,
	.emmc_dreq = BCM_DMA_DREQ_EMMC,
	};

	struct bcm_mmc_conf bcm2838_emmc2_conf = {
	.clock_id = BCM2838_MBOX_CLOCK_ID_EMMC2,
	.clock_src = -1,
	.default_freq = BCM2838_DEFAULT_SDHCI_FREQ,
	.quirks = 0,
	.emmc_dreq = BCM_DMA_DREQ_NONE,
	};

	static struct ofw_compat_data compat_data[] = {
	{"broadcom,bcm2835-sdhci", (uintptr_t)&bcm2835_sdhci_conf},
	{"brcm,bcm2835-sdhci", (uintptr_t)&bcm2835_sdhci_conf},
	{"brcm,bcm2835-mmc", (uintptr_t)&bcm2835_sdhci_conf},
	{"brcm,bcm2711-emmc2", (uintptr_t)&bcm2838_emmc2_conf},
	{"brcm,bcm2838-emmc2", (uintptr_t)&bcm2838_emmc2_conf},
	{NULL, 0}
	};

	TUNABLE_INT("hw.bcm2835.sdhci.hs", &bcm2835_sdhci_hs);
	TUNABLE_INT("hw.bcm2835.sdhci.pio_mode", &bcm2835_sdhci_pio_mode);

	struct bcm_sdhci_softc {
	device_t sc_dev;
	struct resource * sc_mem_res;
	struct resource * sc_irq_res;
	bus_space_tag_t sc_bst;
	bus_space_handle_t sc_bsh;
	void * sc_intrhand;
	struct mmc_request * sc_req;
	struct sdhci_slot sc_slot;
	+ struct mmc_fdt_helper sc_mmc_helper;
	int sc_dma_ch;
	bus_dma_tag_t sc_dma_tag;
	bus_dmamap_t sc_dma_map;
	vm_paddr_t sc_sdhci_buffer_phys;
	bus_addr_t dmamap_seg_addrs[ALLOCATED_DMA_SEGS];
	bus_size_t dmamap_seg_sizes[ALLOCATED_DMA_SEGS];
	int dmamap_seg_count;
	int dmamap_seg_index;
	int dmamap_status;
	uint32_t blksz_and_count;
	uint32_t cmd_and_mode;
	bool need_update_blk;
	#ifdef NOTYET
	device_t clkman;
	#endif
	struct bcm_mmc_conf * conf;
	};

	static int bcm_sdhci_probe(device_t);
	static int bcm_sdhci_attach(device_t);
	static int bcm_sdhci_detach(device_t);
	static void bcm_sdhci_intr(void *);

	static int bcm_sdhci_get_ro(device_t, device_t);
	static void bcm_sdhci_dma_intr(int ch, void *arg);
	static void bcm_sdhci_start_dma(struct sdhci_slot *slot);

	static void
	bcm_sdhci_dmacb(void arg, bus_dma_segment_t segs, int nseg, int err)
	{
	struct bcm_sdhci_softc *sc = arg;
	int i;

	/* Sanity check: we can only ever have one mapping at a time. */
	KASSERT(sc->dmamap_seg_count == 0, ("leaked DMA segment"));
	sc->dmamap_status = err;
	sc->dmamap_seg_count = nseg;

	/* Note nseg is guaranteed to be zero if err is non-zero. */
	for (i = 0; i < nseg; i++) {
	sc->dmamap_seg_addrs[i] = segs[i].ds_addr;
	sc->dmamap_seg_sizes[i] = segs[i].ds_len;
	}
	}

	static int
	bcm_sdhci_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	device_set_desc(dev, "Broadcom 2708 SDHCI controller");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	bcm_sdhci_attach(device_t dev)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);
	int rid, err;
	phandle_t node;
	pcell_t cell;
	u_int default_freq;

	sc->sc_dev = dev;
	sc->sc_req = NULL;

	sc->conf = (struct bcm_mmc_conf *)ofw_bus_search_compatible(dev,
	compat_data)->ocd_data;
	if (sc->conf == 0)
	return (ENXIO);

	err = bcm2835_mbox_set_power_state(BCM2835_MBOX_POWER_ID_EMMC, TRUE);
	if (err != 0) {
	if (bootverbose)
	device_printf(dev, "Unable to enable the power\n");
	return (err);
	}

	default_freq = 0;
	err = bcm2835_mbox_get_clock_rate(sc->conf->clock_id, &default_freq);
	if (err == 0) {
	/* Convert to MHz */
	default_freq /= 1000000;
	}
	if (default_freq == 0) {
	node = ofw_bus_get_node(sc->sc_dev);
	if ((OF_getencprop(node, "clock-frequency", &cell,
	sizeof(cell))) > 0)
	default_freq = cell / 1000000;
	}
	if (default_freq == 0)
	default_freq = sc->conf->default_freq;

	if (bootverbose)
	device_printf(dev, "SDHCI frequency: %dMHz\n", default_freq);
	#ifdef NOTYET
	if (sc->conf->clock_src > 0) {
	uint32_t f;
	sc->clkman = devclass_get_device(
	devclass_find("bcm2835_clkman"), 0);
	if (sc->clkman == NULL) {
	device_printf(dev, "cannot find Clock Manager\n");
	return (ENXIO);
	}

	f = bcm2835_clkman_set_frequency(sc->clkman,
	sc->conf->clock_src, default_freq);
	if (f == 0)
	return (EINVAL);

	if (bootverbose)
	device_printf(dev, "Clock source frequency: %dMHz\n",
	f);
	}
	#endif

	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (!sc->sc_mem_res) {
	device_printf(dev, "cannot allocate memory window\n");
	err = ENXIO;
	goto fail;
	}

	sc->sc_bst = rman_get_bustag(sc->sc_mem_res);
	sc->sc_bsh = rman_get_bushandle(sc->sc_mem_res);

	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE \| RF_SHAREABLE);
	if (!sc->sc_irq_res) {
	device_printf(dev, "cannot allocate interrupt\n");
	err = ENXIO;
	goto fail;
	}

	if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO \| INTR_MPSAFE,
	NULL, bcm_sdhci_intr, sc, &sc->sc_intrhand)) {
	device_printf(dev, "cannot setup interrupt handler\n");
	err = ENXIO;
	goto fail;
	}

	if (!bcm2835_sdhci_pio_mode)
	sc->sc_slot.opt = SDHCI_PLATFORM_TRANSFER;

	sc->sc_slot.caps = SDHCI_CAN_VDD_330 \| SDHCI_CAN_VDD_180;
	if (bcm2835_sdhci_hs)
	sc->sc_slot.caps \|= SDHCI_CAN_DO_HISPD;
	sc->sc_slot.caps \|= (default_freq << SDHCI_CLOCK_BASE_SHIFT);
	sc->sc_slot.quirks = sc->conf->quirks;

	sdhci_init_slot(dev, &sc->sc_slot, 0);
	+ mmc_fdt_parse(dev, 0, &sc->sc_mmc_helper, &sc->sc_slot.host);

	sc->sc_dma_ch = bcm_dma_allocate(BCM_DMA_CH_ANY);
	if (sc->sc_dma_ch == BCM_DMA_CH_INVALID)
	goto fail;

	err = bcm_dma_setup_intr(sc->sc_dma_ch, bcm_sdhci_dma_intr, sc);
	if (err != 0) {
	device_printf(dev,
	"cannot setup dma interrupt handler\n");
	err = ENXIO;
	goto fail;
	}

	/* Allocate bus_dma resources. */
	err = bus_dma_tag_create(bus_get_dma_tag(dev),
	1, 0, bcm283x_dmabus_peripheral_lowaddr(),
	BUS_SPACE_MAXADDR, NULL, NULL,
	BCM_DMA_MAXSIZE, ALLOCATED_DMA_SEGS, BCM_SDHCI_BUFFER_SIZE,
	BUS_DMA_ALLOCNOW, NULL, NULL,
	&sc->sc_dma_tag);

	if (err) {
	device_printf(dev, "failed allocate DMA tag");
	goto fail;
	}

	err = bus_dmamap_create(sc->sc_dma_tag, 0, &sc->sc_dma_map);
	if (err) {
	device_printf(dev, "bus_dmamap_create failed\n");
	goto fail;
	}

	/* FIXME: Fix along with other BUS_SPACE_PHYSADDR instances */
	sc->sc_sdhci_buffer_phys = rman_get_start(sc->sc_mem_res) +
	SDHCI_BUFFER;

	bus_generic_probe(dev);
	bus_generic_attach(dev);

	sdhci_start_slot(&sc->sc_slot);

	/* Seed our copies. */
	sc->blksz_and_count = SDHCI_READ_4(dev, &sc->sc_slot, SDHCI_BLOCK_SIZE);
	sc->cmd_and_mode = SDHCI_READ_4(dev, &sc->sc_slot, SDHCI_TRANSFER_MODE);

	return (0);

	fail:
	if (sc->sc_intrhand)
	bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intrhand);
	if (sc->sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);

	return (err);
	}

	static int
	bcm_sdhci_detach(device_t dev)
	{

	return (EBUSY);
	}

	static void
	bcm_sdhci_intr(void *arg)
	{
	struct bcm_sdhci_softc *sc = arg;

	sdhci_generic_intr(&sc->sc_slot);
	}

	+static int
	+bcm_sdhci_update_ios(device_t bus, device_t child)
	+{
	+#ifdef EXT_RESOURCES
	+ struct bcm_sdhci_softc *sc;
	+ struct mmc_ios *ios;
	+#endif
	+ int rv;
	+
	+#ifdef EXT_RESOURCES
	+ sc = device_get_softc(bus);
	+ ios = &sc->sc_slot.host.ios;
	+
	+ if (ios->power_mode == power_up) {
	+ if (sc->sc_mmc_helper.vmmc_supply)
	+ regulator_enable(sc->sc_mmc_helper.vmmc_supply);
	+ if (sc->sc_mmc_helper.vqmmc_supply)
	+ regulator_enable(sc->sc_mmc_helper.vqmmc_supply);
	+ }
	+#endif
	+
	+ rv = sdhci_generic_update_ios(bus, child);
	+ if (rv != 0)
	+ return (rv);
	+
	+#ifdef EXT_RESOURCES
	+ if (ios->power_mode == power_off) {
	+ if (sc->sc_mmc_helper.vmmc_supply)
	+ regulator_disable(sc->sc_mmc_helper.vmmc_supply);
	+ if (sc->sc_mmc_helper.vqmmc_supply)
	+ regulator_disable(sc->sc_mmc_helper.vqmmc_supply);
	+ }
	+#endif
	+
	+ return (0);
	+}
	+
	static int
	bcm_sdhci_get_ro(device_t bus, device_t child)
	{

	return (0);
	}

	static inline uint32_t
	RD4(struct bcm_sdhci_softc *sc, bus_size_t off)
	{
	uint32_t val = bus_space_read_4(sc->sc_bst, sc->sc_bsh, off);
	return val;
	}

	static inline void
	WR4(struct bcm_sdhci_softc *sc, bus_size_t off, uint32_t val)
	{

	bus_space_write_4(sc->sc_bst, sc->sc_bsh, off, val);
	/*
	* The Arasan HC has a bug where it may lose the content of
	* consecutive writes to registers that are within two SD-card
	* clock cycles of each other (a clock domain crossing problem).
	*/
	if (sc->sc_slot.clock > 0)
	DELAY(((2 * 1000000) / sc->sc_slot.clock) + 1);
	}

	static uint8_t
	bcm_sdhci_read_1(device_t dev, struct sdhci_slot *slot, bus_size_t off)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);
	uint32_t val = RD4(sc, off & ~3);

	return ((val >> (off & 3)*8) & 0xff);
	}

	static uint16_t
	bcm_sdhci_read_2(device_t dev, struct sdhci_slot *slot, bus_size_t off)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);
	uint32_t val32;

	/*
	* Standard 32-bit handling of command and transfer mode, as
	* well as block size and count.
	*/
	if ((off == SDHCI_BLOCK_SIZE \|\| off == SDHCI_BLOCK_COUNT) &&
	sc->need_update_blk)
	val32 = sc->blksz_and_count;
	else if (off == SDHCI_TRANSFER_MODE \|\| off == SDHCI_COMMAND_FLAGS)
	val32 = sc->cmd_and_mode;
	else
	val32 = RD4(sc, off & ~3);

	return ((val32 >> (off & 3)*8) & 0xffff);
	}

	static uint32_t
	bcm_sdhci_read_4(device_t dev, struct sdhci_slot *slot, bus_size_t off)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);

	return RD4(sc, off);
	}

	static void
	bcm_sdhci_read_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint32_t *data, bus_size_t count)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);

	bus_space_read_multi_4(sc->sc_bst, sc->sc_bsh, off, data, count);
	}

	static void
	bcm_sdhci_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint8_t val)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);
	uint32_t val32 = RD4(sc, off & ~3);
	val32 &= ~(0xff << (off & 3)*8);
	val32 \|= (val << (off & 3)*8);
	WR4(sc, off & ~3, val32);
	}

	static void
	bcm_sdhci_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint16_t val)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);
	uint32_t val32;

	/*
	* If we have a queued up 16bit value for blk size or count, use and
	* update the saved value rather than doing any real register access.
	* If we did not touch either since the last write, then read from
	* register as at least block count can change.
	* Similarly, if we are about to issue a command, always use the saved
	* value for transfer mode as we can never write that without issuing
	* a command.
	*/
	if ((off == SDHCI_BLOCK_SIZE \|\| off == SDHCI_BLOCK_COUNT) &&
	sc->need_update_blk)
	val32 = sc->blksz_and_count;
	else if (off == SDHCI_COMMAND_FLAGS)
	val32 = sc->cmd_and_mode;
	else
	val32 = RD4(sc, off & ~3);

	val32 &= ~(0xffff << (off & 3)*8);
	val32 \|= (val << (off & 3)*8);

	if (off == SDHCI_TRANSFER_MODE)
	sc->cmd_and_mode = val32;
	else if (off == SDHCI_BLOCK_SIZE \|\| off == SDHCI_BLOCK_COUNT) {
	sc->blksz_and_count = val32;
	sc->need_update_blk = true;
	} else {
	if (off == SDHCI_COMMAND_FLAGS) {
	/* If we saved blk writes, do them now before cmd. */
	if (sc->need_update_blk) {
	WR4(sc, SDHCI_BLOCK_SIZE, sc->blksz_and_count);
	sc->need_update_blk = false;
	}
	/* Always save cmd and mode registers. */
	sc->cmd_and_mode = val32;
	}
	WR4(sc, off & ~3, val32);
	}
	}

	static void
	bcm_sdhci_write_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint32_t val)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);
	WR4(sc, off, val);
	}

	static void
	bcm_sdhci_write_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint32_t *data, bus_size_t count)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(dev);

	bus_space_write_multi_4(sc->sc_bst, sc->sc_bsh, off, data, count);
	}

	static void
	bcm_sdhci_start_dma_seg(struct bcm_sdhci_softc *sc)
	{
	struct sdhci_slot *slot;
	vm_paddr_t pdst, psrc;
	int err, idx, len, sync_op, width;

	slot = &sc->sc_slot;
	mtx_assert(&slot->mtx, MA_OWNED);
	idx = sc->dmamap_seg_index++;
	len = sc->dmamap_seg_sizes[idx];
	slot->offset += len;
	width = (len & 0xf ? BCM_DMA_32BIT : BCM_DMA_128BIT);

	if (slot->curcmd->data->flags & MMC_DATA_READ) {
	/*
	* Peripherals on the AXI bus do not need DREQ pacing for reads
	* from the ARM core, so we can safely set this to NONE.
	*/
	bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE,
	BCM_DMA_SAME_ADDR, BCM_DMA_32BIT);
	bcm_dma_setup_dst(sc->sc_dma_ch, BCM_DMA_DREQ_NONE,
	BCM_DMA_INC_ADDR, width);
	psrc = sc->sc_sdhci_buffer_phys;
	pdst = sc->dmamap_seg_addrs[idx];
	sync_op = BUS_DMASYNC_PREREAD;
	} else {
	/*
	* The ordering here is important, because the last write to
	* dst/src in the dma control block writes the real dreq value.
	*/
	bcm_dma_setup_src(sc->sc_dma_ch, BCM_DMA_DREQ_NONE,
	BCM_DMA_INC_ADDR, width);
	bcm_dma_setup_dst(sc->sc_dma_ch, sc->conf->emmc_dreq,
	BCM_DMA_SAME_ADDR, BCM_DMA_32BIT);
	psrc = sc->dmamap_seg_addrs[idx];
	pdst = sc->sc_sdhci_buffer_phys;
	sync_op = BUS_DMASYNC_PREWRITE;
	}

	/*
	* When starting a new DMA operation do the busdma sync operation, and
	* disable SDCHI data interrrupts because we'll be driven by DMA
	* interrupts (or SDHCI error interrupts) until the IO is done.
	*/
	if (idx == 0) {
	bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map, sync_op);

	slot->intmask &= ~DATA_XFER_MASK;
	bcm_sdhci_write_4(sc->sc_dev, slot, SDHCI_SIGNAL_ENABLE,
	slot->intmask);
	}

	/*
	* Start the DMA transfer. Only programming errors (like failing to
	* allocate a channel) cause a non-zero return from bcm_dma_start().
	*/
	err = bcm_dma_start(sc->sc_dma_ch, psrc, pdst, len);
	KASSERT((err == 0), ("bcm2835_sdhci: failed DMA start"));
	}

	static void
	bcm_sdhci_dma_exit(struct bcm_sdhci_softc *sc)
	{
	struct sdhci_slot *slot = &sc->sc_slot;

	mtx_assert(&slot->mtx, MA_OWNED);

	/* Re-enable interrupts */
	slot->intmask \|= DATA_XFER_MASK;
	bcm_sdhci_write_4(slot->bus, slot, SDHCI_SIGNAL_ENABLE,
	slot->intmask);
	}

	static void
	bcm_sdhci_dma_unload(struct bcm_sdhci_softc *sc)
	{
	struct sdhci_slot *slot = &sc->sc_slot;

	if (sc->dmamap_seg_count == 0)
	return;
	if ((slot->curcmd->data->flags & MMC_DATA_READ) != 0)
	bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map,
	BUS_DMASYNC_POSTREAD);
	else
	bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_map,
	BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(sc->sc_dma_tag, sc->sc_dma_map);

	sc->dmamap_seg_count = 0;
	sc->dmamap_seg_index = 0;
	}

	static void
	bcm_sdhci_dma_intr(int ch, void *arg)
	{
	struct bcm_sdhci_softc sc = (struct bcm_sdhci_softc )arg;
	struct sdhci_slot *slot = &sc->sc_slot;
	uint32_t reg;

	mtx_lock(&slot->mtx);
	if (slot->curcmd == NULL)
	goto out;
	/*
	* If there are more segments for the current dma, start the next one.
	* Otherwise unload the dma map and decide what to do next based on the
	* status of the sdhci controller and whether there's more data left.
	*/
	if (sc->dmamap_seg_index < sc->dmamap_seg_count) {
	bcm_sdhci_start_dma_seg(sc);
	goto out;
	}

	bcm_sdhci_dma_unload(sc);

	/*
	* If we had no further segments pending, we need to determine how to
	* proceed next. If the 'data/space pending' bit is already set and we
	* can continue via DMA, do so. Otherwise, re-enable interrupts and
	* return.
	*/
	reg = bcm_sdhci_read_4(slot->bus, slot, SDHCI_INT_STATUS) &
	DATA_XFER_MASK;
	if ((reg & DATA_PENDING_MASK) != 0 &&
	BCM_SDHCI_SEGSZ_LEFT(slot) >= BCM_SDHCI_BUFFER_SIZE) {
	/* ACK any pending interrupts */
	bcm_sdhci_write_4(slot->bus, slot, SDHCI_INT_STATUS,
	DATA_PENDING_MASK);

	bcm_sdhci_start_dma(slot);
	if (slot->curcmd->error != 0) {
	/* We won't recover from this error for this command. */
	bcm_sdhci_dma_unload(sc);
	bcm_sdhci_dma_exit(sc);
	sdhci_finish_data(slot);
	}
	} else if ((reg & SDHCI_INT_DATA_END) != 0) {
	bcm_sdhci_dma_exit(sc);
	bcm_sdhci_write_4(slot->bus, slot, SDHCI_INT_STATUS,
	reg);
	slot->flags &= ~PLATFORM_DATA_STARTED;
	sdhci_finish_data(slot);
	} else {
	bcm_sdhci_dma_exit(sc);
	}
	out:
	mtx_unlock(&slot->mtx);
	}

	static void
	bcm_sdhci_start_dma(struct sdhci_slot *slot)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(slot->bus);
	uint8_t *buf;
	size_t left;

	mtx_assert(&slot->mtx, MA_OWNED);

	left = BCM_SDHCI_SEGSZ_LEFT(slot);
	buf = (uint8_t *)slot->curcmd->data->data + slot->offset;
	KASSERT(left != 0,
	("%s: DMA handling incorrectly indicated", __func__));

	/*
	* No need to check segment count here; if we've not yet unloaded
	* previous segments, we'll catch that in bcm_sdhci_dmacb.
	*/
	if (bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, buf, left,
	bcm_sdhci_dmacb, sc, BUS_DMA_NOWAIT) != 0 \|\|
	sc->dmamap_status != 0) {
	slot->curcmd->error = MMC_ERR_NO_MEMORY;
	return;
	}

	/* DMA start */
	bcm_sdhci_start_dma_seg(sc);
	}

	static int
	bcm_sdhci_will_handle_transfer(device_t dev, struct sdhci_slot *slot)
	{
	#ifdef INVARIANTS
	struct bcm_sdhci_softc *sc = device_get_softc(slot->bus);
	#endif

	/*
	* This indicates that we somehow let a data interrupt slip by into the
	* SDHCI framework, when it should not have. This really needs to be
	* caught and fixed ASAP, as it really shouldn't happen.
	*/
	KASSERT(sc->dmamap_seg_count == 0,
	("data pending interrupt pushed through SDHCI framework"));

	/*
	* Do not use DMA for transfers less than our block size. Checking
	* alignment serves little benefit, as we round transfer sizes down to
	* a multiple of the block size and push the transfer back to
	* SDHCI-driven PIO once we're below the block size.
	*/
	if (BCM_SDHCI_SEGSZ_LEFT(slot) < BCM_DMA_BLOCK_SIZE)
	return (0);

	return (1);
	}

	static void
	bcm_sdhci_start_transfer(device_t dev, struct sdhci_slot *slot,
	uint32_t *intmask)
	{

	/* DMA transfer FIFO 1KB */
	bcm_sdhci_start_dma(slot);
	}

	static void
	bcm_sdhci_finish_transfer(device_t dev, struct sdhci_slot *slot)
	{
	struct bcm_sdhci_softc *sc = device_get_softc(slot->bus);

	/*
	* Clean up. Interrupts are clearly enabled, because we received an
	* SDHCI_INT_DATA_END to get this far -- just make sure we don't leave
	* anything laying around.
	*/
	if (sc->dmamap_seg_count != 0) {
	/*
	* Our segment math should have worked out such that we would
	* never finish the transfer without having used up all of the
	* segments. If we haven't, that means we must have erroneously
	* regressed to SDHCI-driven PIO to finish the operation and
	* this is certainly caused by developer-error.
	*/
	bcm_sdhci_dma_unload(sc);
	}

	sdhci_finish_data(slot);
	}

	static device_method_t bcm_sdhci_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, bcm_sdhci_probe),
	DEVMETHOD(device_attach, bcm_sdhci_attach),
	DEVMETHOD(device_detach, bcm_sdhci_detach),

	/* Bus interface */
	DEVMETHOD(bus_read_ivar, sdhci_generic_read_ivar),
	DEVMETHOD(bus_write_ivar, sdhci_generic_write_ivar),
	DEVMETHOD(bus_add_child, bus_generic_add_child),

	/* MMC bridge interface */
	- DEVMETHOD(mmcbr_update_ios, sdhci_generic_update_ios),
	+ DEVMETHOD(mmcbr_update_ios, bcm_sdhci_update_ios),
	DEVMETHOD(mmcbr_request, sdhci_generic_request),
	DEVMETHOD(mmcbr_get_ro, bcm_sdhci_get_ro),
	DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host),
	DEVMETHOD(mmcbr_release_host, sdhci_generic_release_host),

	/* Platform transfer methods */
	DEVMETHOD(sdhci_platform_will_handle, bcm_sdhci_will_handle_transfer),
	DEVMETHOD(sdhci_platform_start_transfer, bcm_sdhci_start_transfer),
	DEVMETHOD(sdhci_platform_finish_transfer, bcm_sdhci_finish_transfer),
	/* SDHCI registers accessors */
	DEVMETHOD(sdhci_read_1, bcm_sdhci_read_1),
	DEVMETHOD(sdhci_read_2, bcm_sdhci_read_2),
	DEVMETHOD(sdhci_read_4, bcm_sdhci_read_4),
	DEVMETHOD(sdhci_read_multi_4, bcm_sdhci_read_multi_4),
	DEVMETHOD(sdhci_write_1, bcm_sdhci_write_1),
	DEVMETHOD(sdhci_write_2, bcm_sdhci_write_2),
	DEVMETHOD(sdhci_write_4, bcm_sdhci_write_4),
	DEVMETHOD(sdhci_write_multi_4, bcm_sdhci_write_multi_4),

	DEVMETHOD_END
	};

	static devclass_t bcm_sdhci_devclass;

	static driver_t bcm_sdhci_driver = {
	"sdhci_bcm",
	bcm_sdhci_methods,
	sizeof(struct bcm_sdhci_softc),
	};

	DRIVER_MODULE(sdhci_bcm, simplebus, bcm_sdhci_driver, bcm_sdhci_devclass,
	NULL, NULL);
	#ifdef NOTYET
	MODULE_DEPEND(sdhci_bcm, bcm2835_clkman, 1, 1, 1);
	#endif
	SDHCI_DEPEND(sdhci_bcm);
	#ifndef MMCCAM
	MMC_DECLARE_BRIDGE(sdhci_bcm);
	#endif
	diff --git a/sys/arm/ti/am335x/am3359_cppi41.c b/sys/arm/ti/am335x/am3359_cppi41.c
	new file mode 100644
	index 000000000000..a3e4e8bee155
	--- /dev/null
	+++ b/sys/arm/ti/am335x/am3359_cppi41.c
	@@ -0,0 +1,192 @@
	+/*-
	+ * Copyright (c) 2019 Emmanuel Vadot <manu@FreeBSD.org>
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+/* Based on sys/arm/ti/ti_sysc.c */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/bus.h>
	+#include <sys/resource.h>
	+#include <sys/rman.h>
	+#include <sys/lock.h>
	+#include <sys/mutex.h>
	+
	+#include <machine/bus.h>
	+#include <machine/resource.h>
	+
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/ofw/openfirm.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include <arm/ti/ti_sysc.h>
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+struct ti_am3359_cppi41_softc {
	+ device_t dev;
	+ struct syscon * syscon;
	+ struct resource * res[4];
	+ bus_space_tag_t bst;
	+ bus_space_handle_t bsh;
	+ struct mtx mtx;
	+};
	+
	+static struct resource_spec ti_am3359_cppi41_res_spec[] = {
	+ { SYS_RES_MEMORY, 0, RF_ACTIVE \| RF_SHAREABLE },
	+ { SYS_RES_MEMORY, 1, RF_ACTIVE \| RF_SHAREABLE },
	+ { SYS_RES_MEMORY, 2, RF_ACTIVE \| RF_SHAREABLE },
	+ { SYS_RES_MEMORY, 3, RF_ACTIVE \| RF_SHAREABLE },
	+ { -1, 0 }
	+};
	+
	+/* Device */
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,am3359-cppi41", 1 },
	+ { NULL, 0 }
	+};
	+
	+static int
	+ti_am3359_cppi41_write_4(device_t dev, bus_addr_t addr, uint32_t val)
	+{
	+ struct ti_am3359_cppi41_softc *sc;
	+
	+ sc = device_get_softc(dev);
	+ DPRINTF(sc->dev, "offset=%lx write %x\n", addr, val);
	+ mtx_lock(&sc->mtx);
	+ bus_space_write_4(sc->bst, sc->bsh, addr, val);
	+ mtx_unlock(&sc->mtx);
	+ return (0);
	+}
	+
	+static uint32_t
	+ti_am3359_cppi41_read_4(device_t dev, bus_addr_t addr)
	+{
	+ struct ti_am3359_cppi41_softc *sc;
	+ uint32_t val;
	+
	+ sc = device_get_softc(dev);
	+
	+ mtx_lock(&sc->mtx);
	+ val = bus_space_read_4(sc->bst, sc->bsh, addr);
	+ mtx_unlock(&sc->mtx);
	+ DPRINTF(sc->dev, "offset=%lx Read %x\n", addr, val);
	+ return (val);
	+}
	+
	+/* device interface */
	+static int
	+ti_am3359_cppi41_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI AM3359 CPPI 41");
	+ return(BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+ti_am3359_cppi41_attach(device_t dev)
	+{
	+ struct ti_am3359_cppi41_softc *sc;
	+ phandle_t node;
	+ uint32_t reg, reset_bit, timeout=10;
	+ uint64_t sysc_address;
	+ device_t parent;
	+
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+
	+ if (bus_alloc_resources(dev, ti_am3359_cppi41_res_spec, sc->res)) {
	+ device_printf(sc->dev, "Cant allocate resources\n");
	+ return (ENXIO);
	+ }
	+
	+ sc->dev = dev;
	+ sc->bst = rman_get_bustag(sc->res[0]);
	+ sc->bsh = rman_get_bushandle(sc->res[0]);
	+
	+ mtx_init(&sc->mtx, device_get_nameunit(sc->dev), NULL, MTX_DEF);
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ /* variant of am335x_usbss.c */
	+ DPRINTF(dev, "-- RESET USB --\n");
	+ parent = device_get_parent(dev);
	+ reset_bit = ti_sysc_get_soft_reset_bit(parent);
	+ if (reset_bit == 0) {
	+ DPRINTF(dev, "Dont have reset bit\n");
	+ return (0);
	+ }
	+ sysc_address = ti_sysc_get_sysc_address_offset_host(parent);
	+ DPRINTF(dev, "sysc_address %x\n", sysc_address);
	+ ti_am3359_cppi41_write_4(dev, sysc_address, reset_bit);
	+ DELAY(100);
	+ reg = ti_am3359_cppi41_read_4(dev, sysc_address);
	+ if ((reg & reset_bit) && timeout--) {
	+ DPRINTF(dev, "Reset still ongoing - wait a little bit longer\n");
	+ DELAY(100);
	+ reg = ti_am3359_cppi41_read_4(dev, sysc_address);
	+ }
	+ if (timeout == 0)
	+ device_printf(dev, "USB Reset timeout\n");
	+
	+ return (0);
	+}
	+
	+
	+static device_method_t ti_am3359_cppi41_methods[] = {
	+ DEVMETHOD(device_probe, ti_am3359_cppi41_probe),
	+ DEVMETHOD(device_attach, ti_am3359_cppi41_attach),
	+
	+ DEVMETHOD_END
	+};
	+
	+
	+DEFINE_CLASS_1(ti_am3359_cppi41, ti_am3359_cppi41_driver,
	+ ti_am3359_cppi41_methods,sizeof(struct ti_am3359_cppi41_softc),
	+ simplebus_driver);
	+
	+static devclass_t ti_am3359_cppi41_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_am3359_cppi41, simplebus, ti_am3359_cppi41_driver,
	+ ti_am3359_cppi41_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_am3359_cppi41, 1);
	+MODULE_DEPEND(ti_am3359_cppi41, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_dmtimer.c b/sys/arm/ti/am335x/am335x_dmtimer.c
	index a36c66e56ed0..eb869ca88b94 100644
	--- a/sys/arm/ti/am335x/am335x_dmtimer.c
	+++ b/sys/arm/ti/am335x/am335x_dmtimer.c
	@@ -1,360 +1,408 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/rman.h>
	#include <sys/timeet.h>
	#include <sys/timetc.h>
	#include <machine/bus.h>

	#include <machine/machdep.h> /* For arm_set_delay */

	+#include <dev/extres/clk/clk.h>
	+
	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	+#include <arm/ti/ti_sysc.h>

	#include "am335x_dmtreg.h"

	struct am335x_dmtimer_softc {
	device_t dev;
	int tmr_mem_rid;
	struct resource * tmr_mem_res;
	int tmr_irq_rid;
	struct resource * tmr_irq_res;
	void *tmr_irq_handler;
	- uint32_t sysclk_freq;
	+ clk_t clk_fck;
	+ uint64_t sysclk_freq;
	uint32_t tclr; /* Cached TCLR register. */
	union {
	struct timecounter tc;
	struct eventtimer et;
	} func;
	int tmr_num; /* Hardware unit number. */
	char tmr_name[12]; /* "DMTimerN", N = tmr_num */
	};

	static struct am335x_dmtimer_softc *am335x_dmtimer_et_sc = NULL;
	static struct am335x_dmtimer_softc *am335x_dmtimer_tc_sc = NULL;

	static void am335x_dmtimer_delay(int, void *);

	/*
	* We use dmtimer2 for eventtimer and dmtimer3 for timecounter.
	*/
	#define ET_TMR_NUM 2
	#define TC_TMR_NUM 3

	/* List of compatible strings for FDT tree */
	static struct ofw_compat_data compat_data[] = {
	{"ti,am335x-timer", 1},
	{"ti,am335x-timer-1ms", 1},
	{NULL, 0},
	};

	#define DMTIMER_READ4(sc, reg) bus_read_4((sc)->tmr_mem_res, (reg))
	#define DMTIMER_WRITE4(sc, reg, val) bus_write_4((sc)->tmr_mem_res, (reg), (val))

	static int
	am335x_dmtimer_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
	{
	struct am335x_dmtimer_softc *sc;
	uint32_t initial_count, reload_count;

	sc = et->et_priv;

	/*
	* Stop the timer before changing it. This routine will often be called
	* while the timer is still running, to either lengthen or shorten the
	* current event time. We need to ensure the timer doesn't expire while
	* we're working with it.
	*
	* Also clear any pending interrupt status, because it's at least
	* theoretically possible that we're running in a primary interrupt
	* context now, and a timer interrupt could be pending even before we
	* stopped the timer. The more likely case is that we're being called
	* from the et_event_cb() routine dispatched from our own handler, but
	* it's not clear to me that that's the only case possible.
	*/
	sc->tclr &= ~(DMT_TCLR_START \| DMT_TCLR_AUTOLOAD);
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);
	DMTIMER_WRITE4(sc, DMT_IRQSTATUS, DMT_IRQ_OVF);

	if (period != 0) {
	reload_count = ((uint32_t)et->et_frequency * period) >> 32;
	sc->tclr \|= DMT_TCLR_AUTOLOAD;
	} else {
	reload_count = 0;
	}

	if (first != 0)
	initial_count = ((uint32_t)et->et_frequency * first) >> 32;
	else
	initial_count = reload_count;

	/*
	* Set auto-reload and current-count values. This timer hardware counts
	* up from the initial/reload value and interrupts on the zero rollover.
	*/
	DMTIMER_WRITE4(sc, DMT_TLDR, 0xFFFFFFFF - reload_count);
	DMTIMER_WRITE4(sc, DMT_TCRR, 0xFFFFFFFF - initial_count);

	/* Enable overflow interrupt, and start the timer. */
	DMTIMER_WRITE4(sc, DMT_IRQENABLE_SET, DMT_IRQ_OVF);
	sc->tclr \|= DMT_TCLR_START;
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);

	return (0);
	}

	static int
	am335x_dmtimer_et_stop(struct eventtimer *et)
	{
	struct am335x_dmtimer_softc *sc;

	sc = et->et_priv;

	/* Stop timer, disable and clear interrupt. */
	sc->tclr &= ~(DMT_TCLR_START \| DMT_TCLR_AUTOLOAD);
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);
	DMTIMER_WRITE4(sc, DMT_IRQENABLE_CLR, DMT_IRQ_OVF);
	DMTIMER_WRITE4(sc, DMT_IRQSTATUS, DMT_IRQ_OVF);
	return (0);
	}

	static int
	am335x_dmtimer_et_intr(void *arg)
	{
	struct am335x_dmtimer_softc *sc;

	sc = arg;

	/* Ack the interrupt, and invoke the callback if it's still enabled. */
	DMTIMER_WRITE4(sc, DMT_IRQSTATUS, DMT_IRQ_OVF);
	if (sc->func.et.et_active)
	sc->func.et.et_event_cb(&sc->func.et, sc->func.et.et_arg);

	return (FILTER_HANDLED);
	}

	static int
	am335x_dmtimer_et_init(struct am335x_dmtimer_softc *sc)
	{
	KASSERT(am335x_dmtimer_et_sc == NULL, ("already have an eventtimer"));

	/*
	* Setup eventtimer interrupt handling. Panic if anything goes wrong,
	* because the system just isn't going to run without an eventtimer.
	*/
	sc->tmr_irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
	&sc->tmr_irq_rid, RF_ACTIVE);
	if (sc->tmr_irq_res == NULL)
	panic("am335x_dmtimer: could not allocate irq resources");
	if (bus_setup_intr(sc->dev, sc->tmr_irq_res, INTR_TYPE_CLK,
	am335x_dmtimer_et_intr, NULL, sc, &sc->tmr_irq_handler) != 0)
	panic("am335x_dmtimer: count not setup irq handler");

	sc->func.et.et_name = sc->tmr_name;
	sc->func.et.et_flags = ET_FLAGS_PERIODIC \| ET_FLAGS_ONESHOT;
	sc->func.et.et_quality = 500;
	sc->func.et.et_frequency = sc->sysclk_freq;
	sc->func.et.et_min_period =
	((0x00000005LLU << 32) / sc->func.et.et_frequency);
	sc->func.et.et_max_period =
	(0xfffffffeLLU << 32) / sc->func.et.et_frequency;
	sc->func.et.et_start = am335x_dmtimer_et_start;
	sc->func.et.et_stop = am335x_dmtimer_et_stop;
	sc->func.et.et_priv = sc;

	am335x_dmtimer_et_sc = sc;
	et_register(&sc->func.et);

	return (0);
	}

	static unsigned
	am335x_dmtimer_tc_get_timecount(struct timecounter *tc)
	{
	struct am335x_dmtimer_softc *sc;

	sc = tc->tc_priv;

	return (DMTIMER_READ4(sc, DMT_TCRR));
	}

	static int
	am335x_dmtimer_tc_init(struct am335x_dmtimer_softc *sc)
	{
	KASSERT(am335x_dmtimer_tc_sc == NULL, ("already have a timecounter"));

	/* Set up timecounter, start it, register it. */
	DMTIMER_WRITE4(sc, DMT_TSICR, DMT_TSICR_RESET);
	while (DMTIMER_READ4(sc, DMT_TIOCP_CFG) & DMT_TIOCP_RESET)
	continue;

	sc->tclr \|= DMT_TCLR_START \| DMT_TCLR_AUTOLOAD;
	DMTIMER_WRITE4(sc, DMT_TLDR, 0);
	DMTIMER_WRITE4(sc, DMT_TCRR, 0);
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);

	sc->func.tc.tc_name = sc->tmr_name;
	sc->func.tc.tc_get_timecount = am335x_dmtimer_tc_get_timecount;
	sc->func.tc.tc_counter_mask = ~0u;
	sc->func.tc.tc_frequency = sc->sysclk_freq;
	sc->func.tc.tc_quality = 500;
	sc->func.tc.tc_priv = sc;

	am335x_dmtimer_tc_sc = sc;
	tc_init(&sc->func.tc);

	arm_set_delay(am335x_dmtimer_delay, sc);

	return (0);
	}

	static int
	am335x_dmtimer_probe(device_t dev)
	{
	char strbuf[32];
	int tmr_num;
	+ uint64_t rev_address;

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	/*
	- * Get the hardware unit number (the N from ti,hwmods="timerN").
	+ * Get the hardware unit number from address of rev register.
	* If this isn't the hardware unit we're going to use for either the
	* eventtimer or the timecounter, no point in instantiating the device.
	*/
	- tmr_num = ti_hwmods_get_unit(dev, "timer");
	- if (tmr_num != ET_TMR_NUM && tmr_num != TC_TMR_NUM)
	- return (ENXIO);
	+ rev_address = ti_sysc_get_rev_address(device_get_parent(dev));
	+ switch (rev_address) {
	+ case DMTIMER2_REV:
	+ tmr_num = 2;
	+ break;
	+ case DMTIMER3_REV:
	+ tmr_num = 3;
	+ break;
	+ default:
	+ /* Not DMTIMER2 or DMTIMER3 */
	+ return (ENXIO);
	+ }

	snprintf(strbuf, sizeof(strbuf), "AM335x DMTimer%d", tmr_num);
	device_set_desc_copy(dev, strbuf);

	return(BUS_PROBE_DEFAULT);
	}

	static int
	am335x_dmtimer_attach(device_t dev)
	{
	struct am335x_dmtimer_softc *sc;
	- clk_ident_t timer_id;
	int err;
	+ uint64_t rev_address;
	+ clk_t sys_clkin;

	sc = device_get_softc(dev);
	sc->dev = dev;

	- /* Get the base clock frequency. */
	- if ((err = ti_prcm_clk_get_source_freq(SYS_CLK, &sc->sysclk_freq)) != 0)
	- return (err);
	+ /* expect one clock */
	+ err = clk_get_by_ofw_index(dev, 0, 0, &sc->clk_fck);
	+ if (err != 0) {
	+ device_printf(dev, "Cant find clock index 0. err: %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ err = clk_get_by_name(dev, "sys_clkin_ck@40", &sys_clkin);
	+ if (err != 0) {
	+ device_printf(dev, "Cant find sys_clkin_ck@40 err: %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ /* Select M_OSC as DPLL parent */
	+ err = clk_set_parent_by_clk(sc->clk_fck, sys_clkin);
	+ if (err != 0) {
	+ device_printf(dev, "Cant set mux to CLK_M_OSC\n");
	+ return (ENXIO);
	+ }

	/* Enable clocks and power on the device. */
	- if ((timer_id = ti_hwmods_get_clock(dev)) == INVALID_CLK_IDENT)
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	+ if (err != 0) {
	+ device_printf(dev, "Cant enable sysc clkctrl, err %d\n", err);
	return (ENXIO);
	- if ((err = ti_prcm_clk_set_source(timer_id, SYSCLK_CLK)) != 0)
	- return (err);
	- if ((err = ti_prcm_clk_enable(timer_id)) != 0)
	- return (err);
	+ }
	+
	+ /* Get the base clock frequency. */
	+ err = clk_get_freq(sc->clk_fck, &sc->sysclk_freq);
	+ if (err != 0) {
	+ device_printf(dev, "Cant get sysclk frequency, err %d\n", err);
	+ return (ENXIO);
	+ }

	/* Request the memory resources. */
	sc->tmr_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&sc->tmr_mem_rid, RF_ACTIVE);
	if (sc->tmr_mem_res == NULL) {
	return (ENXIO);
	}

	- sc->tmr_num = ti_hwmods_get_unit(dev, "timer");
	+ rev_address = ti_sysc_get_rev_address(device_get_parent(dev));
	+ switch (rev_address) {
	+ case DMTIMER2_REV:
	+ sc->tmr_num = 2;
	+ break;
	+ case DMTIMER3_REV:
	+ sc->tmr_num = 3;
	+ break;
	+ default:
	+ device_printf(dev, "Not timer 2 or 3! %#jx\n",
	+ rev_address);
	+ return (ENXIO);
	+ }
	+
	snprintf(sc->tmr_name, sizeof(sc->tmr_name), "DMTimer%d", sc->tmr_num);

	/*
	* Go set up either a timecounter or eventtimer. We wouldn't have
	* attached if we weren't one or the other.
	*/
	if (sc->tmr_num == ET_TMR_NUM)
	am335x_dmtimer_et_init(sc);
	else if (sc->tmr_num == TC_TMR_NUM)
	am335x_dmtimer_tc_init(sc);
	else
	panic("am335x_dmtimer: bad timer number %d", sc->tmr_num);

	return (0);
	}

	static device_method_t am335x_dmtimer_methods[] = {
	DEVMETHOD(device_probe, am335x_dmtimer_probe),
	DEVMETHOD(device_attach, am335x_dmtimer_attach),
	{ 0, 0 }
	};

	static driver_t am335x_dmtimer_driver = {
	"am335x_dmtimer",
	am335x_dmtimer_methods,
	sizeof(struct am335x_dmtimer_softc),
	};

	static devclass_t am335x_dmtimer_devclass;

	DRIVER_MODULE(am335x_dmtimer, simplebus, am335x_dmtimer_driver, am335x_dmtimer_devclass, 0, 0);
	-MODULE_DEPEND(am335x_dmtimer, am335x_prcm, 1, 1, 1);
	+MODULE_DEPEND(am335x_dmtimer, ti_sysc, 1, 1, 1);

	static void
	am335x_dmtimer_delay(int usec, void *arg)
	{
	struct am335x_dmtimer_softc *sc = arg;
	int32_t counts;
	uint32_t first, last;

	/* Get the number of times to count */
	counts = (usec + 1) * (sc->sysclk_freq / 1000000);

	first = DMTIMER_READ4(sc, DMT_TCRR);

	while (counts > 0) {
	last = DMTIMER_READ4(sc, DMT_TCRR);
	if (last > first) {
	counts -= (int32_t)(last - first);
	} else {
	counts -= (int32_t)((0xFFFFFFFF - first) + last);
	}
	first = last;
	}
	}
	diff --git a/sys/arm/ti/am335x/am335x_dmtpps.c b/sys/arm/ti/am335x/am335x_dmtpps.c
	index 6aa374a0f76f..24d83f248eef 100644
	--- a/sys/arm/ti/am335x/am335x_dmtpps.c
	+++ b/sys/arm/ti/am335x/am335x_dmtpps.c
	@@ -1,545 +1,621 @@
	/*-
	* Copyright (c) 2015 Ian lepore <ian@freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/*
	* AM335x PPS driver using DMTimer capture.
	*
	* Note that this PPS driver does not use an interrupt. Instead it uses the
	* hardware's ability to latch the timer's count register in response to a
	* signal on an IO pin. Each of timers 4-7 have an associated pin, and this
	* code allows any one of those to be used.
	*
	* The timecounter routines in kern_tc.c call the pps poll routine periodically
	* to see if a new counter value has been latched. When a new value has been
	* latched, the only processing done in the poll routine is to capture the
	* current set of timecounter timehands (done with pps_capture()) and the
	* latched value from the timer. The remaining work (done by pps_event() while
	* holding a mutex) is scheduled to be done later in a non-interrupt context.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	+#include "opt_platform.h"
	+
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/timepps.h>
	#include <sys/timetc.h>
	#include <machine/bus.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>
	+#include <dev/extres/clk/clk.h>

	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_pinmux.h>
	#include <arm/ti/am335x/am335x_scm_padconf.h>

	#include "am335x_dmtreg.h"

	#define PPS_CDEV_NAME "dmtpps"

	struct dmtpps_softc {
	device_t dev;
	int mem_rid;
	struct resource * mem_res;
	int tmr_num; /* N from hwmod str "timerN" */
	char tmr_name[12]; /* "DMTimerN" */
	uint32_t tclr; /* Cached TCLR register. */
	struct timecounter tc;
	int pps_curmode; /* Edge mode now set in hw. */
	struct cdev * pps_cdev;
	struct pps_state pps_state;
	struct mtx pps_mtx;
	+ clk_t clk_fck;
	+ uint64_t sysclk_freq;
	};

	static int dmtpps_tmr_num; /* Set by probe() */

	/* List of compatible strings for FDT tree */
	static struct ofw_compat_data compat_data[] = {
	{"ti,am335x-timer", 1},
	{"ti,am335x-timer-1ms", 1},
	{NULL, 0},
	};
	SIMPLEBUS_PNP_INFO(compat_data);

	/*
	* A table relating pad names to the hardware timer number they can be mux'd to.
	*/
	struct padinfo {
	char * ballname;
	int tmr_num;
	};
	static struct padinfo dmtpps_padinfo[] = {
	{"GPMC_ADVn_ALE", 4},
	{"I2C0_SDA", 4},
	{"MII1_TX_EN", 4},
	{"XDMA_EVENT_INTR0", 4},
	{"GPMC_BEn0_CLE", 5},
	{"MDC", 5},
	{"MMC0_DAT3", 5},
	{"UART1_RTSn", 5},
	{"GPMC_WEn", 6},
	{"MDIO", 6},
	{"MMC0_DAT2", 6},
	{"UART1_CTSn", 6},
	{"GPMC_OEn_REn", 7},
	{"I2C0_SCL", 7},
	{"UART0_CTSn", 7},
	{"XDMA_EVENT_INTR1", 7},
	{NULL, 0}
	};

	/*
	* This is either brilliantly user-friendly, or utterly lame...
	*
	* The am335x chip is used on the popular Beaglebone boards. Those boards have
	* pins for all four capture-capable timers available on the P8 header. Allow
	* users to configure the input pin by giving the name of the header pin.
	*/
	struct nicknames {
	const char * nick;
	const char * name;
	};
	static struct nicknames dmtpps_pin_nicks[] = {
	{"P8-7", "GPMC_ADVn_ALE"},
	{"P8-9", "GPMC_BEn0_CLE"},
	{"P8-10", "GPMC_WEn"},
	{"P8-8", "GPMC_OEn_REn",},
	{NULL, NULL}
	};

	#define DMTIMER_READ4(sc, reg) bus_read_4((sc)->mem_res, (reg))
	#define DMTIMER_WRITE4(sc, reg, val) bus_write_4((sc)->mem_res, (reg), (val))

	/*
	* Translate a short friendly case-insensitive name to its canonical name.
	*/
	static const char *
	dmtpps_translate_nickname(const char *nick)
	{
	struct nicknames *nn;

	for (nn = dmtpps_pin_nicks; nn->nick != NULL; nn++)
	if (strcasecmp(nick, nn->nick) == 0)
	return nn->name;
	return (nick);
	}

	/*
	* See if our tunable is set to the name of the input pin. If not, that's NOT
	* an error, return 0. If so, try to configure that pin as a timer capture
	* input pin, and if that works, then we have our timer unit number and if it
	* fails that IS an error, return -1.
	*/
	static int
	dmtpps_find_tmr_num_by_tunable(void)
	{
	struct padinfo *pi;
	char iname[20];
	char muxmode[12];
	const char * ballname;
	int err;

	if (!TUNABLE_STR_FETCH("hw.am335x_dmtpps.input", iname, sizeof(iname)))
	return (0);
	ballname = dmtpps_translate_nickname(iname);
	for (pi = dmtpps_padinfo; pi->ballname != NULL; pi++) {
	if (strcmp(ballname, pi->ballname) != 0)
	continue;
	snprintf(muxmode, sizeof(muxmode), "timer%d", pi->tmr_num);
	err = ti_pinmux_padconf_set(pi->ballname, muxmode,
	PADCONF_INPUT);
	if (err != 0) {
	printf("am335x_dmtpps: unable to configure capture pin "
	"for %s to input mode\n", muxmode);
	return (-1);
	} else if (bootverbose) {
	printf("am335x_dmtpps: configured pin %s as input "
	"for %s\n", iname, muxmode);
	}
	return (pi->tmr_num);
	}

	/* Invalid name in the tunable, that's an error. */
	printf("am335x_dmtpps: unknown pin name '%s'\n", iname);
	return (-1);
	}

	/*
	* Ask the pinmux driver whether any pin has been configured as a TIMER4..TIMER7
	* input pin. If so, return the timer number, if not return 0.
	*/
	static int
	dmtpps_find_tmr_num_by_padconf(void)
	{
	int err;
	unsigned int padstate;
	const char * padmux;
	struct padinfo *pi;
	char muxmode[12];

	for (pi = dmtpps_padinfo; pi->ballname != NULL; pi++) {
	err = ti_pinmux_padconf_get(pi->ballname, &padmux, &padstate);
	snprintf(muxmode, sizeof(muxmode), "timer%d", pi->tmr_num);
	if (err == 0 && (padstate & RXACTIVE) != 0 &&
	strcmp(muxmode, padmux) == 0)
	return (pi->tmr_num);
	}
	/* Nothing found, not an error. */
	return (0);
	}

	/*
	* Figure out which hardware timer number to use based on input pin
	* configuration. This is done just once, the first time probe() runs.
	*/
	static int
	dmtpps_find_tmr_num(void)
	{
	int tmr_num;

	if ((tmr_num = dmtpps_find_tmr_num_by_tunable()) == 0)
	tmr_num = dmtpps_find_tmr_num_by_padconf();

	if (tmr_num <= 0) {
	printf("am335x_dmtpps: PPS driver not enabled: unable to find "
	"or configure a capture input pin\n");
	tmr_num = -1; /* Must return non-zero to prevent re-probing. */
	}
	return (tmr_num);
	}

	static void
	dmtpps_set_hw_capture(struct dmtpps_softc *sc, bool force_off)
	{
	int newmode;

	if (force_off)
	newmode = 0;
	else
	newmode = sc->pps_state.ppsparam.mode & PPS_CAPTUREASSERT;

	if (newmode == sc->pps_curmode)
	return;
	sc->pps_curmode = newmode;

	if (newmode == PPS_CAPTUREASSERT)
	sc->tclr \|= DMT_TCLR_CAPTRAN_LOHI;
	else
	sc->tclr &= ~DMT_TCLR_CAPTRAN_MASK;
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);
	}

	static unsigned
	dmtpps_get_timecount(struct timecounter *tc)
	{
	struct dmtpps_softc *sc;

	sc = tc->tc_priv;

	return (DMTIMER_READ4(sc, DMT_TCRR));
	}

	static void
	dmtpps_poll(struct timecounter *tc)
	{
	struct dmtpps_softc *sc;

	sc = tc->tc_priv;

	/*
	* If a new value has been latched we've got a PPS event. Capture the
	* timecounter data, then override the capcount field (pps_capture()
	* populates it from the current DMT_TCRR register) with the latched
	* value from the TCAR1 register.
	*
	* Note that we don't have the TCAR interrupt enabled, but the hardware
	* still provides the status bits in the "RAW" status register even when
	* they're masked from generating an irq. However, when clearing the
	* TCAR status to re-arm the capture for the next second, we have to
	* write to the IRQ status register, not the RAW register. Quirky.
	*
	* We do not need to hold a lock while capturing the pps data, because
	* it is captured into an area of the pps_state struct which is read
	* only by pps_event(). We do need to hold a lock while calling
	* pps_event(), because it manipulates data which is also accessed from
	* the ioctl(2) context by userland processes.
	*/
	if (DMTIMER_READ4(sc, DMT_IRQSTATUS_RAW) & DMT_IRQ_TCAR) {
	pps_capture(&sc->pps_state);
	sc->pps_state.capcount = DMTIMER_READ4(sc, DMT_TCAR1);
	DMTIMER_WRITE4(sc, DMT_IRQSTATUS, DMT_IRQ_TCAR);

	mtx_lock_spin(&sc->pps_mtx);
	pps_event(&sc->pps_state, PPS_CAPTUREASSERT);
	mtx_unlock_spin(&sc->pps_mtx);
	}
	}

	static int
	dmtpps_open(struct cdev *dev, int flags, int fmt,
	struct thread *td)
	{
	struct dmtpps_softc *sc;

	sc = dev->si_drv1;

	/*
	* Begin polling for pps and enable capture in the hardware whenever the
	* device is open. Doing this stuff again is harmless if this isn't the
	* first open.
	*/
	sc->tc.tc_poll_pps = dmtpps_poll;
	dmtpps_set_hw_capture(sc, false);

	return 0;
	}

	static int
	dmtpps_close(struct cdev *dev, int flags, int fmt,
	struct thread *td)
	{
	struct dmtpps_softc *sc;

	sc = dev->si_drv1;

	/*
	* Stop polling and disable capture on last close. Use the force-off
	* flag to override the configured mode and turn off the hardware.
	*/
	sc->tc.tc_poll_pps = NULL;
	dmtpps_set_hw_capture(sc, true);

	return 0;
	}

	static int
	dmtpps_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
	int flags, struct thread *td)
	{
	struct dmtpps_softc *sc;
	int err;

	sc = dev->si_drv1;

	/* Let the kernel do the heavy lifting for ioctl. */
	mtx_lock_spin(&sc->pps_mtx);
	err = pps_ioctl(cmd, data, &sc->pps_state);
	mtx_unlock_spin(&sc->pps_mtx);
	if (err != 0)
	return (err);

	/*
	* The capture mode could have changed, set the hardware to whatever
	* mode is now current. Effectively a no-op if nothing changed.
	*/
	dmtpps_set_hw_capture(sc, false);

	return (err);
	}

	static struct cdevsw dmtpps_cdevsw = {
	.d_version = D_VERSION,
	.d_open = dmtpps_open,
	.d_close = dmtpps_close,
	.d_ioctl = dmtpps_ioctl,
	.d_name = PPS_CDEV_NAME,
	};

	static int
	dmtpps_probe(device_t dev)
	{
	char strbuf[64];
	int tmr_num;
	+ uint64_t rev_address;

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	/*
	* If we haven't chosen which hardware timer to use yet, go do that now.
	* We need to know that to decide whether to return success for this
	* hardware timer instance or not.
	*/
	if (dmtpps_tmr_num == 0)
	dmtpps_tmr_num = dmtpps_find_tmr_num();

	/*
	* Figure out which hardware timer is being probed and see if it matches
	* the configured timer number determined earlier.
	*/
	- tmr_num = ti_hwmods_get_unit(dev, "timer");
	+ rev_address = ti_sysc_get_rev_address(device_get_parent(dev));
	+ switch (rev_address) {
	+ case DMTIMER1_1MS_REV:
	+ tmr_num = 1;
	+ break;
	+ case DMTIMER2_REV:
	+ tmr_num = 2;
	+ break;
	+ case DMTIMER3_REV:
	+ tmr_num = 3;
	+ break;
	+ case DMTIMER4_REV:
	+ tmr_num = 4;
	+ break;
	+ case DMTIMER5_REV:
	+ tmr_num = 5;
	+ break;
	+ case DMTIMER6_REV:
	+ tmr_num = 6;
	+ break;
	+ case DMTIMER7_REV:
	+ tmr_num = 7;
	+ break;
	+ default:
	+ return (ENXIO);
	+ }
	+
	if (dmtpps_tmr_num != tmr_num)
	return (ENXIO);

	snprintf(strbuf, sizeof(strbuf), "AM335x PPS-Capture DMTimer%d",
	tmr_num);
	device_set_desc_copy(dev, strbuf);

	return(BUS_PROBE_DEFAULT);
	}

	static int
	dmtpps_attach(device_t dev)
	{
	struct dmtpps_softc *sc;
	struct make_dev_args mda;
	- clk_ident_t timer_id;
	- int err, sysclk_freq;
	+ int err;
	+ clk_t sys_clkin;
	+ uint64_t rev_address;

	sc = device_get_softc(dev);
	sc->dev = dev;

	- /* Get the base clock frequency. */
	- err = ti_prcm_clk_get_source_freq(SYS_CLK, &sysclk_freq);
	+ /* Figure out which hardware timer this is and set the name string. */
	+ rev_address = ti_sysc_get_rev_address(device_get_parent(dev));
	+ switch (rev_address) {
	+ case DMTIMER1_1MS_REV:
	+ sc->tmr_num = 1;
	+ break;
	+ case DMTIMER2_REV:
	+ sc->tmr_num = 2;
	+ break;
	+ case DMTIMER3_REV:
	+ sc->tmr_num = 3;
	+ break;
	+ case DMTIMER4_REV:
	+ sc->tmr_num = 4;
	+ break;
	+ case DMTIMER5_REV:
	+ sc->tmr_num = 5;
	+ break;
	+ case DMTIMER6_REV:
	+ sc->tmr_num = 6;
	+ break;
	+ case DMTIMER7_REV:
	+ sc->tmr_num = 7;
	+ break;
	+ }
	+ snprintf(sc->tmr_name, sizeof(sc->tmr_name), "DMTimer%d", sc->tmr_num);
	+
	+ /* expect one clock */
	+ err = clk_get_by_ofw_index(dev, 0, 0, &sc->clk_fck);
	+ if (err != 0) {
	+ device_printf(dev, "Cant find clock index 0. err: %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ err = clk_get_by_name(dev, "sys_clkin_ck@40", &sys_clkin);
	+ if (err != 0) {
	+ device_printf(dev, "Cant find sys_clkin_ck@40 err: %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ /* Select M_OSC as DPLL parent */
	+ err = clk_set_parent_by_clk(sc->clk_fck, sys_clkin);
	+ if (err != 0) {
	+ device_printf(dev, "Cant set mux to CLK_M_OSC\n");
	+ return (ENXIO);
	+ }

	/* Enable clocks and power on the device. */
	- if ((timer_id = ti_hwmods_get_clock(dev)) == INVALID_CLK_IDENT)
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	+ if (err != 0) {
	+ device_printf(dev, "Cant enable sysc clkctrl, err %d\n", err);
	return (ENXIO);
	- if ((err = ti_prcm_clk_set_source(timer_id, SYSCLK_CLK)) != 0)
	- return (err);
	- if ((err = ti_prcm_clk_enable(timer_id)) != 0)
	- return (err);
	+ }

	+ /* Get the base clock frequency. */
	+ err = clk_get_freq(sc->clk_fck, &sc->sysclk_freq);
	+ if (err != 0) {
	+ device_printf(dev, "Cant get sysclk frequency, err %d\n", err);
	+ return (ENXIO);
	+ }
	/* Request the memory resources. */
	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&sc->mem_rid, RF_ACTIVE);
	if (sc->mem_res == NULL) {
	return (ENXIO);
	}

	- /* Figure out which hardware timer this is and set the name string. */
	- sc->tmr_num = ti_hwmods_get_unit(dev, "timer");
	- snprintf(sc->tmr_name, sizeof(sc->tmr_name), "DMTimer%d", sc->tmr_num);
	-
	/*
	* Configure the timer pulse/capture pin to input/capture mode. This is
	* required in addition to configuring the pin as input with the pinmux
	* controller (which was done via fdt data or tunable at probe time).
	*/
	sc->tclr = DMT_TCLR_GPO_CFG;
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);

	/* Set up timecounter hardware, start it. */
	DMTIMER_WRITE4(sc, DMT_TSICR, DMT_TSICR_RESET);
	while (DMTIMER_READ4(sc, DMT_TIOCP_CFG) & DMT_TIOCP_RESET)
	continue;

	sc->tclr \|= DMT_TCLR_START \| DMT_TCLR_AUTOLOAD;
	DMTIMER_WRITE4(sc, DMT_TLDR, 0);
	DMTIMER_WRITE4(sc, DMT_TCRR, 0);
	DMTIMER_WRITE4(sc, DMT_TCLR, sc->tclr);

	/* Register the timecounter. */
	sc->tc.tc_name = sc->tmr_name;
	sc->tc.tc_get_timecount = dmtpps_get_timecount;
	sc->tc.tc_counter_mask = ~0u;
	- sc->tc.tc_frequency = sysclk_freq;
	+ sc->tc.tc_frequency = sc->sysclk_freq;
	sc->tc.tc_quality = 1000;
	sc->tc.tc_priv = sc;

	tc_init(&sc->tc);

	/*
	* Indicate our PPS capabilities. Have the kernel init its part of the
	* pps_state struct and add its capabilities.
	*
	* While the hardware has a mode to capture each edge, it's not clear we
	* can use it that way, because there's only a single interrupt/status
	* bit to say something was captured, but not which edge it was. For
	* now, just say we can only capture assert events (the positive-going
	* edge of the pulse).
	*/
	mtx_init(&sc->pps_mtx, "dmtpps", NULL, MTX_SPIN);
	sc->pps_state.flags = PPSFLAG_MTX_SPIN;
	sc->pps_state.ppscap = PPS_CAPTUREASSERT;
	sc->pps_state.driver_abi = PPS_ABI_VERSION;
	sc->pps_state.driver_mtx = &sc->pps_mtx;
	pps_init_abi(&sc->pps_state);

	/* Create the PPS cdev. */
	make_dev_args_init(&mda);
	mda.mda_flags = MAKEDEV_WAITOK;
	mda.mda_devsw = &dmtpps_cdevsw;
	mda.mda_cr = NULL;
	mda.mda_uid = UID_ROOT;
	mda.mda_gid = GID_WHEEL;
	mda.mda_mode = 0600;
	mda.mda_unit = device_get_unit(dev);
	mda.mda_si_drv1 = sc;
	if ((err = make_dev_s(&mda, &sc->pps_cdev, PPS_CDEV_NAME)) != 0) {
	device_printf(dev, "Failed to create cdev %s\n", PPS_CDEV_NAME);
	return (err);
	}

	if (bootverbose)
	device_printf(sc->dev, "Using %s for PPS device /dev/%s\n",
	sc->tmr_name, PPS_CDEV_NAME);

	return (0);
	}

	static int
	dmtpps_detach(device_t dev)
	{

	/*
	* There is no way to remove a timecounter once it has been registered,
	* even if it's not in use, so we can never detach. If we were
	* dynamically loaded as a module this will prevent unloading.
	*/
	return (EBUSY);
	}

	static device_method_t dmtpps_methods[] = {
	DEVMETHOD(device_probe, dmtpps_probe),
	DEVMETHOD(device_attach, dmtpps_attach),
	DEVMETHOD(device_detach, dmtpps_detach),
	{ 0, 0 }
	};

	static driver_t dmtpps_driver = {
	"am335x_dmtpps",
	dmtpps_methods,
	sizeof(struct dmtpps_softc),
	};

	static devclass_t dmtpps_devclass;

	DRIVER_MODULE(am335x_dmtpps, simplebus, dmtpps_driver, dmtpps_devclass, 0, 0);
	-MODULE_DEPEND(am335x_dmtpps, am335x_prcm, 1, 1, 1);
	-
	+MODULE_DEPEND(am335x_dmtpps, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_dmtreg.h b/sys/arm/ti/am335x/am335x_dmtreg.h
	index f2ef54ddb917..c91a30570bd8 100644
	--- a/sys/arm/ti/am335x/am335x_dmtreg.h
	+++ b/sys/arm/ti/am335x/am335x_dmtreg.h
	@@ -1,76 +1,88 @@
	/*-
	* Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef AM335X_DMTREG_H
	#define AM335X_DMTREG_H

	#define AM335X_NUM_TIMERS 8

	#define DMT_TIDR 0x00 /* Identification Register */
	#define DMT_TIOCP_CFG 0x10 /* OCP Configuration Reg */
	#define DMT_TIOCP_RESET (1 << 0) /* TIOCP perform soft reset */
	#define DMT_IQR_EOI 0x20 /* IRQ End-Of-Interrupt Reg */
	#define DMT_IRQSTATUS_RAW 0x24 /* IRQSTATUS Raw Reg */
	#define DMT_IRQSTATUS 0x28 /* IRQSTATUS Reg */
	#define DMT_IRQENABLE_SET 0x2c /* IRQSTATUS Set Reg */
	#define DMT_IRQENABLE_CLR 0x30 /* IRQSTATUS Clear Reg */
	#define DMT_IRQWAKEEN 0x34 /* IRQ Wakeup Enable Reg */
	#define DMT_IRQ_MAT (1 << 0) /* IRQ: Match */
	#define DMT_IRQ_OVF (1 << 1) /* IRQ: Overflow */
	#define DMT_IRQ_TCAR (1 << 2) /* IRQ: Capture */
	#define DMT_IRQ_MASK (DMT_IRQ_TCAR \| DMT_IRQ_OVF \| DMT_IRQ_MAT)
	#define DMT_TCLR 0x38 /* Control Register */
	#define DMT_TCLR_START (1 << 0) /* Start timer */
	#define DMT_TCLR_AUTOLOAD (1 << 1) /* Auto-reload on overflow */
	#define DMT_TCLR_PRES_MASK (7 << 2) /* Prescaler mask */
	#define DMT_TCLR_PRES_ENABLE (1 << 5) /* Prescaler enable */
	#define DMT_TCLR_COMP_ENABLE (1 << 6) /* Compare enable */
	#define DMT_TCLR_PWM_HIGH (1 << 7) /* PWM default output high */
	#define DMT_TCLR_CAPTRAN_MASK (3 << 8) /* Capture transition mask */
	#define DMT_TCLR_CAPTRAN_NONE (0 << 8) /* Capture: none */
	#define DMT_TCLR_CAPTRAN_LOHI (1 << 8) /* Capture lo->hi transition */
	#define DMT_TCLR_CAPTRAN_HILO (2 << 8) /* Capture hi->lo transition */
	#define DMT_TCLR_CAPTRAN_BOTH (3 << 8) /* Capture both transitions */
	#define DMT_TCLR_TRGMODE_MASK (3 << 10) /* Trigger output mode mask */
	#define DMT_TCLR_TRGMODE_NONE (0 << 10) /* Trigger off */
	#define DMT_TCLR_TRGMODE_OVFL (1 << 10) /* Trigger on overflow */
	#define DMT_TCLR_TRGMODE_BOTH (2 << 10) /* Trigger on match + ovflow */
	#define DMT_TCLR_PWM_PTOGGLE (1 << 12) /* PWM toggles */
	#define DMT_TCLR_CAP_MODE_2ND (1 << 13) /* Capture second event mode */
	#define DMT_TCLR_GPO_CFG (1 << 14) /* Tmr pin conf, 0=out, 1=in */
	#define DMT_TCRR 0x3C /* Counter Register */
	#define DMT_TLDR 0x40 /* Load Reg */
	#define DMT_TTGR 0x44 /* Trigger Reg */
	#define DMT_TWPS 0x48 /* Write Posted Status Reg */
	#define DMT_TMAR 0x4C /* Match Reg */
	#define DMT_TCAR1 0x50 /* Capture Reg */
	#define DMT_TSICR 0x54 /* Synchr. Interface Ctrl Reg */
	#define DMT_TSICR_RESET (1 << 1) /* TSICR perform soft reset */
	#define DMT_TCAR2 0x48 /* Capture Reg */

	+/* Location of revision register from TRM Memory map chapter 2 */
	+/* L4_WKUP */
	+#define DMTIMER0_REV 0x05000
	+#define DMTIMER1_1MS_REV 0x31000
	+/* L4_PER */
	+#define DMTIMER2_REV 0x40000
	+#define DMTIMER3_REV 0x42000
	+#define DMTIMER4_REV 0x44000
	+#define DMTIMER5_REV 0x46000
	+#define DMTIMER6_REV 0x48000
	+#define DMTIMER7_REV 0x4A000
	+
	#endif /* AM335X_DMTREG_H */
	diff --git a/sys/arm/ti/am335x/am335x_gpio.c b/sys/arm/ti/am335x/am335x_gpio.c
	index cbfde175c2e7..beb169b3e4b5 100644
	--- a/sys/arm/ti/am335x/am335x_gpio.c
	+++ b/sys/arm/ti/am335x/am335x_gpio.c
	@@ -1,157 +1,158 @@
	/*-
	* Copyright (c) 2012 Damjan Marion <dmarion@FreeBSD.org>
	* Copyright (c) 2014 Andrew Turner <andrew@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/resource.h>
	#include <sys/rman.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <machine/intr.h>
	#include <sys/gpio.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <arm/ti/ti_cpuid.h>
	#include <arm/ti/ti_gpio.h>
	#include <arm/ti/ti_pinmux.h>

	#include <arm/ti/am335x/am335x_scm_padconf.h>

	#include "ti_gpio_if.h"

	static struct ofw_compat_data compat_data[] = {
	{"ti,am335x-gpio", 1},
	/* Linux uses ti,omap4-gpio on am335x so we need to support it */
	{"ti,omap4-gpio", 1},
	{"ti,gpio", 1},
	{NULL, 0},
	};

	static int
	am335x_gpio_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	if (ti_chip() != CHIP_AM335X)
	return (ENXIO);

	device_set_desc(dev, "TI AM335x General Purpose I/O (GPIO)");

	return (0);
	}

	static int
	am335x_gpio_set_flags(device_t dev, uint32_t gpio, uint32_t flags)
	{
	unsigned int state = 0;
	struct ti_gpio_softc *sc = device_get_softc(dev);

	if (flags & GPIO_PIN_OUTPUT) {
	if (flags & GPIO_PIN_PULLUP)
	state = PADCONF_OUTPUT_PULLUP;
	else
	state = PADCONF_OUTPUT;
	} else if (flags & GPIO_PIN_INPUT) {
	if (flags & GPIO_PIN_PULLUP)
	state = PADCONF_INPUT_PULLUP;
	else if (flags & GPIO_PIN_PULLDOWN)
	state = PADCONF_INPUT_PULLDOWN;
	else
	state = PADCONF_INPUT;
	}
	return ti_pinmux_padconf_set_gpiomode(sc->sc_bank*32 + gpio, state);
	}

	static int
	am335x_gpio_get_flags(device_t dev, uint32_t gpio, uint32_t *flags)
	{
	unsigned int state;
	struct ti_gpio_softc *sc = device_get_softc(dev);

	if (ti_pinmux_padconf_get_gpiomode(sc->sc_bank*32 + gpio, &state) != 0) {
	*flags = 0;
	return (EINVAL);
	} else {
	switch (state) {
	case PADCONF_OUTPUT:
	*flags = GPIO_PIN_OUTPUT;
	break;
	case PADCONF_OUTPUT_PULLUP:
	*flags = GPIO_PIN_OUTPUT \| GPIO_PIN_PULLUP;
	break;
	case PADCONF_INPUT:
	*flags = GPIO_PIN_INPUT;
	break;
	case PADCONF_INPUT_PULLUP:
	*flags = GPIO_PIN_INPUT \| GPIO_PIN_PULLUP;
	break;
	case PADCONF_INPUT_PULLDOWN:
	*flags = GPIO_PIN_INPUT \| GPIO_PIN_PULLDOWN;
	break;
	default:
	*flags = 0;
	break;
	}
	}

	return (0);
	}

	static device_method_t am335x_gpio_methods[] = {
	/* bus interface */
	DEVMETHOD(device_probe, am335x_gpio_probe),

	/* ti_gpio interface */
	DEVMETHOD(ti_gpio_set_flags, am335x_gpio_set_flags),
	DEVMETHOD(ti_gpio_get_flags, am335x_gpio_get_flags),

	DEVMETHOD_END
	};

	extern driver_t ti_gpio_driver;
	static devclass_t am335x_gpio_devclass;

	DEFINE_CLASS_1(gpio, am335x_gpio_driver, am335x_gpio_methods,
	sizeof(struct ti_gpio_softc), ti_gpio_driver);
	DRIVER_MODULE(am335x_gpio, simplebus, am335x_gpio_driver, am335x_gpio_devclass,
	0, 0);
	+MODULE_DEPEND(am335x_gpio, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_lcd.c b/sys/arm/ti/am335x/am335x_lcd.c
	index 5c60bf4be6bf..387bd37e3ebf 100644
	--- a/sys/arm/ti/am335x/am335x_lcd.c
	+++ b/sys/arm/ti/am335x/am335x_lcd.c
	@@ -1,1083 +1,1104 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright 2013 Oleksandr Tymoshenko <gonzo@freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_syscons.h"
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/clock.h>
	#include <sys/eventhandler.h>
	#include <sys/time.h>
	#include <sys/bus.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/resource.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <sys/fbio.h>
	#include <sys/consio.h>

	#include <machine/bus.h>

	+#include <dev/extres/clk/clk.h>
	+
	#include <dev/fdt/fdt_common.h>
	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/videomode/videomode.h>
	#include <dev/videomode/edidvar.h>

	#include <dev/fb/fbreg.h>
	#ifdef DEV_SC
	#include <dev/syscons/syscons.h>
	#else /* VT */
	#include <dev/vt/vt.h>
	#endif

	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_scm.h>

	#include "am335x_lcd.h"
	#include "am335x_pwm.h"

	#include "fb_if.h"
	#include "hdmi_if.h"

	#define LCD_PID 0x00
	#define LCD_CTRL 0x04
	#define CTRL_DIV_MASK 0xff
	#define CTRL_DIV_SHIFT 8
	#define CTRL_AUTO_UFLOW_RESTART (1 << 1)
	#define CTRL_RASTER_MODE 1
	#define CTRL_LIDD_MODE 0
	#define LCD_LIDD_CTRL 0x0C
	#define LCD_LIDD_CS0_CONF 0x10
	#define LCD_LIDD_CS0_ADDR 0x14
	#define LCD_LIDD_CS0_DATA 0x18
	#define LCD_LIDD_CS1_CONF 0x1C
	#define LCD_LIDD_CS1_ADDR 0x20
	#define LCD_LIDD_CS1_DATA 0x24
	#define LCD_RASTER_CTRL 0x28
	#define RASTER_CTRL_TFT24_UNPACKED (1 << 26)
	#define RASTER_CTRL_TFT24 (1 << 25)
	#define RASTER_CTRL_STN565 (1 << 24)
	#define RASTER_CTRL_TFTPMAP (1 << 23)
	#define RASTER_CTRL_NIBMODE (1 << 22)
	#define RASTER_CTRL_PALMODE_SHIFT 20
	#define PALETTE_PALETTE_AND_DATA 0x00
	#define PALETTE_PALETTE_ONLY 0x01
	#define PALETTE_DATA_ONLY 0x02
	#define RASTER_CTRL_REQDLY_SHIFT 12
	#define RASTER_CTRL_MONO8B (1 << 9)
	#define RASTER_CTRL_RBORDER (1 << 8)
	#define RASTER_CTRL_LCDTFT (1 << 7)
	#define RASTER_CTRL_LCDBW (1 << 1)
	#define RASTER_CTRL_LCDEN (1 << 0)
	#define LCD_RASTER_TIMING_0 0x2C
	#define RASTER_TIMING_0_HBP_SHIFT 24
	#define RASTER_TIMING_0_HFP_SHIFT 16
	#define RASTER_TIMING_0_HSW_SHIFT 10
	#define RASTER_TIMING_0_PPLLSB_SHIFT 4
	#define RASTER_TIMING_0_PPLMSB_SHIFT 3
	#define LCD_RASTER_TIMING_1 0x30
	#define RASTER_TIMING_1_VBP_SHIFT 24
	#define RASTER_TIMING_1_VFP_SHIFT 16
	#define RASTER_TIMING_1_VSW_SHIFT 10
	#define RASTER_TIMING_1_LPP_SHIFT 0
	#define LCD_RASTER_TIMING_2 0x34
	#define RASTER_TIMING_2_HSWHI_SHIFT 27
	#define RASTER_TIMING_2_LPP_B10_SHIFT 26
	#define RASTER_TIMING_2_PHSVS (1 << 25)
	#define RASTER_TIMING_2_PHSVS_RISE (1 << 24)
	#define RASTER_TIMING_2_PHSVS_FALL (0 << 24)
	#define RASTER_TIMING_2_IOE (1 << 23)
	#define RASTER_TIMING_2_IPC (1 << 22)
	#define RASTER_TIMING_2_IHS (1 << 21)
	#define RASTER_TIMING_2_IVS (1 << 20)
	#define RASTER_TIMING_2_ACBI_SHIFT 16
	#define RASTER_TIMING_2_ACB_SHIFT 8
	#define RASTER_TIMING_2_HBPHI_SHIFT 4
	#define RASTER_TIMING_2_HFPHI_SHIFT 0
	#define LCD_RASTER_SUBPANEL 0x38
	#define LCD_RASTER_SUBPANEL2 0x3C
	#define LCD_LCDDMA_CTRL 0x40
	#define LCDDMA_CTRL_DMA_MASTER_PRIO_SHIFT 16
	#define LCDDMA_CTRL_TH_FIFO_RDY_SHIFT 8
	#define LCDDMA_CTRL_BURST_SIZE_SHIFT 4
	#define LCDDMA_CTRL_BYTES_SWAP (1 << 3)
	#define LCDDMA_CTRL_BE (1 << 1)
	#define LCDDMA_CTRL_FB0_ONLY 0
	#define LCDDMA_CTRL_FB0_FB1 (1 << 0)
	#define LCD_LCDDMA_FB0_BASE 0x44
	#define LCD_LCDDMA_FB0_CEILING 0x48
	#define LCD_LCDDMA_FB1_BASE 0x4C
	#define LCD_LCDDMA_FB1_CEILING 0x50
	#define LCD_SYSCONFIG 0x54
	#define SYSCONFIG_STANDBY_FORCE (0 << 4)
	#define SYSCONFIG_STANDBY_NONE (1 << 4)
	#define SYSCONFIG_STANDBY_SMART (2 << 4)
	#define SYSCONFIG_IDLE_FORCE (0 << 2)
	#define SYSCONFIG_IDLE_NONE (1 << 2)
	#define SYSCONFIG_IDLE_SMART (2 << 2)
	#define LCD_IRQSTATUS_RAW 0x58
	#define LCD_IRQSTATUS 0x5C
	#define LCD_IRQENABLE_SET 0x60
	#define LCD_IRQENABLE_CLEAR 0x64
	#define IRQ_EOF1 (1 << 9)
	#define IRQ_EOF0 (1 << 8)
	#define IRQ_PL (1 << 6)
	#define IRQ_FUF (1 << 5)
	#define IRQ_ACB (1 << 3)
	#define IRQ_SYNC_LOST (1 << 2)
	#define IRQ_RASTER_DONE (1 << 1)
	#define IRQ_FRAME_DONE (1 << 0)
	#define LCD_END_OF_INT_IND 0x68
	#define LCD_CLKC_ENABLE 0x6C
	#define CLKC_ENABLE_DMA (1 << 2)
	#define CLKC_ENABLE_LDID (1 << 1)
	#define CLKC_ENABLE_CORE (1 << 0)
	#define LCD_CLKC_RESET 0x70
	#define CLKC_RESET_MAIN (1 << 3)
	#define CLKC_RESET_DMA (1 << 2)
	#define CLKC_RESET_LDID (1 << 1)
	#define CLKC_RESET_CORE (1 << 0)

	#define LCD_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx)
	#define LCD_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx)
	#define LCD_LOCK_INIT(_sc) mtx_init(&(_sc)->sc_mtx, \
	device_get_nameunit(_sc->sc_dev), "am335x_lcd", MTX_DEF)
	#define LCD_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx);

	#define LCD_READ4(_sc, reg) bus_read_4((_sc)->sc_mem_res, reg);
	#define LCD_WRITE4(_sc, reg, value) \
	bus_write_4((_sc)->sc_mem_res, reg, value);

	/* Backlight is controlled by eCAS interface on PWM unit 0 */
	#define PWM_UNIT 0
	#define PWM_PERIOD 100

	#define MODE_HBP(mode) ((mode)->htotal - (mode)->hsync_end)
	#define MODE_HFP(mode) ((mode)->hsync_start - (mode)->hdisplay)
	#define MODE_HSW(mode) ((mode)->hsync_end - (mode)->hsync_start)
	#define MODE_VBP(mode) ((mode)->vtotal - (mode)->vsync_end)
	#define MODE_VFP(mode) ((mode)->vsync_start - (mode)->vdisplay)
	#define MODE_VSW(mode) ((mode)->vsync_end - (mode)->vsync_start)

	#define MAX_PIXEL_CLOCK 126000
	#define MAX_BANDWIDTH (1280102460)

	struct am335x_lcd_softc {
	device_t sc_dev;
	struct fb_info sc_fb_info;
	struct resource *sc_mem_res;
	struct resource *sc_irq_res;
	void *sc_intr_hl;
	struct mtx sc_mtx;
	int sc_backlight;
	struct sysctl_oid *sc_oid;

	struct panel_info sc_panel;

	/* Framebuffer */
	bus_dma_tag_t sc_dma_tag;
	bus_dmamap_t sc_dma_map;
	size_t sc_fb_size;
	bus_addr_t sc_fb_phys;
	uint8_t *sc_fb_base;

	/* HDMI framer */
	phandle_t sc_hdmi_framer;
	eventhandler_tag sc_hdmi_evh;
	+
	+ /* Clock */
	+ clk_t sc_clk_dpll_disp_ck;
	};

	static void
	am335x_fb_dmamap_cb(void arg, bus_dma_segment_t segs, int nseg, int err)
	{
	bus_addr_t *addr;

	if (err)
	return;

	addr = (bus_addr_t*)arg;
	*addr = segs[0].ds_addr;
	}

	static uint32_t
	am335x_lcd_calc_divisor(uint32_t reference, uint32_t freq)
	{
	uint32_t div, i;
	uint32_t delta, min_delta;

	min_delta = freq;
	div = 255;

	/* Raster mode case: divisors are in range from 2 to 255 */
	for (i = 2; i < 255; i++) {
	delta = abs(reference/i - freq);
	if (delta < min_delta) {
	div = i;
	min_delta = delta;
	}
	}

	return (div);
	}

	static int
	am335x_lcd_sysctl_backlight(SYSCTL_HANDLER_ARGS)
	{
	struct am335x_lcd_softc sc = (struct am335x_lcd_softc)arg1;
	int error;
	int backlight;

	backlight = sc->sc_backlight;
	error = sysctl_handle_int(oidp, &backlight, 0, req);

	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	if (backlight < 0)
	backlight = 0;
	if (backlight > 100)
	backlight = 100;

	LCD_LOCK(sc);
	error = am335x_pwm_config_ecap(PWM_UNIT, PWM_PERIOD,
	backlight*PWM_PERIOD/100);
	if (error == 0)
	sc->sc_backlight = backlight;
	LCD_UNLOCK(sc);

	return (error);
	}

	static uint32_t
	am335x_mode_vrefresh(const struct videomode *mode)
	{
	uint32_t refresh;

	/* Calculate vertical refresh rate */
	refresh = (mode->dot_clock * 1000 / mode->htotal);
	refresh = (refresh + mode->vtotal / 2) / mode->vtotal;

	if (mode->flags & VID_INTERLACE)
	refresh *= 2;
	if (mode->flags & VID_DBLSCAN)
	refresh /= 2;

	return refresh;
	}

	static int
	am335x_mode_is_valid(const struct videomode *mode)
	{
	uint32_t hbp, hfp, hsw;
	uint32_t vbp, vfp, vsw;

	if (mode->dot_clock > MAX_PIXEL_CLOCK)
	return (0);

	if (mode->hdisplay & 0xf)
	return (0);

	if (mode->vdisplay > 2048)
	return (0);

	/* Check ranges for timing parameters */
	hbp = MODE_HBP(mode) - 1;
	hfp = MODE_HFP(mode) - 1;
	hsw = MODE_HSW(mode) - 1;
	vbp = MODE_VBP(mode);
	vfp = MODE_VFP(mode);
	vsw = MODE_VSW(mode) - 1;

	if (hbp > 0x3ff)
	return (0);
	if (hfp > 0x3ff)
	return (0);
	if (hsw > 0x3ff)
	return (0);

	if (vbp > 0xff)
	return (0);
	if (vfp > 0xff)
	return (0);
	if (vsw > 0x3f)
	return (0);
	if (mode->vdisplaymode->hdisplayam335x_mode_vrefresh(mode)
	> MAX_BANDWIDTH)
	return (0);

	return (1);
	}

	static void
	am335x_read_hdmi_property(device_t dev)
	{
	phandle_t node, xref;
	phandle_t endpoint;
	phandle_t hdmi_xref;
	struct am335x_lcd_softc *sc;

	sc = device_get_softc(dev);
	node = ofw_bus_get_node(dev);
	sc->sc_hdmi_framer = 0;

	/*
	* Old FreeBSD way of referencing to HDMI framer
	*/
	if (OF_getencprop(node, "hdmi", &hdmi_xref, sizeof(hdmi_xref)) != -1) {
	sc->sc_hdmi_framer = hdmi_xref;
	return;
	}

	/*
	* Use bindings described in Linux docs:
	* bindings/media/video-interfaces.txt
	* We assume that the only endpoint in LCDC node
	* is HDMI framer.
	*/
	node = ofw_bus_find_child(node, "port");

	/* No media bindings */
	if (node == 0)
	return;

	for (endpoint = OF_child(node); endpoint != 0; endpoint = OF_peer(endpoint)) {
	if (OF_getencprop(endpoint, "remote-endpoint", &xref, sizeof(xref)) != -1) {
	/* port/port@0/endpoint@0 */
	node = OF_node_from_xref(xref);
	/* port/port@0 */
	node = OF_parent(node);
	/* port */
	node = OF_parent(node);
	/* actual owner of port, in our case HDMI framer */
	sc->sc_hdmi_framer = OF_xref_from_node(OF_parent(node));
	if (sc->sc_hdmi_framer != 0)
	return;
	}
	}
	}

	static int
	am335x_read_property(device_t dev, phandle_t node, const char name, uint32_t val)
	{
	pcell_t cell;

	if ((OF_getencprop(node, name, &cell, sizeof(cell))) <= 0) {
	device_printf(dev, "missing '%s' attribute in LCD panel info\n",
	name);
	return (ENXIO);
	}

	*val = cell;

	return (0);
	}

	static int
	am335x_read_timing(device_t dev, phandle_t node, struct panel_info *panel)
	{
	int error;
	phandle_t timings_node, timing_node, native;

	timings_node = ofw_bus_find_child(node, "display-timings");
	if (timings_node == 0) {
	device_printf(dev, "no \"display-timings\" node\n");
	return (-1);
	}

	if (OF_searchencprop(timings_node, "native-mode", &native,
	sizeof(native)) == -1) {
	device_printf(dev, "no \"native-mode\" reference in \"timings\" node\n");
	return (-1);
	}

	timing_node = OF_node_from_xref(native);

	error = 0;
	if ((error = am335x_read_property(dev, timing_node,
	"hactive", &panel->panel_width)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"vactive", &panel->panel_height)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"hfront-porch", &panel->panel_hfp)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"hback-porch", &panel->panel_hbp)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"hsync-len", &panel->panel_hsw)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"vfront-porch", &panel->panel_vfp)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"vback-porch", &panel->panel_vbp)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"vsync-len", &panel->panel_vsw)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"clock-frequency", &panel->panel_pxl_clk)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"pixelclk-active", &panel->pixelclk_active)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"hsync-active", &panel->hsync_active)))
	goto out;

	if ((error = am335x_read_property(dev, timing_node,
	"vsync-active", &panel->vsync_active)))
	goto out;

	out:
	return (error);
	}

	static int
	am335x_read_panel_info(device_t dev, phandle_t node, struct panel_info *panel)
	{
	phandle_t panel_info_node;

	panel_info_node = ofw_bus_find_child(node, "panel-info");
	if (panel_info_node == 0)
	return (-1);

	am335x_read_property(dev, panel_info_node,
	"ac-bias", &panel->ac_bias);

	am335x_read_property(dev, panel_info_node,
	"ac-bias-intrpt", &panel->ac_bias_intrpt);

	am335x_read_property(dev, panel_info_node,
	"dma-burst-sz", &panel->dma_burst_sz);

	am335x_read_property(dev, panel_info_node,
	"bpp", &panel->bpp);

	am335x_read_property(dev, panel_info_node,
	"fdd", &panel->fdd);

	am335x_read_property(dev, panel_info_node,
	"sync-edge", &panel->sync_edge);

	am335x_read_property(dev, panel_info_node,
	"sync-ctrl", &panel->sync_ctrl);

	return (0);
	}

	static void
	am335x_lcd_intr(void *arg)
	{
	struct am335x_lcd_softc *sc = arg;
	uint32_t reg;

	reg = LCD_READ4(sc, LCD_IRQSTATUS);
	LCD_WRITE4(sc, LCD_IRQSTATUS, reg);
	/* Read value back to make sure it reached the hardware */
	reg = LCD_READ4(sc, LCD_IRQSTATUS);

	if (reg & IRQ_SYNC_LOST) {
	reg = LCD_READ4(sc, LCD_RASTER_CTRL);
	reg &= ~RASTER_CTRL_LCDEN;
	LCD_WRITE4(sc, LCD_RASTER_CTRL, reg);

	reg = LCD_READ4(sc, LCD_RASTER_CTRL);
	reg \|= RASTER_CTRL_LCDEN;
	LCD_WRITE4(sc, LCD_RASTER_CTRL, reg);
	goto done;
	}

	if (reg & IRQ_PL) {
	reg = LCD_READ4(sc, LCD_RASTER_CTRL);
	reg &= ~RASTER_CTRL_LCDEN;
	LCD_WRITE4(sc, LCD_RASTER_CTRL, reg);

	reg = LCD_READ4(sc, LCD_RASTER_CTRL);
	reg \|= RASTER_CTRL_LCDEN;
	LCD_WRITE4(sc, LCD_RASTER_CTRL, reg);
	goto done;
	}

	if (reg & IRQ_EOF0) {
	LCD_WRITE4(sc, LCD_LCDDMA_FB0_BASE, sc->sc_fb_phys);
	LCD_WRITE4(sc, LCD_LCDDMA_FB0_CEILING, sc->sc_fb_phys + sc->sc_fb_size - 1);
	reg &= ~IRQ_EOF0;
	}

	if (reg & IRQ_EOF1) {
	LCD_WRITE4(sc, LCD_LCDDMA_FB1_BASE, sc->sc_fb_phys);
	LCD_WRITE4(sc, LCD_LCDDMA_FB1_CEILING, sc->sc_fb_phys + sc->sc_fb_size - 1);
	reg &= ~IRQ_EOF1;
	}

	if (reg & IRQ_FUF) {
	/* TODO: Handle FUF */
	}

	if (reg & IRQ_ACB) {
	/* TODO: Handle ACB */
	}

	done:
	LCD_WRITE4(sc, LCD_END_OF_INT_IND, 0);
	/* Read value back to make sure it reached the hardware */
	reg = LCD_READ4(sc, LCD_END_OF_INT_IND);
	}

	static const struct videomode *
	am335x_lcd_pick_mode(struct edid_info *ei)
	{
	const struct videomode *videomode;
	const struct videomode *m;
	int n;

	/* Get standard VGA as default */
	videomode = NULL;

	/*
	* Pick a mode.
	*/
	if (ei->edid_preferred_mode != NULL) {
	if (am335x_mode_is_valid(ei->edid_preferred_mode))
	videomode = ei->edid_preferred_mode;
	}

	if (videomode == NULL) {
	m = ei->edid_modes;

	sort_modes(ei->edid_modes,
	&ei->edid_preferred_mode,
	ei->edid_nmodes);
	for (n = 0; n < ei->edid_nmodes; n++)
	if (am335x_mode_is_valid(&m[n])) {
	videomode = &m[n];
	break;
	}
	}

	return videomode;
	}

	static int
	am335x_lcd_configure(struct am335x_lcd_softc *sc)
	{
	int div;
	uint32_t reg, timing0, timing1, timing2;
	uint32_t burst_log;
	size_t dma_size;
	uint32_t hbp, hfp, hsw;
	uint32_t vbp, vfp, vsw;
	uint32_t width, height;
	- unsigned int ref_freq;
	+ uint64_t ref_freq;
	int err;

	/*
	* try to adjust clock to get double of requested frequency
	* HDMI/DVI displays are very sensitive to error in frequncy value
	*/
	- if (ti_prcm_clk_set_source_freq(LCDC_CLK, sc->sc_panel.panel_pxl_clk*2)) {
	+
	+ err = clk_set_freq(sc->sc_clk_dpll_disp_ck, sc->sc_panel.panel_pxl_clk*2,
	+ CLK_SET_ROUND_ANY);
	+ if (err != 0) {
	device_printf(sc->sc_dev, "can't set source frequency\n");
	return (ENXIO);
	}

	- if (ti_prcm_clk_get_source_freq(LCDC_CLK, &ref_freq)) {
	+ err = clk_get_freq(sc->sc_clk_dpll_disp_ck, &ref_freq);
	+ if (err != 0) {
	device_printf(sc->sc_dev, "can't get reference frequency\n");
	return (ENXIO);
	}

	- /* Panle initialization */
	+ /* Panel initialization */
	dma_size = round_page(sc->sc_panel.panel_widthsc->sc_panel.panel_heightsc->sc_panel.bpp/8);

	/*
	* Now allocate framebuffer memory
	*/
	err = bus_dma_tag_create(
	bus_get_dma_tag(sc->sc_dev),
	4, 0, /* alignment, boundary */
	BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	dma_size, 1, /* maxsize, nsegments */
	dma_size, 0, /* maxsegsize, flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->sc_dma_tag);
	if (err)
	goto done;

	err = bus_dmamem_alloc(sc->sc_dma_tag, (void **)&sc->sc_fb_base,
	BUS_DMA_COHERENT, &sc->sc_dma_map);

	if (err) {
	device_printf(sc->sc_dev, "cannot allocate framebuffer\n");
	goto done;
	}

	err = bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, sc->sc_fb_base,
	dma_size, am335x_fb_dmamap_cb, &sc->sc_fb_phys, BUS_DMA_NOWAIT);

	if (err) {
	device_printf(sc->sc_dev, "cannot load DMA map\n");
	goto done;
	}

	/* Make sure it's blank */
	memset(sc->sc_fb_base, 0x0, dma_size);

	/* Calculate actual FB Size */
	sc->sc_fb_size = sc->sc_panel.panel_widthsc->sc_panel.panel_heightsc->sc_panel.bpp/8;

	/* Only raster mode is supported */
	reg = CTRL_RASTER_MODE;
	div = am335x_lcd_calc_divisor(ref_freq, sc->sc_panel.panel_pxl_clk);
	reg \|= (div << CTRL_DIV_SHIFT);
	LCD_WRITE4(sc, LCD_CTRL, reg);

	/* Set timing */
	timing0 = timing1 = timing2 = 0;

	hbp = sc->sc_panel.panel_hbp - 1;
	hfp = sc->sc_panel.panel_hfp - 1;
	hsw = sc->sc_panel.panel_hsw - 1;

	vbp = sc->sc_panel.panel_vbp;
	vfp = sc->sc_panel.panel_vfp;
	vsw = sc->sc_panel.panel_vsw - 1;

	height = sc->sc_panel.panel_height - 1;
	width = sc->sc_panel.panel_width - 1;

	/* Horizontal back porch */
	timing0 \|= (hbp & 0xff) << RASTER_TIMING_0_HBP_SHIFT;
	timing2 \|= ((hbp >> 8) & 3) << RASTER_TIMING_2_HBPHI_SHIFT;
	/* Horizontal front porch */
	timing0 \|= (hfp & 0xff) << RASTER_TIMING_0_HFP_SHIFT;
	timing2 \|= ((hfp >> 8) & 3) << RASTER_TIMING_2_HFPHI_SHIFT;
	/* Horizontal sync width */
	timing0 \|= (hsw & 0x3f) << RASTER_TIMING_0_HSW_SHIFT;
	timing2 \|= ((hsw >> 6) & 0xf) << RASTER_TIMING_2_HSWHI_SHIFT;

	/* Vertical back porch, front porch, sync width */
	timing1 \|= (vbp & 0xff) << RASTER_TIMING_1_VBP_SHIFT;
	timing1 \|= (vfp & 0xff) << RASTER_TIMING_1_VFP_SHIFT;
	timing1 \|= (vsw & 0x3f) << RASTER_TIMING_1_VSW_SHIFT;

	/* Pixels per line */
	timing0 \|= ((width >> 10) & 1)
	<< RASTER_TIMING_0_PPLMSB_SHIFT;
	timing0 \|= ((width >> 4) & 0x3f)
	<< RASTER_TIMING_0_PPLLSB_SHIFT;

	/* Lines per panel */
	timing1 \|= (height & 0x3ff)
	<< RASTER_TIMING_1_LPP_SHIFT;
	timing2 \|= ((height >> 10 ) & 1)
	<< RASTER_TIMING_2_LPP_B10_SHIFT;

	/* clock signal settings */
	if (sc->sc_panel.sync_ctrl)
	timing2 \|= RASTER_TIMING_2_PHSVS;
	if (sc->sc_panel.sync_edge)
	timing2 \|= RASTER_TIMING_2_PHSVS_RISE;
	else
	timing2 \|= RASTER_TIMING_2_PHSVS_FALL;
	if (sc->sc_panel.hsync_active == 0)
	timing2 \|= RASTER_TIMING_2_IHS;
	if (sc->sc_panel.vsync_active == 0)
	timing2 \|= RASTER_TIMING_2_IVS;
	if (sc->sc_panel.pixelclk_active == 0)
	timing2 \|= RASTER_TIMING_2_IPC;

	/* AC bias */
	timing2 \|= (sc->sc_panel.ac_bias << RASTER_TIMING_2_ACB_SHIFT);
	timing2 \|= (sc->sc_panel.ac_bias_intrpt << RASTER_TIMING_2_ACBI_SHIFT);

	LCD_WRITE4(sc, LCD_RASTER_TIMING_0, timing0);
	LCD_WRITE4(sc, LCD_RASTER_TIMING_1, timing1);
	LCD_WRITE4(sc, LCD_RASTER_TIMING_2, timing2);

	/* DMA settings */
	reg = LCDDMA_CTRL_FB0_FB1;
	/* Find power of 2 for current burst size */
	switch (sc->sc_panel.dma_burst_sz) {
	case 1:
	burst_log = 0;
	break;
	case 2:
	burst_log = 1;
	break;
	case 4:
	burst_log = 2;
	break;
	case 8:
	burst_log = 3;
	break;
	case 16:
	default:
	burst_log = 4;
	break;
	}
	reg \|= (burst_log << LCDDMA_CTRL_BURST_SIZE_SHIFT);
	/* XXX: FIFO TH */
	reg \|= (0 << LCDDMA_CTRL_TH_FIFO_RDY_SHIFT);
	LCD_WRITE4(sc, LCD_LCDDMA_CTRL, reg);

	LCD_WRITE4(sc, LCD_LCDDMA_FB0_BASE, sc->sc_fb_phys);
	LCD_WRITE4(sc, LCD_LCDDMA_FB0_CEILING, sc->sc_fb_phys + sc->sc_fb_size - 1);
	LCD_WRITE4(sc, LCD_LCDDMA_FB1_BASE, sc->sc_fb_phys);
	LCD_WRITE4(sc, LCD_LCDDMA_FB1_CEILING, sc->sc_fb_phys + sc->sc_fb_size - 1);

	/* Enable LCD */
	reg = RASTER_CTRL_LCDTFT;
	reg \|= (sc->sc_panel.fdd << RASTER_CTRL_REQDLY_SHIFT);
	reg \|= (PALETTE_DATA_ONLY << RASTER_CTRL_PALMODE_SHIFT);
	if (sc->sc_panel.bpp >= 24)
	reg \|= RASTER_CTRL_TFT24;
	if (sc->sc_panel.bpp == 32)
	reg \|= RASTER_CTRL_TFT24_UNPACKED;
	LCD_WRITE4(sc, LCD_RASTER_CTRL, reg);

	LCD_WRITE4(sc, LCD_CLKC_ENABLE,
	CLKC_ENABLE_DMA \| CLKC_ENABLE_LDID \| CLKC_ENABLE_CORE);

	LCD_WRITE4(sc, LCD_CLKC_RESET, CLKC_RESET_MAIN);
	DELAY(100);
	LCD_WRITE4(sc, LCD_CLKC_RESET, 0);

	reg = IRQ_EOF1 \| IRQ_EOF0 \| IRQ_FUF \| IRQ_PL \|
	IRQ_ACB \| IRQ_SYNC_LOST \| IRQ_RASTER_DONE \|
	IRQ_FRAME_DONE;
	LCD_WRITE4(sc, LCD_IRQENABLE_SET, reg);

	reg = LCD_READ4(sc, LCD_RASTER_CTRL);
	reg \|= RASTER_CTRL_LCDEN;
	LCD_WRITE4(sc, LCD_RASTER_CTRL, reg);

	LCD_WRITE4(sc, LCD_SYSCONFIG,
	SYSCONFIG_STANDBY_SMART \| SYSCONFIG_IDLE_SMART);

	sc->sc_fb_info.fb_name = device_get_nameunit(sc->sc_dev);
	sc->sc_fb_info.fb_vbase = (intptr_t)sc->sc_fb_base;
	sc->sc_fb_info.fb_pbase = sc->sc_fb_phys;
	sc->sc_fb_info.fb_size = sc->sc_fb_size;
	sc->sc_fb_info.fb_bpp = sc->sc_fb_info.fb_depth = sc->sc_panel.bpp;
	sc->sc_fb_info.fb_stride = sc->sc_panel.panel_width*sc->sc_panel.bpp / 8;
	sc->sc_fb_info.fb_width = sc->sc_panel.panel_width;
	sc->sc_fb_info.fb_height = sc->sc_panel.panel_height;

	#ifdef DEV_SC
	err = (sc_attach_unit(device_get_unit(sc->sc_dev),
	device_get_flags(sc->sc_dev) \| SC_AUTODETECT_KBD));

	if (err) {
	device_printf(sc->sc_dev, "failed to attach syscons\n");
	goto fail;
	}

	am335x_lcd_syscons_setup((vm_offset_t)sc->sc_fb_base, sc->sc_fb_phys, &panel);
	#else /* VT */
	device_t fbd = device_add_child(sc->sc_dev, "fbd",
	device_get_unit(sc->sc_dev));
	if (fbd != NULL) {
	if (device_probe_and_attach(fbd) != 0)
	device_printf(sc->sc_dev, "failed to attach fbd device\n");
	} else
	device_printf(sc->sc_dev, "failed to add fbd child\n");
	#endif

	done:
	return (err);
	}

	static void
	am335x_lcd_hdmi_event(void *arg, device_t hdmi, int event)
	{
	struct am335x_lcd_softc *sc;
	const struct videomode *videomode;
	struct videomode hdmi_mode;
	device_t hdmi_dev;
	uint8_t *edid;
	uint32_t edid_len;
	struct edid_info ei;

	sc = arg;

	/* Nothing to work with */
	if (!sc->sc_hdmi_framer) {
	device_printf(sc->sc_dev, "HDMI event without HDMI framer set\n");
	return;
	}

	hdmi_dev = OF_device_from_xref(sc->sc_hdmi_framer);
	if (!hdmi_dev) {
	device_printf(sc->sc_dev, "no actual device for \"hdmi\" property\n");
	return;
	}

	edid = NULL;
	edid_len = 0;
	if (HDMI_GET_EDID(hdmi_dev, &edid, &edid_len) != 0) {
	device_printf(sc->sc_dev, "failed to get EDID info from HDMI framer\n");
	return;
	}

	videomode = NULL;

	if (edid_parse(edid, &ei) == 0) {
	edid_print(&ei);
	videomode = am335x_lcd_pick_mode(&ei);
	} else
	device_printf(sc->sc_dev, "failed to parse EDID\n");

	/* Use standard VGA as fallback */
	if (videomode == NULL)
	videomode = pick_mode_by_ref(640, 480, 60);

	if (videomode == NULL) {
	device_printf(sc->sc_dev, "failed to find usable videomode");
	return;
	}

	device_printf(sc->sc_dev, "detected videomode: %dx%d @ %dKHz\n", videomode->hdisplay,
	videomode->vdisplay, am335x_mode_vrefresh(videomode));

	sc->sc_panel.panel_width = videomode->hdisplay;
	sc->sc_panel.panel_height = videomode->vdisplay;
	sc->sc_panel.panel_hfp = videomode->hsync_start - videomode->hdisplay;
	sc->sc_panel.panel_hbp = videomode->htotal - videomode->hsync_end;
	sc->sc_panel.panel_hsw = videomode->hsync_end - videomode->hsync_start;
	sc->sc_panel.panel_vfp = videomode->vsync_start - videomode->vdisplay;
	sc->sc_panel.panel_vbp = videomode->vtotal - videomode->vsync_end;
	sc->sc_panel.panel_vsw = videomode->vsync_end - videomode->vsync_start;
	sc->sc_panel.pixelclk_active = 1;

	/* logic for HSYNC should be reversed */
	if (videomode->flags & VID_NHSYNC)
	sc->sc_panel.hsync_active = 1;
	else
	sc->sc_panel.hsync_active = 0;

	if (videomode->flags & VID_NVSYNC)
	sc->sc_panel.vsync_active = 0;
	else
	sc->sc_panel.vsync_active = 1;

	sc->sc_panel.panel_pxl_clk = videomode->dot_clock * 1000;

	am335x_lcd_configure(sc);

	memcpy(&hdmi_mode, videomode, sizeof(hdmi_mode));
	hdmi_mode.hskew = videomode->hsync_end - videomode->hsync_start;
	hdmi_mode.flags \|= VID_HSKEW;

	HDMI_SET_VIDEOMODE(hdmi_dev, &hdmi_mode);
	}

	static int
	am335x_lcd_probe(device_t dev)
	{
	#ifdef DEV_SC
	int err;
	#endif

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,am33xx-tilcdc"))
	return (ENXIO);

	device_set_desc(dev, "AM335x LCD controller");

	#ifdef DEV_SC
	err = sc_probe_unit(device_get_unit(dev),
	device_get_flags(dev) \| SC_AUTODETECT_KBD);
	if (err != 0)
	return (err);
	#endif

	return (BUS_PROBE_DEFAULT);
	}

	static int
	am335x_lcd_attach(device_t dev)
	{
	struct am335x_lcd_softc *sc;

	int err;
	int rid;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *tree;
	phandle_t root, panel_node;

	err = 0;
	sc = device_get_softc(dev);
	sc->sc_dev = dev;

	am335x_read_hdmi_property(dev);

	root = OF_finddevice("/");
	if (root == -1) {
	device_printf(dev, "failed to get FDT root node\n");
	return (ENXIO);
	}

	+ /* Fixme: Cant find any reference in DTS for dpll_disp_ck@498 for now. */
	+ err = clk_get_by_name(dev, "dpll_disp_ck@498", &sc->sc_clk_dpll_disp_ck);
	+ if (err != 0) {
	+ device_printf(dev, "Cant get dpll_disp_ck@49\n");
	+ return (ENXIO);
	+ }
	+
	sc->sc_panel.ac_bias = 255;
	sc->sc_panel.ac_bias_intrpt = 0;
	sc->sc_panel.dma_burst_sz = 16;
	sc->sc_panel.bpp = 16;
	sc->sc_panel.fdd = 128;
	sc->sc_panel.sync_edge = 0;
	sc->sc_panel.sync_ctrl = 1;

	panel_node = fdt_find_compatible(root, "ti,tilcdc,panel", 1);
	if (panel_node != 0) {
	device_printf(dev, "using static panel info\n");
	if (am335x_read_panel_info(dev, panel_node, &sc->sc_panel)) {
	device_printf(dev, "failed to read panel info\n");
	return (ENXIO);
	}

	if (am335x_read_timing(dev, panel_node, &sc->sc_panel)) {
	device_printf(dev, "failed to read timings\n");
	return (ENXIO);
	}
	}

	- ti_prcm_clk_enable(LCDC_CLK);
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	+ if (err != 0) {
	+ device_printf(dev, "Failed to enable sysc clkctrl, err %d\n", err);
	+ return (ENXIO);
	+ }

	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (!sc->sc_mem_res) {
	device_printf(dev, "cannot allocate memory window\n");
	return (ENXIO);
	}

	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE);
	if (!sc->sc_irq_res) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "cannot allocate interrupt\n");
	return (ENXIO);
	}

	if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC \| INTR_MPSAFE,
	NULL, am335x_lcd_intr, sc,
	&sc->sc_intr_hl) != 0) {
	bus_release_resource(dev, SYS_RES_IRQ, rid,
	sc->sc_irq_res);
	bus_release_resource(dev, SYS_RES_MEMORY, rid,
	sc->sc_mem_res);
	device_printf(dev, "Unable to setup the irq handler.\n");
	return (ENXIO);
	}

	LCD_LOCK_INIT(sc);

	/* Init backlight interface */
	ctx = device_get_sysctl_ctx(sc->sc_dev);
	tree = device_get_sysctl_tree(sc->sc_dev);
	sc->sc_oid = SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
	"backlight", CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	am335x_lcd_sysctl_backlight, "I", "LCD backlight");
	sc->sc_backlight = 0;
	/* Check if eCAS interface is available at this point */
	if (am335x_pwm_config_ecap(PWM_UNIT,
	PWM_PERIOD, PWM_PERIOD) == 0)
	sc->sc_backlight = 100;

	if (panel_node != 0)
	am335x_lcd_configure(sc);
	else
	sc->sc_hdmi_evh = EVENTHANDLER_REGISTER(hdmi_event,
	am335x_lcd_hdmi_event, sc, EVENTHANDLER_PRI_ANY);

	return (0);
	}

	static int
	am335x_lcd_detach(device_t dev)
	{
	/* Do not let unload driver */
	return (EBUSY);
	}

	static struct fb_info *
	am335x_lcd_fb_getinfo(device_t dev)
	{
	struct am335x_lcd_softc *sc;

	sc = device_get_softc(dev);

	return (&sc->sc_fb_info);
	}

	static device_method_t am335x_lcd_methods[] = {
	DEVMETHOD(device_probe, am335x_lcd_probe),
	DEVMETHOD(device_attach, am335x_lcd_attach),
	DEVMETHOD(device_detach, am335x_lcd_detach),

	/* Framebuffer service methods */
	DEVMETHOD(fb_getinfo, am335x_lcd_fb_getinfo),

	DEVMETHOD_END
	};

	static driver_t am335x_lcd_driver = {
	"fb",
	am335x_lcd_methods,
	sizeof(struct am335x_lcd_softc),
	};

	static devclass_t am335x_lcd_devclass;

	DRIVER_MODULE(am335x_lcd, simplebus, am335x_lcd_driver, am335x_lcd_devclass, 0, 0);
	MODULE_VERSION(am335x_lcd, 1);
	MODULE_DEPEND(am335x_lcd, simplebus, 1, 1, 1);
	+MODULE_DEPEND(am335x_lcd, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_musb.c b/sys/arm/ti/am335x/am335x_musb.c
	index cbe4d06338c9..d868e8a64f8b 100644
	--- a/sys/arm/ti/am335x/am335x_musb.c
	+++ b/sys/arm/ti/am335x/am335x_musb.c
	@@ -1,421 +1,462 @@
	/*-
	* Copyright (c) 2013 Oleksandr Tymoshenko <gonzo@freebsd.org>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/stdint.h>
	#include <sys/stddef.h>
	#include <sys/param.h>
	#include <sys/queue.h>
	#include <sys/types.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/bus.h>
	#include <sys/module.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/condvar.h>
	#include <sys/sysctl.h>
	#include <sys/sx.h>
	#include <sys/unistd.h>
	#include <sys/callout.h>
	#include <sys/malloc.h>
	#include <sys/priv.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/usb/usb.h>
	#include <dev/usb/usbdi.h>

	#include <dev/usb/usb_core.h>
	#include <dev/usb/usb_busdma.h>
	#include <dev/usb/usb_process.h>
	#include <dev/usb/usb_util.h>

	#define USB_DEBUG_VAR usbssdebug

	#include <dev/usb/usb_controller.h>
	#include <dev/usb/usb_bus.h>
	#include <dev/usb/controller/musb_otg.h>
	#include <dev/usb/usb_debug.h>

	#include <sys/rman.h>

	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_scm.h>
	#include <arm/ti/am335x/am335x_scm.h>
	+#include <arm/ti/ti_sysc.h>
	+#include <dev/extres/clk/clk.h>
	+#include <dev/extres/syscon/syscon.h>
	+#include "syscon_if.h"

	#define USBCTRL_REV 0x00
	#define USBCTRL_CTRL 0x14
	#define USBCTRL_STAT 0x18
	#define USBCTRL_IRQ_STAT0 0x30
	#define IRQ_STAT0_RXSHIFT 16
	#define IRQ_STAT0_TXSHIFT 0
	#define USBCTRL_IRQ_STAT1 0x34
	#define IRQ_STAT1_DRVVBUS (1 << 8)
	#define USBCTRL_INTEN_SET0 0x38
	#define USBCTRL_INTEN_SET1 0x3C
	#define USBCTRL_INTEN_USB_ALL 0x1ff
	#define USBCTRL_INTEN_USB_SOF (1 << 3)
	#define USBCTRL_INTEN_CLR0 0x40
	#define USBCTRL_INTEN_CLR1 0x44
	#define USBCTRL_UTMI 0xE0
	#define USBCTRL_UTMI_FSDATAEXT (1 << 1)
	#define USBCTRL_MODE 0xE8
	#define USBCTRL_MODE_IDDIG (1 << 8)
	#define USBCTRL_MODE_IDDIGMUX (1 << 7)

	/* USBSS resource + 2 MUSB ports */

	#define RES_USBCORE 0
	#define RES_USBCTRL 1

	#define USB_WRITE4(sc, idx, reg, val) do { \
	bus_write_4((sc)->sc_mem_res[idx], (reg), (val)); \
	} while (0)

	#define USB_READ4(sc, idx, reg) bus_read_4((sc)->sc_mem_res[idx], (reg))

	#define USBCTRL_WRITE4(sc, reg, val) \
	USB_WRITE4((sc), RES_USBCTRL, (reg), (val))
	#define USBCTRL_READ4(sc, reg) \
	USB_READ4((sc), RES_USBCTRL, (reg))

	static struct resource_spec am335x_musbotg_mem_spec[] = {
	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
	{ SYS_RES_MEMORY, 1, RF_ACTIVE },
	{ -1, 0, 0 }
	};

	#ifdef USB_DEBUG
	static int usbssdebug = 0;

	static SYSCTL_NODE(_hw_usb, OID_AUTO, am335x_usbss,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"AM335x USBSS");
	SYSCTL_INT(_hw_usb_am335x_usbss, OID_AUTO, debug, CTLFLAG_RW,
	&usbssdebug, 0, "Debug level");
	#endif

	static device_probe_t musbotg_probe;
	static device_attach_t musbotg_attach;
	static device_detach_t musbotg_detach;

	struct musbotg_super_softc {
	struct musbotg_softc sc_otg;
	struct resource *sc_mem_res[2];
	int sc_irq_rid;
	+ struct syscon *syscon;
	};

	static void
	musbotg_vbus_poll(struct musbotg_super_softc *sc)
	{
	uint32_t stat;

	if (sc->sc_otg.sc_mode == MUSB2_DEVICE_MODE)
	musbotg_vbus_interrupt(&sc->sc_otg, 1);
	else {
	stat = USBCTRL_READ4(sc, USBCTRL_STAT);
	musbotg_vbus_interrupt(&sc->sc_otg, stat & 1);
	}
	}

	/*
	* Arg to musbotg_clocks_on and musbot_clocks_off is
	* a uint32_t * pointing to the SCM register offset.
	*/
	static uint32_t USB_CTRL[] = {SCM_USB_CTRL0, SCM_USB_CTRL1};

	static void
	musbotg_clocks_on(void *arg)
	{
	struct musbotg_softc *sc;
	- uint32_t c, reg;
	+ struct musbotg_super_softc *ssc;
	+ uint32_t reg;

	sc = arg;
	- reg = USB_CTRL[sc->sc_id];
	+ ssc = sc->sc_platform_data;
	+
	+ reg = SYSCON_READ_4(ssc->syscon, USB_CTRL[sc->sc_id]);
	+ reg &= ~3; /* Enable power */
	+ reg \|= 1 << 19; /* VBUS detect enable */
	+ reg \|= 1 << 20; /* Session end enable */

	- ti_scm_reg_read_4(reg, &c);
	- c &= ~3; /* Enable power */
	- c \|= 1 << 19; /* VBUS detect enable */
	- c \|= 1 << 20; /* Session end enable */
	- ti_scm_reg_write_4(reg, c);
	+ SYSCON_WRITE_4(ssc->syscon, USB_CTRL[sc->sc_id], reg);
	}

	static void
	musbotg_clocks_off(void *arg)
	{
	struct musbotg_softc *sc;
	- uint32_t c, reg;
	+ struct musbotg_super_softc *ssc;
	+ uint32_t reg;

	sc = arg;
	- reg = USB_CTRL[sc->sc_id];
	+ ssc = sc->sc_platform_data;

	/* Disable power to PHY */
	- ti_scm_reg_read_4(reg, &c);
	- ti_scm_reg_write_4(reg, c \| 3);
	+ reg = SYSCON_READ_4(ssc->syscon, USB_CTRL[sc->sc_id]);
	+ SYSCON_WRITE_4(ssc->syscon, USB_CTRL[sc->sc_id], reg \| 3);
	}

	static void
	musbotg_ep_int_set(struct musbotg_softc *sc, int ep, int on)
	{
	struct musbotg_super_softc *ssc = sc->sc_platform_data;
	uint32_t epmask;

	epmask = ((1 << ep) << IRQ_STAT0_RXSHIFT);
	epmask \|= ((1 << ep) << IRQ_STAT0_TXSHIFT);
	if (on)
	USBCTRL_WRITE4(ssc, USBCTRL_INTEN_SET0, epmask);
	else
	USBCTRL_WRITE4(ssc, USBCTRL_INTEN_CLR0, epmask);
	}

	static void
	musbotg_wrapper_interrupt(void *arg)
	{
	struct musbotg_softc *sc = arg;
	struct musbotg_super_softc *ssc = sc->sc_platform_data;
	uint32_t stat, stat0, stat1;

	stat = USBCTRL_READ4(ssc, USBCTRL_STAT);
	stat0 = USBCTRL_READ4(ssc, USBCTRL_IRQ_STAT0);
	stat1 = USBCTRL_READ4(ssc, USBCTRL_IRQ_STAT1);
	if (stat0)
	USBCTRL_WRITE4(ssc, USBCTRL_IRQ_STAT0, stat0);
	if (stat1)
	USBCTRL_WRITE4(ssc, USBCTRL_IRQ_STAT1, stat1);

	DPRINTFN(4, "port%d: stat0=%08x stat1=%08x, stat=%08x\n",
	sc->sc_id, stat0, stat1, stat);

	if (stat1 & IRQ_STAT1_DRVVBUS)
	musbotg_vbus_interrupt(sc, stat & 1);

	musbotg_interrupt(arg, ((stat0 >> 16) & 0xffff),
	stat0 & 0xffff, stat1 & 0xff);
	}

	static int
	musbotg_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,musb-am33xx"))
	return (ENXIO);

	device_set_desc(dev, "TI AM33xx integrated USB OTG controller");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	musbotg_attach(device_t dev)
	{
	struct musbotg_super_softc *sc = device_get_softc(dev);
	char mode[16];
	int err;
	uint32_t reg;
	+ phandle_t opp_table;
	+ clk_t clk_usbotg_fck;

	sc->sc_otg.sc_id = device_get_unit(dev);

	+ /* FIXME: The devicetree needs to be updated to get a handle to the gate
	+ * usbotg_fck@47c. see TRM 8.1.12.2 CM_WKUP CM_CLKDCOLDO_DPLL_PER.
	+ */
	+ err = clk_get_by_name(dev, "usbotg_fck@47c", &clk_usbotg_fck);
	+ if (err) {
	+ device_printf(dev, "Can not find usbotg_fck@47c\n");
	+ return (ENXIO);
	+ }
	+
	+ err = clk_enable(clk_usbotg_fck);
	+ if (err) {
	+ device_printf(dev, "Can not enable usbotg_fck@47c\n");
	+ return (ENXIO);
	+ }
	+
	+ /* FIXME: For now; Go and kidnap syscon from opp-table */
	+ opp_table = OF_finddevice("/opp-table");
	+ if (opp_table == -1) {
	+ device_printf(dev, "Cant find /opp-table\n");
	+ return (ENXIO);
	+ }
	+ if (!OF_hasprop(opp_table, "syscon")) {
	+ device_printf(dev, "/opp-table missing syscon property\n");
	+ return (ENXIO);
	+ }
	+ err = syscon_get_by_ofw_property(dev, opp_table, "syscon", &sc->syscon);
	+ if (err) {
	+ device_printf(dev, "Failed to get syscon\n");
	+ return (ENXIO);
	+ }
	+
	/* Request the memory resources */
	err = bus_alloc_resources(dev, am335x_musbotg_mem_spec,
	sc->sc_mem_res);
	if (err) {
	device_printf(dev,
	"Error: could not allocate mem resources\n");
	return (ENXIO);
	}

	/* Request the IRQ resources */
	sc->sc_otg.sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
	&sc->sc_irq_rid, RF_ACTIVE);
	if (sc->sc_otg.sc_irq_res == NULL) {
	device_printf(dev,
	"Error: could not allocate irq resources\n");
	return (ENXIO);
	}

	/* setup MUSB OTG USB controller interface softc */
	sc->sc_otg.sc_clocks_on = &musbotg_clocks_on;
	sc->sc_otg.sc_clocks_off = &musbotg_clocks_off;
	sc->sc_otg.sc_clocks_arg = &sc->sc_otg;

	sc->sc_otg.sc_ep_int_set = musbotg_ep_int_set;

	/* initialise some bus fields */
	sc->sc_otg.sc_bus.parent = dev;
	sc->sc_otg.sc_bus.devices = sc->sc_otg.sc_devices;
	sc->sc_otg.sc_bus.devices_max = MUSB2_MAX_DEVICES;
	sc->sc_otg.sc_bus.dma_bits = 32;

	/* get all DMA memory */
	if (usb_bus_mem_alloc_all(&sc->sc_otg.sc_bus,
	USB_GET_DMA_TAG(dev), NULL)) {
	device_printf(dev,
	"Failed allocate bus mem for musb\n");
	return (ENOMEM);
	}
	sc->sc_otg.sc_io_res = sc->sc_mem_res[RES_USBCORE];
	sc->sc_otg.sc_io_tag =
	rman_get_bustag(sc->sc_otg.sc_io_res);
	sc->sc_otg.sc_io_hdl =
	rman_get_bushandle(sc->sc_otg.sc_io_res);
	sc->sc_otg.sc_io_size =
	rman_get_size(sc->sc_otg.sc_io_res);

	sc->sc_otg.sc_bus.bdev = device_add_child(dev, "usbus", -1);
	if (!(sc->sc_otg.sc_bus.bdev)) {
	device_printf(dev, "No busdev for musb\n");
	goto error;
	}
	device_set_ivars(sc->sc_otg.sc_bus.bdev,
	&sc->sc_otg.sc_bus);

	err = bus_setup_intr(dev, sc->sc_otg.sc_irq_res,
	INTR_TYPE_BIO \| INTR_MPSAFE,
	NULL, (driver_intr_t *)musbotg_wrapper_interrupt,
	&sc->sc_otg, &sc->sc_otg.sc_intr_hdl);
	if (err) {
	sc->sc_otg.sc_intr_hdl = NULL;
	device_printf(dev,
	"Failed to setup interrupt for musb\n");
	goto error;
	}

	sc->sc_otg.sc_platform_data = sc;
	if (OF_getprop(ofw_bus_get_node(dev), "dr_mode", mode,
	sizeof(mode)) > 0) {
	if (strcasecmp(mode, "host") == 0)
	sc->sc_otg.sc_mode = MUSB2_HOST_MODE;
	else
	sc->sc_otg.sc_mode = MUSB2_DEVICE_MODE;
	} else {
	/* Beaglebone defaults: USB0 device, USB1 HOST. */
	if (sc->sc_otg.sc_id == 0)
	sc->sc_otg.sc_mode = MUSB2_DEVICE_MODE;
	else
	sc->sc_otg.sc_mode = MUSB2_HOST_MODE;
	}

	/*
	* software-controlled function
	*/

	if (sc->sc_otg.sc_mode == MUSB2_HOST_MODE) {
	reg = USBCTRL_READ4(sc, USBCTRL_MODE);
	reg \|= USBCTRL_MODE_IDDIGMUX;
	reg &= ~USBCTRL_MODE_IDDIG;
	USBCTRL_WRITE4(sc, USBCTRL_MODE, reg);
	USBCTRL_WRITE4(sc, USBCTRL_UTMI,
	USBCTRL_UTMI_FSDATAEXT);
	} else {
	reg = USBCTRL_READ4(sc, USBCTRL_MODE);
	reg \|= USBCTRL_MODE_IDDIGMUX;
	reg \|= USBCTRL_MODE_IDDIG;
	USBCTRL_WRITE4(sc, USBCTRL_MODE, reg);
	}

	reg = USBCTRL_INTEN_USB_ALL & ~USBCTRL_INTEN_USB_SOF;
	USBCTRL_WRITE4(sc, USBCTRL_INTEN_SET1, reg);
	USBCTRL_WRITE4(sc, USBCTRL_INTEN_CLR0, 0xffffffff);

	err = musbotg_init(&sc->sc_otg);
	if (!err)
	err = device_probe_and_attach(sc->sc_otg.sc_bus.bdev);

	if (err)
	goto error;

	/* poll VBUS one time */
	musbotg_vbus_poll(sc);

	return (0);

	error:
	musbotg_detach(dev);
	return (ENXIO);
	}

	static int
	musbotg_detach(device_t dev)
	{
	struct musbotg_super_softc *sc = device_get_softc(dev);
	int err;

	/* during module unload there are lots of children leftover */
	device_delete_children(dev);

	if (sc->sc_otg.sc_irq_res && sc->sc_otg.sc_intr_hdl) {
	/*
	* only call musbotg_uninit() after musbotg_init()
	*/
	musbotg_uninit(&sc->sc_otg);

	err = bus_teardown_intr(dev, sc->sc_otg.sc_irq_res,
	sc->sc_otg.sc_intr_hdl);
	sc->sc_otg.sc_intr_hdl = NULL;
	}

	usb_bus_mem_free_all(&sc->sc_otg.sc_bus, NULL);

	/* Free resources if any */
	if (sc->sc_mem_res[0])
	bus_release_resources(dev, am335x_musbotg_mem_spec,
	sc->sc_mem_res);

	if (sc->sc_otg.sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid,
	sc->sc_otg.sc_irq_res);

	return (0);
	}

	static device_method_t musbotg_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, musbotg_probe),
	DEVMETHOD(device_attach, musbotg_attach),
	DEVMETHOD(device_detach, musbotg_detach),
	DEVMETHOD(device_suspend, bus_generic_suspend),
	DEVMETHOD(device_resume, bus_generic_resume),
	DEVMETHOD(device_shutdown, bus_generic_shutdown),

	DEVMETHOD_END
	};

	static driver_t musbotg_driver = {
	.name = "musbotg",
	.methods = musbotg_methods,
	.size = sizeof(struct musbotg_super_softc),
	};

	static devclass_t musbotg_devclass;

	-DRIVER_MODULE(musbotg, usbss, musbotg_driver, musbotg_devclass, 0, 0);
	-MODULE_DEPEND(musbotg, usbss, 1, 1, 1);
	+DRIVER_MODULE(musbotg, ti_sysc, musbotg_driver, musbotg_devclass, 0, 0);
	+MODULE_DEPEND(musbotg, ti_sysc, 1, 1, 1);
	+MODULE_DEPEND(musbotg, ti_am3359_cppi41, 1, 1, 1);
	+MODULE_DEPEND(usbss, usb, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_prcm.c b/sys/arm/ti/am335x/am335x_prcm.c
	deleted file mode 100644
	index 875be72f4937..000000000000
	--- a/sys/arm/ti/am335x/am335x_prcm.c
	+++ /dev/null
	@@ -1,884 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/bus.h>
	-#include <sys/kernel.h>
	-#include <sys/module.h>
	-#include <sys/malloc.h>
	-#include <sys/rman.h>
	-#include <sys/timeet.h>
	-#include <sys/timetc.h>
	-#include <sys/watchdog.h>
	-#include <machine/bus.h>
	-#include <machine/cpu.h>
	-#include <machine/intr.h>
	-
	-#include <arm/ti/tivar.h>
	-#include <arm/ti/ti_scm.h>
	-#include <arm/ti/ti_prcm.h>
	-
	-#include <dev/ofw/openfirm.h>
	-#include <dev/ofw/ofw_bus.h>
	-#include <dev/ofw/ofw_bus_subr.h>
	-
	-#include <machine/bus.h>
	-
	-#include "am335x_scm.h"
	-
	-#define CM_PER 0
	-#define CM_PER_L4LS_CLKSTCTRL (CM_PER + 0x000)
	-#define CM_PER_L3S_CLKSTCTRL (CM_PER + 0x004)
	-#define CM_PER_L3_CLKSTCTRL (CM_PER + 0x00C)
	-#define CM_PER_CPGMAC0_CLKCTRL (CM_PER + 0x014)
	-#define CM_PER_LCDC_CLKCTRL (CM_PER + 0x018)
	-#define CM_PER_USB0_CLKCTRL (CM_PER + 0x01C)
	-#define CM_PER_TPTC0_CLKCTRL (CM_PER + 0x024)
	-#define CM_PER_UART5_CLKCTRL (CM_PER + 0x038)
	-#define CM_PER_MMC0_CLKCTRL (CM_PER + 0x03C)
	-#define CM_PER_I2C2_CLKCTRL (CM_PER + 0x044)
	-#define CM_PER_I2C1_CLKCTRL (CM_PER + 0x048)
	-#define CM_PER_SPI0_CLKCTRL (CM_PER + 0x04C)
	-#define CM_PER_SPI1_CLKCTRL (CM_PER + 0x050)
	-#define CM_PER_UART1_CLKCTRL (CM_PER + 0x06C)
	-#define CM_PER_UART2_CLKCTRL (CM_PER + 0x070)
	-#define CM_PER_UART3_CLKCTRL (CM_PER + 0x074)
	-#define CM_PER_UART4_CLKCTRL (CM_PER + 0x078)
	-#define CM_PER_TIMER7_CLKCTRL (CM_PER + 0x07C)
	-#define CM_PER_TIMER2_CLKCTRL (CM_PER + 0x080)
	-#define CM_PER_TIMER3_CLKCTRL (CM_PER + 0x084)
	-#define CM_PER_TIMER4_CLKCTRL (CM_PER + 0x088)
	-#define CM_PER_GPIO1_CLKCTRL (CM_PER + 0x0AC)
	-#define CM_PER_GPIO2_CLKCTRL (CM_PER + 0x0B0)
	-#define CM_PER_GPIO3_CLKCTRL (CM_PER + 0x0B4)
	-#define CM_PER_TPCC_CLKCTRL (CM_PER + 0x0BC)
	-#define CM_PER_EPWMSS1_CLKCTRL (CM_PER + 0x0CC)
	-#define CM_PER_EPWMSS0_CLKCTRL (CM_PER + 0x0D4)
	-#define CM_PER_EPWMSS2_CLKCTRL (CM_PER + 0x0D8)
	-#define CM_PER_L3_INSTR_CLKCTRL (CM_PER + 0x0DC)
	-#define CM_PER_L3_CLKCTRL (CM_PER + 0x0E0)
	-#define CM_PER_PRUSS_CLKCTRL (CM_PER + 0x0E8)
	-#define CM_PER_TIMER5_CLKCTRL (CM_PER + 0x0EC)
	-#define CM_PER_TIMER6_CLKCTRL (CM_PER + 0x0F0)
	-#define CM_PER_MMC1_CLKCTRL (CM_PER + 0x0F4)
	-#define CM_PER_MMC2_CLKCTRL (CM_PER + 0x0F8)
	-#define CM_PER_TPTC1_CLKCTRL (CM_PER + 0x0FC)
	-#define CM_PER_TPTC2_CLKCTRL (CM_PER + 0x100)
	-#define CM_PER_SPINLOCK0_CLKCTRL (CM_PER + 0x10C)
	-#define CM_PER_MAILBOX0_CLKCTRL (CM_PER + 0x110)
	-#define CM_PER_OCPWP_L3_CLKSTCTRL (CM_PER + 0x12C)
	-#define CM_PER_OCPWP_CLKCTRL (CM_PER + 0x130)
	-#define CM_PER_CPSW_CLKSTCTRL (CM_PER + 0x144)
	-#define CM_PER_PRUSS_CLKSTCTRL (CM_PER + 0x140)
	-
	-#define CM_WKUP 0x400
	-#define CM_WKUP_CLKSTCTRL (CM_WKUP + 0x000)
	-#define CM_WKUP_CONTROL_CLKCTRL (CM_WKUP + 0x004)
	-#define CM_WKUP_GPIO0_CLKCTRL (CM_WKUP + 0x008)
	-#define CM_WKUP_CM_L3_AON_CLKSTCTRL (CM_WKUP + 0x01C)
	-#define CM_WKUP_CM_CLKSEL_DPLL_MPU (CM_WKUP + 0x02C)
	-#define CM_WKUP_CM_IDLEST_DPLL_DISP (CM_WKUP + 0x048)
	-#define CM_WKUP_CM_CLKSEL_DPLL_DISP (CM_WKUP + 0x054)
	-#define CM_WKUP_CM_CLKDCOLDO_DPLL_PER (CM_WKUP + 0x07C)
	-#define CM_WKUP_CM_CLKMODE_DPLL_DISP (CM_WKUP + 0x098)
	-#define CM_WKUP_I2C0_CLKCTRL (CM_WKUP + 0x0B8)
	-#define CM_WKUP_ADC_TSC_CLKCTRL (CM_WKUP + 0x0BC)
	-
	-#define CM_DPLL 0x500
	-#define CLKSEL_TIMER7_CLK (CM_DPLL + 0x004)
	-#define CLKSEL_TIMER2_CLK (CM_DPLL + 0x008)
	-#define CLKSEL_TIMER3_CLK (CM_DPLL + 0x00C)
	-#define CLKSEL_TIMER4_CLK (CM_DPLL + 0x010)
	-#define CLKSEL_TIMER5_CLK (CM_DPLL + 0x018)
	-#define CLKSEL_TIMER6_CLK (CM_DPLL + 0x01C)
	-#define CLKSEL_PRUSS_OCP_CLK (CM_DPLL + 0x030)
	-
	-#define CM_RTC 0x800
	-#define CM_RTC_RTC_CLKCTRL (CM_RTC + 0x000)
	-#define CM_RTC_CLKSTCTRL (CM_RTC + 0x004)
	-
	-#define PRM_PER 0xC00
	-#define PRM_PER_RSTCTRL (PRM_PER + 0x00)
	-
	-#define PRM_DEVICE_OFFSET 0xF00
	-#define PRM_RSTCTRL (PRM_DEVICE_OFFSET + 0x00)
	-
	-struct am335x_prcm_softc {
	- struct resource * res[2];
	- bus_space_tag_t bst;
	- bus_space_handle_t bsh;
	- int attach_done;
	-};
	-
	-static struct resource_spec am335x_prcm_spec[] = {
	- { SYS_RES_MEMORY, 0, RF_ACTIVE },
	- { -1, 0 }
	-};
	-
	-static struct am335x_prcm_softc *am335x_prcm_sc = NULL;
	-
	-static int am335x_clk_noop_activate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_generic_activate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_gpio_activate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_noop_deactivate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_generic_deactivate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_noop_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc);
	-static int am335x_clk_generic_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc);
	-static int am335x_clk_hsmmc_get_source_freq(struct ti_clock_dev clkdev, unsigned int freq);
	-static int am335x_clk_get_sysclk_freq(struct ti_clock_dev clkdev, unsigned int freq);
	-static int am335x_clk_get_arm_fclk_freq(struct ti_clock_dev clkdev, unsigned int freq);
	-static int am335x_clk_get_arm_disp_freq(struct ti_clock_dev clkdev, unsigned int freq);
	-static int am335x_clk_set_arm_disp_freq(struct ti_clock_dev *clkdev, unsigned int freq);
	-static void am335x_prcm_reset(void);
	-static int am335x_clk_cpsw_activate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_musb0_activate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_lcdc_activate(struct ti_clock_dev *clkdev);
	-static int am335x_clk_pruss_activate(struct ti_clock_dev *clkdev);
	-
	-#define AM335X_NOOP_CLOCK_DEV(i) \
	- { .id = (i), \
	- .clk_activate = am335x_clk_noop_activate, \
	- .clk_deactivate = am335x_clk_noop_deactivate, \
	- .clk_set_source = am335x_clk_noop_set_source, \
	- .clk_accessible = NULL, \
	- .clk_get_source_freq = NULL, \
	- .clk_set_source_freq = NULL \
	- }
	-
	-#define AM335X_GENERIC_CLOCK_DEV(i) \
	- { .id = (i), \
	- .clk_activate = am335x_clk_generic_activate, \
	- .clk_deactivate = am335x_clk_generic_deactivate, \
	- .clk_set_source = am335x_clk_generic_set_source, \
	- .clk_accessible = NULL, \
	- .clk_get_source_freq = NULL, \
	- .clk_set_source_freq = NULL \
	- }
	-
	-#define AM335X_GPIO_CLOCK_DEV(i) \
	- { .id = (i), \
	- .clk_activate = am335x_clk_gpio_activate, \
	- .clk_deactivate = am335x_clk_generic_deactivate, \
	- .clk_set_source = am335x_clk_generic_set_source, \
	- .clk_accessible = NULL, \
	- .clk_get_source_freq = NULL, \
	- .clk_set_source_freq = NULL \
	- }
	-
	-#define AM335X_MMCHS_CLOCK_DEV(i) \
	- { .id = (i), \
	- .clk_activate = am335x_clk_generic_activate, \
	- .clk_deactivate = am335x_clk_generic_deactivate, \
	- .clk_set_source = am335x_clk_generic_set_source, \
	- .clk_accessible = NULL, \
	- .clk_get_source_freq = am335x_clk_hsmmc_get_source_freq, \
	- .clk_set_source_freq = NULL \
	- }
	-
	-struct ti_clock_dev ti_am335x_clk_devmap[] = {
	- /* System clocks */
	- { .id = SYS_CLK,
	- .clk_activate = NULL,
	- .clk_deactivate = NULL,
	- .clk_set_source = NULL,
	- .clk_accessible = NULL,
	- .clk_get_source_freq = am335x_clk_get_sysclk_freq,
	- .clk_set_source_freq = NULL,
	- },
	- /* MPU (ARM) core clocks */
	- { .id = MPU_CLK,
	- .clk_activate = NULL,
	- .clk_deactivate = NULL,
	- .clk_set_source = NULL,
	- .clk_accessible = NULL,
	- .clk_get_source_freq = am335x_clk_get_arm_fclk_freq,
	- .clk_set_source_freq = NULL,
	- },
	- /* CPSW Ethernet Switch core clocks */
	- { .id = CPSW_CLK,
	- .clk_activate = am335x_clk_cpsw_activate,
	- .clk_deactivate = NULL,
	- .clk_set_source = NULL,
	- .clk_accessible = NULL,
	- .clk_get_source_freq = NULL,
	- .clk_set_source_freq = NULL,
	- },
	-
	- /* Mentor USB HS controller core clocks */
	- { .id = MUSB0_CLK,
	- .clk_activate = am335x_clk_musb0_activate,
	- .clk_deactivate = NULL,
	- .clk_set_source = NULL,
	- .clk_accessible = NULL,
	- .clk_get_source_freq = NULL,
	- .clk_set_source_freq = NULL,
	- },
	-
	- /* LCD controller clocks */
	- { .id = LCDC_CLK,
	- .clk_activate = am335x_clk_lcdc_activate,
	- .clk_deactivate = NULL,
	- .clk_set_source = NULL,
	- .clk_accessible = NULL,
	- .clk_get_source_freq = am335x_clk_get_arm_disp_freq,
	- .clk_set_source_freq = am335x_clk_set_arm_disp_freq,
	- },
	-
	- /* UART */
	- AM335X_NOOP_CLOCK_DEV(UART1_CLK),
	- AM335X_GENERIC_CLOCK_DEV(UART2_CLK),
	- AM335X_GENERIC_CLOCK_DEV(UART3_CLK),
	- AM335X_GENERIC_CLOCK_DEV(UART4_CLK),
	- AM335X_GENERIC_CLOCK_DEV(UART5_CLK),
	- AM335X_GENERIC_CLOCK_DEV(UART6_CLK),
	-
	- /* DMTimer */
	- AM335X_GENERIC_CLOCK_DEV(TIMER2_CLK),
	- AM335X_GENERIC_CLOCK_DEV(TIMER3_CLK),
	- AM335X_GENERIC_CLOCK_DEV(TIMER4_CLK),
	- AM335X_GENERIC_CLOCK_DEV(TIMER5_CLK),
	- AM335X_GENERIC_CLOCK_DEV(TIMER6_CLK),
	- AM335X_GENERIC_CLOCK_DEV(TIMER7_CLK),
	-
	- /* GPIO, we use hwmods as reference, not units in spec */
	- AM335X_GPIO_CLOCK_DEV(GPIO1_CLK),
	- AM335X_GPIO_CLOCK_DEV(GPIO2_CLK),
	- AM335X_GPIO_CLOCK_DEV(GPIO3_CLK),
	- AM335X_GPIO_CLOCK_DEV(GPIO4_CLK),
	-
	- /* I2C we use hwmods as reference, not units in spec */
	- AM335X_GENERIC_CLOCK_DEV(I2C1_CLK),
	- AM335X_GENERIC_CLOCK_DEV(I2C2_CLK),
	- AM335X_GENERIC_CLOCK_DEV(I2C3_CLK),
	-
	- /* McSPI we use hwmods as reference, not units in spec */
	- AM335X_GENERIC_CLOCK_DEV(SPI0_CLK),
	- AM335X_GENERIC_CLOCK_DEV(SPI1_CLK),
	-
	- /* TSC_ADC */
	- AM335X_GENERIC_CLOCK_DEV(TSC_ADC_CLK),
	-
	- /* EDMA */
	- AM335X_GENERIC_CLOCK_DEV(EDMA_TPCC_CLK),
	- AM335X_GENERIC_CLOCK_DEV(EDMA_TPTC0_CLK),
	- AM335X_GENERIC_CLOCK_DEV(EDMA_TPTC1_CLK),
	- AM335X_GENERIC_CLOCK_DEV(EDMA_TPTC2_CLK),
	-
	- /* MMCHS */
	- AM335X_MMCHS_CLOCK_DEV(MMC1_CLK),
	- AM335X_MMCHS_CLOCK_DEV(MMC2_CLK),
	- AM335X_MMCHS_CLOCK_DEV(MMC3_CLK),
	-
	- /* PWMSS */
	- AM335X_GENERIC_CLOCK_DEV(PWMSS0_CLK),
	- AM335X_GENERIC_CLOCK_DEV(PWMSS1_CLK),
	- AM335X_GENERIC_CLOCK_DEV(PWMSS2_CLK),
	-
	- /* System Mailbox clock */
	- AM335X_GENERIC_CLOCK_DEV(MAILBOX0_CLK),
	-
	- /* SPINLOCK */
	- AM335X_GENERIC_CLOCK_DEV(SPINLOCK0_CLK),
	-
	- /* PRU-ICSS */
	- { .id = PRUSS_CLK,
	- .clk_activate = am335x_clk_pruss_activate,
	- .clk_deactivate = NULL,
	- .clk_set_source = NULL,
	- .clk_accessible = NULL,
	- .clk_get_source_freq = NULL,
	- .clk_set_source_freq = NULL,
	- },
	-
	- /* RTC */
	- AM335X_GENERIC_CLOCK_DEV(RTC_CLK),
	-
	- { INVALID_CLK_IDENT, NULL, NULL, NULL, NULL }
	-};
	-
	-struct am335x_clk_details {
	- clk_ident_t id;
	- uint32_t clkctrl_reg;
	- uint32_t clksel_reg;
	-};
	-
	-#define _CLK_DETAIL(i, c, s) \
	- { .id = (i), \
	- .clkctrl_reg = (c), \
	- .clksel_reg = (s), \
	- }
	-
	-static struct am335x_clk_details g_am335x_clk_details[] = {
	-
	- /* UART. UART0 clock not controllable. */
	- _CLK_DETAIL(UART1_CLK, 0, 0),
	- _CLK_DETAIL(UART2_CLK, CM_PER_UART1_CLKCTRL, 0),
	- _CLK_DETAIL(UART3_CLK, CM_PER_UART2_CLKCTRL, 0),
	- _CLK_DETAIL(UART4_CLK, CM_PER_UART3_CLKCTRL, 0),
	- _CLK_DETAIL(UART5_CLK, CM_PER_UART4_CLKCTRL, 0),
	- _CLK_DETAIL(UART6_CLK, CM_PER_UART5_CLKCTRL, 0),
	-
	- /* DMTimer modules */
	- _CLK_DETAIL(TIMER2_CLK, CM_PER_TIMER2_CLKCTRL, CLKSEL_TIMER2_CLK),
	- _CLK_DETAIL(TIMER3_CLK, CM_PER_TIMER3_CLKCTRL, CLKSEL_TIMER3_CLK),
	- _CLK_DETAIL(TIMER4_CLK, CM_PER_TIMER4_CLKCTRL, CLKSEL_TIMER4_CLK),
	- _CLK_DETAIL(TIMER5_CLK, CM_PER_TIMER5_CLKCTRL, CLKSEL_TIMER5_CLK),
	- _CLK_DETAIL(TIMER6_CLK, CM_PER_TIMER6_CLKCTRL, CLKSEL_TIMER6_CLK),
	- _CLK_DETAIL(TIMER7_CLK, CM_PER_TIMER7_CLKCTRL, CLKSEL_TIMER7_CLK),
	-
	- /* GPIO modules, hwmods start with gpio1 */
	- _CLK_DETAIL(GPIO1_CLK, CM_WKUP_GPIO0_CLKCTRL, 0),
	- _CLK_DETAIL(GPIO2_CLK, CM_PER_GPIO1_CLKCTRL, 0),
	- _CLK_DETAIL(GPIO3_CLK, CM_PER_GPIO2_CLKCTRL, 0),
	- _CLK_DETAIL(GPIO4_CLK, CM_PER_GPIO3_CLKCTRL, 0),
	-
	- /* I2C modules, hwmods start with i2c1 */
	- _CLK_DETAIL(I2C1_CLK, CM_WKUP_I2C0_CLKCTRL, 0),
	- _CLK_DETAIL(I2C2_CLK, CM_PER_I2C1_CLKCTRL, 0),
	- _CLK_DETAIL(I2C3_CLK, CM_PER_I2C2_CLKCTRL, 0),
	-
	- /* McSPI modules, hwmods start with spi0 */
	- _CLK_DETAIL(SPI0_CLK, CM_PER_SPI0_CLKCTRL, 0),
	- _CLK_DETAIL(SPI1_CLK, CM_PER_SPI1_CLKCTRL, 0),
	-
	- /* TSC_ADC module */
	- _CLK_DETAIL(TSC_ADC_CLK, CM_WKUP_ADC_TSC_CLKCTRL, 0),
	-
	- /* EDMA modules */
	- _CLK_DETAIL(EDMA_TPCC_CLK, CM_PER_TPCC_CLKCTRL, 0),
	- _CLK_DETAIL(EDMA_TPTC0_CLK, CM_PER_TPTC0_CLKCTRL, 0),
	- _CLK_DETAIL(EDMA_TPTC1_CLK, CM_PER_TPTC1_CLKCTRL, 0),
	- _CLK_DETAIL(EDMA_TPTC2_CLK, CM_PER_TPTC2_CLKCTRL, 0),
	-
	- /* MMCHS modules, hwmods start with mmc1*/
	- _CLK_DETAIL(MMC1_CLK, CM_PER_MMC0_CLKCTRL, 0),
	- _CLK_DETAIL(MMC2_CLK, CM_PER_MMC1_CLKCTRL, 0),
	- _CLK_DETAIL(MMC3_CLK, CM_PER_MMC1_CLKCTRL, 0),
	-
	- /* PWMSS modules */
	- _CLK_DETAIL(PWMSS0_CLK, CM_PER_EPWMSS0_CLKCTRL, 0),
	- _CLK_DETAIL(PWMSS1_CLK, CM_PER_EPWMSS1_CLKCTRL, 0),
	- _CLK_DETAIL(PWMSS2_CLK, CM_PER_EPWMSS2_CLKCTRL, 0),
	-
	- _CLK_DETAIL(MAILBOX0_CLK, CM_PER_MAILBOX0_CLKCTRL, 0),
	- _CLK_DETAIL(SPINLOCK0_CLK, CM_PER_SPINLOCK0_CLKCTRL, 0),
	-
	- /* RTC module */
	- _CLK_DETAIL(RTC_CLK, CM_RTC_RTC_CLKCTRL, 0),
	-
	- { INVALID_CLK_IDENT, 0},
	-};
	-
	-/* Read/Write macros */
	-#define prcm_read_4(reg) \
	- bus_space_read_4(am335x_prcm_sc->bst, am335x_prcm_sc->bsh, reg)
	-#define prcm_write_4(reg, val) \
	- bus_space_write_4(am335x_prcm_sc->bst, am335x_prcm_sc->bsh, reg, val)
	-
	-void am335x_prcm_setup_dmtimer(int);
	-
	-static int
	-am335x_prcm_probe(device_t dev)
	-{
	-
	- if (!ofw_bus_status_okay(dev))
	- return (ENXIO);
	-
	- if (ofw_bus_is_compatible(dev, "ti,am3-prcm")) {
	- device_set_desc(dev, "AM335x Power and Clock Management");
	- return(BUS_PROBE_DEFAULT);
	- }
	-
	- return (ENXIO);
	-}
	-
	-static int
	-am335x_prcm_attach(device_t dev)
	-{
	- struct am335x_prcm_softc *sc = device_get_softc(dev);
	-
	- if (am335x_prcm_sc)
	- return (ENXIO);
	-
	- if (bus_alloc_resources(dev, am335x_prcm_spec, sc->res)) {
	- device_printf(dev, "could not allocate resources\n");
	- return (ENXIO);
	- }
	-
	- sc->bst = rman_get_bustag(sc->res[0]);
	- sc->bsh = rman_get_bushandle(sc->res[0]);
	-
	- am335x_prcm_sc = sc;
	- ti_cpu_reset = am335x_prcm_reset;
	-
	- return (0);
	-}
	-
	-static void
	-am335x_prcm_new_pass(device_t dev)
	-{
	- struct am335x_prcm_softc *sc = device_get_softc(dev);
	- unsigned int sysclk, fclk;
	-
	- sc = device_get_softc(dev);
	- if (sc->attach_done \|\|
	- bus_current_pass < (BUS_PASS_TIMER + BUS_PASS_ORDER_EARLY)) {
	- bus_generic_new_pass(dev);
	- return;
	- }
	-
	- sc->attach_done = 1;
	-
	- if (am335x_clk_get_sysclk_freq(NULL, &sysclk) != 0)
	- sysclk = 0;
	- if (am335x_clk_get_arm_fclk_freq(NULL, &fclk) != 0)
	- fclk = 0;
	- if (sysclk && fclk)
	- device_printf(dev, "Clocks: System %u.%01u MHz, CPU %u MHz\n",
	- sysclk/1000000, (sysclk % 1000000)/100000, fclk/1000000);
	- else {
	- device_printf(dev, "can't read frequencies yet (SCM device not ready?)\n");
	- goto fail;
	- }
	-
	- return;
	-
	-fail:
	- device_detach(dev);
	- return;
	-}
	-
	-static device_method_t am335x_prcm_methods[] = {
	- DEVMETHOD(device_probe, am335x_prcm_probe),
	- DEVMETHOD(device_attach, am335x_prcm_attach),
	-
	- /* Bus interface */
	- DEVMETHOD(bus_new_pass, am335x_prcm_new_pass),
	- { 0, 0 }
	-};
	-
	-static driver_t am335x_prcm_driver = {
	- "am335x_prcm",
	- am335x_prcm_methods,
	- sizeof(struct am335x_prcm_softc),
	-};
	-
	-static devclass_t am335x_prcm_devclass;
	-
	-EARLY_DRIVER_MODULE(am335x_prcm, simplebus, am335x_prcm_driver,
	- am335x_prcm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	-MODULE_VERSION(am335x_prcm, 1);
	-MODULE_DEPEND(am335x_prcm, ti_scm, 1, 1, 1);
	-
	-static struct am335x_clk_details*
	-am335x_clk_details(clk_ident_t id)
	-{
	- struct am335x_clk_details *walker;
	-
	- for (walker = g_am335x_clk_details; walker->id != INVALID_CLK_IDENT; walker++) {
	- if (id == walker->id)
	- return (walker);
	- }
	-
	- return NULL;
	-}
	-
	-static int
	-am335x_clk_noop_activate(struct ti_clock_dev *clkdev)
	-{
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_generic_activate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	- struct am335x_clk_details* clk_details;
	-
	- if (sc == NULL)
	- return ENXIO;
	-
	- clk_details = am335x_clk_details(clkdev->id);
	-
	- if (clk_details == NULL)
	- return (ENXIO);
	-
	- /* set _CLKCTRL register MODULEMODE[1:0] to enable(2) /
	- prcm_write_4(clk_details->clkctrl_reg, 2);
	- while ((prcm_read_4(clk_details->clkctrl_reg) & 0x3) != 2)
	- DELAY(10);
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_gpio_activate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	- struct am335x_clk_details* clk_details;
	-
	- if (sc == NULL)
	- return ENXIO;
	-
	- clk_details = am335x_clk_details(clkdev->id);
	-
	- if (clk_details == NULL)
	- return (ENXIO);
	-
	- /* set _CLKCTRL register MODULEMODE[1:0] to enable(2) /
	- /* set _CLKCTRL register OPTFCLKEN_GPIO_1_G DBCLK[18] to FCLK_EN(1) /
	- prcm_write_4(clk_details->clkctrl_reg, 2 \| (1 << 18));
	- while ((prcm_read_4(clk_details->clkctrl_reg) &
	- (3 \| (1 << 18) )) != (2 \| (1 << 18)))
	- DELAY(10);
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_noop_deactivate(struct ti_clock_dev *clkdev)
	-{
	-
	- return(0);
	-}
	-
	-static int
	-am335x_clk_generic_deactivate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	- struct am335x_clk_details* clk_details;
	-
	- if (sc == NULL)
	- return ENXIO;
	-
	- clk_details = am335x_clk_details(clkdev->id);
	-
	- if (clk_details == NULL)
	- return (ENXIO);
	-
	- /* set _CLKCTRL register MODULEMODE[1:0] to disable(0) /
	- prcm_write_4(clk_details->clkctrl_reg, 0);
	- while ((prcm_read_4(clk_details->clkctrl_reg) & 0x3) != 0)
	- DELAY(10);
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_noop_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc)
	-{
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_generic_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	- struct am335x_clk_details* clk_details;
	- uint32_t reg;
	-
	- if (sc == NULL)
	- return ENXIO;
	-
	- clk_details = am335x_clk_details(clkdev->id);
	-
	- if (clk_details == NULL)
	- return (ENXIO);
	-
	- switch (clksrc) {
	- case EXT_CLK:
	- reg = 0; /* SEL2: TCLKIN clock */
	- break;
	- case SYSCLK_CLK:
	- reg = 1; /* SEL1: CLK_M_OSC clock */
	- break;
	- case F32KHZ_CLK:
	- reg = 2; /* SEL3: CLK_32KHZ clock */
	- break;
	- default:
	- return (ENXIO);
	- }
	-
	- prcm_write_4(clk_details->clksel_reg, reg);
	- while ((prcm_read_4(clk_details->clksel_reg) & 0x3) != reg)
	- DELAY(10);
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_hsmmc_get_source_freq(struct ti_clock_dev clkdev, unsigned int freq)
	-{
	- *freq = 96000000;
	- return (0);
	-}
	-
	-static int
	-am335x_clk_get_sysclk_freq(struct ti_clock_dev clkdev, unsigned int freq)
	-{
	- uint32_t ctrl_status;
	-
	- /* Read the input clock freq from the control module. */
	- if (ti_scm_reg_read_4(SCM_CTRL_STATUS, &ctrl_status))
	- return (ENXIO);
	-
	- switch ((ctrl_status>>22) & 0x3) {
	- case 0x0:
	- /* 19.2Mhz */
	- *freq = 19200000;
	- break;
	- case 0x1:
	- /* 24Mhz */
	- *freq = 24000000;
	- break;
	- case 0x2:
	- /* 25Mhz */
	- *freq = 25000000;
	- break;
	- case 0x3:
	- /* 26Mhz */
	- *freq = 26000000;
	- break;
	- }
	-
	- return (0);
	-}
	-
	-#define DPLL_BYP_CLKSEL(reg) ((reg>>23) & 1)
	-#define DPLL_DIV(reg) ((reg & 0x7f)+1)
	-#define DPLL_MULT(reg) ((reg>>8) & 0x7FF)
	-#define DPLL_MAX_MUL 0x800
	-#define DPLL_MAX_DIV 0x80
	-
	-static int
	-am335x_clk_get_arm_fclk_freq(struct ti_clock_dev clkdev, unsigned int freq)
	-{
	- uint32_t reg;
	- uint32_t sysclk;
	-
	- reg = prcm_read_4(CM_WKUP_CM_CLKSEL_DPLL_MPU);
	-
	- /Check if we are running in bypass /
	- if (DPLL_BYP_CLKSEL(reg))
	- return ENXIO;
	-
	- am335x_clk_get_sysclk_freq(NULL, &sysclk);
	- freq = DPLL_MULT(reg) (sysclk / DPLL_DIV(reg));
	- return(0);
	-}
	-
	-static int
	-am335x_clk_get_arm_disp_freq(struct ti_clock_dev clkdev, unsigned int freq)
	-{
	- uint32_t reg;
	- uint32_t sysclk;
	-
	- reg = prcm_read_4(CM_WKUP_CM_CLKSEL_DPLL_DISP);
	-
	- /Check if we are running in bypass /
	- if (DPLL_BYP_CLKSEL(reg))
	- return ENXIO;
	-
	- am335x_clk_get_sysclk_freq(NULL, &sysclk);
	- freq = DPLL_MULT(reg) (sysclk / DPLL_DIV(reg));
	- return(0);
	-}
	-
	-static int
	-am335x_clk_set_arm_disp_freq(struct ti_clock_dev *clkdev, unsigned int freq)
	-{
	- uint32_t sysclk;
	- uint32_t mul, div;
	- uint32_t i, j;
	- unsigned int delta, min_delta;
	-
	- am335x_clk_get_sysclk_freq(NULL, &sysclk);
	-
	- /* Bypass mode */
	- prcm_write_4(CM_WKUP_CM_CLKMODE_DPLL_DISP, 0x4);
	-
	- /* Make sure it's in bypass mode */
	- while (!(prcm_read_4(CM_WKUP_CM_IDLEST_DPLL_DISP)
	- & (1 << 8)))
	- DELAY(10);
	-
	- /* Dumb and non-optimal implementation */
	- min_delta = freq;
	- for (i = 1; i < DPLL_MAX_MUL; i++) {
	- for (j = 1; j < DPLL_MAX_DIV; j++) {
	- delta = abs(freq - i*(sysclk/j));
	- if (delta < min_delta) {
	- mul = i;
	- div = j;
	- min_delta = delta;
	- }
	- if (min_delta == 0)
	- break;
	- }
	- }
	-
	- prcm_write_4(CM_WKUP_CM_CLKSEL_DPLL_DISP, (mul << 8) \| (div - 1));
	-
	- /* Locked mode */
	- prcm_write_4(CM_WKUP_CM_CLKMODE_DPLL_DISP, 0x7);
	-
	- int timeout = 10000;
	- while ((!(prcm_read_4(CM_WKUP_CM_IDLEST_DPLL_DISP)
	- & (1 << 0))) && timeout--)
	- DELAY(10);
	-
	- return(0);
	-}
	-
	-static void
	-am335x_prcm_reset(void)
	-{
	- prcm_write_4(PRM_RSTCTRL, (1<<1));
	-}
	-
	-static int
	-am335x_clk_cpsw_activate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	-
	- if (sc == NULL)
	- return ENXIO;
	-
	- /* set MODULENAME to ENABLE */
	- prcm_write_4(CM_PER_CPGMAC0_CLKCTRL, 2);
	-
	- /* wait for IDLEST to become Func(0) */
	- while(prcm_read_4(CM_PER_CPGMAC0_CLKCTRL) & (3<<16));
	-
	- /set CLKTRCTRL to SW_WKUP(2) /
	- prcm_write_4(CM_PER_CPSW_CLKSTCTRL, 2);
	-
	- /* wait for 125 MHz OCP clock to become active */
	- while((prcm_read_4(CM_PER_CPSW_CLKSTCTRL) & (1<<4)) == 0);
	- return(0);
	-}
	-
	-static int
	-am335x_clk_musb0_activate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	-
	- if (sc == NULL)
	- return ENXIO;
	-
	- /* set ST_DPLL_CLKDCOLDO(9) to CLK_GATED(1) */
	- /* set DPLL_CLKDCOLDO_GATE_CTRL(8) to CLK_ENABLE(1)*/
	- prcm_write_4(CM_WKUP_CM_CLKDCOLDO_DPLL_PER, 0x300);
	-
	- /set MODULEMODE to ENABLE(2) /
	- prcm_write_4(CM_PER_USB0_CLKCTRL, 2);
	-
	- /* wait for MODULEMODE to become ENABLE(2) */
	- while ((prcm_read_4(CM_PER_USB0_CLKCTRL) & 0x3) != 2)
	- DELAY(10);
	-
	- /* wait for IDLEST to become Func(0) */
	- while(prcm_read_4(CM_PER_USB0_CLKCTRL) & (3<<16))
	- DELAY(10);
	-
	- return(0);
	-}
	-
	-static int
	-am335x_clk_lcdc_activate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	-
	- if (sc == NULL)
	- return (ENXIO);
	-
	- /*
	- * For now set frequency to 2*VGA_PIXEL_CLOCK
	- */
	- am335x_clk_set_arm_disp_freq(clkdev, 25175000*2);
	-
	- /set MODULEMODE to ENABLE(2) /
	- prcm_write_4(CM_PER_LCDC_CLKCTRL, 2);
	-
	- /* wait for MODULEMODE to become ENABLE(2) */
	- while ((prcm_read_4(CM_PER_LCDC_CLKCTRL) & 0x3) != 2)
	- DELAY(10);
	-
	- /* wait for IDLEST to become Func(0) */
	- while(prcm_read_4(CM_PER_LCDC_CLKCTRL) & (3<<16))
	- DELAY(10);
	-
	- return (0);
	-}
	-
	-static int
	-am335x_clk_pruss_activate(struct ti_clock_dev *clkdev)
	-{
	- struct am335x_prcm_softc *sc = am335x_prcm_sc;
	-
	- if (sc == NULL)
	- return (ENXIO);
	-
	- /* Set MODULEMODE to ENABLE(2) */
	- prcm_write_4(CM_PER_PRUSS_CLKCTRL, 2);
	-
	- /* Wait for MODULEMODE to become ENABLE(2) */
	- while ((prcm_read_4(CM_PER_PRUSS_CLKCTRL) & 0x3) != 2)
	- DELAY(10);
	-
	- /* Set CLKTRCTRL to SW_WKUP(2) */
	- prcm_write_4(CM_PER_PRUSS_CLKSTCTRL, 2);
	-
	- /* Wait for the 200 MHz OCP clock to become active */
	- while ((prcm_read_4(CM_PER_PRUSS_CLKSTCTRL) & (1<<4)) == 0)
	- DELAY(10);
	-
	- /* Wait for the 200 MHz IEP clock to become active */
	- while ((prcm_read_4(CM_PER_PRUSS_CLKSTCTRL) & (1<<5)) == 0)
	- DELAY(10);
	-
	- /* Wait for the 192 MHz UART clock to become active */
	- while ((prcm_read_4(CM_PER_PRUSS_CLKSTCTRL) & (1<<6)) == 0)
	- DELAY(10);
	-
	- /* Select L3F as OCP clock */
	- prcm_write_4(CLKSEL_PRUSS_OCP_CLK, 0);
	- while ((prcm_read_4(CLKSEL_PRUSS_OCP_CLK) & 0x3) != 0)
	- DELAY(10);
	-
	- /* Clear the RESET bit */
	- prcm_write_4(PRM_PER_RSTCTRL, prcm_read_4(PRM_PER_RSTCTRL) & ~2);
	-
	- return (0);
	-}
	diff --git a/sys/arm/ti/am335x/am335x_pwmss.c b/sys/arm/ti/am335x/am335x_pwmss.c
	index 814865d6b762..287b6ce9e8fe 100644
	--- a/sys/arm/ti/am335x/am335x_pwmss.c
	+++ b/sys/arm/ti/am335x/am335x_pwmss.c
	@@ -1,163 +1,179 @@
	/*-
	* Copyright (c) 2013 Oleksandr Tymoshenko <gonzo@freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/resource.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>

	#include <machine/bus.h>

	#include <dev/fdt/simplebus.h>
	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	-#include <arm/ti/ti_scm.h>
	+#include <arm/ti/ti_sysc.h>
	+
	+#include <dev/extres/syscon/syscon.h>
	+#include "syscon_if.h"

	#include "am335x_pwm.h"
	#include "am335x_scm.h"

	#define PWMSS_IDVER 0x00
	#define PWMSS_SYSCONFIG 0x04
	#define PWMSS_CLKCONFIG 0x08
	#define CLKCONFIG_EPWMCLK_EN (1 << 8)
	#define PWMSS_CLKSTATUS 0x0C

	+/* TRM chapter 2 memory map table 2-3 + VER register location */
	+#define PWMSS_REV_0 0x0000
	+#define PWMSS_REV_1 0x2000
	+#define PWMSS_REV_2 0x4000
	+
	static device_probe_t am335x_pwmss_probe;
	static device_attach_t am335x_pwmss_attach;
	static device_detach_t am335x_pwmss_detach;

	struct am335x_pwmss_softc {
	struct simplebus_softc sc_simplebus;
	device_t sc_dev;
	- clk_ident_t sc_clk;
	+ struct syscon *syscon;
	};

	static device_method_t am335x_pwmss_methods[] = {
	DEVMETHOD(device_probe, am335x_pwmss_probe),
	DEVMETHOD(device_attach, am335x_pwmss_attach),
	DEVMETHOD(device_detach, am335x_pwmss_detach),

	DEVMETHOD_END
	};

	static int
	am335x_pwmss_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,am33xx-pwmss"))
	return (ENXIO);

	device_set_desc(dev, "AM335x PWM");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	am335x_pwmss_attach(device_t dev)
	{
	struct am335x_pwmss_softc *sc;
	uint32_t reg, id;
	- phandle_t node;
	+ uint64_t rev_address;
	+ phandle_t node, opp_table;

	sc = device_get_softc(dev);
	sc->sc_dev = dev;

	- sc->sc_clk = ti_hwmods_get_clock(dev);
	- if (sc->sc_clk == INVALID_CLK_IDENT) {
	- device_printf(dev, "failed to get device id based on ti,hwmods\n");
	- return (EINVAL);
	+ /* FIXME: For now; Go and kidnap syscon from opp-table */
	+ opp_table = OF_finddevice("/opp-table");
	+ if (opp_table == -1) {
	+ device_printf(dev, "Cant find /opp-table\n");
	+ return (ENXIO);
	+ }
	+ if (!OF_hasprop(opp_table, "syscon")) {
	+ device_printf(dev, "/opp-table doesnt have required syscon property\n");
	+ return (ENXIO);
	+ }
	+ if (syscon_get_by_ofw_property(dev, opp_table, "syscon", &sc->syscon) != 0) {
	+ device_printf(dev, "Failed to get syscon\n");
	+ return (ENXIO);
	}

	- ti_prcm_clk_enable(sc->sc_clk);
	- ti_scm_reg_read_4(SCM_PWMSS_CTRL, &reg);
	- switch (sc->sc_clk) {
	- case PWMSS0_CLK:
	- id = 0;
	- break;
	- case PWMSS1_CLK:
	- id = 1;
	- break;
	-
	- case PWMSS2_CLK:
	- id = 2;
	- break;
	- default:
	- device_printf(dev, "unknown pwmss clock id: %d\n", sc->sc_clk);
	- return (EINVAL);
	+ ti_sysc_clock_enable(device_get_parent(dev));
	+
	+ rev_address = ti_sysc_get_rev_address(device_get_parent(dev));
	+ switch (rev_address) {
	+ case PWMSS_REV_0:
	+ id = 0;
	+ break;
	+ case PWMSS_REV_1:
	+ id = 1;
	+ break;
	+ case PWMSS_REV_2:
	+ id = 2;
	+ break;
	}
	+
	+ reg = SYSCON_READ_4(sc->syscon, SCM_PWMSS_CTRL);
	reg \|= (1 << id);
	- ti_scm_reg_write_4(SCM_PWMSS_CTRL, reg);
	+ SYSCON_WRITE_4(sc->syscon, SCM_PWMSS_CTRL, reg);

	node = ofw_bus_get_node(dev);

	if (node == -1)
	return (ENXIO);

	simplebus_init(dev, node);

	/*
	* Allow devices to identify.
	*/
	bus_generic_probe(dev);

	/*
	* Now walk the OFW tree and attach top-level devices.
	*/
	for (node = OF_child(node); node > 0; node = OF_peer(node))
	simplebus_add_device(dev, node, 0, NULL, -1, NULL);

	return (bus_generic_attach(dev));
	}

	static int
	am335x_pwmss_detach(device_t dev)
	{

	return (0);
	}

	DEFINE_CLASS_1(am335x_pwmss, am335x_pwmss_driver, am335x_pwmss_methods,
	sizeof(struct am335x_pwmss_softc), simplebus_driver);
	static devclass_t am335x_pwmss_devclass;
	DRIVER_MODULE(am335x_pwmss, simplebus, am335x_pwmss_driver, am335x_pwmss_devclass, 0, 0);
	MODULE_VERSION(am335x_pwmss, 1);
	+MODULE_DEPEND(am335x_pwmss, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_rtc.c b/sys/arm/ti/am335x/am335x_rtc.c
	index 9620f5284e84..c2612d2fc7a9 100644
	--- a/sys/arm/ti/am335x/am335x_rtc.c
	+++ b/sys/arm/ti/am335x/am335x_rtc.c
	@@ -1,211 +1,213 @@
	/*-
	* Copyright (c) 2015 Luiz Otavio O Souza <loos@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/clock.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>

	#include <machine/bus.h>

	+#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>
	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/am335x/am335x_rtcvar.h>
	#include <arm/ti/am335x/am335x_rtcreg.h>

	#define RTC_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx)
	#define RTC_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx)
	#define RTC_LOCK_INIT(_sc) mtx_init(&(_sc)->sc_mtx, \
	device_get_nameunit(_sc->sc_dev), "am335x_rtc", MTX_DEF)
	#define RTC_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx)

	#define RTC_READ4(_sc, reg) \
	bus_read_4((_sc)->sc_mem_res, reg)
	#define RTC_WRITE4(_sc, reg, value) \
	bus_write_4((_sc)->sc_mem_res, reg, value)

	#define RTC_MAXIRQS 2

	struct am335x_rtc_softc {
	device_t sc_dev;
	struct mtx sc_mtx;
	struct resource *sc_irq_res[RTC_MAXIRQS];
	struct resource *sc_mem_res;
	};

	static struct am335x_rtc_softc *rtc_sc = NULL;
	static struct resource_spec am335x_rtc_irq_spec[] = {
	{ SYS_RES_IRQ, 0, RF_ACTIVE },
	{ SYS_RES_IRQ, 1, RF_ACTIVE },
	{ -1, 0, 0 }
	};

	static int
	am335x_rtc_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);
	if (!ofw_bus_is_compatible(dev, "ti,da830-rtc"))
	return (ENXIO);
	device_set_desc(dev, "AM335x RTC (power management mode)");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	am335x_rtc_attach(device_t dev)
	{
	int rid;
	struct am335x_rtc_softc *sc;
	uint32_t rev;

	if (rtc_sc != NULL)
	return (ENXIO);
	rtc_sc = sc = device_get_softc(dev);
	sc->sc_dev = dev;
	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (!sc->sc_mem_res) {
	device_printf(dev, "cannot allocate memory resources\n");
	return (ENXIO);
	}
	if (bus_alloc_resources(dev, am335x_rtc_irq_spec, sc->sc_irq_res) != 0) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "cannot allocate irq resources\n");
	return (ENXIO);
	}
	RTC_LOCK_INIT(sc);

	/* Enable the RTC module. */
	- ti_prcm_clk_enable(RTC_CLK);
	+ ti_sysc_clock_enable(device_get_parent(dev));
	rev = RTC_READ4(sc, RTC_REVISION);
	device_printf(dev, "AM335X RTC v%d.%d.%d\n",
	(rev >> 8) & 0x7, (rev >> 6) & 0x3, rev & 0x3f);
	/* Unlock the RTC. */
	RTC_WRITE4(sc, RTC_KICK0R, RTC_KICK0R_PASS);
	RTC_WRITE4(sc, RTC_KICK1R, RTC_KICK1R_PASS);
	/* Stop the RTC, we don't need it right now. */
	RTC_WRITE4(sc, RTC_CTRL, 0);
	/* Disable interrupts. */
	RTC_WRITE4(sc, RTC_INTR, 0);
	/* Ack any pending interrupt. */
	RTC_WRITE4(sc, RTC_STATUS, RTC_STATUS_ALARM2 \| RTC_STATUS_ALARM);
	/* Enable external clock (xtal) and 32 kHz clock. */
	RTC_WRITE4(sc, RTC_OSC, RTC_OSC_32KCLK_EN \| RTC_OSC_32KCLK_SEL);
	/* Enable pmic_pwr_enable. */
	RTC_WRITE4(sc, RTC_PMIC, PMIC_PWR_ENABLE);

	return (0);
	}

	static int
	am335x_rtc_detach(device_t dev)
	{
	struct am335x_rtc_softc *sc;

	sc = device_get_softc(dev);
	if (sc->sc_irq_res[0] != NULL)
	bus_release_resources(dev, am335x_rtc_irq_spec, sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	RTC_LOCK_DESTROY(sc);

	return (0);
	}

	void
	am335x_rtc_pmic_pwr_toggle(void)
	{
	int timeout;
	struct clocktime ct;
	struct timespec ts;

	/*
	* We stop the RTC so we don't need to check the STATUS.BUSY bit
	* before update ALARM2 registers.
	*/
	timeout = 10;
	RTC_WRITE4(rtc_sc, RTC_CTRL, 0);
	while (--timeout && RTC_READ4(rtc_sc, RTC_STATUS) & RTC_STATUS_RUN)
	DELAY(100);
	if (timeout == 0) {
	device_printf(rtc_sc->sc_dev, "RTC does not stop.\n");
	return;
	}
	/* Program the ALARM2 to fire in 2 seconds. */
	ct.dow = 0;
	ct.nsec = 0;
	ct.sec = FROMBCD(RTC_READ4(rtc_sc, RTC_SECONDS) & 0x7f);
	ct.min = FROMBCD(RTC_READ4(rtc_sc, RTC_MINUTES) & 0x7f);
	ct.hour = FROMBCD(RTC_READ4(rtc_sc, RTC_HOURS) & 0x3f);
	ct.day = FROMBCD(RTC_READ4(rtc_sc, RTC_DAYS) & 0x3f);
	ct.mon = FROMBCD(RTC_READ4(rtc_sc, RTC_MONTHS) & 0x1f);
	ct.year = FROMBCD(RTC_READ4(rtc_sc, RTC_YEARS) & 0xff);
	ct.year += POSIX_BASE_YEAR;
	clock_ct_to_ts(&ct, &ts);
	ts.tv_sec += 2;
	clock_ts_to_ct(&ts, &ct);
	RTC_WRITE4(rtc_sc, RTC_ALARM2_SECONDS, TOBCD(ct.sec));
	RTC_WRITE4(rtc_sc, RTC_ALARM2_MINUTES, TOBCD(ct.min));
	RTC_WRITE4(rtc_sc, RTC_ALARM2_HOURS, TOBCD(ct.hour));
	RTC_WRITE4(rtc_sc, RTC_ALARM2_DAYS, TOBCD(ct.day));
	RTC_WRITE4(rtc_sc, RTC_ALARM2_MONTHS, TOBCD(ct.mon));
	RTC_WRITE4(rtc_sc, RTC_ALARM2_YEARS, TOBCD(ct.year - POSIX_BASE_YEAR));
	/* Enable ALARM2 interrupt. */
	RTC_WRITE4(rtc_sc, RTC_INTR, RTC_INTR_ALARM2);
	/* Start count. */
	RTC_WRITE4(rtc_sc, RTC_CTRL, RTC_CTRL_RUN);
	}

	static device_method_t am335x_rtc_methods[] = {
	DEVMETHOD(device_probe, am335x_rtc_probe),
	DEVMETHOD(device_attach, am335x_rtc_attach),
	DEVMETHOD(device_detach, am335x_rtc_detach),

	DEVMETHOD_END
	};

	static driver_t am335x_rtc_driver = {
	"am335x_rtc",
	am335x_rtc_methods,
	sizeof(struct am335x_rtc_softc),
	};

	static devclass_t am335x_rtc_devclass;

	DRIVER_MODULE(am335x_rtc, simplebus, am335x_rtc_driver, am335x_rtc_devclass, 0, 0);
	MODULE_VERSION(am335x_rtc, 1);
	MODULE_DEPEND(am335x_rtc, simplebus, 1, 1, 1);
	+MODULE_DEPEND(am335x_rtc, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_scm.c b/sys/arm/ti/am335x/am335x_scm.c
	index 6f040ed74b7d..e72c14ba58ad 100644
	--- a/sys/arm/ti/am335x/am335x_scm.c
	+++ b/sys/arm/ti/am335x/am335x_scm.c
	@@ -1,172 +1,198 @@
	/*-
	* Copyright (c) 2016 Rubicon Communications, LLC (Netgate)
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/sysctl.h>

	#include <machine/bus.h>

	#include <arm/ti/am335x/am335x_scm.h>
	#include <arm/ti/ti_cpuid.h>
	#include <arm/ti/ti_scm.h>

	+#include <dev/extres/syscon/syscon.h>
	+#include "syscon_if.h"
	+
	#define TZ_ZEROC 2731

	struct am335x_scm_softc {
	int sc_last_temp;
	struct sysctl_oid *sc_temp_oid;
	+ struct syscon *syscon;
	};

	static int
	am335x_scm_temp_sysctl(SYSCTL_HANDLER_ARGS)
	{
	device_t dev;
	int i, temp;
	struct am335x_scm_softc *sc;
	uint32_t reg;

	dev = (device_t)arg1;
	sc = device_get_softc(dev);

	/* Read the temperature and convert to Kelvin. */
	for(i = 50; i > 0; i--) {
	- ti_scm_reg_read_4(SCM_BGAP_CTRL, &reg);
	+ reg = SYSCON_READ_4(sc->syscon, SCM_BGAP_CTRL);
	if ((reg & SCM_BGAP_EOCZ) == 0)
	break;
	DELAY(50);
	}
	if ((reg & SCM_BGAP_EOCZ) == 0) {
	sc->sc_last_temp =
	(reg >> SCM_BGAP_TEMP_SHIFT) & SCM_BGAP_TEMP_MASK;
	sc->sc_last_temp *= 10;
	}
	temp = sc->sc_last_temp + TZ_ZEROC;

	return (sysctl_handle_int(oidp, &temp, 0, req));
	}

	static void
	am335x_scm_identify(driver_t *driver, device_t parent)
	{
	device_t child;

	/* AM335x only. */
	if (ti_chip() != CHIP_AM335X)
	return;

	/* Make sure we attach only once. */
	if (device_find_child(parent, "am335x_scm", -1) != NULL)
	return;

	child = device_add_child(parent, "am335x_scm", -1);
	if (child == NULL)
	device_printf(parent, "cannot add ti_scm child\n");
	}

	static int
	am335x_scm_probe(device_t dev)
	{
	+ /* Just allow the first one */
	+ if (strcmp(device_get_nameunit(dev), "am335x_scm0") != 0)
	+ return (ENXIO);

	device_set_desc(dev, "AM335x Control Module Extension");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	am335x_scm_attach(device_t dev)
	{
	struct am335x_scm_softc *sc;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid_list *tree;
	uint32_t reg;
	+ phandle_t opp_table;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+
	+ /* FIXME: For now; Go and kidnap syscon from opp-table */
	+ opp_table = OF_finddevice("/opp-table");
	+ if (opp_table == -1) {
	+ device_printf(dev, "Cant find /opp-table\n");
	+ return (ENXIO);
	+ }
	+ if (!OF_hasprop(opp_table, "syscon")) {
	+ device_printf(dev, "/opp-table missing syscon property\n");
	+ return (ENXIO);
	+ }
	+ err = syscon_get_by_ofw_property(dev, opp_table, "syscon", &sc->syscon);
	+ if (err) {
	+ device_printf(dev, "Failed to get syscon\n");
	+ return (ENXIO);
	+ }

	/* Reset the digital outputs. */
	- ti_scm_reg_write_4(SCM_BGAP_CTRL, 0);
	- ti_scm_reg_read_4(SCM_BGAP_CTRL, &reg);
	+ SYSCON_WRITE_4(sc->syscon, SCM_BGAP_CTRL, 0);
	+ reg = SYSCON_READ_4(sc->syscon, SCM_BGAP_CTRL);
	DELAY(500);
	/* Set continous mode. */
	- ti_scm_reg_write_4(SCM_BGAP_CTRL, SCM_BGAP_CONTCONV);
	- ti_scm_reg_read_4(SCM_BGAP_CTRL, &reg);
	+ SYSCON_WRITE_4(sc->syscon, SCM_BGAP_CTRL, SCM_BGAP_CONTCONV);
	+ reg = SYSCON_READ_4(sc->syscon, SCM_BGAP_CTRL);
	DELAY(500);
	/* Start the ADC conversion. */
	reg = SCM_BGAP_CLRZ \| SCM_BGAP_CONTCONV \| SCM_BGAP_SOC;
	- ti_scm_reg_write_4(SCM_BGAP_CTRL, reg);
	+ SYSCON_WRITE_4(sc->syscon, SCM_BGAP_CTRL, reg);

	/* Temperature sysctl. */
	- sc = device_get_softc(dev);
	ctx = device_get_sysctl_ctx(dev);
	tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
	sc->sc_temp_oid = SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
	"temperature", CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	dev, 0, am335x_scm_temp_sysctl, "IK", "Current temperature");

	return (0);
	}

	static int
	am335x_scm_detach(device_t dev)
	{
	struct am335x_scm_softc *sc;

	sc = device_get_softc(dev);

	/* Remove temperature sysctl. */
	if (sc->sc_temp_oid != NULL)
	sysctl_remove_oid(sc->sc_temp_oid, 1, 0);

	/* Stop the bandgap ADC. */
	- ti_scm_reg_write_4(SCM_BGAP_CTRL, SCM_BGAP_BGOFF);
	+ SYSCON_WRITE_4(sc->syscon, SCM_BGAP_CTRL, SCM_BGAP_BGOFF);

	return (0);
	}

	static device_method_t am335x_scm_methods[] = {
	DEVMETHOD(device_identify, am335x_scm_identify),
	DEVMETHOD(device_probe, am335x_scm_probe),
	DEVMETHOD(device_attach, am335x_scm_attach),
	DEVMETHOD(device_detach, am335x_scm_detach),

	DEVMETHOD_END
	};

	static driver_t am335x_scm_driver = {
	"am335x_scm",
	am335x_scm_methods,
	sizeof(struct am335x_scm_softc),
	};

	static devclass_t am335x_scm_devclass;

	DRIVER_MODULE(am335x_scm, ti_scm, am335x_scm_driver, am335x_scm_devclass, 0, 0);
	MODULE_VERSION(am335x_scm, 1);
	-MODULE_DEPEND(am335x_scm, ti_scm, 1, 1, 1);
	+MODULE_DEPEND(am335x_scm, ti_scm_syscon, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_sysc.c b/sys/arm/ti/am335x/am335x_usb_phy.c
	similarity index 64%
	copy from sys/arm/ti/ti_sysc.c
	copy to sys/arm/ti/am335x/am335x_usb_phy.c
	index d428dd44a1ab..00e28122dcec 100644
	--- a/sys/arm/ti/ti_sysc.c
	+++ b/sys/arm/ti/am335x/am335x_usb_phy.c
	@@ -1,128 +1,121 @@
	/*-
	- * Copyright (c) 2019 Emmanuel Vadot <manu@FreeBSD.org>
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/fbio.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/rman.h>
	#include <sys/resource.h>
	#include <machine/bus.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/pmap.h>

	#include <dev/fdt/simplebus.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	+#define TI_AM335X_USB_PHY 1
	+#define TI_AM335X_USB_PHY_END 0
	+
	static struct ofw_compat_data compat_data[] = {
	- { "ti,sysc", 1 },
	- { NULL, 0 }
	+ { "ti,am335x-usb-phy", TI_AM335X_USB_PHY },
	+ { NULL, TI_AM335X_USB_PHY_END }
	};

	-struct ti_sysc_softc {
	- struct simplebus_softc sc;
	+struct ti_usb_phy_softc {
	device_t dev;
	};

	-static int ti_sysc_probe(device_t dev);
	-static int ti_sysc_attach(device_t dev);
	-static int ti_sysc_detach(device_t dev);
	+static int ti_usb_phy_probe(device_t dev);
	+static int ti_usb_phy_attach(device_t dev);
	+static int ti_usb_phy_detach(device_t dev);

	static int
	-ti_sysc_probe(device_t dev)
	+ti_usb_phy_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	- device_set_desc(dev, "TI SYSC Interconnect");
	+ device_set_desc(dev, "TI AM335x USB PHY");
	if (!bootverbose)
	device_quiet(dev);

	return (BUS_PROBE_DEFAULT);
	}

	static int
	-ti_sysc_attach(device_t dev)
	+ti_usb_phy_attach(device_t dev)
	{
	- struct ti_sysc_softc *sc;
	- device_t cdev;
	- phandle_t node, child;
	+ struct ti_usb_phy_softc *sc;
	+ phandle_t node;

	sc = device_get_softc(dev);
	sc->dev = dev;
	node = ofw_bus_get_node(dev);

	- simplebus_init(dev, node);
	- if (simplebus_fill_ranges(node, &sc->sc) < 0) {
	- device_printf(dev, "could not get ranges\n");
	- return (ENXIO);
	- }
	-
	- for (child = OF_child(node); child > 0; child = OF_peer(child)) {
	- cdev = simplebus_add_device(dev, child, 0, NULL, -1, NULL);
	- if (cdev != NULL)
	- device_probe_and_attach(cdev);
	- }
	+ /* FIXME: Add dev/extres/phy/ interface */

	return (bus_generic_attach(dev));
	}

	static int
	-ti_sysc_detach(device_t dev)
	+ti_usb_phy_detach(device_t dev)
	{
	-
	return (EBUSY);
	}

	-static device_method_t ti_sysc_methods[] = {
	+static device_method_t ti_usb_phy_methods[] = {
	/* Device interface */
	- DEVMETHOD(device_probe, ti_sysc_probe),
	- DEVMETHOD(device_attach, ti_sysc_attach),
	- DEVMETHOD(device_detach, ti_sysc_detach),
	+ DEVMETHOD(device_probe, ti_usb_phy_probe),
	+ DEVMETHOD(device_attach, ti_usb_phy_attach),
	+ DEVMETHOD(device_detach, ti_usb_phy_detach),

	DEVMETHOD_END
	};

	-DEFINE_CLASS_1(ti_sysc, ti_sysc_driver, ti_sysc_methods,
	- sizeof(struct ti_sysc_softc), simplebus_driver);
	+DEFINE_CLASS_1(ti_usb_phy, ti_usb_phy_driver, ti_usb_phy_methods,
	+ sizeof(struct ti_usb_phy_softc), simplebus_driver);

	-static devclass_t ti_sysc_devclass;
	+static devclass_t ti_usb_phy_devclass;

	-EARLY_DRIVER_MODULE(ti_sysc, simplebus, ti_sysc_driver,
	-ti_sysc_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
	+EARLY_DRIVER_MODULE(ti_usb_phy, simplebus, ti_usb_phy_driver,
	+ ti_usb_phy_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
	+MODULE_VERSION(ti_usb_phy, 1);
	+MODULE_DEPEND(ti_usb_phy, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/am335x_usbss.c b/sys/arm/ti/am335x/am335x_usbss.c
	deleted file mode 100644
	index ce78a7b64315..000000000000
	--- a/sys/arm/ti/am335x/am335x_usbss.c
	+++ /dev/null
	@@ -1,226 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 2013 Oleksandr Tymoshenko <gonzo@freebsd.org>
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include <sys/stdint.h>
	-#include <sys/stddef.h>
	-#include <sys/param.h>
	-#include <sys/queue.h>
	-#include <sys/types.h>
	-#include <sys/systm.h>
	-#include <sys/kernel.h>
	-#include <sys/bus.h>
	-#include <sys/module.h>
	-#include <sys/lock.h>
	-#include <sys/mutex.h>
	-#include <sys/condvar.h>
	-#include <sys/sysctl.h>
	-#include <sys/sx.h>
	-#include <sys/unistd.h>
	-#include <sys/callout.h>
	-#include <sys/malloc.h>
	-#include <sys/priv.h>
	-
	-#include <dev/fdt/simplebus.h>
	-#include <dev/ofw/openfirm.h>
	-#include <dev/ofw/ofw_bus.h>
	-#include <dev/ofw/ofw_bus_subr.h>
	-
	-#include <dev/usb/usb.h>
	-#include <dev/usb/usbdi.h>
	-
	-#include <dev/usb/usb_core.h>
	-#include <dev/usb/usb_busdma.h>
	-#include <dev/usb/usb_process.h>
	-#include <dev/usb/usb_util.h>
	-
	-#include <dev/usb/usb_controller.h>
	-#include <dev/usb/usb_bus.h>
	-#include <dev/usb/controller/musb_otg.h>
	-#include <dev/usb/usb_debug.h>
	-
	-#include <sys/rman.h>
	-
	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_scm.h>
	-#include <arm/ti/am335x/am335x_scm.h>
	-
	-#define AM335X_USB_PORTS 2
	-
	-#define USBSS_REVREG 0x00
	-#define USBSS_SYSCONFIG 0x10
	-#define USBSS_SYSCONFIG_SRESET 1
	-
	-#define USBCTRL_REV 0x00
	-#define USBCTRL_CTRL 0x14
	-#define USBCTRL_STAT 0x18
	-#define USBCTRL_IRQ_STAT0 0x30
	-#define IRQ_STAT0_RXSHIFT 16
	-#define IRQ_STAT0_TXSHIFT 0
	-#define USBCTRL_IRQ_STAT1 0x34
	-#define IRQ_STAT1_DRVVBUS (1 << 8)
	-#define USBCTRL_INTEN_SET0 0x38
	-#define USBCTRL_INTEN_SET1 0x3C
	-#define USBCTRL_INTEN_USB_ALL 0x1ff
	-#define USBCTRL_INTEN_USB_SOF (1 << 3)
	-#define USBCTRL_INTEN_CLR0 0x40
	-#define USBCTRL_INTEN_CLR1 0x44
	-#define USBCTRL_UTMI 0xE0
	-#define USBCTRL_UTMI_FSDATAEXT (1 << 1)
	-#define USBCTRL_MODE 0xE8
	-#define USBCTRL_MODE_IDDIG (1 << 8)
	-#define USBCTRL_MODE_IDDIGMUX (1 << 7)
	-
	-#define USBSS_WRITE4(sc, reg, val) \
	- bus_write_4((sc)->sc_mem_res, (reg), (val))
	-#define USBSS_READ4(sc, reg) \
	- bus_read_4((sc)->sc_mem_res, (reg))
	-
	-static device_probe_t usbss_probe;
	-static device_attach_t usbss_attach;
	-static device_detach_t usbss_detach;
	-
	-struct usbss_softc {
	- struct simplebus_softc simplebus_sc;
	- struct resource *sc_mem_res;
	- int sc_mem_rid;
	-};
	-
	-static int
	-usbss_probe(device_t dev)
	-{
	-
	- if (!ofw_bus_status_okay(dev))
	- return (ENXIO);
	-
	- if (!ofw_bus_is_compatible(dev, "ti,am33xx-usb"))
	- return (ENXIO);
	-
	- device_set_desc(dev, "TI AM33xx integrated USB OTG controller");
	-
	- return (BUS_PROBE_DEFAULT);
	-}
	-
	-static int
	-usbss_attach(device_t dev)
	-{
	- struct usbss_softc *sc = device_get_softc(dev);
	- int i;
	- uint32_t rev;
	- phandle_t node;
	-
	- /* Request the memory resources */
	- sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	- &sc->sc_mem_rid, RF_ACTIVE);
	- if (sc->sc_mem_res == NULL) {
	- device_printf(dev,
	- "Error: could not allocate mem resources\n");
	- return (ENXIO);
	- }
	-
	- /* Enable device clocks. */
	- ti_prcm_clk_enable(MUSB0_CLK);
	-
	- /*
	- * Reset USBSS, USB0 and USB1.
	- * The registers of USB subsystem must not be accessed while the
	- * reset pulse is active (200ns).
	- */
	- USBSS_WRITE4(sc, USBSS_SYSCONFIG, USBSS_SYSCONFIG_SRESET);
	- DELAY(100);
	- i = 10;
	- while (USBSS_READ4(sc, USBSS_SYSCONFIG) & USBSS_SYSCONFIG_SRESET) {
	- DELAY(100);
	- if (i-- == 0) {
	- device_printf(dev, "reset timeout.\n");
	- return (ENXIO);
	- }
	- }
	-
	- /* Read the module revision. */
	- rev = USBSS_READ4(sc, USBSS_REVREG);
	- device_printf(dev, "TI AM335X USBSS v%d.%d.%d\n",
	- (rev >> 8) & 7, (rev >> 6) & 3, rev & 63);
	-
	- node = ofw_bus_get_node(dev);
	-
	- if (node == -1) {
	- usbss_detach(dev);
	- return (ENXIO);
	- }
	-
	- simplebus_init(dev, node);
	-
	- /*
	- * Allow devices to identify.
	- */
	- bus_generic_probe(dev);
	-
	- /*
	- * Now walk the OFW tree and attach top-level devices.
	- */
	- for (node = OF_child(node); node > 0; node = OF_peer(node))
	- simplebus_add_device(dev, node, 0, NULL, -1, NULL);
	-
	- return (bus_generic_attach(dev));
	-}
	-
	-static int
	-usbss_detach(device_t dev)
	-{
	- struct usbss_softc *sc = device_get_softc(dev);
	-
	- /* Free resources if any */
	- if (sc->sc_mem_res)
	- bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_mem_rid,
	- sc->sc_mem_res);
	-
	- /* during module unload there are lots of children leftover */
	- device_delete_children(dev);
	-
	- return (0);
	-}
	-
	-static device_method_t usbss_methods[] = {
	- /* Device interface */
	- DEVMETHOD(device_probe, usbss_probe),
	- DEVMETHOD(device_attach, usbss_attach),
	- DEVMETHOD(device_detach, usbss_detach),
	- DEVMETHOD(device_suspend, bus_generic_suspend),
	- DEVMETHOD(device_resume, bus_generic_resume),
	- DEVMETHOD(device_shutdown, bus_generic_shutdown),
	-
	- DEVMETHOD_END
	-};
	-
	-DEFINE_CLASS_1(usbss, usbss_driver, usbss_methods,
	- sizeof(struct usbss_softc), simplebus_driver);
	-static devclass_t usbss_devclass;
	-DRIVER_MODULE(usbss, simplebus, usbss_driver, usbss_devclass, 0, 0);
	-MODULE_DEPEND(usbss, usb, 1, 1, 1);
	diff --git a/sys/arm/ti/am335x/files.am335x b/sys/arm/ti/am335x/files.am335x
	index 66af0d867847..4ecdc1e35b43 100644
	--- a/sys/arm/ti/am335x/files.am335x
	+++ b/sys/arm/ti/am335x/files.am335x
	@@ -1,25 +1,25 @@
	#$FreeBSD$

	arm/ti/aintc.c standard

	arm/ti/am335x/am335x_dmtimer.c standard
	arm/ti/am335x/am335x_dmtpps.c optional am335x_dmtpps
	arm/ti/am335x/am335x_gpio.c optional gpio
	arm/ti/am335x/am335x_lcd.c optional sc \| vt
	arm/ti/am335x/am335x_lcd_syscons.c optional sc
	arm/ti/am335x/am335x_pmic.c optional am335x_pmic
	-arm/ti/am335x/am335x_prcm.c standard
	arm/ti/am335x/am335x_pwmss.c standard
	dev/pwm/pwmbus_if.m standard
	arm/ti/am335x/am335x_ehrpwm.c standard
	arm/ti/am335x/am335x_ecap.c standard
	arm/ti/am335x/am335x_rtc.c optional am335x_rtc
	arm/ti/am335x/am335x_scm.c standard
	arm/ti/am335x/am335x_scm_padconf.c standard
	-arm/ti/am335x/am335x_usbss.c optional musb fdt
	arm/ti/am335x/am335x_musb.c optional musb fdt
	+arm/ti/am335x/am335x_usb_phy.c optional musb fdt
	+arm/ti/am335x/am3359_cppi41.c optional musb fdt

	arm/ti/am335x/tda19988.c optional hdmi

	arm/ti/ti_edma3.c standard
	arm/ti/cpsw/if_cpsw.c optional cpsw
	diff --git a/sys/arm/ti/clk/clock_common.c b/sys/arm/ti/clk/clock_common.c
	new file mode 100644
	index 000000000000..15b0e75a8a1e
	--- /dev/null
	+++ b/sys/arm/ti/clk/clock_common.c
	@@ -0,0 +1,152 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/conf.h>
	+#include <sys/bus.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/systm.h>
	+#include <sys/libkern.h>
	+#include <sys/types.h>
	+#include <sys/malloc.h>
	+
	+#include <machine/bus.h>
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/extres/clk/clk_mux.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include "clock_common.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+void
	+read_clock_cells(device_t dev, struct clock_cell_info *clk) {
	+ ssize_t numbytes_clocks;
	+ phandle_t node, parent, *cells;
	+ int index, ncells, rv;
	+
	+ node = ofw_bus_get_node(dev);
	+
	+ /* Get names of parent clocks */
	+ numbytes_clocks = OF_getproplen(node, "clocks");
	+ clk->num_clock_cells = numbytes_clocks / sizeof(cell_t);
	+
	+ /* Allocate space and get clock cells content */
	+ /* clock_cells / clock_cells_ncells will be freed in
	+ * find_parent_clock_names()
	+ */
	+ clk->clock_cells = malloc(numbytes_clocks, M_DEVBUF, M_WAITOK\|M_ZERO);
	+ clk->clock_cells_ncells = malloc(clk->num_clock_cells*sizeof(uint8_t),
	+ M_DEVBUF, M_WAITOK\|M_ZERO);
	+ OF_getencprop(node, "clocks", clk->clock_cells, numbytes_clocks);
	+
	+ /* Count number of clocks */
	+ clk->num_real_clocks = 0;
	+ for (index = 0; index < clk->num_clock_cells; index++) {
	+ rv = ofw_bus_parse_xref_list_alloc(node, "clocks", "#clock-cells",
	+ clk->num_real_clocks, &parent, &ncells, &cells);
	+ if (rv != 0)
	+ continue;
	+
	+ if (cells != NULL)
	+ OF_prop_free(cells);
	+
	+ clk->clock_cells_ncells[index] = ncells;
	+ index += ncells;
	+ clk->num_real_clocks++;
	+ }
	+}
	+
	+int
	+find_parent_clock_names(device_t dev, struct clock_cell_info clk, struct clknode_init_def def) {
	+ int index, clock_index, err;
	+ bool found_all = true;
	+ clk_t parent;
	+
	+ /* Figure out names */
	+ for (index = 0, clock_index = 0; index < clk->num_clock_cells; index++) {
	+ /* Get name of parent clock */
	+ err = clk_get_by_ofw_index(dev, 0, clock_index, &parent);
	+ if (err != 0) {
	+ clock_index++;
	+ found_all = false;
	+ DPRINTF(dev, "Failed to find clock_cells[%d]=0x%x\n",
	+ index, clk->clock_cells[index]);
	+
	+ index += clk->clock_cells_ncells[index];
	+ continue;
	+ }
	+
	+ def->parent_names[clock_index] = clk_get_name(parent);
	+ clk_release(parent);
	+
	+ DPRINTF(dev, "Found parent clock[%d/%d]: %s\n",
	+ clock_index, clk->num_real_clocks,
	+ def->parent_names[clock_index]);
	+
	+ clock_index++;
	+ index += clk->clock_cells_ncells[index];
	+ }
	+
	+ if (!found_all) {
	+ return 1;
	+ }
	+
	+ free(clk->clock_cells, M_DEVBUF);
	+ free(clk->clock_cells_ncells, M_DEVBUF);
	+ return 0;
	+}
	+
	+void
	+create_clkdef(device_t dev, struct clock_cell_info clk, struct clknode_init_def def) {
	+ def->id = 1;
	+
	+ clk_parse_ofw_clk_name(dev, ofw_bus_get_node(dev), &def->name);
	+
	+ DPRINTF(dev, "node name: %s\n", def->name);
	+
	+ def->parent_cnt = clk->num_real_clocks;
	+ def->parent_names = malloc(clk->num_real_clockssizeof(char ),
	+ M_OFWPROP, M_WAITOK);
	+}
	+void
	+free_clkdef(struct clknode_init_def *def) {
	+ OF_prop_free(__DECONST(char *, def->name));
	+ OF_prop_free(def->parent_names);
	+}
	diff --git a/sys/arm/ti/clk/clock_common.h b/sys/arm/ti/clk/clock_common.h
	new file mode 100644
	index 000000000000..148494f90331
	--- /dev/null
	+++ b/sys/arm/ti/clk/clock_common.h
	@@ -0,0 +1,43 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+struct clock_cell_info {
	+ cell_t *clock_cells;
	+ uint8_t *clock_cells_ncells;
	+ uint32_t num_clock_cells;
	+ uint8_t num_real_clocks;
	+};
	+
	+void read_clock_cells(device_t dev, struct clock_cell_info *clk);
	+int find_parent_clock_names(device_t dev, struct clock_cell_info clk, struct clknode_init_def def);
	+void create_clkdef(device_t dev, struct clock_cell_info clk, struct clknode_init_def def);
	+void free_clkdef(struct clknode_init_def *def);
	diff --git a/sys/arm/ti/clk/ti_clk_clkctrl.c b/sys/arm/ti/clk/ti_clk_clkctrl.c
	new file mode 100644
	index 000000000000..6b2fff5e12bb
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_clk_clkctrl.c
	@@ -0,0 +1,219 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/bus.h>
	+#include <sys/malloc.h>
	+
	+#include <dev/extres/clk/clk.h>
	+
	+#include <arm/ti/clk/ti_clk_clkctrl.h>
	+
	+#include "clkdev_if.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/*
	+ * clknode for clkctrl, implements gate and mux (for gpioc)
	+ */
	+
	+#define GPIO_X_GDBCLK_MASK 0x00040000
	+#define IDLEST_MASK 0x00030000
	+#define MODULEMODE_MASK 0x00000003
	+
	+#define GPIOX_GDBCLK_ENABLE 0x00040000
	+#define GPIOX_GDBCLK_DISABLE 0x00000000
	+#define IDLEST_FUNC 0x00000000
	+#define IDLEST_TRANS 0x00010000
	+#define IDLEST_IDLE 0x00020000
	+#define IDLEST_DISABLE 0x00030000
	+
	+#define MODULEMODE_DISABLE 0x0
	+#define MODULEMODE_ENABLE 0x2
	+
	+struct ti_clkctrl_clknode_sc {
	+ device_t dev;
	+ bool gdbclk;
	+ /* omap4-cm range.host + ti,clkctrl reg[0] */
	+ uint32_t register_offset;
	+};
	+
	+#define WRITE4(_clk, off, val) \
	+ CLKDEV_WRITE_4(clknode_get_device(_clk), off, val)
	+#define READ4(_clk, off, val) \
	+ CLKDEV_READ_4(clknode_get_device(_clk), off, val)
	+#define DEVICE_LOCK(_clk) \
	+ CLKDEV_DEVICE_LOCK(clknode_get_device(_clk))
	+#define DEVICE_UNLOCK(_clk) \
	+ CLKDEV_DEVICE_UNLOCK(clknode_get_device(_clk))
	+
	+static int
	+ti_clkctrl_init(struct clknode *clk, device_t dev)
	+{
	+ struct ti_clkctrl_clknode_sc *sc;
	+
	+ sc = clknode_get_softc(clk);
	+ sc->dev = dev;
	+
	+ clknode_init_parent_idx(clk, 0);
	+ return (0);
	+}
	+
	+static int
	+ti_clkctrl_set_gdbclk_gate(struct clknode *clk, bool enable)
	+{
	+ struct ti_clkctrl_clknode_sc *sc;
	+ uint32_t val, gpio_x_gdbclk;
	+ uint32_t timeout = 100;
	+
	+ sc = clknode_get_softc(clk);
	+
	+ READ4(clk, sc->register_offset, &val);
	+ DPRINTF(sc->dev, "val(%x) & (%x \| %x = %x)\n",
	+ val, GPIO_X_GDBCLK_MASK, MODULEMODE_MASK,
	+ GPIO_X_GDBCLK_MASK \| MODULEMODE_MASK);
	+
	+ if (enable) {
	+ val = val & MODULEMODE_MASK;
	+ val \|= GPIOX_GDBCLK_ENABLE;
	+ } else {
	+ val = val & MODULEMODE_MASK;
	+ val \|= GPIOX_GDBCLK_DISABLE;
	+ }
	+
	+ DPRINTF(sc->dev, "val %x\n", val);
	+ WRITE4(clk, sc->register_offset, val);
	+
	+ /* Wait */
	+ while (timeout) {
	+ READ4(clk, sc->register_offset, &val);
	+ gpio_x_gdbclk = val & GPIO_X_GDBCLK_MASK;
	+ if (enable && (gpio_x_gdbclk == GPIOX_GDBCLK_ENABLE))
	+ break;
	+ else if (!enable && (gpio_x_gdbclk == GPIOX_GDBCLK_DISABLE))
	+ break;
	+ DELAY(10);
	+ timeout--;
	+ }
	+ if (timeout == 0) {
	+ device_printf(sc->dev, "ti_clkctrl_set_gdbclk_gate: Timeout\n");
	+ return (1);
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+ti_clkctrl_set_gate(struct clknode *clk, bool enable)
	+{
	+ struct ti_clkctrl_clknode_sc *sc;
	+ uint32_t val, idlest, module;
	+ uint32_t timeout=100;
	+ int err;
	+
	+ sc = clknode_get_softc(clk);
	+
	+ if (sc->gdbclk) {
	+ err = ti_clkctrl_set_gdbclk_gate(clk, enable);
	+ return (err);
	+ }
	+
	+ READ4(clk, sc->register_offset, &val);
	+
	+ if (enable)
	+ WRITE4(clk, sc->register_offset, MODULEMODE_ENABLE);
	+ else
	+ WRITE4(clk, sc->register_offset, MODULEMODE_DISABLE);
	+
	+ while (timeout) {
	+ READ4(clk, sc->register_offset, &val);
	+ idlest = val & IDLEST_MASK;
	+ module = val & MODULEMODE_MASK;
	+ if (enable &&
	+ (idlest == IDLEST_FUNC \|\| idlest == IDLEST_TRANS) &&
	+ module == MODULEMODE_ENABLE)
	+ break;
	+ else if (!enable &&
	+ idlest == IDLEST_DISABLE &&
	+ module == MODULEMODE_DISABLE)
	+ break;
	+ DELAY(10);
	+ timeout--;
	+ }
	+
	+ if (timeout == 0) {
	+ device_printf(sc->dev, "ti_clkctrl_set_gate: Timeout\n");
	+ return (1);
	+ }
	+
	+ return (0);
	+}
	+
	+static clknode_method_t ti_clkctrl_clknode_methods[] = {
	+ /* Device interface */
	+ CLKNODEMETHOD(clknode_init, ti_clkctrl_init),
	+ CLKNODEMETHOD(clknode_set_gate, ti_clkctrl_set_gate),
	+ CLKNODEMETHOD_END
	+};
	+
	+DEFINE_CLASS_1(ti_clkctrl_clknode, ti_clkctrl_clknode_class,
	+ ti_clkctrl_clknode_methods, sizeof(struct ti_clkctrl_clknode_sc),
	+ clknode_class);
	+
	+int
	+ti_clknode_clkctrl_register(struct clkdom *clkdom,
	+ struct ti_clk_clkctrl_def *clkdef)
	+{
	+ struct clknode *clk;
	+ struct ti_clkctrl_clknode_sc *sc;
	+
	+ clk = clknode_create(clkdom, &ti_clkctrl_clknode_class,
	+ &clkdef->clkdef);
	+
	+ if (clk == NULL) {
	+ return (1);
	+ }
	+
	+ sc = clknode_get_softc(clk);
	+ sc->register_offset = clkdef->register_offset;
	+ sc->gdbclk = clkdef->gdbclk;
	+
	+ if (clknode_register(clkdom, clk) == NULL) {
	+ return (2);
	+ }
	+ return (0);
	+}
	diff --git a/sys/arm/ti/clk/ti_clk_clkctrl.h b/sys/arm/ti/clk/ti_clk_clkctrl.h
	new file mode 100644
	index 000000000000..d78736244815
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_clk_clkctrl.h
	@@ -0,0 +1,43 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#ifndef _TI_CLK_CLKCTRL_H_
	+#define _TI_CLK_CLKCTRL_H_
	+
	+#include <dev/extres/clk/clk.h>
	+
	+struct ti_clk_clkctrl_def {
	+ struct clknode_init_def clkdef;
	+ bool gdbclk;
	+ uint32_t register_offset;
	+};
	+
	+int ti_clknode_clkctrl_register(struct clkdom clkdom, struct ti_clk_clkctrl_def clkdef);
	+
	+#endif /* _TI_CLK_CLKCTRL_H_ */
	diff --git a/sys/arm/ti/clk/ti_clk_dpll.c b/sys/arm/ti/clk/ti_clk_dpll.c
	new file mode 100644
	index 000000000000..14e48dc95026
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_clk_dpll.c
	@@ -0,0 +1,341 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2019 Emmanuel Vadot <manu@freebsd.org>
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * based on sys/arm/allwinner/clkng/aw_clk_np.c
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/bus.h>
	+
	+#include <dev/extres/clk/clk.h>
	+
	+#include <arm/ti/clk/ti_clk_dpll.h>
	+
	+#include "clkdev_if.h"
	+
	+/*
	+ * clknode for clocks matching the formula :
	+ *
	+ * clk = clkin * n / p
	+ *
	+ */
	+
	+struct ti_dpll_clknode_sc {
	+ uint32_t ti_clkmode_offset; /* control */
	+ uint8_t ti_clkmode_flags;
	+
	+ uint32_t ti_idlest_offset;
	+
	+ uint32_t ti_clksel_offset; /* mult-div1 */
	+ struct ti_clk_factor n; /* ti_clksel_mult */
	+ struct ti_clk_factor p; /* ti_clksel_div */
	+
	+ uint32_t ti_autoidle_offset;
	+};
	+
	+#define WRITE4(_clk, off, val) \
	+ CLKDEV_WRITE_4(clknode_get_device(_clk), off, val)
	+#define READ4(_clk, off, val) \
	+ CLKDEV_READ_4(clknode_get_device(_clk), off, val)
	+#define DEVICE_LOCK(_clk) \
	+ CLKDEV_DEVICE_LOCK(clknode_get_device(_clk))
	+#define DEVICE_UNLOCK(_clk) \
	+ CLKDEV_DEVICE_UNLOCK(clknode_get_device(_clk))
	+
	+static int
	+ti_dpll_clk_init(struct clknode *clk, device_t dev)
	+{
	+ struct ti_dpll_clknode_sc *sc;
	+
	+ sc = clknode_get_softc(clk);
	+
	+ clknode_init_parent_idx(clk, 0);
	+ return (0);
	+}
	+
	+/* helper to keep aw_clk_np_find_best "intact" */
	+static inline uint32_t
	+ti_clk_factor_get_max(struct ti_clk_factor *factor)
	+{
	+ uint32_t max;
	+
	+ if (factor->flags & TI_CLK_FACTOR_FIXED)
	+ max = factor->value;
	+ else {
	+ max = (1 << factor->width);
	+ }
	+
	+ return (max);
	+}
	+
	+static inline uint32_t
	+ti_clk_factor_get_min(struct ti_clk_factor *factor)
	+{
	+ uint32_t min;
	+
	+ if (factor->flags & TI_CLK_FACTOR_FIXED)
	+ min = factor->value;
	+ else if (factor->flags & TI_CLK_FACTOR_ZERO_BASED)
	+ min = 0;
	+ else if (factor->flags & TI_CLK_FACTOR_MIN_VALUE)
	+ min = factor->min_value;
	+ else
	+ min = 1;
	+
	+ return (min);
	+}
	+
	+static uint64_t
	+ti_dpll_clk_find_best(struct ti_dpll_clknode_sc *sc, uint64_t fparent,
	+ uint64_t fout, uint32_t factor_n, uint32_t *factor_p)
	+{
	+ uint64_t cur, best;
	+ uint32_t n, p, max_n, max_p, min_n, min_p;
	+
	+ factor_n = factor_p = 0;
	+
	+ max_n = ti_clk_factor_get_max(&sc->n);
	+ max_p = ti_clk_factor_get_max(&sc->p);
	+ min_n = ti_clk_factor_get_min(&sc->n);
	+ min_p = ti_clk_factor_get_min(&sc->p);
	+
	+ for (p = min_p; p <= max_p; ) {
	+ for (n = min_n; n <= max_n; ) {
	+ cur = fparent * n / p;
	+ if (abs(fout - cur) < abs(fout - best)) {
	+ best = cur;
	+ *factor_n = n;
	+ *factor_p = p;
	+ }
	+
	+ n++;
	+ }
	+ p++;
	+ }
	+
	+ return (best);
	+}
	+
	+static inline uint32_t
	+ti_clk_get_factor(uint32_t val, struct ti_clk_factor *factor)
	+{
	+ uint32_t factor_val;
	+
	+ if (factor->flags & TI_CLK_FACTOR_FIXED)
	+ return (factor->value);
	+
	+ factor_val = (val & factor->mask) >> factor->shift;
	+ if (!(factor->flags & TI_CLK_FACTOR_ZERO_BASED))
	+ factor_val += 1;
	+
	+ return (factor_val);
	+}
	+
	+static inline uint32_t
	+ti_clk_factor_get_value(struct ti_clk_factor *factor, uint32_t raw)
	+{
	+ uint32_t val;
	+
	+ if (factor->flags & TI_CLK_FACTOR_FIXED)
	+ return (factor->value);
	+
	+ if (factor->flags & TI_CLK_FACTOR_ZERO_BASED)
	+ val = raw;
	+ else if (factor->flags & TI_CLK_FACTOR_MAX_VALUE &&
	+ raw > factor->max_value)
	+ val = factor->max_value;
	+ else
	+ val = raw - 1;
	+
	+ return (val);
	+}
	+
	+
	+static int
	+ti_dpll_clk_set_freq(struct clknode clk, uint64_t fparent, uint64_t fout,
	+ int flags, int *stop)
	+{
	+ struct ti_dpll_clknode_sc *sc;
	+ uint64_t cur, best;
	+ uint32_t val, n, p, best_n, best_p, timeout;
	+
	+ sc = clknode_get_softc(clk);
	+
	+ best = cur = 0;
	+
	+ best = ti_dpll_clk_find_best(sc, fparent, fout,
	+ &best_n, &best_p);
	+
	+ if ((flags & CLK_SET_DRYRUN) != 0) {
	+ *fout = best;
	+ *stop = 1;
	+ return (0);
	+ }
	+
	+ if ((best < *fout) &&
	+ (flags == CLK_SET_ROUND_DOWN)) {
	+ *stop = 1;
	+ return (ERANGE);
	+ }
	+ if ((best > *fout) &&
	+ (flags == CLK_SET_ROUND_UP)) {
	+ *stop = 1;
	+ return (ERANGE);
	+ }
	+
	+ DEVICE_LOCK(clk);
	+ /* 1 switch PLL to bypass mode */
	+ WRITE4(clk, sc->ti_clkmode_offset, DPLL_EN_MN_BYPASS_MODE);
	+
	+ /* 2 Ensure PLL is in bypass */
	+ timeout = 10000;
	+ do {
	+ DELAY(10);
	+ READ4(clk, sc->ti_idlest_offset, &val);
	+ } while (!(val & ST_MN_BYPASS_MASK) && timeout--);
	+
	+ if (timeout == 0) {
	+ DEVICE_UNLOCK(clk);
	+ return (ERANGE); // FIXME: Better return value?
	+ }
	+
	+ /* 3 Set DPLL_MULT & DPLL_DIV bits */
	+ READ4(clk, sc->ti_clksel_offset, &val);
	+
	+ n = ti_clk_factor_get_value(&sc->n, best_n);
	+ p = ti_clk_factor_get_value(&sc->p, best_p);
	+ val &= ~sc->n.mask;
	+ val &= ~sc->p.mask;
	+ val \|= n << sc->n.shift;
	+ val \|= p << sc->p.shift;
	+
	+ WRITE4(clk, sc->ti_clksel_offset, val);
	+
	+ /* 4. configure M2, M4, M5 and M6 */
	+ /*
	+ * FIXME: According to documentation M2/M4/M5/M6 can be set "later"
	+ * See note in TRM 8.1.6.7.1
	+ */
	+
	+ /* 5 Switch over to lock mode */
	+ WRITE4(clk, sc->ti_clkmode_offset, DPLL_EN_LOCK_MODE);
	+
	+ /* 6 Ensure PLL is locked */
	+ timeout = 10000;
	+ do {
	+ DELAY(10);
	+ READ4(clk, sc->ti_idlest_offset, &val);
	+ } while (!(val & ST_DPLL_CLK_MASK) && timeout--);
	+
	+ DEVICE_UNLOCK(clk);
	+ if (timeout == 0) {
	+ return (ERANGE); // FIXME: Better return value?
	+ }
	+
	+ *fout = best;
	+ *stop = 1;
	+
	+ return (0);
	+}
	+
	+static int
	+ti_dpll_clk_recalc(struct clknode clk, uint64_t freq)
	+{
	+ struct ti_dpll_clknode_sc *sc;
	+ uint32_t val, n, p;
	+
	+ sc = clknode_get_softc(clk);
	+
	+ DEVICE_LOCK(clk);
	+ READ4(clk, sc->ti_clksel_offset, &val);
	+ DEVICE_UNLOCK(clk);
	+
	+ n = ti_clk_get_factor(val, &sc->n);
	+ p = ti_clk_get_factor(val, &sc->p);
	+
	+ freq = freq * n / p;
	+
	+ return (0);
	+}
	+
	+static clknode_method_t ti_dpll_clknode_methods[] = {
	+ /* Device interface */
	+ CLKNODEMETHOD(clknode_init, ti_dpll_clk_init),
	+ CLKNODEMETHOD(clknode_recalc_freq, ti_dpll_clk_recalc),
	+ CLKNODEMETHOD(clknode_set_freq, ti_dpll_clk_set_freq),
	+ CLKNODEMETHOD_END
	+};
	+
	+DEFINE_CLASS_1(ti_dpll_clknode, ti_dpll_clknode_class, ti_dpll_clknode_methods,
	+ sizeof(struct ti_dpll_clknode_sc), clknode_class);
	+
	+int
	+ti_clknode_dpll_register(struct clkdom clkdom, struct ti_clk_dpll_def clkdef)
	+{
	+ struct clknode *clk;
	+ struct ti_dpll_clknode_sc *sc;
	+
	+ clk = clknode_create(clkdom, &ti_dpll_clknode_class, &clkdef->clkdef);
	+ if (clk == NULL)
	+ return (1);
	+
	+ sc = clknode_get_softc(clk);
	+
	+ sc->ti_clkmode_offset = clkdef->ti_clkmode_offset;
	+ sc->ti_clkmode_flags = clkdef->ti_clkmode_flags;
	+ sc->ti_idlest_offset = clkdef->ti_idlest_offset;
	+ sc->ti_clksel_offset = clkdef->ti_clksel_offset;
	+
	+ sc->n.shift = clkdef->ti_clksel_mult.shift;
	+ sc->n.mask = clkdef->ti_clksel_mult.mask;
	+ sc->n.width = clkdef->ti_clksel_mult.width;
	+ sc->n.value = clkdef->ti_clksel_mult.value;
	+ sc->n.min_value = clkdef->ti_clksel_mult.min_value;
	+ sc->n.max_value = clkdef->ti_clksel_mult.max_value;
	+ sc->n.flags = clkdef->ti_clksel_mult.flags;
	+
	+ sc->p.shift = clkdef->ti_clksel_div.shift;
	+ sc->p.mask = clkdef->ti_clksel_div.mask;
	+ sc->p.width = clkdef->ti_clksel_div.width;
	+ sc->p.value = clkdef->ti_clksel_div.value;
	+ sc->p.min_value = clkdef->ti_clksel_div.min_value;
	+ sc->p.max_value = clkdef->ti_clksel_div.max_value;
	+ sc->p.flags = clkdef->ti_clksel_div.flags;
	+
	+ sc->ti_autoidle_offset = clkdef->ti_autoidle_offset;
	+
	+ clknode_register(clkdom, clk);
	+
	+ return (0);
	+}
	diff --git a/sys/arm/ti/clk/ti_clk_dpll.h b/sys/arm/ti/clk/ti_clk_dpll.h
	new file mode 100644
	index 000000000000..54bc0b988d6e
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_clk_dpll.h
	@@ -0,0 +1,97 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2017 Emmanuel Vadot <manu@freebsd.org>
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#ifndef _TI_DPLL_CLOCK_H_
	+#define _TI_DPLL_CLOCK_H_
	+
	+#include <dev/extres/clk/clk.h>
	+
	+/* Registers are described in AM335x TRM chapter 8.1.12.2.* */
	+
	+/* Register offsets */
	+#define CM_CLKSEL_DPLL_PERIPH 0x49C
	+
	+/* CM_IDLEST_DPLL_xxx */
	+#define ST_MN_BYPASS_MASK 0x0100
	+#define ST_MN_BYPASS_SHIFT 8
	+#define ST_DPLL_CLK_MASK 0x0001
	+
	+/* CM_CLKMODE_DPLL_DPLL_EN feature flag */
	+#define LOW_POWER_STOP_MODE_FLAG 0x01
	+#define MN_BYPASS_MODE_FLAG 0x02
	+#define IDLE_BYPASS_LOW_POWER_MODE_FLAG 0x04
	+#define IDLE_BYPASS_FAST_RELOCK_MODE_FLAG 0x08
	+#define LOCK_MODE_FLAG 0x10
	+
	+/* CM_CLKMODE_DPLL_xxx */
	+#define DPLL_EN_LOW_POWER_STOP_MODE 0x01
	+#define DPLL_EN_MN_BYPASS_MODE 0x04
	+#define DPLL_EN_IDLE_BYPASS_LOW_POWER_MODE 0x05
	+#define DPLL_EN_IDLE_BYPASS_FAST_RELOCK_MODE 0x06
	+#define DPLL_EN_LOCK_MODE 0x07
	+
	+
	+#define TI_CLK_FACTOR_ZERO_BASED 0x0002
	+#define TI_CLK_FACTOR_FIXED 0x0008
	+#define TI_CLK_FACTOR_MIN_VALUE 0x0020
	+#define TI_CLK_FACTOR_MAX_VALUE 0x0040
	+
	+/* Based on aw_clk_factor sys/arm/allwinner/clkng/aw_clk.h */
	+struct ti_clk_factor {
	+ uint32_t shift; /* Shift bits for the factor */
	+ uint32_t mask; /* Mask to get the factor */
	+ uint32_t width; /* Number of bits for the factor */
	+ uint32_t value; /* Fixed value */
	+
	+ uint32_t min_value;
	+ uint32_t max_value;
	+
	+ uint32_t flags; /* Flags */
	+};
	+
	+struct ti_clk_dpll_def {
	+ struct clknode_init_def clkdef;
	+
	+ uint32_t ti_clkmode_offset; /* control */
	+ uint8_t ti_clkmode_flags;
	+
	+ uint32_t ti_idlest_offset;
	+
	+ uint32_t ti_clksel_offset; /* mult-div1 */
	+ struct ti_clk_factor ti_clksel_mult;
	+ struct ti_clk_factor ti_clksel_div;
	+
	+ uint32_t ti_autoidle_offset;
	+};
	+
	+int ti_clknode_dpll_register(struct clkdom clkdom, struct ti_clk_dpll_def clkdef);
	+
	+#endif /* _TI_DPLL_CLOCK_H_ */
	diff --git a/sys/arm/ti/clk/ti_clkctrl.c b/sys/arm/ti/clk/ti_clkctrl.c
	new file mode 100644
	index 000000000000..5ba0dbe19b79
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_clkctrl.c
	@@ -0,0 +1,353 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright 2016 Michal Meloun <mmel@FreeBSD.org>
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/bus.h>
	+#include <sys/fbio.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/rman.h>
	+#include <sys/resource.h>
	+#include <machine/bus.h>
	+#include <vm/vm.h>
	+#include <vm/vm_extern.h>
	+#include <vm/vm_kern.h>
	+#include <vm/pmap.h>
	+
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include <arm/ti/clk/ti_clk_clkctrl.h>
	+#include <arm/ti/ti_omap4_cm.h>
	+#include <arm/ti/ti_cpuid.h>
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+#define L4LS_CLKCTRL_38 2
	+#define L4_WKUP_CLKCTRL_0 1
	+#define NO_SPECIAL_REG 0
	+
	+/* Documentation/devicetree/bindings/clock/ti-clkctrl.txt */
	+
	+#define TI_CLKCTRL_L4_WKUP 5
	+#define TI_CLKCTRL_L4_SECURE 4
	+#define TI_CLKCTRL_L4_PER 3
	+#define TI_CLKCTRL_L4_CFG 2
	+#define TI_CLKCTRL 1
	+#define TI_CLKCTRL_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,clkctrl-l4-wkup", TI_CLKCTRL_L4_WKUP },
	+ { "ti,clkctrl-l4-secure", TI_CLKCTRL_L4_SECURE },
	+ { "ti,clkctrl-l4-per", TI_CLKCTRL_L4_PER },
	+ { "ti,clkctrl-l4-cfg", TI_CLKCTRL_L4_CFG },
	+ { "ti,clkctrl", TI_CLKCTRL },
	+ { NULL, TI_CLKCTRL_END }
	+};
	+
	+struct ti_clkctrl_softc {
	+ device_t dev;
	+
	+ struct clkdom *clkdom;
	+};
	+
	+static int ti_clkctrl_probe(device_t dev);
	+static int ti_clkctrl_attach(device_t dev);
	+static int ti_clkctrl_detach(device_t dev);
	+int clkctrl_ofw_map(struct clkdom *clkdom, uint32_t ncells,
	+ phandle_t cells, struct clknode *clk);
	+static int
	+create_clkctrl(struct ti_clkctrl_softc sc, cell_t reg, uint32_t index, uint32_t reg_offset,
	+ uint64_t parent_offset, const char *org_name, bool special_gdbclk_reg);
	+
	+static int
	+ti_clkctrl_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI clkctrl");
	+
	+ return (BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+ti_clkctrl_attach(device_t dev)
	+{
	+ struct ti_clkctrl_softc *sc;
	+ phandle_t node;
	+ cell_t *reg;
	+ ssize_t numbytes_reg;
	+ int num_reg, err, ti_clock_cells;
	+ uint32_t index, reg_offset, reg_address;
	+ const char *org_name;
	+ uint64_t parent_offset;
	+ uint8_t special_reg = NO_SPECIAL_REG;
	+
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+ node = ofw_bus_get_node(dev);
	+
	+ /* Sanity check */
	+ err = OF_searchencprop(node, "#clock-cells",
	+ &ti_clock_cells, sizeof(ti_clock_cells));
	+ if (err == -1) {
	+ device_printf(sc->dev, "Failed to get #clock-cells\n");
	+ return (ENXIO);
	+ }
	+
	+ if (ti_clock_cells != 2) {
	+ device_printf(sc->dev, "clock cells(%d) != 2\n",
	+ ti_clock_cells);
	+ return (ENXIO);
	+ }
	+
	+ /* Grab the content of reg properties */
	+ numbytes_reg = OF_getproplen(node, "reg");
	+ if (numbytes_reg == 0) {
	+ device_printf(sc->dev, "reg property empty - check your devicetree\n");
	+ return (ENXIO);
	+ }
	+ num_reg = numbytes_reg / sizeof(cell_t);
	+
	+ reg = malloc(numbytes_reg, M_DEVBUF, M_WAITOK);
	+ OF_getencprop(node, "reg", reg, numbytes_reg);
	+
	+ /* Create clock domain */
	+ sc->clkdom = clkdom_create(sc->dev);
	+ if (sc->clkdom == NULL) {
	+ free(reg, M_DEVBUF);
	+ DPRINTF(sc->dev, "Failed to create clkdom\n");
	+ return (ENXIO);
	+ }
	+ clkdom_set_ofw_mapper(sc->clkdom, clkctrl_ofw_map);
	+
	+ /* Create clock nodes */
	+ /* name */
	+ clk_parse_ofw_clk_name(sc->dev, node, &org_name);
	+
	+ /* Get parent range */
	+ parent_offset = ti_omap4_cm_get_simplebus_base_host(device_get_parent(dev));
	+
	+ /* Check if this is a clkctrl with special registers like gpio */
	+ switch (ti_chip()) {
	+#ifdef SOC_OMAP4
	+ case CHIP_OMAP_4:
	+ /* FIXME: Todo */
	+ break;
	+
	+#endif /* SOC_OMAP4 */
	+#ifdef SOC_TI_AM335X
	+ /* Checkout TRM 8.1.12.1.29 - 8.1.12.31 and 8.1.12.2.3
	+ * and the DTS.
	+ */
	+ case CHIP_AM335X:
	+ if (strcmp(org_name, "l4ls-clkctrl@38") == 0)
	+ special_reg = L4LS_CLKCTRL_38;
	+ else if (strcmp(org_name, "l4-wkup-clkctrl@0") == 0)
	+ special_reg = L4_WKUP_CLKCTRL_0;
	+ break;
	+#endif /* SOC_TI_AM335X */
	+ default:
	+ break;
	+ }
	+
	+ /* reg property has a pair of (base address, length) */
	+ for (index = 0; index < num_reg; index += 2) {
	+ for (reg_offset = 0; reg_offset < reg[index+1]; reg_offset += sizeof(cell_t)) {
	+
	+ err = create_clkctrl(sc, reg, index, reg_offset, parent_offset,
	+ org_name, false);
	+ if (err)
	+ goto cleanup;
	+
	+ /* Create special clkctrl for GDBCLK in GPIO registers */
	+ switch (special_reg) {
	+ case NO_SPECIAL_REG:
	+ break;
	+ case L4LS_CLKCTRL_38:
	+ reg_address = reg[index] + reg_offset-reg[0];
	+ if (reg_address == 0x74 \|\|
	+ reg_address == 0x78 \|\|
	+ reg_address == 0x7C)
	+ {
	+ err = create_clkctrl(sc, reg, index, reg_offset,
	+ parent_offset, org_name, true);
	+ if (err)
	+ goto cleanup;
	+ }
	+ break;
	+ case L4_WKUP_CLKCTRL_0:
	+ reg_address = reg[index] + reg_offset - reg[0];
	+ if (reg_address == 0x8)
	+ {
	+ err = create_clkctrl(sc, reg, index, reg_offset,
	+ parent_offset, org_name, true);
	+ if (err)
	+ goto cleanup;
	+ }
	+ break;
	+ } /* switch (special_reg) */
	+ } /* inner for */
	+ } /* for */
	+
	+ err = clkdom_finit(sc->clkdom);
	+ if (err) {
	+ DPRINTF(sc->dev, "Clk domain finit fails %x.\n", err);
	+ err = ENXIO;
	+ goto cleanup;
	+ }
	+
	+cleanup:
	+ OF_prop_free(__DECONST(char *, org_name));
	+
	+ free(reg, M_DEVBUF);
	+
	+ if (err)
	+ return (err);
	+
	+ return (bus_generic_attach(dev));
	+}
	+
	+static int
	+ti_clkctrl_detach(device_t dev)
	+{
	+ return (EBUSY);
	+}
	+
	+/* modified version of default mapper from clk.c */
	+int
	+clkctrl_ofw_map(struct clkdom *clkdom, uint32_t ncells,
	+ phandle_t cells, struct clknode *clk) {
	+ if (ncells == 0)
	+ *clk = clknode_find_by_id(clkdom, 1);
	+ else if (ncells == 1)
	+ *clk = clknode_find_by_id(clkdom, cells[0]);
	+ else if (ncells == 2) {
	+ /* To avoid collision with other IDs just add one.
	+ * All other registers has an offset of 4 from each other.
	+ */
	+ if (cells[1])
	+ *clk = clknode_find_by_id(clkdom, cells[0]+1);
	+ else
	+ *clk = clknode_find_by_id(clkdom, cells[0]);
	+ }
	+ else
	+ return (ERANGE);
	+
	+ if (*clk == NULL)
	+ return (ENXIO);
	+
	+ return (0);
	+}
	+
	+static int
	+create_clkctrl(struct ti_clkctrl_softc sc, cell_t reg, uint32_t index, uint32_t reg_offset,
	+ uint64_t parent_offset, const char *org_name, bool special_gdbclk_reg) {
	+ struct ti_clk_clkctrl_def def;
	+ char *name;
	+ size_t name_len;
	+ int err;
	+
	+ name_len = strlen(org_name) + 1 + 5; /* 5 = _xxxx */
	+ name = malloc(name_len, M_OFWPROP, M_WAITOK);
	+
	+ /*
	+ * Check out XX_CLKCTRL-INDEX(offset)-macro dance in
	+ * sys/gnu/dts/dts/include/dt-bindings/clock/am3.h
	+ * sys/gnu/dts/dts/include/dt-bindings/clock/am4.h
	+ * sys/gnu/dts/dts/include/dt-bindings/clock/dra7.h
	+ * reg[0] are in practice the same as the offset described in the dts.
	+ */
	+ /* special_gdbclk_reg are 0 or 1 */
	+ def.clkdef.id = reg[index] + reg_offset - reg[0] + special_gdbclk_reg;
	+ def.register_offset = parent_offset + reg[index] + reg_offset;
	+
	+ /* Indicate this clkctrl is special and dont use IDLEST/MODULEMODE */
	+ def.gdbclk = special_gdbclk_reg;
	+
	+ /* Make up an uniq name in the namespace for each clkctrl */
	+ snprintf(name, name_len, "%s_%x",
	+ org_name, def.clkdef.id);
	+ def.clkdef.name = (const char *) name;
	+
	+ DPRINTF(sc->dev, "ti_clkctrl_attach: reg[%d]: %s %x\n",
	+ index, def.clkdef.name, def.clkdef.id);
	+
	+ /* No parent name */
	+ def.clkdef.parent_cnt = 0;
	+
	+ /* set flags */
	+ def.clkdef.flags = 0x0;
	+
	+ /* Register the clkctrl */
	+ err = ti_clknode_clkctrl_register(sc->clkdom, &def);
	+ if (err) {
	+ DPRINTF(sc->dev,
	+ "ti_clknode_clkctrl_register[%d:%d] failed %x\n",
	+ index, reg_offset, err);
	+ err = ENXIO;
	+ }
	+ OF_prop_free(name);
	+ return (err);
	+}
	+
	+static device_method_t ti_clkctrl_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_probe, ti_clkctrl_probe),
	+ DEVMETHOD(device_attach, ti_clkctrl_attach),
	+ DEVMETHOD(device_detach, ti_clkctrl_detach),
	+
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_0(ti_clkctrl, ti_clkctrl_driver, ti_clkctrl_methods,
	+ sizeof(struct ti_clkctrl_softc));
	+
	+static devclass_t ti_clkctrl_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_clkctrl, simplebus, ti_clkctrl_driver,
	+ti_clkctrl_devclass, 0, 0, BUS_PASS_BUS+BUS_PASS_ORDER_MIDDLE);
	+
	+MODULE_VERSION(ti_clkctrl, 1);
	diff --git a/sys/arm/ti/clk/ti_divider_clock.c b/sys/arm/ti/clk/ti_divider_clock.c
	new file mode 100644
	index 000000000000..753b5f535d29
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_divider_clock.c
	@@ -0,0 +1,264 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/conf.h>
	+#include <sys/bus.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/systm.h>
	+#include <sys/libkern.h>
	+
	+#include <machine/bus.h>
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/extres/clk/clk_div.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include "clock_common.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/*
	+ * Devicetree description
	+ * Documentation/devicetree/bindings/clock/ti/divider.txt
	+ */
	+
	+struct ti_divider_softc {
	+ device_t sc_dev;
	+ bool attach_done;
	+ struct clk_div_def div_def;
	+
	+ struct clock_cell_info clock_cell;
	+ struct clkdom *clkdom;
	+};
	+
	+static int ti_divider_probe(device_t dev);
	+static int ti_divider_attach(device_t dev);
	+static int ti_divider_detach(device_t dev);
	+
	+#define TI_DIVIDER_CLOCK 2
	+#define TI_COMPOSITE_DIVIDER_CLOCK 1
	+#define TI_DIVIDER_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,divider-clock", TI_DIVIDER_CLOCK },
	+ { "ti,composite-divider-clock", TI_COMPOSITE_DIVIDER_CLOCK },
	+ { NULL, TI_DIVIDER_END }
	+};
	+
	+static int
	+register_clk(struct ti_divider_softc *sc) {
	+ int err;
	+
	+ sc->clkdom = clkdom_create(sc->sc_dev);
	+ if (sc->clkdom == NULL) {
	+ DPRINTF(sc->sc_dev, "Failed to create clkdom\n");
	+ return (ENXIO);
	+ }
	+
	+ err = clknode_div_register(sc->clkdom, &sc->div_def);
	+ if (err) {
	+ DPRINTF(sc->sc_dev, "clknode_div_register failed %x\n", err);
	+ return (ENXIO);
	+ }
	+
	+ err = clkdom_finit(sc->clkdom);
	+ if (err) {
	+ DPRINTF(sc->sc_dev, "Clk domain finit fails %x.\n", err);
	+ return (ENXIO);
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+ti_divider_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI Divider Clock");
	+
	+ return (BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+ti_divider_attach(device_t dev)
	+{
	+ struct ti_divider_softc *sc;
	+ phandle_t node;
	+ int err;
	+ cell_t value;
	+ uint32_t ti_max_div;
	+
	+ sc = device_get_softc(dev);
	+ sc->sc_dev = dev;
	+ node = ofw_bus_get_node(dev);
	+
	+ /* Grab the content of reg properties */
	+ OF_getencprop(node, "reg", &value, sizeof(value));
	+ sc->div_def.offset = value;
	+
	+ if (OF_hasprop(node, "ti,bit-shift")) {
	+ OF_getencprop(node, "ti,bit-shift", &value, sizeof(value));
	+ sc->div_def.i_shift = value;
	+ }
	+
	+ if (OF_hasprop(node, "ti,index-starts-at-one")) {
	+ sc->div_def.div_flags = CLK_DIV_ZERO_BASED;
	+ }
	+
	+ if (OF_hasprop(node, "ti,index-power-of-two")) {
	+ /* FIXME: later */
	+ device_printf(sc->sc_dev, "ti,index-power-of-two - Not implemented\n");
	+ /* remember to update i_width a few lines below */
	+ }
	+ if (OF_hasprop(node, "ti,max-div")) {
	+ OF_getencprop(node, "ti,max-div", &value, sizeof(value));
	+ ti_max_div = value;
	+ }
	+
	+ if (OF_hasprop(node, "clock-output-names"))
	+ device_printf(sc->sc_dev, "clock-output-names\n");
	+ if (OF_hasprop(node, "ti,dividers"))
	+ device_printf(sc->sc_dev, "ti,dividers\n");
	+ if (OF_hasprop(node, "ti,min-div"))
	+ device_printf(sc->sc_dev, "ti,min-div - Not implemented\n");
	+
	+ if (OF_hasprop(node, "ti,autoidle-shift"))
	+ device_printf(sc->sc_dev, "ti,autoidle-shift - Not implemented\n");
	+ if (OF_hasprop(node, "ti,set-rate-parent"))
	+ device_printf(sc->sc_dev, "ti,set-rate-parent - Not implemented\n");
	+ if (OF_hasprop(node, "ti,latch-bit"))
	+ device_printf(sc->sc_dev, "ti,latch-bit - Not implemented\n");
	+
	+ /* Figure out the width from ti_max_div */
	+ if (sc->div_def.div_flags)
	+ sc->div_def.i_width = fls(ti_max_div-1);
	+ else
	+ sc->div_def.i_width = fls(ti_max_div);
	+
	+ DPRINTF(sc->sc_dev, "div_def.i_width %x\n", sc->div_def.i_width);
	+
	+ read_clock_cells(sc->sc_dev, &sc->clock_cell);
	+
	+ create_clkdef(sc->sc_dev, &sc->clock_cell, &sc->div_def.clkdef);
	+
	+ err = find_parent_clock_names(sc->sc_dev, &sc->clock_cell, &sc->div_def.clkdef);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_divider_new_pass */
	+ DPRINTF(sc->sc_dev, "find_parent_clock_names failed\n");
	+ return (bus_generic_attach(sc->sc_dev));
	+ }
	+
	+ err = register_clk(sc);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_divider_new_pass */
	+ DPRINTF(sc->sc_dev, "register_clk failed\n");
	+ return (bus_generic_attach(sc->sc_dev));
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->div_def.clkdef);
	+
	+ return (bus_generic_attach(sc->sc_dev));
	+}
	+
	+static int
	+ti_divider_detach(device_t dev)
	+{
	+ return (EBUSY);
	+}
	+
	+static void
	+ti_divider_new_pass(device_t dev)
	+{
	+ struct ti_divider_softc *sc;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+
	+ if (sc->attach_done) {
	+ return;
	+ }
	+
	+ err = find_parent_clock_names(sc->sc_dev, &sc->clock_cell, &sc->div_def.clkdef);
	+ if (err) {
	+ /* free_clkdef will be called in a later call to ti_divider_new_pass */
	+ DPRINTF(sc->sc_dev, "new_pass find_parent_clock_names failed\n");
	+ return;
	+ }
	+
	+ err = register_clk(sc);
	+ if (err) {
	+ /* free_clkdef will be called in a later call to ti_divider_new_pass */
	+ DPRINTF(sc->sc_dev, "new_pass register_clk failed\n");
	+ return;
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->div_def.clkdef);
	+}
	+
	+static device_method_t ti_divider_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_probe, ti_divider_probe),
	+ DEVMETHOD(device_attach, ti_divider_attach),
	+ DEVMETHOD(device_detach, ti_divider_detach),
	+
	+ /* Bus interface */
	+ DEVMETHOD(bus_new_pass, ti_divider_new_pass),
	+
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_0(ti_divider, ti_divider_driver, ti_divider_methods,
	+ sizeof(struct ti_divider_softc));
	+
	+static devclass_t ti_divider_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_divider, simplebus, ti_divider_driver,
	+ ti_divider_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_divider, 1);
	diff --git a/sys/arm/ti/clk/ti_dpll_clock.c b/sys/arm/ti/clk/ti_dpll_clock.c
	new file mode 100644
	index 000000000000..91127c570c4d
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_dpll_clock.c
	@@ -0,0 +1,375 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/conf.h>
	+#include <sys/bus.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/systm.h>
	+#include <sys/libkern.h>
	+
	+#include <machine/bus.h>
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/extres/clk/clk_div.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include <arm/ti/clk/ti_clk_dpll.h>
	+#include "clock_common.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/*
	+ * Devicetree description
	+ * Documentation/devicetree/bindings/clock/ti/dpll.txt
	+ */
	+
	+struct ti_dpll_softc {
	+ device_t dev;
	+ uint8_t dpll_type;
	+
	+ bool attach_done;
	+ struct ti_clk_dpll_def dpll_def;
	+
	+ struct clock_cell_info clock_cell;
	+ struct clkdom *clkdom;
	+};
	+
	+static int ti_dpll_probe(device_t dev);
	+static int ti_dpll_attach(device_t dev);
	+static int ti_dpll_detach(device_t dev);
	+
	+#define TI_OMAP3_DPLL_CLOCK 17
	+#define TI_OMAP3_DPLL_CORE_CLOCK 16
	+#define TI_OMAP3_DPLL_PER_CLOCK 15
	+#define TI_OMAP3_DPLL_PER_J_TYPE_CLOCK 14
	+#define TI_OMAP4_DPLL_CLOCK 13
	+#define TI_OMAP4_DPLL_X2_CLOCK 12
	+#define TI_OMAP4_DPLL_CORE_CLOCK 11
	+#define TI_OMAP4_DPLL_M4XEN_CLOCK 10
	+#define TI_OMAP4_DPLL_J_TYPE_CLOCK 9
	+#define TI_OMAP5_MPU_DPLL_CLOCK 8
	+#define TI_AM3_DPLL_NO_GATE_CLOCK 7
	+#define TI_AM3_DPLL_J_TYPE_CLOCK 6
	+#define TI_AM3_DPLL_NO_GATE_J_TYPE_CLOCK 5
	+#define TI_AM3_DPLL_CLOCK 4
	+#define TI_AM3_DPLL_CORE_CLOCK 3
	+#define TI_AM3_DPLL_X2_CLOCK 2
	+#define TI_OMAP2_DPLL_CORE_CLOCK 1
	+#define TI_DPLL_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,omap3-dpll-clock", TI_OMAP3_DPLL_CLOCK },
	+ { "ti,omap3-dpll-core-clock", TI_OMAP3_DPLL_CORE_CLOCK },
	+ { "ti,omap3-dpll-per-clock", TI_OMAP3_DPLL_PER_CLOCK },
	+ { "ti,omap3-dpll-per-j-type-clock",TI_OMAP3_DPLL_PER_J_TYPE_CLOCK },
	+ { "ti,omap4-dpll-clock", TI_OMAP4_DPLL_CLOCK },
	+ { "ti,omap4-dpll-x2-clock", TI_OMAP4_DPLL_X2_CLOCK },
	+ { "ti,omap4-dpll-core-clock", TI_OMAP4_DPLL_CORE_CLOCK },
	+ { "ti,omap4-dpll-m4xen-clock", TI_OMAP4_DPLL_M4XEN_CLOCK },
	+ { "ti,omap4-dpll-j-type-clock", TI_OMAP4_DPLL_J_TYPE_CLOCK },
	+ { "ti,omap5-mpu-dpll-clock", TI_OMAP5_MPU_DPLL_CLOCK },
	+ { "ti,am3-dpll-no-gate-clock", TI_AM3_DPLL_NO_GATE_CLOCK },
	+ { "ti,am3-dpll-j-type-clock", TI_AM3_DPLL_J_TYPE_CLOCK },
	+ { "ti,am3-dpll-no-gate-j-type-clock",TI_AM3_DPLL_NO_GATE_J_TYPE_CLOCK },
	+ { "ti,am3-dpll-clock", TI_AM3_DPLL_CLOCK },
	+ { "ti,am3-dpll-core-clock", TI_AM3_DPLL_CORE_CLOCK },
	+ { "ti,am3-dpll-x2-clock", TI_AM3_DPLL_X2_CLOCK },
	+ { "ti,omap2-dpll-core-clock", TI_OMAP2_DPLL_CORE_CLOCK },
	+ { NULL, TI_DPLL_END }
	+};
	+
	+static int
	+register_clk(struct ti_dpll_softc *sc) {
	+ int err;
	+
	+ sc->clkdom = clkdom_create(sc->dev);
	+ if (sc->clkdom == NULL) {
	+ DPRINTF(sc->dev, "Failed to create clkdom\n");
	+ return (ENXIO);
	+ }
	+
	+ err = ti_clknode_dpll_register(sc->clkdom, &sc->dpll_def);
	+ if (err) {
	+ DPRINTF(sc->dev,
	+ "ti_clknode_dpll_register failed %x\n", err);
	+ return (ENXIO);
	+ }
	+
	+ err = clkdom_finit(sc->clkdom);
	+ if (err) {
	+ DPRINTF(sc->dev, "Clk domain finit fails %x.\n", err);
	+ return (ENXIO);
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+ti_dpll_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI DPLL Clock");
	+
	+ return (BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+parse_dpll_reg(struct ti_dpll_softc *sc) {
	+ ssize_t numbytes_regs;
	+ uint32_t num_regs;
	+ phandle_t node;
	+ cell_t reg_cells[4];
	+
	+ if (sc->dpll_type == TI_AM3_DPLL_X2_CLOCK \|\|
	+ sc->dpll_type == TI_OMAP4_DPLL_X2_CLOCK) {
	+ sc->dpll_def.ti_clksel_mult.value = 2;
	+ sc->dpll_def.ti_clksel_mult.flags = TI_CLK_FACTOR_FIXED;
	+
	+ sc->dpll_def.ti_clksel_div.value = 1;
	+ sc->dpll_def.ti_clksel_div.flags = TI_CLK_FACTOR_FIXED;
	+ return (0);
	+ }
	+
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ numbytes_regs = OF_getproplen(node, "reg");
	+ num_regs = numbytes_regs / sizeof(cell_t);
	+
	+ /* Sanity check */
	+ if (num_regs > 4)
	+ return (ENXIO);
	+
	+ OF_getencprop(node, "reg", reg_cells, numbytes_regs);
	+
	+ switch (sc->dpll_type) {
	+ case TI_AM3_DPLL_NO_GATE_CLOCK:
	+ case TI_AM3_DPLL_J_TYPE_CLOCK:
	+ case TI_AM3_DPLL_NO_GATE_J_TYPE_CLOCK:
	+ case TI_AM3_DPLL_CLOCK:
	+ case TI_AM3_DPLL_CORE_CLOCK:
	+ case TI_AM3_DPLL_X2_CLOCK:
	+ if (num_regs != 3)
	+ return (ENXIO);
	+ sc->dpll_def.ti_clkmode_offset = reg_cells[0];
	+ sc->dpll_def.ti_idlest_offset = reg_cells[1];
	+ sc->dpll_def.ti_clksel_offset = reg_cells[2];
	+ break;
	+
	+ case TI_OMAP2_DPLL_CORE_CLOCK:
	+ if (num_regs != 2)
	+ return (ENXIO);
	+ sc->dpll_def.ti_clkmode_offset = reg_cells[0];
	+ sc->dpll_def.ti_clksel_offset = reg_cells[1];
	+ break;
	+
	+ default:
	+ sc->dpll_def.ti_clkmode_offset = reg_cells[0];
	+ sc->dpll_def.ti_idlest_offset = reg_cells[1];
	+ sc->dpll_def.ti_clksel_offset = reg_cells[2];
	+ sc->dpll_def.ti_autoidle_offset = reg_cells[3];
	+ break;
	+ }
	+
	+ /* AM335x */
	+ if (sc->dpll_def.ti_clksel_offset == CM_CLKSEL_DPLL_PERIPH) {
	+ sc->dpll_def.ti_clksel_mult.shift = 8;
	+ sc->dpll_def.ti_clksel_mult.mask = 0x000FFF00;
	+ sc->dpll_def.ti_clksel_mult.width = 12;
	+ sc->dpll_def.ti_clksel_mult.value = 0;
	+ sc->dpll_def.ti_clksel_mult.min_value = 2;
	+ sc->dpll_def.ti_clksel_mult.max_value = 4095;
	+ sc->dpll_def.ti_clksel_mult.flags = TI_CLK_FACTOR_ZERO_BASED \|
	+ TI_CLK_FACTOR_MIN_VALUE \|
	+ TI_CLK_FACTOR_MAX_VALUE;
	+
	+ sc->dpll_def.ti_clksel_div.shift = 0;
	+ sc->dpll_def.ti_clksel_div.mask = 0x000000FF;
	+ sc->dpll_def.ti_clksel_div.width = 8;
	+ sc->dpll_def.ti_clksel_div.value = 0;
	+ sc->dpll_def.ti_clksel_div.min_value = 0;
	+ sc->dpll_def.ti_clksel_div.max_value = 255;
	+ sc->dpll_def.ti_clksel_div.flags = TI_CLK_FACTOR_MIN_VALUE \|
	+ TI_CLK_FACTOR_MAX_VALUE;
	+ } else {
	+ sc->dpll_def.ti_clksel_mult.shift = 8;
	+ sc->dpll_def.ti_clksel_mult.mask = 0x0007FF00;
	+ sc->dpll_def.ti_clksel_mult.width = 11;
	+ sc->dpll_def.ti_clksel_mult.value = 0;
	+ sc->dpll_def.ti_clksel_mult.min_value = 2;
	+ sc->dpll_def.ti_clksel_mult.max_value = 2047;
	+ sc->dpll_def.ti_clksel_mult.flags = TI_CLK_FACTOR_ZERO_BASED \|
	+ TI_CLK_FACTOR_MIN_VALUE \|
	+ TI_CLK_FACTOR_MAX_VALUE;
	+
	+ sc->dpll_def.ti_clksel_div.shift = 0;
	+ sc->dpll_def.ti_clksel_div.mask = 0x0000007F;
	+ sc->dpll_def.ti_clksel_div.width = 7;
	+ sc->dpll_def.ti_clksel_div.value = 0;
	+ sc->dpll_def.ti_clksel_div.min_value = 0;
	+ sc->dpll_def.ti_clksel_div.max_value = 127;
	+ sc->dpll_def.ti_clksel_div.flags = TI_CLK_FACTOR_MIN_VALUE \|
	+ TI_CLK_FACTOR_MAX_VALUE;
	+ }
	+ DPRINTF(sc->dev, "clkmode %x idlest %x clksel %x autoidle %x\n",
	+ sc->dpll_def.ti_clkmode_offset, sc->dpll_def.ti_idlest_offset,
	+ sc->dpll_def.ti_clksel_offset,
	+ sc->dpll_def.ti_autoidle_offset);
	+
	+ return (0);
	+}
	+static int
	+ti_dpll_attach(device_t dev)
	+{
	+ struct ti_dpll_softc *sc;
	+ phandle_t node;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+
	+ sc->dpll_type = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
	+ node = ofw_bus_get_node(dev);
	+
	+ /* Grab the content of reg properties */
	+ parse_dpll_reg(sc);
	+
	+ /* default flags (OMAP4&AM335x) not present in the dts at moment */
	+ sc->dpll_def.ti_clkmode_flags = MN_BYPASS_MODE_FLAG \| LOCK_MODE_FLAG;
	+
	+ if (OF_hasprop(node, "ti,low-power-stop")) {
	+ sc->dpll_def.ti_clkmode_flags \|= LOW_POWER_STOP_MODE_FLAG;
	+ }
	+ if (OF_hasprop(node, "ti,low-power-bypass")) {
	+ sc->dpll_def.ti_clkmode_flags \|= IDLE_BYPASS_LOW_POWER_MODE_FLAG;
	+ }
	+ if (OF_hasprop(node, "ti,lock")) {
	+ sc->dpll_def.ti_clkmode_flags \|= LOCK_MODE_FLAG;
	+ }
	+
	+ read_clock_cells(sc->dev, &sc->clock_cell);
	+
	+ create_clkdef(sc->dev, &sc->clock_cell, &sc->dpll_def.clkdef);
	+
	+ err = find_parent_clock_names(sc->dev, &sc->clock_cell,
	+ &sc->dpll_def.clkdef);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_dpll_new_pass */
	+ DPRINTF(sc->dev, "find_parent_clock_names failed\n");
	+ return (bus_generic_attach(sc->dev));
	+ }
	+
	+ err = register_clk(sc);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_dpll_new_pass */
	+ DPRINTF(sc->dev, "register_clk failed\n");
	+ return (bus_generic_attach(sc->dev));
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->dpll_def.clkdef);
	+
	+ return (bus_generic_attach(sc->dev));
	+}
	+
	+static int
	+ti_dpll_detach(device_t dev)
	+{
	+ return (EBUSY);
	+}
	+
	+static void
	+ti_dpll_new_pass(device_t dev)
	+{
	+ struct ti_dpll_softc *sc;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+
	+ if (sc->attach_done) {
	+ return;
	+ }
	+
	+ err = find_parent_clock_names(sc->dev, &sc->clock_cell,
	+ &sc->dpll_def.clkdef);
	+ if (err) {
	+ /* free_clkdef will be called in a later call to ti_dpll_new_pass */
	+ DPRINTF(sc->dev,
	+ "new_pass find_parent_clock_names failed\n");
	+ return;
	+ }
	+
	+ err = register_clk(sc);
	+ if (err) {
	+ /* free_clkdef will be called in a later call to ti_dpll_new_pass */
	+ DPRINTF(sc->dev, "new_pass register_clk failed\n");
	+ return;
	+ }
	+
	+ sc->attach_done = true;
	+ free_clkdef(&sc->dpll_def.clkdef);
	+}
	+
	+static device_method_t ti_dpll_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_probe, ti_dpll_probe),
	+ DEVMETHOD(device_attach, ti_dpll_attach),
	+ DEVMETHOD(device_detach, ti_dpll_detach),
	+
	+ /* Bus interface */
	+ DEVMETHOD(bus_new_pass, ti_dpll_new_pass),
	+
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_0(ti_dpll, ti_dpll_driver, ti_dpll_methods,
	+ sizeof(struct ti_dpll_softc));
	+
	+static devclass_t ti_dpll_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_dpll, simplebus, ti_dpll_driver,
	+ ti_dpll_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_dpll, 1);
	diff --git a/sys/arm/ti/clk/ti_gate_clock.c b/sys/arm/ti/clk/ti_gate_clock.c
	new file mode 100644
	index 000000000000..b4fb65995e74
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_gate_clock.c
	@@ -0,0 +1,266 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/conf.h>
	+#include <sys/bus.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/systm.h>
	+#include <sys/libkern.h>
	+
	+#include <machine/bus.h>
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/extres/clk/clk_gate.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include "clock_common.h"
	+
	+#define DEBUG_GATE 0
	+
	+#if DEBUG_GATE
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/*
	+ * Devicetree description
	+ * Documentation/devicetree/bindings/clock/ti/gate.txt
	+ */
	+
	+struct ti_gate_softc {
	+ device_t sc_dev;
	+ bool attach_done;
	+ uint8_t sc_type;
	+
	+ struct clk_gate_def gate_def;
	+ struct clock_cell_info clock_cell;
	+ struct clkdom *clkdom;
	+};
	+
	+static int ti_gate_probe(device_t dev);
	+static int ti_gate_attach(device_t dev);
	+static int ti_gate_detach(device_t dev);
	+
	+#define TI_GATE_CLOCK 7
	+#define TI_WAIT_GATE_CLOCK 6
	+#define TI_DSS_GATE_CLOCK 5
	+#define TI_AM35XX_GATE_CLOCK 4
	+#define TI_CLKDM_GATE_CLOCK 3
	+#define TI_HSDIV_GATE_CLOCK 2
	+#define TI_COMPOSITE_NO_WAIT_GATE_CLOCK 1
	+#define TI_GATE_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,gate-clock", TI_GATE_CLOCK },
	+ { "ti,wait-gate-clock", TI_WAIT_GATE_CLOCK },
	+ { "ti,dss-gate-clock", TI_DSS_GATE_CLOCK },
	+ { "ti,am35xx-gate-clock", TI_AM35XX_GATE_CLOCK },
	+ { "ti,clkdm-gate-clock", TI_CLKDM_GATE_CLOCK },
	+ { "ti,hsdiv-gate-cloc", TI_HSDIV_GATE_CLOCK },
	+ { "ti,composite-no-wait-gate-clock", TI_COMPOSITE_NO_WAIT_GATE_CLOCK },
	+ { NULL, TI_GATE_END }
	+};
	+
	+static int
	+ti_gate_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI Gate Clock");
	+
	+ return (BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+register_clk(struct ti_gate_softc *sc) {
	+ int err;
	+ sc->clkdom = clkdom_create(sc->sc_dev);
	+ if (sc->clkdom == NULL) {
	+ DPRINTF(sc->sc_dev, "Failed to create clkdom\n");
	+ return ENXIO;
	+ }
	+
	+ err = clknode_gate_register(sc->clkdom, &sc->gate_def);
	+ if (err) {
	+ DPRINTF(sc->sc_dev, "clknode_gate_register failed %x\n", err);
	+ return ENXIO;
	+ }
	+
	+ err = clkdom_finit(sc->clkdom);
	+ if (err) {
	+ DPRINTF(sc->sc_dev, "Clk domain finit fails %x.\n", err);
	+ return ENXIO;
	+ }
	+
	+ return (0);
	+}
	+
	+static int
	+ti_gate_attach(device_t dev)
	+{
	+ struct ti_gate_softc *sc;
	+ phandle_t node;
	+ int err;
	+ cell_t value;
	+
	+ sc = device_get_softc(dev);
	+ sc->sc_dev = dev;
	+ node = ofw_bus_get_node(dev);
	+
	+ /* Get the compatible type */
	+ sc->sc_type = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
	+
	+ /* Get the content of reg properties */
	+ if (sc->sc_type != TI_CLKDM_GATE_CLOCK) {
	+ OF_getencprop(node, "reg", &value, sizeof(value));
	+ sc->gate_def.offset = value;
	+ }
	+#if DEBUG_GATE
	+ else {
	+ DPRINTF(sc->sc_dev, "no reg (TI_CLKDM_GATE_CLOCK)\n");
	+ }
	+#endif
	+
	+ if (OF_hasprop(node, "ti,bit-shift")) {
	+ OF_getencprop(node, "ti,bit-shift", &value, sizeof(value));
	+ sc->gate_def.shift = value;
	+ DPRINTF(sc->sc_dev, "ti,bit-shift => shift %x\n", sc->gate_def.shift);
	+ }
	+ if (OF_hasprop(node, "ti,set-bit-to-disable")) {
	+ sc->gate_def.on_value = 0;
	+ sc->gate_def.off_value = 1;
	+ DPRINTF(sc->sc_dev,
	+ "on_value = 0, off_value = 1 (ti,set-bit-to-disable)\n");
	+ } else {
	+ sc->gate_def.on_value = 1;
	+ sc->gate_def.off_value = 0;
	+ DPRINTF(sc->sc_dev, "on_value = 1, off_value = 0\n");
	+ }
	+
	+ sc->gate_def.gate_flags = 0x0;
	+
	+ read_clock_cells(sc->sc_dev, &sc->clock_cell);
	+
	+ create_clkdef(sc->sc_dev, &sc->clock_cell, &sc->gate_def.clkdef);
	+
	+ /* Calculate mask */
	+ sc->gate_def.mask = (1 << fls(sc->clock_cell.num_real_clocks)) - 1;
	+ DPRINTF(sc->sc_dev, "num_real_clocks %x gate_def.mask %x\n",
	+ sc->clock_cell.num_real_clocks, sc->gate_def.mask);
	+
	+ err = find_parent_clock_names(sc->sc_dev, &sc->clock_cell, &sc->gate_def.clkdef);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_gate_new_pass */
	+ DPRINTF(sc->sc_dev, "find_parent_clock_names failed\n");
	+ return (bus_generic_attach(sc->sc_dev));
	+ }
	+
	+ err = register_clk(sc);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_gate_new_pass */
	+ DPRINTF(sc->sc_dev, "register_clk failed\n");
	+ return (bus_generic_attach(sc->sc_dev));
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->gate_def.clkdef);
	+
	+ return (bus_generic_attach(sc->sc_dev));
	+}
	+
	+static int
	+ti_gate_detach(device_t dev)
	+{
	+ return (EBUSY);
	+}
	+
	+static void
	+ti_gate_new_pass(device_t dev) {
	+ struct ti_gate_softc *sc;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+
	+ if (sc->attach_done) {
	+ return;
	+ }
	+
	+ err = find_parent_clock_names(sc->sc_dev, &sc->clock_cell, &sc->gate_def.clkdef);
	+ if (err) {
	+ /* free_clkdef will be called in later call to ti_gate_new_pass */
	+ DPRINTF(sc->sc_dev, "new_pass find_parent_clock_names failed\n");
	+ return;
	+ }
	+
	+ err = register_clk(sc);
	+ if (err) {
	+ /* free_clkdef will be called in later call to ti_gate_new_pass */
	+ DPRINTF(sc->sc_dev, "new_pass register_clk failed\n");
	+ return;
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->gate_def.clkdef);
	+}
	+
	+static device_method_t ti_gate_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_probe, ti_gate_probe),
	+ DEVMETHOD(device_attach, ti_gate_attach),
	+ DEVMETHOD(device_detach, ti_gate_detach),
	+
	+ /* Bus interface */
	+ DEVMETHOD(bus_new_pass, ti_gate_new_pass),
	+
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_0(ti_gate, ti_gate_driver, ti_gate_methods,
	+ sizeof(struct ti_gate_softc));
	+
	+static devclass_t ti_gate_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_gate, simplebus, ti_gate_driver,
	+ ti_gate_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_gate, 1);
	diff --git a/sys/arm/ti/clk/ti_mux_clock.c b/sys/arm/ti/clk/ti_mux_clock.c
	new file mode 100644
	index 000000000000..bd232290e6a0
	--- /dev/null
	+++ b/sys/arm/ti/clk/ti_mux_clock.c
	@@ -0,0 +1,249 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/conf.h>
	+#include <sys/bus.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/systm.h>
	+#include <sys/libkern.h>
	+#include <sys/types.h>
	+#include <sys/malloc.h>
	+
	+#include <machine/bus.h>
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/extres/clk/clk_mux.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include "clock_common.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/*
	+ * Devicetree description
	+ * Documentation/devicetree/bindings/clock/ti/mux.txt
	+ */
	+
	+struct ti_mux_softc {
	+ device_t sc_dev;
	+ bool attach_done;
	+
	+ struct clk_mux_def mux_def;
	+ struct clock_cell_info clock_cell;
	+ struct clkdom *clkdom;
	+};
	+
	+static int ti_mux_probe(device_t dev);
	+static int ti_mux_attach(device_t dev);
	+static int ti_mux_detach(device_t dev);
	+
	+#define TI_MUX_CLOCK 2
	+#define TI_COMPOSITE_MUX_CLOCK 1
	+#define TI_MUX_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,mux-clock", TI_MUX_CLOCK },
	+ { "ti,composite-mux-clock", TI_COMPOSITE_MUX_CLOCK },
	+ { NULL, TI_MUX_END }
	+};
	+
	+static int
	+ti_mux_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI Mux Clock");
	+
	+ return (BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+register_clk(struct ti_mux_softc *sc) {
	+ int err;
	+
	+ sc->clkdom = clkdom_create(sc->sc_dev);
	+ if (sc->clkdom == NULL) {
	+ DPRINTF(sc->sc_dev, "Failed to create clkdom\n");
	+ return ENXIO;
	+ }
	+
	+ err = clknode_mux_register(sc->clkdom, &sc->mux_def);
	+ if (err) {
	+ DPRINTF(sc->sc_dev, "clknode_mux_register failed %x\n", err);
	+ return ENXIO;
	+ }
	+
	+ err = clkdom_finit(sc->clkdom);
	+ if (err) {
	+ DPRINTF(sc->sc_dev, "Clk domain finit fails %x.\n", err);
	+ return ENXIO;
	+ }
	+
	+ return 0;
	+}
	+
	+static int
	+ti_mux_attach(device_t dev)
	+{
	+ struct ti_mux_softc *sc;
	+ phandle_t node;
	+ int err;
	+ cell_t value;
	+
	+ sc = device_get_softc(dev);
	+ sc->sc_dev = dev;
	+ node = ofw_bus_get_node(dev);
	+
	+ /* Grab the content of reg properties */
	+ OF_getencprop(node, "reg", &value, sizeof(value));
	+ sc->mux_def.offset = value;
	+
	+ if (OF_hasprop(node, "ti,bit-shift")) {
	+ OF_getencprop(node, "ti,bit-shift", &value, sizeof(value));
	+ sc->mux_def.shift = value;
	+ DPRINTF(sc->sc_dev, "ti,bit-shift => shift %x\n", sc->mux_def.shift);
	+ }
	+ if (OF_hasprop(node, "ti,index-starts-at-one")) {
	+ /* FIXME: Add support in dev/extres/clk */
	+ /sc->mux_def.mux_flags = ... /
	+ device_printf(sc->sc_dev, "ti,index-starts-at-one - Not implemented\n");
	+ }
	+
	+ if (OF_hasprop(node, "ti,set-rate-parent"))
	+ device_printf(sc->sc_dev, "ti,set-rate-parent - Not implemented\n");
	+ if (OF_hasprop(node, "ti,latch-bit"))
	+ device_printf(sc->sc_dev, "ti,latch-bit - Not implemented\n");
	+
	+ read_clock_cells(sc->sc_dev, &sc->clock_cell);
	+
	+ create_clkdef(sc->sc_dev, &sc->clock_cell, &sc->mux_def.clkdef);
	+
	+ /* Figure out the width from ti_max_div */
	+ if (sc->mux_def.mux_flags)
	+ sc->mux_def.width = fls(sc->clock_cell.num_real_clocks-1);
	+ else
	+ sc->mux_def.width = fls(sc->clock_cell.num_real_clocks);
	+
	+ DPRINTF(sc->sc_dev, "sc->clock_cell.num_real_clocks %x def.width %x\n",
	+ sc->clock_cell.num_real_clocks, sc->mux_def.width);
	+
	+ err = find_parent_clock_names(sc->sc_dev, &sc->clock_cell, &sc->mux_def.clkdef);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_mux_new_pass */
	+ DPRINTF(sc->sc_dev, "find_parent_clock_names failed\n");
	+ return (bus_generic_attach(sc->sc_dev));
	+ }
	+
	+ err = register_clk(sc);
	+
	+ if (err) {
	+ /* free_clkdef will be called in ti_mux_new_pass */
	+ DPRINTF(sc->sc_dev, "register_clk failed\n");
	+ return (bus_generic_attach(sc->sc_dev));
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->mux_def.clkdef);
	+
	+ return (bus_generic_attach(sc->sc_dev));
	+}
	+
	+static void
	+ti_mux_new_pass(device_t dev)
	+{
	+ struct ti_mux_softc *sc;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+
	+ if (sc->attach_done) {
	+ return;
	+ }
	+
	+ err = find_parent_clock_names(sc->sc_dev, &sc->clock_cell, &sc->mux_def.clkdef);
	+ if (err) {
	+ /* free_clkdef will be called in later call to ti_mux_new_pass */
	+ DPRINTF(sc->sc_dev, "ti_mux_new_pass find_parent_clock_names failed\n");
	+ return;
	+ }
	+
	+ err = register_clk(sc);
	+ if (err) {
	+ /* free_clkdef will be called in later call to ti_mux_new_pass */
	+ DPRINTF(sc->sc_dev, "ti_mux_new_pass register_clk failed\n");
	+ return;
	+ }
	+
	+ sc->attach_done = true;
	+
	+ free_clkdef(&sc->mux_def.clkdef);
	+}
	+
	+static int
	+ti_mux_detach(device_t dev)
	+{
	+ return (EBUSY);
	+}
	+
	+static device_method_t ti_mux_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_probe, ti_mux_probe),
	+ DEVMETHOD(device_attach, ti_mux_attach),
	+ DEVMETHOD(device_detach, ti_mux_detach),
	+
	+ /* Bus interface */
	+ DEVMETHOD(bus_new_pass, ti_mux_new_pass),
	+
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_0(ti_mux, ti_mux_driver, ti_mux_methods,
	+ sizeof(struct ti_mux_softc));
	+
	+static devclass_t ti_mux_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_mux, simplebus, ti_mux_driver,
	+ ti_mux_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_mux, 1);
	diff --git a/sys/arm/ti/cpsw/if_cpsw.c b/sys/arm/ti/cpsw/if_cpsw.c
	index 4503304e7b5b..9c43419d568b 100644
	--- a/sys/arm/ti/cpsw/if_cpsw.c
	+++ b/sys/arm/ti/cpsw/if_cpsw.c
	@@ -1,3006 +1,3028 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	* Copyright (c) 2016 Rubicon Communications, LLC (Netgate)
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/*
	* TI Common Platform Ethernet Switch (CPSW) Driver
	* Found in TI8148 "DaVinci" and AM335x "Sitara" SoCs.
	*
	* This controller is documented in the AM335x Technical Reference
	* Manual, in the TMS320DM814x DaVinci Digital Video Processors TRM
	* and in the TMS320C6452 3 Port Switch Ethernet Subsystem TRM.
	*
	* It is basically a single Ethernet port (port 0) wired internally to
	* a 3-port store-and-forward switch connected to two independent
	* "sliver" controllers (port 1 and port 2). You can operate the
	* controller in a variety of different ways by suitably configuring
	* the slivers and the Address Lookup Engine (ALE) that routes packets
	* between the ports.
	*
	* This code was developed and tested on a BeagleBone with
	* an AM335x SoC.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_cpsw.h"

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/mbuf.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/socket.h>
	#include <sys/sockio.h>
	#include <sys/sysctl.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <machine/stdarg.h>

	#include <net/ethernet.h>
	#include <net/bpf.h>
	#include <net/if.h>
	#include <net/if_dl.h>
	#include <net/if_media.h>
	#include <net/if_types.h>

	-#include <arm/ti/ti_scm.h>
	+#include <dev/extres/syscon/syscon.h>
	+#include "syscon_if.h"
	#include <arm/ti/am335x/am335x_scm.h>

	#include <dev/mii/mii.h>
	#include <dev/mii/miivar.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/fdt/fdt_common.h>

	#ifdef CPSW_ETHERSWITCH
	#include <dev/etherswitch/etherswitch.h>
	#include "etherswitch_if.h"
	#endif

	#include "if_cpswreg.h"
	#include "if_cpswvar.h"

	#include "miibus_if.h"

	/* Device probe/attach/detach. */
	static int cpsw_probe(device_t);
	static int cpsw_attach(device_t);
	static int cpsw_detach(device_t);
	static int cpswp_probe(device_t);
	static int cpswp_attach(device_t);
	static int cpswp_detach(device_t);

	static phandle_t cpsw_get_node(device_t, device_t);

	/* Device Init/shutdown. */
	static int cpsw_shutdown(device_t);
	static void cpswp_init(void *);
	static void cpswp_init_locked(void *);
	static void cpswp_stop_locked(struct cpswp_softc *);

	/* Device Suspend/Resume. */
	static int cpsw_suspend(device_t);
	static int cpsw_resume(device_t);

	/* Ioctl. */
	static int cpswp_ioctl(struct ifnet *, u_long command, caddr_t data);

	static int cpswp_miibus_readreg(device_t, int phy, int reg);
	static int cpswp_miibus_writereg(device_t, int phy, int reg, int value);
	static void cpswp_miibus_statchg(device_t);

	/* Send/Receive packets. */
	static void cpsw_intr_rx(void *arg);
	static struct mbuf cpsw_rx_dequeue(struct cpsw_softc );
	static void cpsw_rx_enqueue(struct cpsw_softc *);
	static void cpswp_start(struct ifnet *);
	static void cpsw_intr_tx(void *);
	static void cpswp_tx_enqueue(struct cpswp_softc *);
	static int cpsw_tx_dequeue(struct cpsw_softc *);

	/* Misc interrupts and watchdog. */
	static void cpsw_intr_rx_thresh(void *);
	static void cpsw_intr_misc(void *);
	static void cpswp_tick(void *);
	static void cpswp_ifmedia_sts(struct ifnet , struct ifmediareq );
	static int cpswp_ifmedia_upd(struct ifnet *);
	static void cpsw_tx_watchdog(void *);

	/* ALE support */
	static void cpsw_ale_read_entry(struct cpsw_softc , uint16_t, uint32_t );
	static void cpsw_ale_write_entry(struct cpsw_softc , uint16_t, uint32_t );
	static int cpsw_ale_mc_entry_set(struct cpsw_softc , uint8_t, int, uint8_t );
	static void cpsw_ale_dump_table(struct cpsw_softc *);
	static int cpsw_ale_update_vlan_table(struct cpsw_softc *, int, int, int, int,
	int);
	static int cpswp_ale_update_addresses(struct cpswp_softc *, int);

	/* Statistics and sysctls. */
	static void cpsw_add_sysctls(struct cpsw_softc *);
	static void cpsw_stats_collect(struct cpsw_softc *);
	static int cpsw_stats_sysctl(SYSCTL_HANDLER_ARGS);

	#ifdef CPSW_ETHERSWITCH
	static etherswitch_info_t *cpsw_getinfo(device_t);
	static int cpsw_getport(device_t, etherswitch_port_t *);
	static int cpsw_setport(device_t, etherswitch_port_t *);
	static int cpsw_getconf(device_t, etherswitch_conf_t *);
	static int cpsw_getvgroup(device_t, etherswitch_vlangroup_t *);
	static int cpsw_setvgroup(device_t, etherswitch_vlangroup_t *);
	static int cpsw_readreg(device_t, int);
	static int cpsw_writereg(device_t, int, int);
	static int cpsw_readphy(device_t, int, int);
	static int cpsw_writephy(device_t, int, int, int);
	#endif

	/*
	* Arbitrary limit on number of segments in an mbuf to be transmitted.
	* Packets with more segments than this will be defragmented before
	* they are queued.
	*/
	#define CPSW_TXFRAGS 16

	/* Shared resources. */
	static device_method_t cpsw_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, cpsw_probe),
	DEVMETHOD(device_attach, cpsw_attach),
	DEVMETHOD(device_detach, cpsw_detach),
	DEVMETHOD(device_shutdown, cpsw_shutdown),
	DEVMETHOD(device_suspend, cpsw_suspend),
	DEVMETHOD(device_resume, cpsw_resume),
	/* Bus interface */
	DEVMETHOD(bus_add_child, device_add_child_ordered),
	/* OFW methods */
	DEVMETHOD(ofw_bus_get_node, cpsw_get_node),
	#ifdef CPSW_ETHERSWITCH
	/* etherswitch interface */
	DEVMETHOD(etherswitch_getinfo, cpsw_getinfo),
	DEVMETHOD(etherswitch_readreg, cpsw_readreg),
	DEVMETHOD(etherswitch_writereg, cpsw_writereg),
	DEVMETHOD(etherswitch_readphyreg, cpsw_readphy),
	DEVMETHOD(etherswitch_writephyreg, cpsw_writephy),
	DEVMETHOD(etherswitch_getport, cpsw_getport),
	DEVMETHOD(etherswitch_setport, cpsw_setport),
	DEVMETHOD(etherswitch_getvgroup, cpsw_getvgroup),
	DEVMETHOD(etherswitch_setvgroup, cpsw_setvgroup),
	DEVMETHOD(etherswitch_getconf, cpsw_getconf),
	#endif
	DEVMETHOD_END
	};

	static driver_t cpsw_driver = {
	"cpswss",
	cpsw_methods,
	sizeof(struct cpsw_softc),
	};

	static devclass_t cpsw_devclass;

	DRIVER_MODULE(cpswss, simplebus, cpsw_driver, cpsw_devclass, 0, 0);

	/* Port/Slave resources. */
	static device_method_t cpswp_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, cpswp_probe),
	DEVMETHOD(device_attach, cpswp_attach),
	DEVMETHOD(device_detach, cpswp_detach),
	/* MII interface */
	DEVMETHOD(miibus_readreg, cpswp_miibus_readreg),
	DEVMETHOD(miibus_writereg, cpswp_miibus_writereg),
	DEVMETHOD(miibus_statchg, cpswp_miibus_statchg),
	DEVMETHOD_END
	};

	static driver_t cpswp_driver = {
	"cpsw",
	cpswp_methods,
	sizeof(struct cpswp_softc),
	};

	static devclass_t cpswp_devclass;

	#ifdef CPSW_ETHERSWITCH
	DRIVER_MODULE(etherswitch, cpswss, etherswitch_driver, etherswitch_devclass, 0, 0);
	MODULE_DEPEND(cpswss, etherswitch, 1, 1, 1);
	#endif

	DRIVER_MODULE(cpsw, cpswss, cpswp_driver, cpswp_devclass, 0, 0);
	DRIVER_MODULE(miibus, cpsw, miibus_driver, miibus_devclass, 0, 0);
	MODULE_DEPEND(cpsw, ether, 1, 1, 1);
	MODULE_DEPEND(cpsw, miibus, 1, 1, 1);

	#ifdef CPSW_ETHERSWITCH
	static struct cpsw_vlangroups cpsw_vgroups[CPSW_VLANS];
	#endif

	static uint32_t slave_mdio_addr[] = { 0x4a100200, 0x4a100300 };

	static struct resource_spec irq_res_spec[] = {
	{ SYS_RES_IRQ, 0, RF_ACTIVE \| RF_SHAREABLE },
	{ SYS_RES_IRQ, 1, RF_ACTIVE \| RF_SHAREABLE },
	{ SYS_RES_IRQ, 2, RF_ACTIVE \| RF_SHAREABLE },
	{ SYS_RES_IRQ, 3, RF_ACTIVE \| RF_SHAREABLE },
	{ -1, 0 }
	};

	static struct {
	void (cb)(void );
	} cpsw_intr_cb[] = {
	{ cpsw_intr_rx_thresh },
	{ cpsw_intr_rx },
	{ cpsw_intr_tx },
	{ cpsw_intr_misc },
	};

	/* Number of entries here must match size of stats
	* array in struct cpswp_softc. */
	static struct cpsw_stat {
	int reg;
	char *oid;
	} cpsw_stat_sysctls[CPSW_SYSCTL_COUNT] = {
	{0x00, "GoodRxFrames"},
	{0x04, "BroadcastRxFrames"},
	{0x08, "MulticastRxFrames"},
	{0x0C, "PauseRxFrames"},
	{0x10, "RxCrcErrors"},
	{0x14, "RxAlignErrors"},
	{0x18, "OversizeRxFrames"},
	{0x1c, "RxJabbers"},
	{0x20, "ShortRxFrames"},
	{0x24, "RxFragments"},
	{0x30, "RxOctets"},
	{0x34, "GoodTxFrames"},
	{0x38, "BroadcastTxFrames"},
	{0x3c, "MulticastTxFrames"},
	{0x40, "PauseTxFrames"},
	{0x44, "DeferredTxFrames"},
	{0x48, "CollisionsTxFrames"},
	{0x4c, "SingleCollisionTxFrames"},
	{0x50, "MultipleCollisionTxFrames"},
	{0x54, "ExcessiveCollisions"},
	{0x58, "LateCollisions"},
	{0x5c, "TxUnderrun"},
	{0x60, "CarrierSenseErrors"},
	{0x64, "TxOctets"},
	{0x68, "RxTx64OctetFrames"},
	{0x6c, "RxTx65to127OctetFrames"},
	{0x70, "RxTx128to255OctetFrames"},
	{0x74, "RxTx256to511OctetFrames"},
	{0x78, "RxTx512to1024OctetFrames"},
	{0x7c, "RxTx1024upOctetFrames"},
	{0x80, "NetOctets"},
	{0x84, "RxStartOfFrameOverruns"},
	{0x88, "RxMiddleOfFrameOverruns"},
	{0x8c, "RxDmaOverruns"}
	};

	/*
	* Basic debug support.
	*/

	static void
	cpsw_debugf_head(const char *funcname)
	{
	int t = (int)(time_second % (24 * 60 * 60));

	printf("%02d:%02d:%02d %s ", t / (60 * 60), (t / 60) % 60, t % 60, funcname);
	}

	static void
	cpsw_debugf(const char *fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);
	vprintf(fmt, ap);
	va_end(ap);
	printf("\n");

	}

	#define CPSW_DEBUGF(_sc, a) do { \
	if ((_sc)->debug) { \
	cpsw_debugf_head(__func__); \
	cpsw_debugf a; \
	} \
	} while (0)

	/*
	* Locking macros
	*/
	#define CPSW_TX_LOCK(sc) do { \
	mtx_assert(&(sc)->rx.lock, MA_NOTOWNED); \
	mtx_lock(&(sc)->tx.lock); \
	} while (0)

	#define CPSW_TX_UNLOCK(sc) mtx_unlock(&(sc)->tx.lock)
	#define CPSW_TX_LOCK_ASSERT(sc) mtx_assert(&(sc)->tx.lock, MA_OWNED)

	#define CPSW_RX_LOCK(sc) do { \
	mtx_assert(&(sc)->tx.lock, MA_NOTOWNED); \
	mtx_lock(&(sc)->rx.lock); \
	} while (0)

	#define CPSW_RX_UNLOCK(sc) mtx_unlock(&(sc)->rx.lock)
	#define CPSW_RX_LOCK_ASSERT(sc) mtx_assert(&(sc)->rx.lock, MA_OWNED)

	#define CPSW_PORT_LOCK(_sc) do { \
	mtx_assert(&(_sc)->lock, MA_NOTOWNED); \
	mtx_lock(&(_sc)->lock); \
	} while (0)

	#define CPSW_PORT_UNLOCK(_sc) mtx_unlock(&(_sc)->lock)
	#define CPSW_PORT_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->lock, MA_OWNED)

	/*
	* Read/Write macros
	*/
	#define cpsw_read_4(_sc, _reg) bus_read_4((_sc)->mem_res, (_reg))
	#define cpsw_write_4(_sc, _reg, _val) \
	bus_write_4((_sc)->mem_res, (_reg), (_val))

	#define cpsw_cpdma_bd_offset(i) (CPSW_CPPI_RAM_OFFSET + ((i)*16))

	#define cpsw_cpdma_bd_paddr(sc, slot) \
	BUS_SPACE_PHYSADDR(sc->mem_res, slot->bd_offset)
	#define cpsw_cpdma_read_bd(sc, slot, val) \
	bus_read_region_4(sc->mem_res, slot->bd_offset, (uint32_t *) val, 4)
	#define cpsw_cpdma_write_bd(sc, slot, val) \
	bus_write_region_4(sc->mem_res, slot->bd_offset, (uint32_t *) val, 4)
	#define cpsw_cpdma_write_bd_next(sc, slot, next_slot) \
	cpsw_write_4(sc, slot->bd_offset, cpsw_cpdma_bd_paddr(sc, next_slot))
	#define cpsw_cpdma_write_bd_flags(sc, slot, val) \
	bus_write_2(sc->mem_res, slot->bd_offset + 14, val)
	#define cpsw_cpdma_read_bd_flags(sc, slot) \
	bus_read_2(sc->mem_res, slot->bd_offset + 14)
	#define cpsw_write_hdp_slot(sc, queue, slot) \
	cpsw_write_4(sc, (queue)->hdp_offset, cpsw_cpdma_bd_paddr(sc, slot))
	#define CP_OFFSET (CPSW_CPDMA_TX_CP(0) - CPSW_CPDMA_TX_HDP(0))
	#define cpsw_read_cp(sc, queue) \
	cpsw_read_4(sc, (queue)->hdp_offset + CP_OFFSET)
	#define cpsw_write_cp(sc, queue, val) \
	cpsw_write_4(sc, (queue)->hdp_offset + CP_OFFSET, (val))
	#define cpsw_write_cp_slot(sc, queue, slot) \
	cpsw_write_cp(sc, queue, cpsw_cpdma_bd_paddr(sc, slot))

	#if 0
	/* XXX temporary function versions for debugging. */
	static void
	cpsw_write_hdp_slotX(struct cpsw_softc sc, struct cpsw_queue queue, struct cpsw_slot *slot)
	{
	uint32_t reg = queue->hdp_offset;
	uint32_t v = cpsw_cpdma_bd_paddr(sc, slot);
	CPSW_DEBUGF(("HDP <=== 0x%08x (was 0x%08x)", v, cpsw_read_4(sc, reg)));
	cpsw_write_4(sc, reg, v);
	}

	static void
	cpsw_write_cp_slotX(struct cpsw_softc sc, struct cpsw_queue queue, struct cpsw_slot *slot)
	{
	uint32_t v = cpsw_cpdma_bd_paddr(sc, slot);
	CPSW_DEBUGF(("CP <=== 0x%08x (expecting 0x%08x)", v, cpsw_read_cp(sc, queue)));
	cpsw_write_cp(sc, queue, v);
	}
	#endif

	/*
	* Expanded dump routines for verbose debugging.
	*/
	static void
	cpsw_dump_slot(struct cpsw_softc sc, struct cpsw_slot slot)
	{
	static const char *flags[] = {"SOP", "EOP", "Owner", "EOQ",
	"TDownCmplt", "PassCRC", "Long", "Short", "MacCtl", "Overrun",
	"PktErr1", "PortEn/PktErr0", "RxVlanEncap", "Port2", "Port1",
	"Port0"};
	struct cpsw_cpdma_bd bd;
	const char *sep;
	int i;

	cpsw_cpdma_read_bd(sc, slot, &bd);
	printf("BD Addr : 0x%08x Next : 0x%08x\n",
	cpsw_cpdma_bd_paddr(sc, slot), bd.next);
	printf(" BufPtr: 0x%08x BufLen: 0x%08x\n", bd.bufptr, bd.buflen);
	printf(" BufOff: 0x%08x PktLen: 0x%08x\n", bd.bufoff, bd.pktlen);
	printf(" Flags: ");
	sep = "";
	for (i = 0; i < 16; ++i) {
	if (bd.flags & (1 << (15 - i))) {
	printf("%s%s", sep, flags[i]);
	sep = ",";
	}
	}
	printf("\n");
	if (slot->mbuf) {
	printf(" Ether: %14D\n",
	(char *)(slot->mbuf->m_data), " ");
	printf(" Packet: %16D\n",
	(char *)(slot->mbuf->m_data) + 14, " ");
	}
	}

	#define CPSW_DUMP_SLOT(cs, slot) do { \
	IF_DEBUG(sc) { \
	cpsw_dump_slot(sc, slot); \
	} \
	} while (0)

	static void
	cpsw_dump_queue(struct cpsw_softc sc, struct cpsw_slots q)
	{
	struct cpsw_slot *slot;
	int i = 0;
	int others = 0;

	STAILQ_FOREACH(slot, q, next) {
	if (i > CPSW_TXFRAGS)
	++others;
	else
	cpsw_dump_slot(sc, slot);
	++i;
	}
	if (others)
	printf(" ... and %d more.\n", others);
	printf("\n");
	}

	#define CPSW_DUMP_QUEUE(sc, q) do { \
	IF_DEBUG(sc) { \
	cpsw_dump_queue(sc, q); \
	} \
	} while (0)

	static void
	cpsw_init_slots(struct cpsw_softc *sc)
	{
	struct cpsw_slot *slot;
	int i;

	STAILQ_INIT(&sc->avail);

	/* Put the slot descriptors onto the global avail list. */
	for (i = 0; i < nitems(sc->_slots); i++) {
	slot = &sc->_slots[i];
	slot->bd_offset = cpsw_cpdma_bd_offset(i);
	STAILQ_INSERT_TAIL(&sc->avail, slot, next);
	}
	}

	static int
	cpsw_add_slots(struct cpsw_softc sc, struct cpsw_queue queue, int requested)
	{
	const int max_slots = nitems(sc->_slots);
	struct cpsw_slot *slot;
	int i;

	if (requested < 0)
	requested = max_slots;

	for (i = 0; i < requested; ++i) {
	slot = STAILQ_FIRST(&sc->avail);
	if (slot == NULL)
	return (0);
	if (bus_dmamap_create(sc->mbuf_dtag, 0, &slot->dmamap)) {
	device_printf(sc->dev, "failed to create dmamap\n");
	return (ENOMEM);
	}
	STAILQ_REMOVE_HEAD(&sc->avail, next);
	STAILQ_INSERT_TAIL(&queue->avail, slot, next);
	++queue->avail_queue_len;
	++queue->queue_slots;
	}
	return (0);
	}

	static void
	cpsw_free_slot(struct cpsw_softc sc, struct cpsw_slot slot)
	{
	int error;

	if (slot->dmamap) {
	if (slot->mbuf)
	bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
	error = bus_dmamap_destroy(sc->mbuf_dtag, slot->dmamap);
	KASSERT(error == 0, ("Mapping still active"));
	slot->dmamap = NULL;
	}
	if (slot->mbuf) {
	m_freem(slot->mbuf);
	slot->mbuf = NULL;
	}
	}

	static void
	cpsw_reset(struct cpsw_softc *sc)
	{
	int i;

	callout_stop(&sc->watchdog.callout);

	/* Reset RMII/RGMII wrapper. */
	cpsw_write_4(sc, CPSW_WR_SOFT_RESET, 1);
	while (cpsw_read_4(sc, CPSW_WR_SOFT_RESET) & 1)
	;

	/* Disable TX and RX interrupts for all cores. */
	for (i = 0; i < 3; ++i) {
	cpsw_write_4(sc, CPSW_WR_C_RX_THRESH_EN(i), 0x00);
	cpsw_write_4(sc, CPSW_WR_C_TX_EN(i), 0x00);
	cpsw_write_4(sc, CPSW_WR_C_RX_EN(i), 0x00);
	cpsw_write_4(sc, CPSW_WR_C_MISC_EN(i), 0x00);
	}

	/* Reset CPSW subsystem. */
	cpsw_write_4(sc, CPSW_SS_SOFT_RESET, 1);
	while (cpsw_read_4(sc, CPSW_SS_SOFT_RESET) & 1)
	;

	/* Reset Sliver port 1 and 2 */
	for (i = 0; i < 2; i++) {
	/* Reset */
	cpsw_write_4(sc, CPSW_SL_SOFT_RESET(i), 1);
	while (cpsw_read_4(sc, CPSW_SL_SOFT_RESET(i)) & 1)
	;
	}

	/* Reset DMA controller. */
	cpsw_write_4(sc, CPSW_CPDMA_SOFT_RESET, 1);
	while (cpsw_read_4(sc, CPSW_CPDMA_SOFT_RESET) & 1)
	;

	/* Disable TX & RX DMA */
	cpsw_write_4(sc, CPSW_CPDMA_TX_CONTROL, 0);
	cpsw_write_4(sc, CPSW_CPDMA_RX_CONTROL, 0);

	/* Clear all queues. */
	for (i = 0; i < 8; i++) {
	cpsw_write_4(sc, CPSW_CPDMA_TX_HDP(i), 0);
	cpsw_write_4(sc, CPSW_CPDMA_RX_HDP(i), 0);
	cpsw_write_4(sc, CPSW_CPDMA_TX_CP(i), 0);
	cpsw_write_4(sc, CPSW_CPDMA_RX_CP(i), 0);
	}

	/* Clear all interrupt Masks */
	cpsw_write_4(sc, CPSW_CPDMA_RX_INTMASK_CLEAR, 0xFFFFFFFF);
	cpsw_write_4(sc, CPSW_CPDMA_TX_INTMASK_CLEAR, 0xFFFFFFFF);
	}

	static void
	cpsw_init(struct cpsw_softc *sc)
	{
	struct cpsw_slot *slot;
	uint32_t reg;

	/* Disable the interrupt pacing. */
	reg = cpsw_read_4(sc, CPSW_WR_INT_CONTROL);
	reg &= ~(CPSW_WR_INT_PACE_EN \| CPSW_WR_INT_PRESCALE_MASK);
	cpsw_write_4(sc, CPSW_WR_INT_CONTROL, reg);

	/* Clear ALE */
	cpsw_write_4(sc, CPSW_ALE_CONTROL, CPSW_ALE_CTL_CLEAR_TBL);

	/* Enable ALE */
	reg = CPSW_ALE_CTL_ENABLE;
	if (sc->dualemac)
	reg \|= CPSW_ALE_CTL_VLAN_AWARE;
	cpsw_write_4(sc, CPSW_ALE_CONTROL, reg);

	/* Set Host Port Mapping. */
	cpsw_write_4(sc, CPSW_PORT_P0_CPDMA_TX_PRI_MAP, 0x76543210);
	cpsw_write_4(sc, CPSW_PORT_P0_CPDMA_RX_CH_MAP, 0);

	/* Initialize ALE: set host port to forwarding(3). */
	cpsw_write_4(sc, CPSW_ALE_PORTCTL(0),
	ALE_PORTCTL_INGRESS \| ALE_PORTCTL_FORWARD);

	cpsw_write_4(sc, CPSW_SS_PTYPE, 0);

	/* Enable statistics for ports 0, 1 and 2 */
	cpsw_write_4(sc, CPSW_SS_STAT_PORT_EN, 7);

	/* Turn off flow control. */
	cpsw_write_4(sc, CPSW_SS_FLOW_CONTROL, 0);

	/* Make IP hdr aligned with 4 */
	cpsw_write_4(sc, CPSW_CPDMA_RX_BUFFER_OFFSET, 2);

	/* Initialize RX Buffer Descriptors */
	cpsw_write_4(sc, CPSW_CPDMA_RX_PENDTHRESH(0), 0);
	cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), 0);

	/* Enable TX & RX DMA */
	cpsw_write_4(sc, CPSW_CPDMA_TX_CONTROL, 1);
	cpsw_write_4(sc, CPSW_CPDMA_RX_CONTROL, 1);

	/* Enable Interrupts for core 0 */
	cpsw_write_4(sc, CPSW_WR_C_RX_THRESH_EN(0), 0xFF);
	cpsw_write_4(sc, CPSW_WR_C_RX_EN(0), 0xFF);
	cpsw_write_4(sc, CPSW_WR_C_TX_EN(0), 0xFF);
	cpsw_write_4(sc, CPSW_WR_C_MISC_EN(0), 0x1F);

	/* Enable host Error Interrupt */
	cpsw_write_4(sc, CPSW_CPDMA_DMA_INTMASK_SET, 3);

	/* Enable interrupts for RX and TX on Channel 0 */
	cpsw_write_4(sc, CPSW_CPDMA_RX_INTMASK_SET,
	CPSW_CPDMA_RX_INT(0) \| CPSW_CPDMA_RX_INT_THRESH(0));
	cpsw_write_4(sc, CPSW_CPDMA_TX_INTMASK_SET, 1);

	/* Initialze MDIO - ENABLE, PREAMBLE=0, FAULTENB, CLKDIV=0xFF */
	/* TODO Calculate MDCLK=CLK/(CLKDIV+1) */
	cpsw_write_4(sc, MDIOCONTROL, MDIOCTL_ENABLE \| MDIOCTL_FAULTENB \| 0xff);

	/* Select MII in GMII_SEL, Internal Delay mode */
	//ti_scm_reg_write_4(0x650, 0);

	/* Initialize active queues. */
	slot = STAILQ_FIRST(&sc->tx.active);
	if (slot != NULL)
	cpsw_write_hdp_slot(sc, &sc->tx, slot);
	slot = STAILQ_FIRST(&sc->rx.active);
	if (slot != NULL)
	cpsw_write_hdp_slot(sc, &sc->rx, slot);
	cpsw_rx_enqueue(sc);
	cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), sc->rx.active_queue_len);
	cpsw_write_4(sc, CPSW_CPDMA_RX_PENDTHRESH(0), CPSW_TXFRAGS);

	/* Activate network interface. */
	sc->rx.running = 1;
	sc->tx.running = 1;
	sc->watchdog.timer = 0;
	callout_init(&sc->watchdog.callout, 0);
	callout_reset(&sc->watchdog.callout, hz, cpsw_tx_watchdog, sc);
	}

	/*
	*
	* Device Probe, Attach, Detach.
	*
	*/

	static int
	cpsw_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,cpsw"))
	return (ENXIO);

	device_set_desc(dev, "3-port Switch Ethernet Subsystem");
	return (BUS_PROBE_DEFAULT);
	}

	static int
	cpsw_intr_attach(struct cpsw_softc *sc)
	{
	int i;

	for (i = 0; i < CPSW_INTR_COUNT; i++) {
	if (bus_setup_intr(sc->dev, sc->irq_res[i],
	INTR_TYPE_NET \| INTR_MPSAFE, NULL,
	cpsw_intr_cb[i].cb, sc, &sc->ih_cookie[i]) != 0) {
	return (-1);
	}
	}

	return (0);
	}

	static void
	cpsw_intr_detach(struct cpsw_softc *sc)
	{
	int i;

	for (i = 0; i < CPSW_INTR_COUNT; i++) {
	if (sc->ih_cookie[i]) {
	bus_teardown_intr(sc->dev, sc->irq_res[i],
	sc->ih_cookie[i]);
	}
	}
	}

	static int
	cpsw_get_fdt_data(struct cpsw_softc *sc, int port)
	{
	char *name;
	int len, phy, vlan;
	pcell_t phy_id[3], vlan_id;
	phandle_t child;
	unsigned long mdio_child_addr;

	/* Find any slave with phy-handle/phy_id */
	phy = -1;
	vlan = -1;
	for (child = OF_child(sc->node); child != 0; child = OF_peer(child)) {
	if (OF_getprop_alloc(child, "name", (void **)&name) < 0)
	continue;
	if (sscanf(name, "slave@%lx", &mdio_child_addr) != 1) {
	OF_prop_free(name);
	continue;
	}
	OF_prop_free(name);

	if (mdio_child_addr != slave_mdio_addr[port] &&
	mdio_child_addr != (slave_mdio_addr[port] & 0xFFF))
	continue;

	if (fdt_get_phyaddr(child, NULL, &phy, NULL) != 0){
	/* Users with old DTB will have phy_id instead */
	phy = -1;
	len = OF_getproplen(child, "phy_id");
	if (len / sizeof(pcell_t) == 2) {
	/* Get phy address from fdt */
	if (OF_getencprop(child, "phy_id", phy_id, len) > 0)
	phy = phy_id[1];
	}
	}

	len = OF_getproplen(child, "dual_emac_res_vlan");
	if (len / sizeof(pcell_t) == 1) {
	/* Get phy address from fdt */
	if (OF_getencprop(child, "dual_emac_res_vlan",
	&vlan_id, len) > 0) {
	vlan = vlan_id;
	}
	}

	break;
	}
	if (phy == -1)
	return (ENXIO);
	sc->port[port].phy = phy;
	sc->port[port].vlan = vlan;

	return (0);
	}

	static int
	cpsw_attach(device_t dev)
	{
	int error, i;
	struct cpsw_softc *sc;
	uint32_t reg;

	sc = device_get_softc(dev);
	sc->dev = dev;
	sc->node = ofw_bus_get_node(dev);
	getbinuptime(&sc->attach_uptime);

	if (OF_getencprop(sc->node, "active_slave", &sc->active_slave,
	sizeof(sc->active_slave)) <= 0) {
	sc->active_slave = 0;
	}
	if (sc->active_slave > 1)
	sc->active_slave = 1;

	if (OF_hasprop(sc->node, "dual_emac"))
	sc->dualemac = 1;

	for (i = 0; i < CPSW_PORTS; i++) {
	if (!sc->dualemac && i != sc->active_slave)
	continue;
	if (cpsw_get_fdt_data(sc, i) != 0) {
	device_printf(dev,
	"failed to get PHY address from FDT\n");
	return (ENXIO);
	}
	}

	/* Initialize mutexes */
	mtx_init(&sc->tx.lock, device_get_nameunit(dev),
	"cpsw TX lock", MTX_DEF);
	mtx_init(&sc->rx.lock, device_get_nameunit(dev),
	"cpsw RX lock", MTX_DEF);

	/* Allocate IRQ resources */
	error = bus_alloc_resources(dev, irq_res_spec, sc->irq_res);
	if (error) {
	device_printf(dev, "could not allocate IRQ resources\n");
	cpsw_detach(dev);
	return (ENXIO);
	}

	sc->mem_rid = 0;
	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&sc->mem_rid, RF_ACTIVE);
	if (sc->mem_res == NULL) {
	device_printf(sc->dev, "failed to allocate memory resource\n");
	cpsw_detach(dev);
	return (ENXIO);
	}

	reg = cpsw_read_4(sc, CPSW_SS_IDVER);
	device_printf(dev, "CPSW SS Version %d.%d (%d)\n", (reg >> 8 & 0x7),
	reg & 0xFF, (reg >> 11) & 0x1F);

	cpsw_add_sysctls(sc);

	/* Allocate a busdma tag and DMA safe memory for mbufs. */
	error = bus_dma_tag_create(
	bus_get_dma_tag(sc->dev), /* parent */
	1, 0, /* alignment, boundary */
	BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filtfunc, filtfuncarg */
	MCLBYTES, CPSW_TXFRAGS, /* maxsize, nsegments */
	MCLBYTES, 0, /* maxsegsz, flags */
	NULL, NULL, /* lockfunc, lockfuncarg */
	&sc->mbuf_dtag); /* dmatag */
	if (error) {
	device_printf(dev, "bus_dma_tag_create failed\n");
	cpsw_detach(dev);
	return (error);
	}

	/* Allocate a NULL buffer for padding. */
	sc->nullpad = malloc(ETHER_MIN_LEN, M_DEVBUF, M_WAITOK \| M_ZERO);

	cpsw_init_slots(sc);

	/* Allocate slots to TX and RX queues. */
	STAILQ_INIT(&sc->rx.avail);
	STAILQ_INIT(&sc->rx.active);
	STAILQ_INIT(&sc->tx.avail);
	STAILQ_INIT(&sc->tx.active);
	// For now: 128 slots to TX, rest to RX.
	// XXX TODO: start with 32/64 and grow dynamically based on demand.
	if (cpsw_add_slots(sc, &sc->tx, 128) \|\|
	cpsw_add_slots(sc, &sc->rx, -1)) {
	device_printf(dev, "failed to allocate dmamaps\n");
	cpsw_detach(dev);
	return (ENOMEM);
	}
	device_printf(dev, "Initial queue size TX=%d RX=%d\n",
	sc->tx.queue_slots, sc->rx.queue_slots);

	sc->tx.hdp_offset = CPSW_CPDMA_TX_HDP(0);
	sc->rx.hdp_offset = CPSW_CPDMA_RX_HDP(0);

	if (cpsw_intr_attach(sc) == -1) {
	device_printf(dev, "failed to setup interrupts\n");
	cpsw_detach(dev);
	return (ENXIO);
	}

	#ifdef CPSW_ETHERSWITCH
	for (i = 0; i < CPSW_VLANS; i++)
	cpsw_vgroups[i].vid = -1;
	#endif

	/* Reset the controller. */
	cpsw_reset(sc);
	cpsw_init(sc);

	for (i = 0; i < CPSW_PORTS; i++) {
	if (!sc->dualemac && i != sc->active_slave)
	continue;
	sc->port[i].dev = device_add_child(dev, "cpsw", i);
	if (sc->port[i].dev == NULL) {
	cpsw_detach(dev);
	return (ENXIO);
	}
	}
	bus_generic_probe(dev);
	bus_generic_attach(dev);

	return (0);
	}

	static int
	cpsw_detach(device_t dev)
	{
	struct cpsw_softc *sc;
	int error, i;

	bus_generic_detach(dev);
	sc = device_get_softc(dev);

	for (i = 0; i < CPSW_PORTS; i++) {
	if (sc->port[i].dev)
	device_delete_child(dev, sc->port[i].dev);
	}

	if (device_is_attached(dev)) {
	callout_stop(&sc->watchdog.callout);
	callout_drain(&sc->watchdog.callout);
	}

	/* Stop and release all interrupts */
	cpsw_intr_detach(sc);

	/* Free dmamaps and mbufs */
	for (i = 0; i < nitems(sc->_slots); ++i)
	cpsw_free_slot(sc, &sc->_slots[i]);

	/* Free null padding buffer. */
	if (sc->nullpad)
	free(sc->nullpad, M_DEVBUF);

	/* Free DMA tag */
	if (sc->mbuf_dtag) {
	error = bus_dma_tag_destroy(sc->mbuf_dtag);
	KASSERT(error == 0, ("Unable to destroy DMA tag"));
	}

	/* Free IO memory handler */
	if (sc->mem_res != NULL)
	bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res);
	bus_release_resources(dev, irq_res_spec, sc->irq_res);

	/* Destroy mutexes */
	mtx_destroy(&sc->rx.lock);
	mtx_destroy(&sc->tx.lock);

	/* Detach the switch device, if present. */
	error = bus_generic_detach(dev);
	if (error != 0)
	return (error);

	return (device_delete_children(dev));
	}

	static phandle_t
	cpsw_get_node(device_t bus, device_t dev)
	{

	/* Share controller node with port device. */
	return (ofw_bus_get_node(bus));
	}

	static int
	cpswp_probe(device_t dev)
	{

	if (device_get_unit(dev) > 1) {
	device_printf(dev, "Only two ports are supported.\n");
	return (ENXIO);
	}
	device_set_desc(dev, "Ethernet Switch Port");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	cpswp_attach(device_t dev)
	{
	int error;
	struct ifnet *ifp;
	struct cpswp_softc *sc;
	uint32_t reg;
	uint8_t mac_addr[ETHER_ADDR_LEN];
	+ phandle_t opp_table;
	+ struct syscon *syscon;

	sc = device_get_softc(dev);
	sc->dev = dev;
	sc->pdev = device_get_parent(dev);
	sc->swsc = device_get_softc(sc->pdev);
	sc->unit = device_get_unit(dev);
	sc->phy = sc->swsc->port[sc->unit].phy;
	sc->vlan = sc->swsc->port[sc->unit].vlan;
	if (sc->swsc->dualemac && sc->vlan == -1)
	sc->vlan = sc->unit + 1;

	if (sc->unit == 0) {
	sc->physel = MDIOUSERPHYSEL0;
	sc->phyaccess = MDIOUSERACCESS0;
	} else {
	sc->physel = MDIOUSERPHYSEL1;
	sc->phyaccess = MDIOUSERACCESS1;
	}

	mtx_init(&sc->lock, device_get_nameunit(dev), "cpsw port lock",
	MTX_DEF);

	/* Allocate network interface */
	ifp = sc->ifp = if_alloc(IFT_ETHER);
	if (ifp == NULL) {
	cpswp_detach(dev);
	return (ENXIO);
	}

	if_initname(ifp, device_get_name(sc->dev), sc->unit);
	ifp->if_softc = sc;
	ifp->if_flags = IFF_SIMPLEX \| IFF_MULTICAST \| IFF_BROADCAST;
	ifp->if_capabilities = IFCAP_VLAN_MTU \| IFCAP_HWCSUM; //FIXME VLAN?
	ifp->if_capenable = ifp->if_capabilities;

	ifp->if_init = cpswp_init;
	ifp->if_start = cpswp_start;
	ifp->if_ioctl = cpswp_ioctl;

	ifp->if_snd.ifq_drv_maxlen = sc->swsc->tx.queue_slots;
	IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
	IFQ_SET_READY(&ifp->if_snd);

	+ /* FIXME: For now; Go and kidnap syscon from opp-table */
	+ /* ti,cpsw actually have an optional syscon reference but only for am33xx?? */
	+ opp_table = OF_finddevice("/opp-table");
	+ if (opp_table == -1) {
	+ device_printf(dev, "Cant find /opp-table\n");
	+ cpswp_detach(dev);
	+ return (ENXIO);
	+ }
	+ if (!OF_hasprop(opp_table, "syscon")) {
	+ device_printf(dev, "/opp-table doesnt have required syscon property\n");
	+ cpswp_detach(dev);
	+ return (ENXIO);
	+ }
	+ if (syscon_get_by_ofw_property(dev, opp_table, "syscon", &syscon) != 0) {
	+ device_printf(dev, "Failed to get syscon\n");
	+ cpswp_detach(dev);
	+ return (ENXIO);
	+ }
	+
	/* Get high part of MAC address from control module (mac_id[0\|1]_hi) */
	- ti_scm_reg_read_4(SCM_MAC_ID0_HI + sc->unit * 8, &reg);
	+ reg = SYSCON_READ_4(syscon, SCM_MAC_ID0_HI + sc->unit * 8);
	mac_addr[0] = reg & 0xFF;
	mac_addr[1] = (reg >> 8) & 0xFF;
	mac_addr[2] = (reg >> 16) & 0xFF;
	mac_addr[3] = (reg >> 24) & 0xFF;

	/* Get low part of MAC address from control module (mac_id[0\|1]_lo) */
	- ti_scm_reg_read_4(SCM_MAC_ID0_LO + sc->unit * 8, &reg);
	+ reg = SYSCON_READ_4(syscon, SCM_MAC_ID0_LO + sc->unit * 8);
	mac_addr[4] = reg & 0xFF;
	mac_addr[5] = (reg >> 8) & 0xFF;

	error = mii_attach(dev, &sc->miibus, ifp, cpswp_ifmedia_upd,
	cpswp_ifmedia_sts, BMSR_DEFCAPMASK, sc->phy, MII_OFFSET_ANY, 0);
	if (error) {
	device_printf(dev, "attaching PHYs failed\n");
	cpswp_detach(dev);
	return (error);
	}
	sc->mii = device_get_softc(sc->miibus);

	/* Select PHY and enable interrupts */
	cpsw_write_4(sc->swsc, sc->physel,
	MDIO_PHYSEL_LINKINTENB \| (sc->phy & 0x1F));

	ether_ifattach(sc->ifp, mac_addr);
	callout_init(&sc->mii_callout, 0);

	return (0);
	}

	static int
	cpswp_detach(device_t dev)
	{
	struct cpswp_softc *sc;

	sc = device_get_softc(dev);
	CPSW_DEBUGF(sc->swsc, (""));
	if (device_is_attached(dev)) {
	ether_ifdetach(sc->ifp);
	CPSW_PORT_LOCK(sc);
	cpswp_stop_locked(sc);
	CPSW_PORT_UNLOCK(sc);
	callout_drain(&sc->mii_callout);
	}

	bus_generic_detach(dev);

	if_free(sc->ifp);
	mtx_destroy(&sc->lock);

	return (0);
	}

	/*
	*
	* Init/Shutdown.
	*
	*/

	static int
	cpsw_ports_down(struct cpsw_softc *sc)
	{
	struct cpswp_softc *psc;
	struct ifnet ifp1, ifp2;

	if (!sc->dualemac)
	return (1);
	psc = device_get_softc(sc->port[0].dev);
	ifp1 = psc->ifp;
	psc = device_get_softc(sc->port[1].dev);
	ifp2 = psc->ifp;
	if ((ifp1->if_flags & IFF_UP) == 0 && (ifp2->if_flags & IFF_UP) == 0)
	return (1);

	return (0);
	}

	static void
	cpswp_init(void *arg)
	{
	struct cpswp_softc *sc = arg;

	CPSW_DEBUGF(sc->swsc, (""));
	CPSW_PORT_LOCK(sc);
	cpswp_init_locked(arg);
	CPSW_PORT_UNLOCK(sc);
	}

	static void
	cpswp_init_locked(void *arg)
	{
	#ifdef CPSW_ETHERSWITCH
	int i;
	#endif
	struct cpswp_softc *sc = arg;
	struct ifnet *ifp;
	uint32_t reg;

	CPSW_DEBUGF(sc->swsc, (""));
	CPSW_PORT_LOCK_ASSERT(sc);
	ifp = sc->ifp;
	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
	return;

	getbinuptime(&sc->init_uptime);

	if (!sc->swsc->rx.running && !sc->swsc->tx.running) {
	/* Reset the controller. */
	cpsw_reset(sc->swsc);
	cpsw_init(sc->swsc);
	}

	/* Set Slave Mapping. */
	cpsw_write_4(sc->swsc, CPSW_SL_RX_PRI_MAP(sc->unit), 0x76543210);
	cpsw_write_4(sc->swsc, CPSW_PORT_P_TX_PRI_MAP(sc->unit + 1),
	0x33221100);
	cpsw_write_4(sc->swsc, CPSW_SL_RX_MAXLEN(sc->unit), 0x5f2);
	/* Enable MAC RX/TX modules. */
	/* TODO: Docs claim that IFCTL_B and IFCTL_A do the same thing? */
	/* Huh? Docs call bit 0 "Loopback" some places, "FullDuplex" others. */
	reg = cpsw_read_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit));
	reg \|= CPSW_SL_MACTL_GMII_ENABLE;
	cpsw_write_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit), reg);

	/* Initialize ALE: set port to forwarding, initialize addrs */
	cpsw_write_4(sc->swsc, CPSW_ALE_PORTCTL(sc->unit + 1),
	ALE_PORTCTL_INGRESS \| ALE_PORTCTL_FORWARD);
	cpswp_ale_update_addresses(sc, 1);

	if (sc->swsc->dualemac) {
	/* Set Port VID. */
	cpsw_write_4(sc->swsc, CPSW_PORT_P_VLAN(sc->unit + 1),
	sc->vlan & 0xfff);
	cpsw_ale_update_vlan_table(sc->swsc, sc->vlan,
	(1 << (sc->unit + 1)) \| (1 << 0), /* Member list */
	(1 << (sc->unit + 1)) \| (1 << 0), /* Untagged egress */
	(1 << (sc->unit + 1)) \| (1 << 0), 0); /* mcast reg flood */
	#ifdef CPSW_ETHERSWITCH
	for (i = 0; i < CPSW_VLANS; i++) {
	if (cpsw_vgroups[i].vid != -1)
	continue;
	cpsw_vgroups[i].vid = sc->vlan;
	break;
	}
	#endif
	}

	mii_mediachg(sc->mii);
	callout_reset(&sc->mii_callout, hz, cpswp_tick, sc);
	ifp->if_drv_flags \|= IFF_DRV_RUNNING;
	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
	}

	static int
	cpsw_shutdown(device_t dev)
	{
	struct cpsw_softc *sc;
	struct cpswp_softc *psc;
	int i;

	sc = device_get_softc(dev);
	CPSW_DEBUGF(sc, (""));
	for (i = 0; i < CPSW_PORTS; i++) {
	if (!sc->dualemac && i != sc->active_slave)
	continue;
	psc = device_get_softc(sc->port[i].dev);
	CPSW_PORT_LOCK(psc);
	cpswp_stop_locked(psc);
	CPSW_PORT_UNLOCK(psc);
	}

	return (0);
	}

	static void
	cpsw_rx_teardown(struct cpsw_softc *sc)
	{
	int i = 0;

	CPSW_RX_LOCK(sc);
	CPSW_DEBUGF(sc, ("starting RX teardown"));
	sc->rx.teardown = 1;
	cpsw_write_4(sc, CPSW_CPDMA_RX_TEARDOWN, 0);
	CPSW_RX_UNLOCK(sc);
	while (sc->rx.running) {
	if (++i > 10) {
	device_printf(sc->dev,
	"Unable to cleanly shutdown receiver\n");
	return;
	}
	DELAY(200);
	}
	if (!sc->rx.running)
	CPSW_DEBUGF(sc, ("finished RX teardown (%d retries)", i));
	}

	static void
	cpsw_tx_teardown(struct cpsw_softc *sc)
	{
	int i = 0;

	CPSW_TX_LOCK(sc);
	CPSW_DEBUGF(sc, ("starting TX teardown"));
	/* Start the TX queue teardown if queue is not empty. */
	if (STAILQ_FIRST(&sc->tx.active) != NULL)
	cpsw_write_4(sc, CPSW_CPDMA_TX_TEARDOWN, 0);
	else
	sc->tx.teardown = 1;
	cpsw_tx_dequeue(sc);
	while (sc->tx.running && ++i < 10) {
	DELAY(200);
	cpsw_tx_dequeue(sc);
	}
	if (sc->tx.running) {
	device_printf(sc->dev,
	"Unable to cleanly shutdown transmitter\n");
	}
	CPSW_DEBUGF(sc,
	("finished TX teardown (%d retries, %d idle buffers)", i,
	sc->tx.active_queue_len));
	CPSW_TX_UNLOCK(sc);
	}

	static void
	cpswp_stop_locked(struct cpswp_softc *sc)
	{
	struct ifnet *ifp;
	uint32_t reg;

	ifp = sc->ifp;
	CPSW_DEBUGF(sc->swsc, (""));
	CPSW_PORT_LOCK_ASSERT(sc);

	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
	return;

	/* Disable interface */
	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
	ifp->if_drv_flags \|= IFF_DRV_OACTIVE;

	/* Stop ticker */
	callout_stop(&sc->mii_callout);

	/* Tear down the RX/TX queues. */
	if (cpsw_ports_down(sc->swsc)) {
	cpsw_rx_teardown(sc->swsc);
	cpsw_tx_teardown(sc->swsc);
	}

	/* Stop MAC RX/TX modules. */
	reg = cpsw_read_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit));
	reg &= ~CPSW_SL_MACTL_GMII_ENABLE;
	cpsw_write_4(sc->swsc, CPSW_SL_MACCONTROL(sc->unit), reg);

	if (cpsw_ports_down(sc->swsc)) {
	/* Capture stats before we reset controller. */
	cpsw_stats_collect(sc->swsc);

	cpsw_reset(sc->swsc);
	cpsw_init(sc->swsc);
	}
	}

	/*
	* Suspend/Resume.
	*/

	static int
	cpsw_suspend(device_t dev)
	{
	struct cpsw_softc *sc;
	struct cpswp_softc *psc;
	int i;

	sc = device_get_softc(dev);
	CPSW_DEBUGF(sc, (""));
	for (i = 0; i < CPSW_PORTS; i++) {
	if (!sc->dualemac && i != sc->active_slave)
	continue;
	psc = device_get_softc(sc->port[i].dev);
	CPSW_PORT_LOCK(psc);
	cpswp_stop_locked(psc);
	CPSW_PORT_UNLOCK(psc);
	}

	return (0);
	}

	static int
	cpsw_resume(device_t dev)
	{
	struct cpsw_softc *sc;

	sc = device_get_softc(dev);
	CPSW_DEBUGF(sc, ("UNIMPLEMENTED"));

	return (0);
	}

	/*
	*
	* IOCTL
	*
	*/

	static void
	cpsw_set_promisc(struct cpswp_softc *sc, int set)
	{
	uint32_t reg;

	/*
	* Enabling promiscuous mode requires ALE_BYPASS to be enabled.
	* That disables the ALE forwarding logic and causes every
	* packet to be sent only to the host port. In bypass mode,
	* the ALE processes host port transmit packets the same as in
	* normal mode.
	*/
	reg = cpsw_read_4(sc->swsc, CPSW_ALE_CONTROL);
	reg &= ~CPSW_ALE_CTL_BYPASS;
	if (set)
	reg \|= CPSW_ALE_CTL_BYPASS;
	cpsw_write_4(sc->swsc, CPSW_ALE_CONTROL, reg);
	}

	static void
	cpsw_set_allmulti(struct cpswp_softc *sc, int set)
	{
	if (set) {
	printf("All-multicast mode unimplemented\n");
	}
	}

	static int
	cpswp_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
	{
	struct cpswp_softc *sc;
	struct ifreq *ifr;
	int error;
	uint32_t changed;

	error = 0;
	sc = ifp->if_softc;
	ifr = (struct ifreq *)data;

	switch (command) {
	case SIOCSIFCAP:
	changed = ifp->if_capenable ^ ifr->ifr_reqcap;
	if (changed & IFCAP_HWCSUM) {
	if ((ifr->ifr_reqcap & changed) & IFCAP_HWCSUM)
	ifp->if_capenable \|= IFCAP_HWCSUM;
	else
	ifp->if_capenable &= ~IFCAP_HWCSUM;
	}
	error = 0;
	break;
	case SIOCSIFFLAGS:
	CPSW_PORT_LOCK(sc);
	if (ifp->if_flags & IFF_UP) {
	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
	changed = ifp->if_flags ^ sc->if_flags;
	CPSW_DEBUGF(sc->swsc,
	("SIOCSIFFLAGS: UP & RUNNING (changed=0x%x)",
	changed));
	if (changed & IFF_PROMISC)
	cpsw_set_promisc(sc,
	ifp->if_flags & IFF_PROMISC);
	if (changed & IFF_ALLMULTI)
	cpsw_set_allmulti(sc,
	ifp->if_flags & IFF_ALLMULTI);
	} else {
	CPSW_DEBUGF(sc->swsc,
	("SIOCSIFFLAGS: starting up"));
	cpswp_init_locked(sc);
	}
	} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
	CPSW_DEBUGF(sc->swsc, ("SIOCSIFFLAGS: shutting down"));
	cpswp_stop_locked(sc);
	}

	sc->if_flags = ifp->if_flags;
	CPSW_PORT_UNLOCK(sc);
	break;
	case SIOCADDMULTI:
	cpswp_ale_update_addresses(sc, 0);
	break;
	case SIOCDELMULTI:
	/* Ugh. DELMULTI doesn't provide the specific address
	being removed, so the best we can do is remove
	everything and rebuild it all. */
	cpswp_ale_update_addresses(sc, 1);
	break;
	case SIOCGIFMEDIA:
	case SIOCSIFMEDIA:
	error = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, command);
	break;
	default:
	error = ether_ioctl(ifp, command, data);
	}
	return (error);
	}

	/*
	*
	* MIIBUS
	*
	*/
	static int
	cpswp_miibus_ready(struct cpsw_softc *sc, uint32_t reg)
	{
	uint32_t r, retries = CPSW_MIIBUS_RETRIES;

	while (--retries) {
	r = cpsw_read_4(sc, reg);
	if ((r & MDIO_PHYACCESS_GO) == 0)
	return (1);
	DELAY(CPSW_MIIBUS_DELAY);
	}

	return (0);
	}

	static int
	cpswp_miibus_readreg(device_t dev, int phy, int reg)
	{
	struct cpswp_softc *sc;
	uint32_t cmd, r;

	sc = device_get_softc(dev);
	if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) {
	device_printf(dev, "MDIO not ready to read\n");
	return (0);
	}

	/* Set GO, reg, phy */
	cmd = MDIO_PHYACCESS_GO \| (reg & 0x1F) << 21 \| (phy & 0x1F) << 16;
	cpsw_write_4(sc->swsc, sc->phyaccess, cmd);

	if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) {
	device_printf(dev, "MDIO timed out during read\n");
	return (0);
	}

	r = cpsw_read_4(sc->swsc, sc->phyaccess);
	if ((r & MDIO_PHYACCESS_ACK) == 0) {
	device_printf(dev, "Failed to read from PHY.\n");
	r = 0;
	}
	return (r & 0xFFFF);
	}

	static int
	cpswp_miibus_writereg(device_t dev, int phy, int reg, int value)
	{
	struct cpswp_softc *sc;
	uint32_t cmd;

	sc = device_get_softc(dev);
	if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) {
	device_printf(dev, "MDIO not ready to write\n");
	return (0);
	}

	/* Set GO, WRITE, reg, phy, and value */
	cmd = MDIO_PHYACCESS_GO \| MDIO_PHYACCESS_WRITE \|
	(reg & 0x1F) << 21 \| (phy & 0x1F) << 16 \| (value & 0xFFFF);
	cpsw_write_4(sc->swsc, sc->phyaccess, cmd);

	if (!cpswp_miibus_ready(sc->swsc, sc->phyaccess)) {
	device_printf(dev, "MDIO timed out during write\n");
	return (0);
	}

	return (0);
	}

	static void
	cpswp_miibus_statchg(device_t dev)
	{
	struct cpswp_softc *sc;
	uint32_t mac_control, reg;

	sc = device_get_softc(dev);
	CPSW_DEBUGF(sc->swsc, (""));

	reg = CPSW_SL_MACCONTROL(sc->unit);
	mac_control = cpsw_read_4(sc->swsc, reg);
	mac_control &= ~(CPSW_SL_MACTL_GIG \| CPSW_SL_MACTL_IFCTL_A \|
	CPSW_SL_MACTL_IFCTL_B \| CPSW_SL_MACTL_FULLDUPLEX);

	switch(IFM_SUBTYPE(sc->mii->mii_media_active)) {
	case IFM_1000_SX:
	case IFM_1000_LX:
	case IFM_1000_CX:
	case IFM_1000_T:
	mac_control \|= CPSW_SL_MACTL_GIG;
	break;

	case IFM_100_TX:
	mac_control \|= CPSW_SL_MACTL_IFCTL_A;
	break;
	}
	if (sc->mii->mii_media_active & IFM_FDX)
	mac_control \|= CPSW_SL_MACTL_FULLDUPLEX;

	cpsw_write_4(sc->swsc, reg, mac_control);
	}

	/*
	*
	* Transmit/Receive Packets.
	*
	*/
	static void
	cpsw_intr_rx(void *arg)
	{
	struct cpsw_softc *sc;
	struct ifnet *ifp;
	struct mbuf received, next;

	sc = (struct cpsw_softc *)arg;
	CPSW_RX_LOCK(sc);
	if (sc->rx.teardown) {
	sc->rx.running = 0;
	sc->rx.teardown = 0;
	cpsw_write_cp(sc, &sc->rx, 0xfffffffc);
	}
	received = cpsw_rx_dequeue(sc);
	cpsw_rx_enqueue(sc);
	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 1);
	CPSW_RX_UNLOCK(sc);

	while (received != NULL) {
	next = received->m_nextpkt;
	received->m_nextpkt = NULL;
	ifp = received->m_pkthdr.rcvif;
	(*ifp->if_input)(ifp, received);
	if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
	received = next;
	}
	}

	static struct mbuf *
	cpsw_rx_dequeue(struct cpsw_softc *sc)
	{
	int nsegs, port, removed;
	struct cpsw_cpdma_bd bd;
	struct cpsw_slot last, slot;
	struct cpswp_softc *psc;
	struct mbuf m, m0, mb_head, mb_tail;
	uint16_t m0_flags;

	nsegs = 0;
	m0 = NULL;
	last = NULL;
	mb_head = NULL;
	mb_tail = NULL;
	removed = 0;

	/* Pull completed packets off hardware RX queue. */
	while ((slot = STAILQ_FIRST(&sc->rx.active)) != NULL) {
	cpsw_cpdma_read_bd(sc, slot, &bd);

	/*
	* Stop on packets still in use by hardware, but do not stop
	* on packets with the teardown complete flag, they will be
	* discarded later.
	*/
	if ((bd.flags & (CPDMA_BD_OWNER \| CPDMA_BD_TDOWNCMPLT)) ==
	CPDMA_BD_OWNER)
	break;

	last = slot;
	++removed;
	STAILQ_REMOVE_HEAD(&sc->rx.active, next);
	STAILQ_INSERT_TAIL(&sc->rx.avail, slot, next);

	bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTREAD);
	bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);

	m = slot->mbuf;
	slot->mbuf = NULL;

	if (bd.flags & CPDMA_BD_TDOWNCMPLT) {
	CPSW_DEBUGF(sc, ("RX teardown is complete"));
	m_freem(m);
	sc->rx.running = 0;
	sc->rx.teardown = 0;
	break;
	}

	port = (bd.flags & CPDMA_BD_PORT_MASK) - 1;
	KASSERT(port >= 0 && port <= 1,
	("patcket received with invalid port: %d", port));
	psc = device_get_softc(sc->port[port].dev);

	/* Set up mbuf */
	m->m_data += bd.bufoff;
	m->m_len = bd.buflen;
	if (bd.flags & CPDMA_BD_SOP) {
	m->m_pkthdr.len = bd.pktlen;
	m->m_pkthdr.rcvif = psc->ifp;
	m->m_flags \|= M_PKTHDR;
	m0_flags = bd.flags;
	m0 = m;
	}
	nsegs++;
	m->m_next = NULL;
	m->m_nextpkt = NULL;
	if (bd.flags & CPDMA_BD_EOP && m0 != NULL) {
	if (m0_flags & CPDMA_BD_PASS_CRC)
	m_adj(m0, -ETHER_CRC_LEN);
	m0_flags = 0;
	m0 = NULL;
	if (nsegs > sc->rx.longest_chain)
	sc->rx.longest_chain = nsegs;
	nsegs = 0;
	}

	if ((psc->ifp->if_capenable & IFCAP_RXCSUM) != 0) {
	/* check for valid CRC by looking into pkt_err[5:4] */
	if ((bd.flags &
	(CPDMA_BD_SOP \| CPDMA_BD_PKT_ERR_MASK)) ==
	CPDMA_BD_SOP) {
	m->m_pkthdr.csum_flags \|= CSUM_IP_CHECKED;
	m->m_pkthdr.csum_flags \|= CSUM_IP_VALID;
	m->m_pkthdr.csum_data = 0xffff;
	}
	}

	if (STAILQ_FIRST(&sc->rx.active) != NULL &&
	(bd.flags & (CPDMA_BD_EOP \| CPDMA_BD_EOQ)) ==
	(CPDMA_BD_EOP \| CPDMA_BD_EOQ)) {
	cpsw_write_hdp_slot(sc, &sc->rx,
	STAILQ_FIRST(&sc->rx.active));
	sc->rx.queue_restart++;
	}

	/* Add mbuf to packet list to be returned. */
	if (mb_tail != NULL && (bd.flags & CPDMA_BD_SOP)) {
	mb_tail->m_nextpkt = m;
	} else if (mb_tail != NULL) {
	mb_tail->m_next = m;
	} else if (mb_tail == NULL && (bd.flags & CPDMA_BD_SOP) == 0) {
	if (bootverbose)
	printf(
	"%s: %s: discanding fragment packet w/o header\n",
	__func__, psc->ifp->if_xname);
	m_freem(m);
	continue;
	} else {
	mb_head = m;
	}
	mb_tail = m;
	}

	if (removed != 0) {
	cpsw_write_cp_slot(sc, &sc->rx, last);
	sc->rx.queue_removes += removed;
	sc->rx.avail_queue_len += removed;
	sc->rx.active_queue_len -= removed;
	if (sc->rx.avail_queue_len > sc->rx.max_avail_queue_len)
	sc->rx.max_avail_queue_len = sc->rx.avail_queue_len;
	CPSW_DEBUGF(sc, ("Removed %d received packet(s) from RX queue", removed));
	}

	return (mb_head);
	}

	static void
	cpsw_rx_enqueue(struct cpsw_softc *sc)
	{
	bus_dma_segment_t seg[1];
	struct cpsw_cpdma_bd bd;
	struct cpsw_slot first_new_slot, last_old_slot, next, slot;
	int error, nsegs, added = 0;

	/* Register new mbufs with hardware. */
	first_new_slot = NULL;
	last_old_slot = STAILQ_LAST(&sc->rx.active, cpsw_slot, next);
	while ((slot = STAILQ_FIRST(&sc->rx.avail)) != NULL) {
	if (first_new_slot == NULL)
	first_new_slot = slot;
	if (slot->mbuf == NULL) {
	slot->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
	if (slot->mbuf == NULL) {
	device_printf(sc->dev,
	"Unable to fill RX queue\n");
	break;
	}
	slot->mbuf->m_len =
	slot->mbuf->m_pkthdr.len =
	slot->mbuf->m_ext.ext_size;
	}

	error = bus_dmamap_load_mbuf_sg(sc->mbuf_dtag, slot->dmamap,
	slot->mbuf, seg, &nsegs, BUS_DMA_NOWAIT);

	KASSERT(nsegs == 1, ("More than one segment (nsegs=%d)", nsegs));
	KASSERT(error == 0, ("DMA error (error=%d)", error));
	if (error != 0 \|\| nsegs != 1) {
	device_printf(sc->dev,
	"%s: Can't prep RX buf for DMA (nsegs=%d, error=%d)\n",
	__func__, nsegs, error);
	bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
	m_freem(slot->mbuf);
	slot->mbuf = NULL;
	break;
	}

	bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_PREREAD);

	/* Create and submit new rx descriptor. */
	if ((next = STAILQ_NEXT(slot, next)) != NULL)
	bd.next = cpsw_cpdma_bd_paddr(sc, next);
	else
	bd.next = 0;
	bd.bufptr = seg->ds_addr;
	bd.bufoff = 0;
	bd.buflen = MCLBYTES - 1;
	bd.pktlen = bd.buflen;
	bd.flags = CPDMA_BD_OWNER;
	cpsw_cpdma_write_bd(sc, slot, &bd);
	++added;

	STAILQ_REMOVE_HEAD(&sc->rx.avail, next);
	STAILQ_INSERT_TAIL(&sc->rx.active, slot, next);
	}

	if (added == 0 \|\| first_new_slot == NULL)
	return;

	CPSW_DEBUGF(sc, ("Adding %d buffers to RX queue", added));

	/* Link new entries to hardware RX queue. */
	if (last_old_slot == NULL) {
	/* Start a fresh queue. */
	cpsw_write_hdp_slot(sc, &sc->rx, first_new_slot);
	} else {
	/* Add buffers to end of current queue. */
	cpsw_cpdma_write_bd_next(sc, last_old_slot, first_new_slot);
	}
	sc->rx.queue_adds += added;
	sc->rx.avail_queue_len -= added;
	sc->rx.active_queue_len += added;
	cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), added);
	if (sc->rx.active_queue_len > sc->rx.max_active_queue_len)
	sc->rx.max_active_queue_len = sc->rx.active_queue_len;
	}

	static void
	cpswp_start(struct ifnet *ifp)
	{
	struct cpswp_softc *sc;

	sc = ifp->if_softc;
	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 \|\|
	sc->swsc->tx.running == 0) {
	return;
	}
	CPSW_TX_LOCK(sc->swsc);
	cpswp_tx_enqueue(sc);
	cpsw_tx_dequeue(sc->swsc);
	CPSW_TX_UNLOCK(sc->swsc);
	}

	static void
	cpsw_intr_tx(void *arg)
	{
	struct cpsw_softc *sc;

	sc = (struct cpsw_softc *)arg;
	CPSW_TX_LOCK(sc);
	if (cpsw_read_4(sc, CPSW_CPDMA_TX_CP(0)) == 0xfffffffc)
	cpsw_write_cp(sc, &sc->tx, 0xfffffffc);
	cpsw_tx_dequeue(sc);
	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 2);
	CPSW_TX_UNLOCK(sc);
	}

	static void
	cpswp_tx_enqueue(struct cpswp_softc *sc)
	{
	bus_dma_segment_t segs[CPSW_TXFRAGS];
	struct cpsw_cpdma_bd bd;
	struct cpsw_slot first_new_slot, last, last_old_slot, next, *slot;
	struct mbuf *m0;
	int error, nsegs, seg, added = 0, padlen;

	/* Pull pending packets from IF queue and prep them for DMA. */
	last = NULL;
	first_new_slot = NULL;
	last_old_slot = STAILQ_LAST(&sc->swsc->tx.active, cpsw_slot, next);
	while ((slot = STAILQ_FIRST(&sc->swsc->tx.avail)) != NULL) {
	IF_DEQUEUE(&sc->ifp->if_snd, m0);
	if (m0 == NULL)
	break;

	slot->mbuf = m0;
	padlen = ETHER_MIN_LEN - ETHER_CRC_LEN - m0->m_pkthdr.len;
	if (padlen < 0)
	padlen = 0;
	else if (padlen > 0)
	m_append(slot->mbuf, padlen, sc->swsc->nullpad);

	/* Create mapping in DMA memory */
	error = bus_dmamap_load_mbuf_sg(sc->swsc->mbuf_dtag,
	slot->dmamap, slot->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
	/* If the packet is too fragmented, try to simplify. */
	if (error == EFBIG \|\|
	(error == 0 && nsegs > sc->swsc->tx.avail_queue_len)) {
	bus_dmamap_unload(sc->swsc->mbuf_dtag, slot->dmamap);
	m0 = m_defrag(slot->mbuf, M_NOWAIT);
	if (m0 == NULL) {
	device_printf(sc->dev,
	"Can't defragment packet; dropping\n");
	m_freem(slot->mbuf);
	} else {
	CPSW_DEBUGF(sc->swsc,
	("Requeueing defragmented packet"));
	IF_PREPEND(&sc->ifp->if_snd, m0);
	}
	slot->mbuf = NULL;
	continue;
	}
	if (error != 0) {
	device_printf(sc->dev,
	"%s: Can't setup DMA (error=%d), dropping packet\n",
	__func__, error);
	bus_dmamap_unload(sc->swsc->mbuf_dtag, slot->dmamap);
	m_freem(slot->mbuf);
	slot->mbuf = NULL;
	break;
	}

	bus_dmamap_sync(sc->swsc->mbuf_dtag, slot->dmamap,
	BUS_DMASYNC_PREWRITE);

	CPSW_DEBUGF(sc->swsc,
	("Queueing TX packet: %d segments + %d pad bytes",
	nsegs, padlen));

	if (first_new_slot == NULL)
	first_new_slot = slot;

	/* Link from the previous descriptor. */
	if (last != NULL)
	cpsw_cpdma_write_bd_next(sc->swsc, last, slot);

	slot->ifp = sc->ifp;

	/* If there is only one segment, the for() loop
	* gets skipped and the single buffer gets set up
	* as both SOP and EOP. */
	if (nsegs > 1) {
	next = STAILQ_NEXT(slot, next);
	bd.next = cpsw_cpdma_bd_paddr(sc->swsc, next);
	} else
	bd.next = 0;
	/* Start by setting up the first buffer. */
	bd.bufptr = segs[0].ds_addr;
	bd.bufoff = 0;
	bd.buflen = segs[0].ds_len;
	bd.pktlen = m_length(slot->mbuf, NULL);
	bd.flags = CPDMA_BD_SOP \| CPDMA_BD_OWNER;
	if (sc->swsc->dualemac) {
	bd.flags \|= CPDMA_BD_TO_PORT;
	bd.flags \|= ((sc->unit + 1) & CPDMA_BD_PORT_MASK);
	}
	for (seg = 1; seg < nsegs; ++seg) {
	/* Save the previous buffer (which isn't EOP) */
	cpsw_cpdma_write_bd(sc->swsc, slot, &bd);
	STAILQ_REMOVE_HEAD(&sc->swsc->tx.avail, next);
	STAILQ_INSERT_TAIL(&sc->swsc->tx.active, slot, next);
	slot = STAILQ_FIRST(&sc->swsc->tx.avail);

	/* Setup next buffer (which isn't SOP) */
	if (nsegs > seg + 1) {
	next = STAILQ_NEXT(slot, next);
	bd.next = cpsw_cpdma_bd_paddr(sc->swsc, next);
	} else
	bd.next = 0;
	bd.bufptr = segs[seg].ds_addr;
	bd.bufoff = 0;
	bd.buflen = segs[seg].ds_len;
	bd.pktlen = 0;
	bd.flags = CPDMA_BD_OWNER;
	}

	/* Save the final buffer. */
	bd.flags \|= CPDMA_BD_EOP;
	cpsw_cpdma_write_bd(sc->swsc, slot, &bd);
	STAILQ_REMOVE_HEAD(&sc->swsc->tx.avail, next);
	STAILQ_INSERT_TAIL(&sc->swsc->tx.active, slot, next);

	last = slot;
	added += nsegs;
	if (nsegs > sc->swsc->tx.longest_chain)
	sc->swsc->tx.longest_chain = nsegs;

	BPF_MTAP(sc->ifp, m0);
	}

	if (first_new_slot == NULL)
	return;

	/* Attach the list of new buffers to the hardware TX queue. */
	if (last_old_slot != NULL &&
	(cpsw_cpdma_read_bd_flags(sc->swsc, last_old_slot) &
	CPDMA_BD_EOQ) == 0) {
	/* Add buffers to end of current queue. */
	cpsw_cpdma_write_bd_next(sc->swsc, last_old_slot,
	first_new_slot);
	} else {
	/* Start a fresh queue. */
	cpsw_write_hdp_slot(sc->swsc, &sc->swsc->tx, first_new_slot);
	}
	sc->swsc->tx.queue_adds += added;
	sc->swsc->tx.avail_queue_len -= added;
	sc->swsc->tx.active_queue_len += added;
	if (sc->swsc->tx.active_queue_len > sc->swsc->tx.max_active_queue_len) {
	sc->swsc->tx.max_active_queue_len = sc->swsc->tx.active_queue_len;
	}
	CPSW_DEBUGF(sc->swsc, ("Queued %d TX packet(s)", added));
	}

	static int
	cpsw_tx_dequeue(struct cpsw_softc *sc)
	{
	struct cpsw_slot slot, last_removed_slot = NULL;
	struct cpsw_cpdma_bd bd;
	uint32_t flags, removed = 0;

	/* Pull completed buffers off the hardware TX queue. */
	slot = STAILQ_FIRST(&sc->tx.active);
	while (slot != NULL) {
	flags = cpsw_cpdma_read_bd_flags(sc, slot);

	/* TearDown complete is only marked on the SOP for the packet. */
	if ((flags & (CPDMA_BD_SOP \| CPDMA_BD_TDOWNCMPLT)) ==
	(CPDMA_BD_SOP \| CPDMA_BD_TDOWNCMPLT)) {
	sc->tx.teardown = 1;
	}

	if ((flags & (CPDMA_BD_SOP \| CPDMA_BD_OWNER)) ==
	(CPDMA_BD_SOP \| CPDMA_BD_OWNER) && sc->tx.teardown == 0)
	break; /* Hardware is still using this packet. */

	bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
	m_freem(slot->mbuf);
	slot->mbuf = NULL;

	if (slot->ifp) {
	if (sc->tx.teardown == 0)
	if_inc_counter(slot->ifp, IFCOUNTER_OPACKETS, 1);
	else
	if_inc_counter(slot->ifp, IFCOUNTER_OQDROPS, 1);
	}

	/* Dequeue any additional buffers used by this packet. */
	while (slot != NULL && slot->mbuf == NULL) {
	STAILQ_REMOVE_HEAD(&sc->tx.active, next);
	STAILQ_INSERT_TAIL(&sc->tx.avail, slot, next);
	++removed;
	last_removed_slot = slot;
	slot = STAILQ_FIRST(&sc->tx.active);
	}

	cpsw_write_cp_slot(sc, &sc->tx, last_removed_slot);

	/* Restart the TX queue if necessary. */
	cpsw_cpdma_read_bd(sc, last_removed_slot, &bd);
	if (slot != NULL && bd.next != 0 && (bd.flags &
	(CPDMA_BD_EOP \| CPDMA_BD_OWNER \| CPDMA_BD_EOQ)) ==
	(CPDMA_BD_EOP \| CPDMA_BD_EOQ)) {
	cpsw_write_hdp_slot(sc, &sc->tx, slot);
	sc->tx.queue_restart++;
	break;
	}
	}

	if (removed != 0) {
	sc->tx.queue_removes += removed;
	sc->tx.active_queue_len -= removed;
	sc->tx.avail_queue_len += removed;
	if (sc->tx.avail_queue_len > sc->tx.max_avail_queue_len)
	sc->tx.max_avail_queue_len = sc->tx.avail_queue_len;
	CPSW_DEBUGF(sc, ("TX removed %d completed packet(s)", removed));
	}

	if (sc->tx.teardown && STAILQ_EMPTY(&sc->tx.active)) {
	CPSW_DEBUGF(sc, ("TX teardown is complete"));
	sc->tx.teardown = 0;
	sc->tx.running = 0;
	}

	return (removed);
	}

	/*
	*
	* Miscellaneous interrupts.
	*
	*/

	static void
	cpsw_intr_rx_thresh(void *arg)
	{
	struct cpsw_softc *sc;
	struct ifnet *ifp;
	struct mbuf received, next;

	sc = (struct cpsw_softc *)arg;
	CPSW_RX_LOCK(sc);
	received = cpsw_rx_dequeue(sc);
	cpsw_rx_enqueue(sc);
	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 0);
	CPSW_RX_UNLOCK(sc);

	while (received != NULL) {
	next = received->m_nextpkt;
	received->m_nextpkt = NULL;
	ifp = received->m_pkthdr.rcvif;
	(*ifp->if_input)(ifp, received);
	if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
	received = next;
	}
	}

	static void
	cpsw_intr_misc_host_error(struct cpsw_softc *sc)
	{
	uint32_t intstat;
	uint32_t dmastat;
	int txerr, rxerr, txchan, rxchan;

	printf("\n\n");
	device_printf(sc->dev,
	"HOST ERROR: PROGRAMMING ERROR DETECTED BY HARDWARE\n");
	printf("\n\n");
	intstat = cpsw_read_4(sc, CPSW_CPDMA_DMA_INTSTAT_MASKED);
	device_printf(sc->dev, "CPSW_CPDMA_DMA_INTSTAT_MASKED=0x%x\n", intstat);
	dmastat = cpsw_read_4(sc, CPSW_CPDMA_DMASTATUS);
	device_printf(sc->dev, "CPSW_CPDMA_DMASTATUS=0x%x\n", dmastat);

	txerr = (dmastat >> 20) & 15;
	txchan = (dmastat >> 16) & 7;
	rxerr = (dmastat >> 12) & 15;
	rxchan = (dmastat >> 8) & 7;

	switch (txerr) {
	case 0: break;
	case 1: printf("SOP error on TX channel %d\n", txchan);
	break;
	case 2: printf("Ownership bit not set on SOP buffer on TX channel %d\n", txchan);
	break;
	case 3: printf("Zero Next Buffer but not EOP on TX channel %d\n", txchan);
	break;
	case 4: printf("Zero Buffer Pointer on TX channel %d\n", txchan);
	break;
	case 5: printf("Zero Buffer Length on TX channel %d\n", txchan);
	break;
	case 6: printf("Packet length error on TX channel %d\n", txchan);
	break;
	default: printf("Unknown error on TX channel %d\n", txchan);
	break;
	}

	if (txerr != 0) {
	printf("CPSW_CPDMA_TX%d_HDP=0x%x\n",
	txchan, cpsw_read_4(sc, CPSW_CPDMA_TX_HDP(txchan)));
	printf("CPSW_CPDMA_TX%d_CP=0x%x\n",
	txchan, cpsw_read_4(sc, CPSW_CPDMA_TX_CP(txchan)));
	cpsw_dump_queue(sc, &sc->tx.active);
	}

	switch (rxerr) {
	case 0: break;
	case 2: printf("Ownership bit not set on RX channel %d\n", rxchan);
	break;
	case 4: printf("Zero Buffer Pointer on RX channel %d\n", rxchan);
	break;
	case 5: printf("Zero Buffer Length on RX channel %d\n", rxchan);
	break;
	case 6: printf("Buffer offset too big on RX channel %d\n", rxchan);
	break;
	default: printf("Unknown RX error on RX channel %d\n", rxchan);
	break;
	}

	if (rxerr != 0) {
	printf("CPSW_CPDMA_RX%d_HDP=0x%x\n",
	rxchan, cpsw_read_4(sc,CPSW_CPDMA_RX_HDP(rxchan)));
	printf("CPSW_CPDMA_RX%d_CP=0x%x\n",
	rxchan, cpsw_read_4(sc, CPSW_CPDMA_RX_CP(rxchan)));
	cpsw_dump_queue(sc, &sc->rx.active);
	}

	printf("\nALE Table\n");
	cpsw_ale_dump_table(sc);

	// XXX do something useful here??
	panic("CPSW HOST ERROR INTERRUPT");

	// Suppress this interrupt in the future.
	cpsw_write_4(sc, CPSW_CPDMA_DMA_INTMASK_CLEAR, intstat);
	printf("XXX HOST ERROR INTERRUPT SUPPRESSED\n");
	// The watchdog will probably reset the controller
	// in a little while. It will probably fail again.
	}

	static void
	cpsw_intr_misc(void *arg)
	{
	struct cpsw_softc *sc = arg;
	uint32_t stat = cpsw_read_4(sc, CPSW_WR_C_MISC_STAT(0));

	if (stat & CPSW_WR_C_MISC_EVNT_PEND)
	CPSW_DEBUGF(sc, ("Time sync event interrupt unimplemented"));
	if (stat & CPSW_WR_C_MISC_STAT_PEND)
	cpsw_stats_collect(sc);
	if (stat & CPSW_WR_C_MISC_HOST_PEND)
	cpsw_intr_misc_host_error(sc);
	if (stat & CPSW_WR_C_MISC_MDIOLINK) {
	cpsw_write_4(sc, MDIOLINKINTMASKED,
	cpsw_read_4(sc, MDIOLINKINTMASKED));
	}
	if (stat & CPSW_WR_C_MISC_MDIOUSER) {
	CPSW_DEBUGF(sc,
	("MDIO operation completed interrupt unimplemented"));
	}
	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 3);
	}

	/*
	*
	* Periodic Checks and Watchdog.
	*
	*/

	static void
	cpswp_tick(void *msc)
	{
	struct cpswp_softc *sc = msc;

	/* Check for media type change */
	mii_tick(sc->mii);
	if (sc->media_status != sc->mii->mii_media.ifm_media) {
	printf("%s: media type changed (ifm_media=%x)\n", __func__,
	sc->mii->mii_media.ifm_media);
	cpswp_ifmedia_upd(sc->ifp);
	}

	/* Schedule another timeout one second from now */
	callout_reset(&sc->mii_callout, hz, cpswp_tick, sc);
	}

	static void
	cpswp_ifmedia_sts(struct ifnet ifp, struct ifmediareq ifmr)
	{
	struct cpswp_softc *sc;
	struct mii_data *mii;

	sc = ifp->if_softc;
	CPSW_DEBUGF(sc->swsc, (""));
	CPSW_PORT_LOCK(sc);

	mii = sc->mii;
	mii_pollstat(mii);

	ifmr->ifm_active = mii->mii_media_active;
	ifmr->ifm_status = mii->mii_media_status;
	CPSW_PORT_UNLOCK(sc);
	}

	static int
	cpswp_ifmedia_upd(struct ifnet *ifp)
	{
	struct cpswp_softc *sc;

	sc = ifp->if_softc;
	CPSW_DEBUGF(sc->swsc, (""));
	CPSW_PORT_LOCK(sc);
	mii_mediachg(sc->mii);
	sc->media_status = sc->mii->mii_media.ifm_media;
	CPSW_PORT_UNLOCK(sc);

	return (0);
	}

	static void
	cpsw_tx_watchdog_full_reset(struct cpsw_softc *sc)
	{
	struct cpswp_softc *psc;
	int i;

	cpsw_debugf_head("CPSW watchdog");
	device_printf(sc->dev, "watchdog timeout\n");
	printf("CPSW_CPDMA_TX%d_HDP=0x%x\n", 0,
	cpsw_read_4(sc, CPSW_CPDMA_TX_HDP(0)));
	printf("CPSW_CPDMA_TX%d_CP=0x%x\n", 0,
	cpsw_read_4(sc, CPSW_CPDMA_TX_CP(0)));
	cpsw_dump_queue(sc, &sc->tx.active);
	for (i = 0; i < CPSW_PORTS; i++) {
	if (!sc->dualemac && i != sc->active_slave)
	continue;
	psc = device_get_softc(sc->port[i].dev);
	CPSW_PORT_LOCK(psc);
	cpswp_stop_locked(psc);
	CPSW_PORT_UNLOCK(psc);
	}
	}

	static void
	cpsw_tx_watchdog(void *msc)
	{
	struct cpsw_softc *sc;

	sc = msc;
	CPSW_TX_LOCK(sc);
	if (sc->tx.active_queue_len == 0 \|\| !sc->tx.running) {
	sc->watchdog.timer = 0; /* Nothing to do. */
	} else if (sc->tx.queue_removes > sc->tx.queue_removes_at_last_tick) {
	sc->watchdog.timer = 0; /* Stuff done while we weren't looking. */
	} else if (cpsw_tx_dequeue(sc) > 0) {
	sc->watchdog.timer = 0; /* We just did something. */
	} else {
	/* There was something to do but it didn't get done. */
	++sc->watchdog.timer;
	if (sc->watchdog.timer > 5) {
	sc->watchdog.timer = 0;
	++sc->watchdog.resets;
	cpsw_tx_watchdog_full_reset(sc);
	}
	}
	sc->tx.queue_removes_at_last_tick = sc->tx.queue_removes;
	CPSW_TX_UNLOCK(sc);

	/* Schedule another timeout one second from now */
	callout_reset(&sc->watchdog.callout, hz, cpsw_tx_watchdog, sc);
	}

	/*
	*
	* ALE support routines.
	*
	*/

	static void
	cpsw_ale_read_entry(struct cpsw_softc sc, uint16_t idx, uint32_t ale_entry)
	{
	cpsw_write_4(sc, CPSW_ALE_TBLCTL, idx & 1023);
	ale_entry[0] = cpsw_read_4(sc, CPSW_ALE_TBLW0);
	ale_entry[1] = cpsw_read_4(sc, CPSW_ALE_TBLW1);
	ale_entry[2] = cpsw_read_4(sc, CPSW_ALE_TBLW2);
	}

	static void
	cpsw_ale_write_entry(struct cpsw_softc sc, uint16_t idx, uint32_t ale_entry)
	{
	cpsw_write_4(sc, CPSW_ALE_TBLW0, ale_entry[0]);
	cpsw_write_4(sc, CPSW_ALE_TBLW1, ale_entry[1]);
	cpsw_write_4(sc, CPSW_ALE_TBLW2, ale_entry[2]);
	cpsw_write_4(sc, CPSW_ALE_TBLCTL, 1 << 31 \| (idx & 1023));
	}

	static void
	cpsw_ale_remove_all_mc_entries(struct cpsw_softc *sc)
	{
	int i;
	uint32_t ale_entry[3];

	/* First four entries are link address and broadcast. */
	for (i = 10; i < CPSW_MAX_ALE_ENTRIES; i++) {
	cpsw_ale_read_entry(sc, i, ale_entry);
	if ((ALE_TYPE(ale_entry) == ALE_TYPE_ADDR \|\|
	ALE_TYPE(ale_entry) == ALE_TYPE_VLAN_ADDR) &&
	ALE_MCAST(ale_entry) == 1) { /* MCast link addr */
	ale_entry[0] = ale_entry[1] = ale_entry[2] = 0;
	cpsw_ale_write_entry(sc, i, ale_entry);
	}
	}
	}

	static int
	cpsw_ale_mc_entry_set(struct cpsw_softc *sc, uint8_t portmap, int vlan,
	uint8_t *mac)
	{
	int free_index = -1, matching_index = -1, i;
	uint32_t ale_entry[3], ale_type;

	/* Find a matching entry or a free entry. */
	for (i = 10; i < CPSW_MAX_ALE_ENTRIES; i++) {
	cpsw_ale_read_entry(sc, i, ale_entry);

	/* Entry Type[61:60] is 0 for free entry */
	if (free_index < 0 && ALE_TYPE(ale_entry) == 0)
	free_index = i;

	if ((((ale_entry[1] >> 8) & 0xFF) == mac[0]) &&
	(((ale_entry[1] >> 0) & 0xFF) == mac[1]) &&
	(((ale_entry[0] >>24) & 0xFF) == mac[2]) &&
	(((ale_entry[0] >>16) & 0xFF) == mac[3]) &&
	(((ale_entry[0] >> 8) & 0xFF) == mac[4]) &&
	(((ale_entry[0] >> 0) & 0xFF) == mac[5])) {
	matching_index = i;
	break;
	}
	}

	if (matching_index < 0) {
	if (free_index < 0)
	return (ENOMEM);
	i = free_index;
	}

	if (vlan != -1)
	ale_type = ALE_TYPE_VLAN_ADDR << 28 \| vlan << 16;
	else
	ale_type = ALE_TYPE_ADDR << 28;

	/* Set MAC address */
	ale_entry[0] = mac[2] << 24 \| mac[3] << 16 \| mac[4] << 8 \| mac[5];
	ale_entry[1] = mac[0] << 8 \| mac[1];

	/* Entry type[61:60] and Mcast fwd state[63:62] is fw(3). */
	ale_entry[1] \|= ALE_MCAST_FWD \| ale_type;

	/* Set portmask [68:66] */
	ale_entry[2] = (portmap & 7) << 2;

	cpsw_ale_write_entry(sc, i, ale_entry);

	return 0;
	}

	static void
	cpsw_ale_dump_table(struct cpsw_softc *sc) {
	int i;
	uint32_t ale_entry[3];
	for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) {
	cpsw_ale_read_entry(sc, i, ale_entry);
	switch (ALE_TYPE(ale_entry)) {
	case ALE_TYPE_VLAN:
	printf("ALE[%4u] %08x %08x %08x ", i, ale_entry[2],
	ale_entry[1], ale_entry[0]);
	printf("type: %u ", ALE_TYPE(ale_entry));
	printf("vlan: %u ", ALE_VLAN(ale_entry));
	printf("untag: %u ", ALE_VLAN_UNTAG(ale_entry));
	printf("reg flood: %u ", ALE_VLAN_REGFLOOD(ale_entry));
	printf("unreg flood: %u ", ALE_VLAN_UNREGFLOOD(ale_entry));
	printf("members: %u ", ALE_VLAN_MEMBERS(ale_entry));
	printf("\n");
	break;
	case ALE_TYPE_ADDR:
	case ALE_TYPE_VLAN_ADDR:
	printf("ALE[%4u] %08x %08x %08x ", i, ale_entry[2],
	ale_entry[1], ale_entry[0]);
	printf("type: %u ", ALE_TYPE(ale_entry));
	printf("mac: %02x:%02x:%02x:%02x:%02x:%02x ",
	(ale_entry[1] >> 8) & 0xFF,
	(ale_entry[1] >> 0) & 0xFF,
	(ale_entry[0] >>24) & 0xFF,
	(ale_entry[0] >>16) & 0xFF,
	(ale_entry[0] >> 8) & 0xFF,
	(ale_entry[0] >> 0) & 0xFF);
	printf(ALE_MCAST(ale_entry) ? "mcast " : "ucast ");
	if (ALE_TYPE(ale_entry) == ALE_TYPE_VLAN_ADDR)
	printf("vlan: %u ", ALE_VLAN(ale_entry));
	printf("port: %u ", ALE_PORTS(ale_entry));
	printf("\n");
	break;
	}
	}
	printf("\n");
	}

	static u_int
	cpswp_set_maddr(void arg, struct sockaddr_dl sdl, u_int cnt)
	{
	struct cpswp_softc *sc = arg;
	uint32_t portmask;

	if (sc->swsc->dualemac)
	portmask = 1 << (sc->unit + 1) \| 1 << 0;
	else
	portmask = 7;

	cpsw_ale_mc_entry_set(sc->swsc, portmask, sc->vlan, LLADDR(sdl));

	return (1);
	}

	static int
	cpswp_ale_update_addresses(struct cpswp_softc *sc, int purge)
	{
	uint8_t *mac;
	uint32_t ale_entry[3], ale_type, portmask;

	if (sc->swsc->dualemac) {
	ale_type = ALE_TYPE_VLAN_ADDR << 28 \| sc->vlan << 16;
	portmask = 1 << (sc->unit + 1) \| 1 << 0;
	} else {
	ale_type = ALE_TYPE_ADDR << 28;
	portmask = 7;
	}

	/*
	* Route incoming packets for our MAC address to Port 0 (host).
	* For simplicity, keep this entry at table index 0 for port 1 and
	* at index 2 for port 2 in the ALE.
	*/
	mac = LLADDR((struct sockaddr_dl *)sc->ifp->if_addr->ifa_addr);
	ale_entry[0] = mac[2] << 24 \| mac[3] << 16 \| mac[4] << 8 \| mac[5];
	ale_entry[1] = ale_type \| mac[0] << 8 \| mac[1]; /* addr entry + mac */
	ale_entry[2] = 0; /* port = 0 */
	cpsw_ale_write_entry(sc->swsc, 0 + 2 * sc->unit, ale_entry);

	/* Set outgoing MAC Address for slave port. */
	cpsw_write_4(sc->swsc, CPSW_PORT_P_SA_HI(sc->unit + 1),
	mac[3] << 24 \| mac[2] << 16 \| mac[1] << 8 \| mac[0]);
	cpsw_write_4(sc->swsc, CPSW_PORT_P_SA_LO(sc->unit + 1),
	mac[5] << 8 \| mac[4]);

	/* Keep the broadcast address at table entry 1 (or 3). */
	ale_entry[0] = 0xffffffff; /* Lower 32 bits of MAC */
	/* ALE_MCAST_FWD, Addr type, upper 16 bits of Mac */
	ale_entry[1] = ALE_MCAST_FWD \| ale_type \| 0xffff;
	ale_entry[2] = portmask << 2;
	cpsw_ale_write_entry(sc->swsc, 1 + 2 * sc->unit, ale_entry);

	/* SIOCDELMULTI doesn't specify the particular address
	being removed, so we have to remove all and rebuild. */
	if (purge)
	cpsw_ale_remove_all_mc_entries(sc->swsc);

	/* Set other multicast addrs desired. */
	if_foreach_llmaddr(sc->ifp, cpswp_set_maddr, sc);

	return (0);
	}

	static int
	cpsw_ale_update_vlan_table(struct cpsw_softc *sc, int vlan, int ports,
	int untag, int mcregflood, int mcunregflood)
	{
	int free_index, i, matching_index;
	uint32_t ale_entry[3];

	free_index = matching_index = -1;
	/* Find a matching entry or a free entry. */
	for (i = 5; i < CPSW_MAX_ALE_ENTRIES; i++) {
	cpsw_ale_read_entry(sc, i, ale_entry);

	/* Entry Type[61:60] is 0 for free entry */
	if (free_index < 0 && ALE_TYPE(ale_entry) == 0)
	free_index = i;

	if (ALE_VLAN(ale_entry) == vlan) {
	matching_index = i;
	break;
	}
	}

	if (matching_index < 0) {
	if (free_index < 0)
	return (-1);
	i = free_index;
	}

	ale_entry[0] = (untag & 7) << 24 \| (mcregflood & 7) << 16 \|
	(mcunregflood & 7) << 8 \| (ports & 7);
	ale_entry[1] = ALE_TYPE_VLAN << 28 \| vlan << 16;
	ale_entry[2] = 0;
	cpsw_ale_write_entry(sc, i, ale_entry);

	return (0);
	}

	/*
	*
	* Statistics and Sysctls.
	*
	*/

	#if 0
	static void
	cpsw_stats_dump(struct cpsw_softc *sc)
	{
	int i;
	uint32_t r;

	for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) {
	r = cpsw_read_4(sc, CPSW_STATS_OFFSET +
	cpsw_stat_sysctls[i].reg);
	CPSW_DEBUGF(sc, ("%s: %ju + %u = %ju", cpsw_stat_sysctls[i].oid,
	(intmax_t)sc->shadow_stats[i], r,
	(intmax_t)sc->shadow_stats[i] + r));
	}
	}
	#endif

	static void
	cpsw_stats_collect(struct cpsw_softc *sc)
	{
	int i;
	uint32_t r;

	CPSW_DEBUGF(sc, ("Controller shadow statistics updated."));

	for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) {
	r = cpsw_read_4(sc, CPSW_STATS_OFFSET +
	cpsw_stat_sysctls[i].reg);
	sc->shadow_stats[i] += r;
	cpsw_write_4(sc, CPSW_STATS_OFFSET + cpsw_stat_sysctls[i].reg,
	r);
	}
	}

	static int
	cpsw_stats_sysctl(SYSCTL_HANDLER_ARGS)
	{
	struct cpsw_softc *sc;
	struct cpsw_stat *stat;
	uint64_t result;

	sc = (struct cpsw_softc *)arg1;
	stat = &cpsw_stat_sysctls[oidp->oid_number];
	result = sc->shadow_stats[oidp->oid_number];
	result += cpsw_read_4(sc, CPSW_STATS_OFFSET + stat->reg);
	return (sysctl_handle_64(oidp, &result, 0, req));
	}

	static int
	cpsw_stat_attached(SYSCTL_HANDLER_ARGS)
	{
	struct cpsw_softc *sc;
	struct bintime t;
	unsigned result;

	sc = (struct cpsw_softc *)arg1;
	getbinuptime(&t);
	bintime_sub(&t, &sc->attach_uptime);
	result = t.sec;
	return (sysctl_handle_int(oidp, &result, 0, req));
	}

	static int
	cpsw_intr_coalesce(SYSCTL_HANDLER_ARGS)
	{
	int error;
	struct cpsw_softc *sc;
	uint32_t ctrl, intr_per_ms;

	sc = (struct cpsw_softc *)arg1;
	error = sysctl_handle_int(oidp, &sc->coal_us, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	ctrl = cpsw_read_4(sc, CPSW_WR_INT_CONTROL);
	ctrl &= ~(CPSW_WR_INT_PACE_EN \| CPSW_WR_INT_PRESCALE_MASK);
	if (sc->coal_us == 0) {
	/* Disable the interrupt pace hardware. */
	cpsw_write_4(sc, CPSW_WR_INT_CONTROL, ctrl);
	cpsw_write_4(sc, CPSW_WR_C_RX_IMAX(0), 0);
	cpsw_write_4(sc, CPSW_WR_C_TX_IMAX(0), 0);
	return (0);
	}

	if (sc->coal_us > CPSW_WR_C_IMAX_US_MAX)
	sc->coal_us = CPSW_WR_C_IMAX_US_MAX;
	if (sc->coal_us < CPSW_WR_C_IMAX_US_MIN)
	sc->coal_us = CPSW_WR_C_IMAX_US_MIN;
	intr_per_ms = 1000 / sc->coal_us;
	/* Just to make sure... */
	if (intr_per_ms > CPSW_WR_C_IMAX_MAX)
	intr_per_ms = CPSW_WR_C_IMAX_MAX;
	if (intr_per_ms < CPSW_WR_C_IMAX_MIN)
	intr_per_ms = CPSW_WR_C_IMAX_MIN;

	/* Set the prescale to produce 4us pulses from the 125 Mhz clock. */
	ctrl \|= (125 * 4) & CPSW_WR_INT_PRESCALE_MASK;

	/* Enable the interrupt pace hardware. */
	cpsw_write_4(sc, CPSW_WR_C_RX_IMAX(0), intr_per_ms);
	cpsw_write_4(sc, CPSW_WR_C_TX_IMAX(0), intr_per_ms);
	ctrl \|= CPSW_WR_INT_C0_RX_PULSE \| CPSW_WR_INT_C0_TX_PULSE;
	cpsw_write_4(sc, CPSW_WR_INT_CONTROL, ctrl);

	return (0);
	}

	static int
	cpsw_stat_uptime(SYSCTL_HANDLER_ARGS)
	{
	struct cpsw_softc *swsc;
	struct cpswp_softc *sc;
	struct bintime t;
	unsigned result;

	swsc = arg1;
	sc = device_get_softc(swsc->port[arg2].dev);
	if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) {
	getbinuptime(&t);
	bintime_sub(&t, &sc->init_uptime);
	result = t.sec;
	} else
	result = 0;
	return (sysctl_handle_int(oidp, &result, 0, req));
	}

	static void
	cpsw_add_queue_sysctls(struct sysctl_ctx_list ctx, struct sysctl_oid node,
	struct cpsw_queue *queue)
	{
	struct sysctl_oid_list *parent;

	parent = SYSCTL_CHILDREN(node);
	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "totalBuffers",
	CTLFLAG_RD, &queue->queue_slots, 0,
	"Total buffers currently assigned to this queue");
	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "activeBuffers",
	CTLFLAG_RD, &queue->active_queue_len, 0,
	"Buffers currently registered with hardware controller");
	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "maxActiveBuffers",
	CTLFLAG_RD, &queue->max_active_queue_len, 0,
	"Max value of activeBuffers since last driver reset");
	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "availBuffers",
	CTLFLAG_RD, &queue->avail_queue_len, 0,
	"Buffers allocated to this queue but not currently "
	"registered with hardware controller");
	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "maxAvailBuffers",
	CTLFLAG_RD, &queue->max_avail_queue_len, 0,
	"Max value of availBuffers since last driver reset");
	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "totalEnqueued",
	CTLFLAG_RD, &queue->queue_adds, 0,
	"Total buffers added to queue");
	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "totalDequeued",
	CTLFLAG_RD, &queue->queue_removes, 0,
	"Total buffers removed from queue");
	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "queueRestart",
	CTLFLAG_RD, &queue->queue_restart, 0,
	"Total times the queue has been restarted");
	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "longestChain",
	CTLFLAG_RD, &queue->longest_chain, 0,
	"Max buffers used for a single packet");
	}

	static void
	cpsw_add_watchdog_sysctls(struct sysctl_ctx_list ctx, struct sysctl_oid node,
	struct cpsw_softc *sc)
	{
	struct sysctl_oid_list *parent;

	parent = SYSCTL_CHILDREN(node);
	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "resets",
	CTLFLAG_RD, &sc->watchdog.resets, 0,
	"Total number of watchdog resets");
	}

	static void
	cpsw_add_sysctls(struct cpsw_softc *sc)
	{
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid stats_node, queue_node, *node;
	struct sysctl_oid_list parent, stats_parent, *queue_parent;
	struct sysctl_oid_list ports_parent, port_parent;
	char port[16];
	int i;

	ctx = device_get_sysctl_ctx(sc->dev);
	parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));

	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "debug",
	CTLFLAG_RW, &sc->debug, 0, "Enable switch debug messages");

	SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "attachedSecs",
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	sc, 0, cpsw_stat_attached, "IU",
	"Time since driver attach");

	SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "intr_coalesce_us",
	CTLTYPE_UINT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	sc, 0, cpsw_intr_coalesce, "IU",
	"minimum time between interrupts");

	node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "ports",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "CPSW Ports Statistics");
	ports_parent = SYSCTL_CHILDREN(node);
	for (i = 0; i < CPSW_PORTS; i++) {
	if (!sc->dualemac && i != sc->active_slave)
	continue;
	port[0] = '0' + i;
	port[1] = '\0';
	node = SYSCTL_ADD_NODE(ctx, ports_parent, OID_AUTO,
	port, CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"CPSW Port Statistics");
	port_parent = SYSCTL_CHILDREN(node);
	SYSCTL_ADD_PROC(ctx, port_parent, OID_AUTO, "uptime",
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, i,
	cpsw_stat_uptime, "IU", "Seconds since driver init");
	}

	stats_node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "CPSW Statistics");
	stats_parent = SYSCTL_CHILDREN(stats_node);
	for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) {
	SYSCTL_ADD_PROC(ctx, stats_parent, i,
	cpsw_stat_sysctls[i].oid,
	CTLTYPE_U64 \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	sc, 0, cpsw_stats_sysctl, "IU",
	cpsw_stat_sysctls[i].oid);
	}

	queue_node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "queue",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "CPSW Queue Statistics");
	queue_parent = SYSCTL_CHILDREN(queue_node);

	node = SYSCTL_ADD_NODE(ctx, queue_parent, OID_AUTO, "tx",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "TX Queue Statistics");
	cpsw_add_queue_sysctls(ctx, node, &sc->tx);

	node = SYSCTL_ADD_NODE(ctx, queue_parent, OID_AUTO, "rx",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "RX Queue Statistics");
	cpsw_add_queue_sysctls(ctx, node, &sc->rx);

	node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "watchdog",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "Watchdog Statistics");
	cpsw_add_watchdog_sysctls(ctx, node, sc);
	}

	#ifdef CPSW_ETHERSWITCH
	static etherswitch_info_t etherswitch_info = {
	.es_nports = CPSW_PORTS + 1,
	.es_nvlangroups = CPSW_VLANS,
	.es_name = "TI Common Platform Ethernet Switch (CPSW)",
	.es_vlan_caps = ETHERSWITCH_VLAN_DOT1Q,
	};

	static etherswitch_info_t *
	cpsw_getinfo(device_t dev)
	{
	return (&etherswitch_info);
	}

	static int
	cpsw_getport(device_t dev, etherswitch_port_t *p)
	{
	int err;
	struct cpsw_softc *sc;
	struct cpswp_softc *psc;
	struct ifmediareq *ifmr;
	uint32_t reg;

	if (p->es_port < 0 \|\| p->es_port > CPSW_PORTS)
	return (ENXIO);

	err = 0;
	sc = device_get_softc(dev);
	if (p->es_port == CPSW_CPU_PORT) {
	p->es_flags \|= ETHERSWITCH_PORT_CPU;
	ifmr = &p->es_ifmr;
	ifmr->ifm_current = ifmr->ifm_active =
	IFM_ETHER \| IFM_1000_T \| IFM_FDX;
	ifmr->ifm_mask = 0;
	ifmr->ifm_status = IFM_ACTIVE \| IFM_AVALID;
	ifmr->ifm_count = 0;
	} else {
	psc = device_get_softc(sc->port[p->es_port - 1].dev);
	err = ifmedia_ioctl(psc->ifp, &p->es_ifr,
	&psc->mii->mii_media, SIOCGIFMEDIA);
	}
	reg = cpsw_read_4(sc, CPSW_PORT_P_VLAN(p->es_port));
	p->es_pvid = reg & ETHERSWITCH_VID_MASK;

	reg = cpsw_read_4(sc, CPSW_ALE_PORTCTL(p->es_port));
	if (reg & ALE_PORTCTL_DROP_UNTAGGED)
	p->es_flags \|= ETHERSWITCH_PORT_DROPUNTAGGED;
	if (reg & ALE_PORTCTL_INGRESS)
	p->es_flags \|= ETHERSWITCH_PORT_INGRESS;

	return (err);
	}

	static int
	cpsw_setport(device_t dev, etherswitch_port_t *p)
	{
	struct cpsw_softc *sc;
	struct cpswp_softc *psc;
	struct ifmedia *ifm;
	uint32_t reg;

	if (p->es_port < 0 \|\| p->es_port > CPSW_PORTS)
	return (ENXIO);

	sc = device_get_softc(dev);
	if (p->es_pvid != 0) {
	cpsw_write_4(sc, CPSW_PORT_P_VLAN(p->es_port),
	p->es_pvid & ETHERSWITCH_VID_MASK);
	}

	reg = cpsw_read_4(sc, CPSW_ALE_PORTCTL(p->es_port));
	if (p->es_flags & ETHERSWITCH_PORT_DROPUNTAGGED)
	reg \|= ALE_PORTCTL_DROP_UNTAGGED;
	else
	reg &= ~ALE_PORTCTL_DROP_UNTAGGED;
	if (p->es_flags & ETHERSWITCH_PORT_INGRESS)
	reg \|= ALE_PORTCTL_INGRESS;
	else
	reg &= ~ALE_PORTCTL_INGRESS;
	cpsw_write_4(sc, CPSW_ALE_PORTCTL(p->es_port), reg);

	/* CPU port does not allow media settings. */
	if (p->es_port == CPSW_CPU_PORT)
	return (0);

	psc = device_get_softc(sc->port[p->es_port - 1].dev);
	ifm = &psc->mii->mii_media;

	return (ifmedia_ioctl(psc->ifp, &p->es_ifr, ifm, SIOCSIFMEDIA));
	}

	static int
	cpsw_getconf(device_t dev, etherswitch_conf_t *conf)
	{

	/* Return the VLAN mode. */
	conf->cmd = ETHERSWITCH_CONF_VLAN_MODE;
	conf->vlan_mode = ETHERSWITCH_VLAN_DOT1Q;

	return (0);
	}

	static int
	cpsw_getvgroup(device_t dev, etherswitch_vlangroup_t *vg)
	{
	int i, vid;
	uint32_t ale_entry[3];
	struct cpsw_softc *sc;

	sc = device_get_softc(dev);

	if (vg->es_vlangroup >= CPSW_VLANS)
	return (EINVAL);

	vg->es_vid = 0;
	vid = cpsw_vgroups[vg->es_vlangroup].vid;
	if (vid == -1)
	return (0);

	for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) {
	cpsw_ale_read_entry(sc, i, ale_entry);
	if (ALE_TYPE(ale_entry) != ALE_TYPE_VLAN)
	continue;
	if (vid != ALE_VLAN(ale_entry))
	continue;

	vg->es_fid = 0;
	vg->es_vid = ALE_VLAN(ale_entry) \| ETHERSWITCH_VID_VALID;
	vg->es_member_ports = ALE_VLAN_MEMBERS(ale_entry);
	vg->es_untagged_ports = ALE_VLAN_UNTAG(ale_entry);
	}

	return (0);
	}

	static void
	cpsw_remove_vlan(struct cpsw_softc *sc, int vlan)
	{
	int i;
	uint32_t ale_entry[3];

	for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) {
	cpsw_ale_read_entry(sc, i, ale_entry);
	if (ALE_TYPE(ale_entry) != ALE_TYPE_VLAN)
	continue;
	if (vlan != ALE_VLAN(ale_entry))
	continue;
	ale_entry[0] = ale_entry[1] = ale_entry[2] = 0;
	cpsw_ale_write_entry(sc, i, ale_entry);
	break;
	}
	}

	static int
	cpsw_setvgroup(device_t dev, etherswitch_vlangroup_t *vg)
	{
	int i;
	struct cpsw_softc *sc;

	sc = device_get_softc(dev);

	for (i = 0; i < CPSW_VLANS; i++) {
	/* Is this Vlan ID in use by another vlangroup ? */
	if (vg->es_vlangroup != i && cpsw_vgroups[i].vid == vg->es_vid)
	return (EINVAL);
	}

	if (vg->es_vid == 0) {
	if (cpsw_vgroups[vg->es_vlangroup].vid == -1)
	return (0);
	cpsw_remove_vlan(sc, cpsw_vgroups[vg->es_vlangroup].vid);
	cpsw_vgroups[vg->es_vlangroup].vid = -1;
	vg->es_untagged_ports = 0;
	vg->es_member_ports = 0;
	vg->es_vid = 0;
	return (0);
	}

	vg->es_vid &= ETHERSWITCH_VID_MASK;
	vg->es_member_ports &= CPSW_PORTS_MASK;
	vg->es_untagged_ports &= CPSW_PORTS_MASK;

	if (cpsw_vgroups[vg->es_vlangroup].vid != -1 &&
	cpsw_vgroups[vg->es_vlangroup].vid != vg->es_vid)
	return (EINVAL);

	cpsw_vgroups[vg->es_vlangroup].vid = vg->es_vid;
	cpsw_ale_update_vlan_table(sc, vg->es_vid, vg->es_member_ports,
	vg->es_untagged_ports, vg->es_member_ports, 0);

	return (0);
	}

	static int
	cpsw_readreg(device_t dev, int addr)
	{

	/* Not supported. */
	return (0);
	}

	static int
	cpsw_writereg(device_t dev, int addr, int value)
	{

	/* Not supported. */
	return (0);
	}

	static int
	cpsw_readphy(device_t dev, int phy, int reg)
	{

	/* Not supported. */
	return (0);
	}

	static int
	cpsw_writephy(device_t dev, int phy, int reg, int data)
	{

	/* Not supported. */
	return (0);
	}
	#endif
	diff --git a/sys/arm/ti/files.ti b/sys/arm/ti/files.ti
	index 83b3c2ed8d89..87beccd120a0 100644
	--- a/sys/arm/ti/files.ti
	+++ b/sys/arm/ti/files.ti
	@@ -1,23 +1,34 @@
	#$FreeBSD$

	arm/ti/ti_cpuid.c standard
	-arm/ti/ti_hwmods.c standard
	arm/ti/ti_machdep.c standard
	arm/ti/ti_prcm.c standard
	+arm/ti/ti_omap4_cm.c standard
	arm/ti/ti_scm.c standard
	+arm/ti/ti_scm_syscon.c standard
	arm/ti/ti_pinmux.c standard
	dev/mbox/mbox_if.m optional ti_mbox
	arm/ti/ti_mbox.c optional ti_mbox
	arm/ti/ti_pruss.c optional ti_pruss
	+arm/ti/ti_prm.c optional ti_pruss
	arm/ti/ti_wdt.c optional ti_wdt
	arm/ti/ti_adc.c optional ti_adc
	arm/ti/ti_gpio.c optional gpio
	arm/ti/ti_gpio_if.m optional gpio
	arm/ti/ti_i2c.c optional ti_i2c
	arm/ti/ti_sdhci.c optional sdhci
	arm/ti/ti_spi.c optional ti_spi
	arm/ti/ti_sysc.c standard

	+arm/ti/clk/clock_common.c standard
	+arm/ti/clk/ti_clk_clkctrl.c standard
	+arm/ti/clk/ti_clkctrl.c standard
	+arm/ti/clk/ti_clk_dpll.c standard
	+arm/ti/clk/ti_dpll_clock.c standard
	+arm/ti/clk/ti_mux_clock.c standard
	+arm/ti/clk/ti_divider_clock.c standard
	+arm/ti/clk/ti_gate_clock.c standard
	+
	dev/uart/uart_dev_ti8250.c optional uart
	dev/uart/uart_dev_ns8250.c optional uart

	diff --git a/sys/arm/ti/omap4/files.omap4 b/sys/arm/ti/omap4/files.omap4
	index ddae056d3fc4..0b2f2d3bf26d 100644
	--- a/sys/arm/ti/omap4/files.omap4
	+++ b/sys/arm/ti/omap4/files.omap4
	@@ -1,22 +1,22 @@
	#$FreeBSD$

	arm/ti/ti_smc.S standard

	arm/ti/usb/omap_ehci.c optional usb ehci
	arm/ti/usb/omap_host.c optional usb
	arm/ti/usb/omap_tll.c optional usb
	arm/ti/ti_sdma.c optional ti_sdma

	arm/ti/omap4/omap4_gpio.c optional gpio
	arm/ti/omap4/omap4_l2cache.c optional pl310
	-arm/ti/omap4/omap4_prcm_clks.c standard
	+#arm/ti/omap4/omap4_prcm_clks.c standard
	arm/ti/omap4/omap4_scm_padconf.c standard
	arm/ti/omap4/omap4_mp.c optional smp
	arm/ti/omap4/omap4_wugen.c standard

	arm/ti/omap4/pandaboard/pandaboard.c standard

	arm/ti/twl/twl.c optional twl
	arm/ti/twl/twl_vreg.c optional twl twl_vreg
	arm/ti/twl/twl_clks.c optional twl twl_clks

	diff --git a/sys/arm/ti/ti_adc.c b/sys/arm/ti/ti_adc.c
	index 7d7f1deae580..3c67500f3ebc 100644
	--- a/sys/arm/ti/ti_adc.c
	+++ b/sys/arm/ti/ti_adc.c
	@@ -1,969 +1,970 @@
	/*-
	* Copyright 2014 Luiz Otavio O Souza <loos@freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_evdev.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>

	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/condvar.h>
	#include <sys/resource.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	#include <sys/selinfo.h>
	#include <sys/poll.h>
	#include <sys/uio.h>

	#include <machine/bus.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#ifdef EVDEV_SUPPORT
	#include <dev/evdev/input.h>
	#include <dev/evdev/evdev.h>
	#endif

	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_adcreg.h>
	#include <arm/ti/ti_adcvar.h>

	#undef DEBUG_TSC

	#define DEFAULT_CHARGE_DELAY 0x400
	#define STEPDLY_OPEN 0x98

	#define ORDER_XP 0
	#define ORDER_XN 1
	#define ORDER_YP 2
	#define ORDER_YN 3

	/* Define our 8 steps, one for each input channel. */
	static struct ti_adc_input ti_adc_inputs[TI_ADC_NPINS] = {
	{ .stepconfig = ADC_STEPCFG(1), .stepdelay = ADC_STEPDLY(1) },
	{ .stepconfig = ADC_STEPCFG(2), .stepdelay = ADC_STEPDLY(2) },
	{ .stepconfig = ADC_STEPCFG(3), .stepdelay = ADC_STEPDLY(3) },
	{ .stepconfig = ADC_STEPCFG(4), .stepdelay = ADC_STEPDLY(4) },
	{ .stepconfig = ADC_STEPCFG(5), .stepdelay = ADC_STEPDLY(5) },
	{ .stepconfig = ADC_STEPCFG(6), .stepdelay = ADC_STEPDLY(6) },
	{ .stepconfig = ADC_STEPCFG(7), .stepdelay = ADC_STEPDLY(7) },
	{ .stepconfig = ADC_STEPCFG(8), .stepdelay = ADC_STEPDLY(8) },
	};

	static int ti_adc_samples[5] = { 0, 2, 4, 8, 16 };

	static int ti_adc_detach(device_t dev);

	#ifdef EVDEV_SUPPORT
	static void
	ti_adc_ev_report(struct ti_adc_softc *sc)
	{

	evdev_push_event(sc->sc_evdev, EV_ABS, ABS_X, sc->sc_x);
	evdev_push_event(sc->sc_evdev, EV_ABS, ABS_Y, sc->sc_y);
	evdev_push_event(sc->sc_evdev, EV_KEY, BTN_TOUCH, sc->sc_pen_down);
	evdev_sync(sc->sc_evdev);
	}
	#endif /* EVDEV */

	static void
	ti_adc_enable(struct ti_adc_softc *sc)
	{
	uint32_t reg;

	TI_ADC_LOCK_ASSERT(sc);

	if (sc->sc_last_state == 1)
	return;

	/* Enable the FIFO0 threshold and the end of sequence interrupt. */
	ADC_WRITE4(sc, ADC_IRQENABLE_SET,
	ADC_IRQ_FIFO0_THRES \| ADC_IRQ_FIFO1_THRES \| ADC_IRQ_END_OF_SEQ);

	reg = ADC_CTRL_STEP_WP \| ADC_CTRL_STEP_ID;
	if (sc->sc_tsc_wires > 0) {
	reg \|= ADC_CTRL_TSC_ENABLE;
	switch (sc->sc_tsc_wires) {
	case 4:
	reg \|= ADC_CTRL_TSC_4WIRE;
	break;
	case 5:
	reg \|= ADC_CTRL_TSC_5WIRE;
	break;
	case 8:
	reg \|= ADC_CTRL_TSC_8WIRE;
	break;
	default:
	break;
	}
	}
	reg \|= ADC_CTRL_ENABLE;
	/* Enable the ADC. Run thru enabled steps, start the conversions. */
	ADC_WRITE4(sc, ADC_CTRL, reg);

	sc->sc_last_state = 1;
	}

	static void
	ti_adc_disable(struct ti_adc_softc *sc)
	{
	int count;
	uint32_t data;

	TI_ADC_LOCK_ASSERT(sc);

	if (sc->sc_last_state == 0)
	return;

	/* Disable all the enabled steps. */
	ADC_WRITE4(sc, ADC_STEPENABLE, 0);

	/* Disable the ADC. */
	ADC_WRITE4(sc, ADC_CTRL, ADC_READ4(sc, ADC_CTRL) & ~ADC_CTRL_ENABLE);

	/* Disable the FIFO0 threshold and the end of sequence interrupt. */
	ADC_WRITE4(sc, ADC_IRQENABLE_CLR,
	ADC_IRQ_FIFO0_THRES \| ADC_IRQ_FIFO1_THRES \| ADC_IRQ_END_OF_SEQ);

	/* ACK any pending interrupt. */
	ADC_WRITE4(sc, ADC_IRQSTATUS, ADC_READ4(sc, ADC_IRQSTATUS));

	/* Drain the FIFO data. */
	count = ADC_READ4(sc, ADC_FIFO0COUNT) & ADC_FIFO_COUNT_MSK;
	while (count > 0) {
	data = ADC_READ4(sc, ADC_FIFO0DATA);
	count = ADC_READ4(sc, ADC_FIFO0COUNT) & ADC_FIFO_COUNT_MSK;
	}

	count = ADC_READ4(sc, ADC_FIFO1COUNT) & ADC_FIFO_COUNT_MSK;
	while (count > 0) {
	data = ADC_READ4(sc, ADC_FIFO1DATA);
	count = ADC_READ4(sc, ADC_FIFO1COUNT) & ADC_FIFO_COUNT_MSK;
	}

	sc->sc_last_state = 0;
	}

	static int
	ti_adc_setup(struct ti_adc_softc *sc)
	{
	int ain, i;
	uint32_t enabled;

	TI_ADC_LOCK_ASSERT(sc);

	/* Check for enabled inputs. */
	enabled = sc->sc_tsc_enabled;
	for (i = 0; i < sc->sc_adc_nchannels; i++) {
	ain = sc->sc_adc_channels[i];
	if (ti_adc_inputs[ain].enable)
	enabled \|= (1U << (ain + 1));
	}

	/* Set the ADC global status. */
	if (enabled != 0) {
	ti_adc_enable(sc);
	/* Update the enabled steps. */
	if (enabled != ADC_READ4(sc, ADC_STEPENABLE))
	ADC_WRITE4(sc, ADC_STEPENABLE, enabled);
	} else
	ti_adc_disable(sc);

	return (0);
	}

	static void
	ti_adc_input_setup(struct ti_adc_softc *sc, int32_t ain)
	{
	struct ti_adc_input *input;
	uint32_t reg, val;

	TI_ADC_LOCK_ASSERT(sc);

	input = &ti_adc_inputs[ain];
	reg = input->stepconfig;
	val = ADC_READ4(sc, reg);

	/* Set single ended operation. */
	val &= ~ADC_STEP_DIFF_CNTRL;

	/* Set the negative voltage reference. */
	val &= ~ADC_STEP_RFM_MSK;

	/* Set the positive voltage reference. */
	val &= ~ADC_STEP_RFP_MSK;

	/* Set the samples average. */
	val &= ~ADC_STEP_AVG_MSK;
	val \|= input->samples << ADC_STEP_AVG_SHIFT;

	/* Select the desired input. */
	val &= ~ADC_STEP_INP_MSK;
	val \|= ain << ADC_STEP_INP_SHIFT;

	/* Set the ADC to one-shot mode. */
	val &= ~ADC_STEP_MODE_MSK;

	ADC_WRITE4(sc, reg, val);
	}

	static void
	ti_adc_reset(struct ti_adc_softc *sc)
	{
	int ain, i;

	TI_ADC_LOCK_ASSERT(sc);

	/* Disable all the inputs. */
	for (i = 0; i < sc->sc_adc_nchannels; i++) {
	ain = sc->sc_adc_channels[i];
	ti_adc_inputs[ain].enable = 0;
	}
	}

	static int
	ti_adc_clockdiv_proc(SYSCTL_HANDLER_ARGS)
	{
	int error, reg;
	struct ti_adc_softc *sc;

	sc = (struct ti_adc_softc *)arg1;

	TI_ADC_LOCK(sc);
	reg = (int)ADC_READ4(sc, ADC_CLKDIV) + 1;
	TI_ADC_UNLOCK(sc);

	error = sysctl_handle_int(oidp, &reg, sizeof(reg), req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	/*
	* The actual written value is the prescaler setting - 1.
	* Enforce a minimum value of 10 (i.e. 9) which limits the maximum
	* ADC clock to ~2.4Mhz (CLK_M_OSC / 10).
	*/
	reg--;
	if (reg < 9)
	reg = 9;
	if (reg > USHRT_MAX)
	reg = USHRT_MAX;

	TI_ADC_LOCK(sc);
	/* Disable the ADC. */
	ti_adc_disable(sc);
	/* Update the ADC prescaler setting. */
	ADC_WRITE4(sc, ADC_CLKDIV, reg);
	/* Enable the ADC again. */
	ti_adc_setup(sc);
	TI_ADC_UNLOCK(sc);

	return (0);
	}

	static int
	ti_adc_enable_proc(SYSCTL_HANDLER_ARGS)
	{
	int error;
	int32_t enable;
	struct ti_adc_softc *sc;
	struct ti_adc_input *input;

	input = (struct ti_adc_input *)arg1;
	sc = input->sc;

	enable = input->enable;
	error = sysctl_handle_int(oidp, &enable, sizeof(enable),
	req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	if (enable)
	enable = 1;

	TI_ADC_LOCK(sc);
	/* Setup the ADC as needed. */
	if (input->enable != enable) {
	input->enable = enable;
	ti_adc_setup(sc);
	if (input->enable == 0)
	input->value = 0;
	}
	TI_ADC_UNLOCK(sc);

	return (0);
	}

	static int
	ti_adc_open_delay_proc(SYSCTL_HANDLER_ARGS)
	{
	int error, reg;
	struct ti_adc_softc *sc;
	struct ti_adc_input *input;

	input = (struct ti_adc_input *)arg1;
	sc = input->sc;

	TI_ADC_LOCK(sc);
	reg = (int)ADC_READ4(sc, input->stepdelay) & ADC_STEP_OPEN_DELAY;
	TI_ADC_UNLOCK(sc);

	error = sysctl_handle_int(oidp, &reg, sizeof(reg), req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	if (reg < 0)
	reg = 0;

	TI_ADC_LOCK(sc);
	ADC_WRITE4(sc, input->stepdelay, reg & ADC_STEP_OPEN_DELAY);
	TI_ADC_UNLOCK(sc);

	return (0);
	}

	static int
	ti_adc_samples_avg_proc(SYSCTL_HANDLER_ARGS)
	{
	int error, samples, i;
	struct ti_adc_softc *sc;
	struct ti_adc_input *input;

	input = (struct ti_adc_input *)arg1;
	sc = input->sc;

	if (input->samples > nitems(ti_adc_samples))
	input->samples = nitems(ti_adc_samples);
	samples = ti_adc_samples[input->samples];

	error = sysctl_handle_int(oidp, &samples, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	TI_ADC_LOCK(sc);
	if (samples != ti_adc_samples[input->samples]) {
	input->samples = 0;
	for (i = 0; i < nitems(ti_adc_samples); i++)
	if (samples >= ti_adc_samples[i])
	input->samples = i;
	ti_adc_input_setup(sc, input->input);
	}
	TI_ADC_UNLOCK(sc);

	return (error);
	}

	static void
	ti_adc_read_data(struct ti_adc_softc *sc)
	{
	int count, ain;
	struct ti_adc_input *input;
	uint32_t data;

	TI_ADC_LOCK_ASSERT(sc);

	/* Read the available data. */
	count = ADC_READ4(sc, ADC_FIFO0COUNT) & ADC_FIFO_COUNT_MSK;
	while (count > 0) {
	data = ADC_READ4(sc, ADC_FIFO0DATA);
	ain = (data & ADC_FIFO_STEP_ID_MSK) >> ADC_FIFO_STEP_ID_SHIFT;
	input = &ti_adc_inputs[ain];
	if (input->enable == 0)
	input->value = 0;
	else
	input->value = (int32_t)(data & ADC_FIFO_DATA_MSK);
	count = ADC_READ4(sc, ADC_FIFO0COUNT) & ADC_FIFO_COUNT_MSK;
	}
	}

	static int
	cmp_values(const void a, const void b)
	{
	const uint32_t v1, v2;
	v1 = a;
	v2 = b;
	if (v1 < v2)
	return -1;
	if (v1 > v2)
	return 1;

	return (0);
	}

	static void
	ti_adc_tsc_read_data(struct ti_adc_softc *sc)
	{
	int count;
	uint32_t data[16];
	uint32_t x, y;
	int i, start, end;

	TI_ADC_LOCK_ASSERT(sc);

	/* Read the available data. */
	count = ADC_READ4(sc, ADC_FIFO1COUNT) & ADC_FIFO_COUNT_MSK;
	if (count == 0)
	return;

	i = 0;
	while (count > 0) {
	data[i++] = ADC_READ4(sc, ADC_FIFO1DATA) & ADC_FIFO_DATA_MSK;
	count = ADC_READ4(sc, ADC_FIFO1COUNT) & ADC_FIFO_COUNT_MSK;
	}

	if (sc->sc_coord_readouts > 3) {
	start = 1;
	end = sc->sc_coord_readouts - 1;
	qsort(data, sc->sc_coord_readouts,
	sizeof(data[0]), &cmp_values);
	qsort(&data[sc->sc_coord_readouts + 2],
	sc->sc_coord_readouts,
	sizeof(data[0]), &cmp_values);
	}
	else {
	start = 0;
	end = sc->sc_coord_readouts;
	}

	x = y = 0;
	for (i = start; i < end; i++)
	y += data[i];
	y /= (end - start);

	for (i = sc->sc_coord_readouts + 2 + start; i < sc->sc_coord_readouts + 2 + end; i++)
	x += data[i];
	x /= (end - start);

	#ifdef DEBUG_TSC
	device_printf(sc->sc_dev, "touchscreen x: %d, y: %d\n", x, y);
	#endif

	#ifdef EVDEV_SUPPORT
	if ((sc->sc_x != x) \|\| (sc->sc_y != y)) {
	sc->sc_x = x;
	sc->sc_y = y;
	ti_adc_ev_report(sc);
	}
	#endif
	}

	static void
	ti_adc_intr_locked(struct ti_adc_softc *sc, uint32_t status)
	{
	/* Read the available data. */
	if (status & ADC_IRQ_FIFO0_THRES)
	ti_adc_read_data(sc);
	}

	static void
	ti_adc_tsc_intr_locked(struct ti_adc_softc *sc, uint32_t status)
	{
	/* Read the available data. */
	if (status & ADC_IRQ_FIFO1_THRES)
	ti_adc_tsc_read_data(sc);

	}

	static void
	ti_adc_intr(void *arg)
	{
	struct ti_adc_softc *sc;
	uint32_t status, rawstatus;

	sc = (struct ti_adc_softc *)arg;

	TI_ADC_LOCK(sc);

	rawstatus = ADC_READ4(sc, ADC_IRQSTATUS_RAW);
	status = ADC_READ4(sc, ADC_IRQSTATUS);

	if (rawstatus & ADC_IRQ_HW_PEN_ASYNC) {
	sc->sc_pen_down = 1;
	status \|= ADC_IRQ_HW_PEN_ASYNC;
	ADC_WRITE4(sc, ADC_IRQENABLE_CLR,
	ADC_IRQ_HW_PEN_ASYNC);
	#ifdef EVDEV_SUPPORT
	ti_adc_ev_report(sc);
	#endif
	}

	if (rawstatus & ADC_IRQ_PEN_UP) {
	sc->sc_pen_down = 0;
	status \|= ADC_IRQ_PEN_UP;
	#ifdef EVDEV_SUPPORT
	ti_adc_ev_report(sc);
	#endif
	}

	if (status & ADC_IRQ_FIFO0_THRES)
	ti_adc_intr_locked(sc, status);

	if (status & ADC_IRQ_FIFO1_THRES)
	ti_adc_tsc_intr_locked(sc, status);

	if (status) {
	/* ACK the interrupt. */
	ADC_WRITE4(sc, ADC_IRQSTATUS, status);
	}

	/* Start the next conversion ? */
	if (status & ADC_IRQ_END_OF_SEQ)
	ti_adc_setup(sc);

	TI_ADC_UNLOCK(sc);
	}

	static void
	ti_adc_sysctl_init(struct ti_adc_softc *sc)
	{
	char pinbuf[3];
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid tree_node, inp_node, *inpN_node;
	struct sysctl_oid_list tree, inp_tree, *inpN_tree;
	int ain, i;

	/*
	* Add per-pin sysctl tree/handlers.
	*/
	ctx = device_get_sysctl_ctx(sc->sc_dev);
	tree_node = device_get_sysctl_tree(sc->sc_dev);
	tree = SYSCTL_CHILDREN(tree_node);
	SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "clockdiv",
	CTLFLAG_RW \| CTLTYPE_UINT \| CTLFLAG_NEEDGIANT, sc, 0,
	ti_adc_clockdiv_proc, "IU", "ADC clock prescaler");
	inp_node = SYSCTL_ADD_NODE(ctx, tree, OID_AUTO, "ain",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "ADC inputs");
	inp_tree = SYSCTL_CHILDREN(inp_node);

	for (i = 0; i < sc->sc_adc_nchannels; i++) {
	ain = sc->sc_adc_channels[i];

	snprintf(pinbuf, sizeof(pinbuf), "%d", ain);
	inpN_node = SYSCTL_ADD_NODE(ctx, inp_tree, OID_AUTO, pinbuf,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "ADC input");
	inpN_tree = SYSCTL_CHILDREN(inpN_node);

	SYSCTL_ADD_PROC(ctx, inpN_tree, OID_AUTO, "enable",
	CTLFLAG_RW \| CTLTYPE_UINT \| CTLFLAG_NEEDGIANT,
	&ti_adc_inputs[ain], 0,
	ti_adc_enable_proc, "IU", "Enable ADC input");
	SYSCTL_ADD_PROC(ctx, inpN_tree, OID_AUTO, "open_delay",
	CTLFLAG_RW \| CTLTYPE_UINT \| CTLFLAG_NEEDGIANT,
	&ti_adc_inputs[ain], 0,
	ti_adc_open_delay_proc, "IU", "ADC open delay");
	SYSCTL_ADD_PROC(ctx, inpN_tree, OID_AUTO, "samples_avg",
	CTLFLAG_RW \| CTLTYPE_UINT \| CTLFLAG_NEEDGIANT,
	&ti_adc_inputs[ain], 0,
	ti_adc_samples_avg_proc, "IU", "ADC samples average");
	SYSCTL_ADD_INT(ctx, inpN_tree, OID_AUTO, "input",
	CTLFLAG_RD, &ti_adc_inputs[ain].value, 0,
	"Converted raw value for the ADC input");
	}
	}

	static void
	ti_adc_inputs_init(struct ti_adc_softc *sc)
	{
	int ain, i;
	struct ti_adc_input *input;

	TI_ADC_LOCK(sc);
	for (i = 0; i < sc->sc_adc_nchannels; i++) {
	ain = sc->sc_adc_channels[i];
	input = &ti_adc_inputs[ain];
	input->sc = sc;
	input->input = ain;
	input->value = 0;
	input->enable = 0;
	input->samples = 0;
	ti_adc_input_setup(sc, ain);
	}
	TI_ADC_UNLOCK(sc);
	}

	static void
	ti_adc_tsc_init(struct ti_adc_softc *sc)
	{
	int i, start_step, end_step;
	uint32_t stepconfig, val;

	TI_ADC_LOCK(sc);

	/* X coordinates */
	stepconfig = ADC_STEP_FIFO1 \| (4 << ADC_STEP_AVG_SHIFT) \|
	ADC_STEP_MODE_HW_ONESHOT \| sc->sc_xp_bit;
	if (sc->sc_tsc_wires == 4)
	stepconfig \|= ADC_STEP_INP(sc->sc_yp_inp) \| sc->sc_xn_bit;
	else if (sc->sc_tsc_wires == 5)
	stepconfig \|= ADC_STEP_INP(4) \|
	sc->sc_xn_bit \| sc->sc_yn_bit \| sc->sc_yp_bit;
	else if (sc->sc_tsc_wires == 8)
	stepconfig \|= ADC_STEP_INP(sc->sc_yp_inp) \| sc->sc_xn_bit;

	start_step = ADC_STEPS - sc->sc_coord_readouts + 1;
	end_step = start_step + sc->sc_coord_readouts - 1;
	for (i = start_step; i <= end_step; i++) {
	ADC_WRITE4(sc, ADC_STEPCFG(i), stepconfig);
	ADC_WRITE4(sc, ADC_STEPDLY(i), STEPDLY_OPEN);
	}

	/* Y coordinates */
	stepconfig = ADC_STEP_FIFO1 \| (4 << ADC_STEP_AVG_SHIFT) \|
	ADC_STEP_MODE_HW_ONESHOT \| sc->sc_yn_bit \|
	ADC_STEP_INM(8);
	if (sc->sc_tsc_wires == 4)
	stepconfig \|= ADC_STEP_INP(sc->sc_xp_inp) \| sc->sc_yp_bit;
	else if (sc->sc_tsc_wires == 5)
	stepconfig \|= ADC_STEP_INP(4) \|
	sc->sc_xp_bit \| sc->sc_xn_bit \| sc->sc_yp_bit;
	else if (sc->sc_tsc_wires == 8)
	stepconfig \|= ADC_STEP_INP(sc->sc_xp_inp) \| sc->sc_yp_bit;

	start_step = ADC_STEPS - (sc->sc_coord_readouts*2 + 2) + 1;
	end_step = start_step + sc->sc_coord_readouts - 1;
	for (i = start_step; i <= end_step; i++) {
	ADC_WRITE4(sc, ADC_STEPCFG(i), stepconfig);
	ADC_WRITE4(sc, ADC_STEPDLY(i), STEPDLY_OPEN);
	}

	/* Charge config */
	val = ADC_READ4(sc, ADC_IDLECONFIG);
	ADC_WRITE4(sc, ADC_TC_CHARGE_STEPCONFIG, val);
	ADC_WRITE4(sc, ADC_TC_CHARGE_DELAY, sc->sc_charge_delay);

	/* 2 steps for Z */
	start_step = ADC_STEPS - (sc->sc_coord_readouts + 2) + 1;
	stepconfig = ADC_STEP_FIFO1 \| (4 << ADC_STEP_AVG_SHIFT) \|
	ADC_STEP_MODE_HW_ONESHOT \| sc->sc_yp_bit \|
	sc->sc_xn_bit \| ADC_STEP_INP(sc->sc_xp_inp) \|
	ADC_STEP_INM(8);
	ADC_WRITE4(sc, ADC_STEPCFG(start_step), stepconfig);
	ADC_WRITE4(sc, ADC_STEPDLY(start_step), STEPDLY_OPEN);
	start_step++;
	stepconfig \|= ADC_STEP_INP(sc->sc_yn_inp);
	ADC_WRITE4(sc, ADC_STEPCFG(start_step), stepconfig);
	ADC_WRITE4(sc, ADC_STEPDLY(start_step), STEPDLY_OPEN);

	ADC_WRITE4(sc, ADC_FIFO1THRESHOLD, (sc->sc_coord_readouts*2 + 2) - 1);

	sc->sc_tsc_enabled = 1;
	start_step = ADC_STEPS - (sc->sc_coord_readouts*2 + 2) + 1;
	end_step = ADC_STEPS;
	for (i = start_step; i <= end_step; i++) {
	sc->sc_tsc_enabled \|= (1 << i);
	}


	TI_ADC_UNLOCK(sc);
	}

	static void
	ti_adc_idlestep_init(struct ti_adc_softc *sc)
	{
	uint32_t val;

	val = ADC_STEP_YNN_SW \| ADC_STEP_INM(8) \| ADC_STEP_INP(8) \| ADC_STEP_YPN_SW;

	ADC_WRITE4(sc, ADC_IDLECONFIG, val);
	}

	static int
	ti_adc_config_wires(struct ti_adc_softc sc, int wire_configs, int nwire_configs)
	{
	int i;
	int wire, ai;

	for (i = 0; i < nwire_configs; i++) {
	wire = wire_configs[i] & 0xf;
	ai = (wire_configs[i] >> 4) & 0xf;
	switch (wire) {
	case ORDER_XP:
	sc->sc_xp_bit = ADC_STEP_XPP_SW;
	sc->sc_xp_inp = ai;
	break;
	case ORDER_XN:
	sc->sc_xn_bit = ADC_STEP_XNN_SW;
	sc->sc_xn_inp = ai;
	break;
	case ORDER_YP:
	sc->sc_yp_bit = ADC_STEP_YPP_SW;
	sc->sc_yp_inp = ai;
	break;
	case ORDER_YN:
	sc->sc_yn_bit = ADC_STEP_YNN_SW;
	sc->sc_yn_inp = ai;
	break;
	default:
	device_printf(sc->sc_dev, "Invalid wire config\n");
	return (-1);
	}
	}
	return (0);
	}

	static int
	ti_adc_probe(device_t dev)
	{

	if (!ofw_bus_is_compatible(dev, "ti,am3359-tscadc"))
	return (ENXIO);
	device_set_desc(dev, "TI ADC controller");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	ti_adc_attach(device_t dev)
	{
	int err, rid, i;
	struct ti_adc_softc *sc;
	uint32_t rev, reg;
	phandle_t node, child;
	pcell_t cell;
	int *channels;
	int nwire_configs;
	int *wire_configs;

	sc = device_get_softc(dev);
	sc->sc_dev = dev;

	node = ofw_bus_get_node(dev);

	sc->sc_tsc_wires = 0;
	sc->sc_coord_readouts = 1;
	sc->sc_x_plate_resistance = 0;
	sc->sc_charge_delay = DEFAULT_CHARGE_DELAY;
	/* Read "tsc" node properties */
	child = ofw_bus_find_child(node, "tsc");
	if (child != 0 && OF_hasprop(child, "ti,wires")) {
	if ((OF_getencprop(child, "ti,wires", &cell, sizeof(cell))) > 0)
	sc->sc_tsc_wires = cell;
	if ((OF_getencprop(child, "ti,coordinate-readouts", &cell,
	sizeof(cell))) > 0)
	sc->sc_coord_readouts = cell;
	if ((OF_getencprop(child, "ti,x-plate-resistance", &cell,
	sizeof(cell))) > 0)
	sc->sc_x_plate_resistance = cell;
	if ((OF_getencprop(child, "ti,charge-delay", &cell,
	sizeof(cell))) > 0)
	sc->sc_charge_delay = cell;
	nwire_configs = OF_getencprop_alloc_multi(child,
	"ti,wire-config", sizeof(*wire_configs),
	(void **)&wire_configs);
	if (nwire_configs != sc->sc_tsc_wires) {
	device_printf(sc->sc_dev,
	"invalid number of ti,wire-config: %d (should be %d)\n",
	nwire_configs, sc->sc_tsc_wires);
	OF_prop_free(wire_configs);
	return (EINVAL);
	}
	err = ti_adc_config_wires(sc, wire_configs, nwire_configs);
	OF_prop_free(wire_configs);
	if (err)
	return (EINVAL);
	}

	/* Read "adc" node properties */
	child = ofw_bus_find_child(node, "adc");
	if (child != 0) {
	sc->sc_adc_nchannels = OF_getencprop_alloc_multi(child,
	"ti,adc-channels", sizeof(channels), (void *)&channels);
	if (sc->sc_adc_nchannels > 0) {
	for (i = 0; i < sc->sc_adc_nchannels; i++)
	sc->sc_adc_channels[i] = channels[i];
	OF_prop_free(channels);
	}
	}

	/* Sanity check FDT data */
	if (sc->sc_tsc_wires + sc->sc_adc_nchannels > TI_ADC_NPINS) {
	device_printf(dev, "total number of chanels (%d) is larger than %d\n",
	sc->sc_tsc_wires + sc->sc_adc_nchannels, TI_ADC_NPINS);
	return (ENXIO);
	}

	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (!sc->sc_mem_res) {
	device_printf(dev, "cannot allocate memory window\n");
	return (ENXIO);
	}

	/* Activate the ADC_TSC module. */
	- err = ti_prcm_clk_enable(TSC_ADC_CLK);
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	if (err)
	return (err);

	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE);
	if (!sc->sc_irq_res) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "cannot allocate interrupt\n");
	return (ENXIO);
	}

	if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC \| INTR_MPSAFE,
	NULL, ti_adc_intr, sc, &sc->sc_intrhand) != 0) {
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "Unable to setup the irq handler.\n");
	return (ENXIO);
	}

	/* Check the ADC revision. */
	- rev = ADC_READ4(sc, ADC_REVISION);
	+ rev = ADC_READ4(sc, ti_sysc_get_rev_address_offset_host(device_get_parent(dev)));
	device_printf(dev,
	"scheme: %#x func: %#x rtl: %d rev: %d.%d custom rev: %d\n",
	(rev & ADC_REV_SCHEME_MSK) >> ADC_REV_SCHEME_SHIFT,
	(rev & ADC_REV_FUNC_MSK) >> ADC_REV_FUNC_SHIFT,
	(rev & ADC_REV_RTL_MSK) >> ADC_REV_RTL_SHIFT,
	(rev & ADC_REV_MAJOR_MSK) >> ADC_REV_MAJOR_SHIFT,
	rev & ADC_REV_MINOR_MSK,
	(rev & ADC_REV_CUSTOM_MSK) >> ADC_REV_CUSTOM_SHIFT);

	reg = ADC_READ4(sc, ADC_CTRL);
	ADC_WRITE4(sc, ADC_CTRL, reg \| ADC_CTRL_STEP_WP \| ADC_CTRL_STEP_ID);

	/*
	* Set the ADC prescaler to 2400 if touchscreen is not enabled
	* and to 24 if it is. This sets the ADC clock to ~10Khz and
	* ~1Mhz respectively (CLK_M_OSC / prescaler).
	*/
	if (sc->sc_tsc_wires)
	ADC_WRITE4(sc, ADC_CLKDIV, 24 - 1);
	else
	ADC_WRITE4(sc, ADC_CLKDIV, 2400 - 1);

	TI_ADC_LOCK_INIT(sc);

	ti_adc_idlestep_init(sc);
	ti_adc_inputs_init(sc);
	ti_adc_sysctl_init(sc);
	ti_adc_tsc_init(sc);

	TI_ADC_LOCK(sc);
	ti_adc_setup(sc);
	TI_ADC_UNLOCK(sc);

	#ifdef EVDEV_SUPPORT
	if (sc->sc_tsc_wires > 0) {
	sc->sc_evdev = evdev_alloc();
	evdev_set_name(sc->sc_evdev, device_get_desc(dev));
	evdev_set_phys(sc->sc_evdev, device_get_nameunit(dev));
	evdev_set_id(sc->sc_evdev, BUS_VIRTUAL, 0, 0, 0);
	evdev_support_prop(sc->sc_evdev, INPUT_PROP_DIRECT);
	evdev_support_event(sc->sc_evdev, EV_SYN);
	evdev_support_event(sc->sc_evdev, EV_ABS);
	evdev_support_event(sc->sc_evdev, EV_KEY);

	evdev_support_abs(sc->sc_evdev, ABS_X, 0, 0,
	ADC_MAX_VALUE, 0, 0, 0);
	evdev_support_abs(sc->sc_evdev, ABS_Y, 0, 0,
	ADC_MAX_VALUE, 0, 0, 0);

	evdev_support_key(sc->sc_evdev, BTN_TOUCH);

	err = evdev_register(sc->sc_evdev);
	if (err) {
	device_printf(dev,
	"failed to register evdev: error=%d\n", err);
	ti_adc_detach(dev);
	return (err);
	}

	sc->sc_pen_down = 0;
	sc->sc_x = -1;
	sc->sc_y = -1;
	}
	#endif /* EVDEV */

	return (0);
	}

	static int
	ti_adc_detach(device_t dev)
	{
	struct ti_adc_softc *sc;

	sc = device_get_softc(dev);

	/* Turn off the ADC. */
	TI_ADC_LOCK(sc);
	ti_adc_reset(sc);
	ti_adc_setup(sc);

	#ifdef EVDEV_SUPPORT
	evdev_free(sc->sc_evdev);
	#endif

	TI_ADC_UNLOCK(sc);

	TI_ADC_LOCK_DESTROY(sc);

	if (sc->sc_intrhand)
	bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intrhand);
	if (sc->sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);

	return (bus_generic_detach(dev));
	}

	static device_method_t ti_adc_methods[] = {
	DEVMETHOD(device_probe, ti_adc_probe),
	DEVMETHOD(device_attach, ti_adc_attach),
	DEVMETHOD(device_detach, ti_adc_detach),

	DEVMETHOD_END
	};

	static driver_t ti_adc_driver = {
	"ti_adc",
	ti_adc_methods,
	sizeof(struct ti_adc_softc),
	};

	static devclass_t ti_adc_devclass;

	DRIVER_MODULE(ti_adc, simplebus, ti_adc_driver, ti_adc_devclass, 0, 0);
	MODULE_VERSION(ti_adc, 1);
	MODULE_DEPEND(ti_adc, simplebus, 1, 1, 1);
	+MODULE_DEPEND(ti_adc, ti_sysc, 1, 1, 1);
	#ifdef EVDEV_SUPPORT
	MODULE_DEPEND(ti_adc, evdev, 1, 1, 1);
	#endif
	diff --git a/sys/arm/ti/ti_edma3.c b/sys/arm/ti/ti_edma3.c
	index f8736e001553..7804ee6b1d8d 100644
	--- a/sys/arm/ti/ti_edma3.c
	+++ b/sys/arm/ti/ti_edma3.c
	@@ -1,428 +1,425 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (C) 2010 Texas Instruments Incorporated - http://www.ti.com/
	* Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of authors nor the names of its contributors may be
	* used to endorse or promote products derived from this software without
	* specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/endian.h>
	#include <sys/mbuf.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/socket.h>
	#include <sys/sysctl.h>

	#include <sys/sockio.h>
	#include <sys/bus.h>
	#include <machine/bus.h>
	#include <sys/rman.h>
	#include <machine/resource.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <arm/ti/ti_scm.h>
	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>

	#include <arm/ti/ti_edma3.h>

	#define TI_EDMA3_NUM_TCS 3
	#define TI_EDMA3_NUM_IRQS 3
	#define TI_EDMA3_NUM_DMA_CHS 64
	#define TI_EDMA3_NUM_QDMA_CHS 8

	#define TI_EDMA3CC_PID 0x000
	#define TI_EDMA3CC_DCHMAP(p) (0x100 + ((p)*4))
	#define TI_EDMA3CC_DMAQNUM(n) (0x240 + ((n)*4))
	#define TI_EDMA3CC_QDMAQNUM 0x260
	#define TI_EDMA3CC_EMCR 0x308
	#define TI_EDMA3CC_EMCRH 0x30C
	#define TI_EDMA3CC_QEMCR 0x314
	#define TI_EDMA3CC_CCERR 0x318
	#define TI_EDMA3CC_CCERRCLR 0x31C
	#define TI_EDMA3CC_DRAE(p) (0x340 + ((p)*8))
	#define TI_EDMA3CC_DRAEH(p) (0x344 + ((p)*8))
	#define TI_EDMA3CC_QRAE(p) (0x380 + ((p)*4))
	#define TI_EDMA3CC_S_ESR(p) (0x2010 + ((p)*0x200))
	#define TI_EDMA3CC_S_ESRH(p) (0x2014 + ((p)*0x200))
	#define TI_EDMA3CC_S_SECR(p) (0x2040 + ((p)*0x200))
	#define TI_EDMA3CC_S_SECRH(p) (0x2044 + ((p)*0x200))
	#define TI_EDMA3CC_S_EESR(p) (0x2030 + ((p)*0x200))
	#define TI_EDMA3CC_S_EESRH(p) (0x2034 + ((p)*0x200))
	#define TI_EDMA3CC_S_IESR(p) (0x2060 + ((p)*0x200))
	#define TI_EDMA3CC_S_IESRH(p) (0x2064 + ((p)*0x200))
	#define TI_EDMA3CC_S_IPR(p) (0x2068 + ((p)*0x200))
	#define TI_EDMA3CC_S_IPRH(p) (0x206C + ((p)*0x200))
	#define TI_EDMA3CC_S_QEESR(p) (0x208C + ((p)*0x200))

	#define TI_EDMA3CC_PARAM_OFFSET 0x4000
	#define TI_EDMA3CC_OPT(p) (TI_EDMA3CC_PARAM_OFFSET + 0x0 + ((p)*0x20))

	#define TI_EDMA3CC_DMAQNUM_SET(c,q) ((0x7 & (q)) << (((c) % 8) * 4))
	#define TI_EDMA3CC_DMAQNUM_CLR(c) (~(0x7 << (((c) % 8) * 4)))
	#define TI_EDMA3CC_QDMAQNUM_SET(c,q) ((0x7 & (q)) << ((c) * 4))
	#define TI_EDMA3CC_QDMAQNUM_CLR(c) (~(0x7 << ((c) * 4)))

	#define TI_EDMA3CC_OPT_TCC_CLR (~(0x3F000))
	#define TI_EDMA3CC_OPT_TCC_SET(p) (((0x3F000 >> 12) & (p)) << 12)

	struct ti_edma3_softc {
	device_t sc_dev;
	/*
	* We use one-element array in case if we need to add
	* mem resources for transfer control windows
	*/
	struct resource * mem_res[1];
	struct resource * irq_res[TI_EDMA3_NUM_IRQS];
	void *ih_cookie[TI_EDMA3_NUM_IRQS];
	};

	static struct ti_edma3_softc *ti_edma3_sc = NULL;

	static struct resource_spec ti_edma3_mem_spec[] = {
	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
	{ -1, 0, 0 }
	};
	static struct resource_spec ti_edma3_irq_spec[] = {
	{ SYS_RES_IRQ, 0, RF_ACTIVE },
	{ SYS_RES_IRQ, 1, RF_ACTIVE },
	{ SYS_RES_IRQ, 2, RF_ACTIVE },
	{ -1, 0, 0 }
	};

	/* Read/Write macros */
	#define ti_edma3_cc_rd_4(reg) bus_read_4(ti_edma3_sc->mem_res[0], reg)
	#define ti_edma3_cc_wr_4(reg, val) bus_write_4(ti_edma3_sc->mem_res[0], reg, val)

	static void ti_edma3_intr_comp(void *arg);
	static void ti_edma3_intr_mperr(void *arg);
	static void ti_edma3_intr_err(void *arg);

	static struct {
	driver_intr_t *handler;
	char * description;
	} ti_edma3_intrs[TI_EDMA3_NUM_IRQS] = {
	{ ti_edma3_intr_comp, "EDMA Completion Interrupt" },
	{ ti_edma3_intr_mperr, "EDMA Memory Protection Error Interrupt" },
	{ ti_edma3_intr_err, "EDMA Error Interrupt" },
	};

	static int
	ti_edma3_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,edma3"))
	return (ENXIO);

	device_set_desc(dev, "TI EDMA Controller");
	return (0);
	}

	static int
	ti_edma3_attach(device_t dev)
	{
	struct ti_edma3_softc *sc = device_get_softc(dev);
	uint32_t reg;
	int err;
	int i;

	if (ti_edma3_sc)
	return (ENXIO);

	ti_edma3_sc = sc;
	sc->sc_dev = dev;

	/* Request the memory resources */
	err = bus_alloc_resources(dev, ti_edma3_mem_spec, sc->mem_res);
	if (err) {
	device_printf(dev, "Error: could not allocate mem resources\n");
	return (ENXIO);
	}

	/* Request the IRQ resources */
	err = bus_alloc_resources(dev, ti_edma3_irq_spec, sc->irq_res);
	if (err) {
	device_printf(dev, "Error: could not allocate irq resources\n");
	return (ENXIO);
	}

	+ /* FIXME: Require DTS from Linux kernel 5.7 */
	+ /* FIXME: OK to enable clkctrl here? */
	/* Enable Channel Controller */
	- ti_prcm_clk_enable(EDMA_TPCC_CLK);
	+ ti_sysc_clock_enable(device_get_parent(dev));

	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_PID);

	device_printf(dev, "EDMA revision %08x\n", reg);


	/* Attach interrupt handlers */
	for (i = 0; i < TI_EDMA3_NUM_IRQS; ++i) {
	err = bus_setup_intr(dev, sc->irq_res[i], INTR_TYPE_MISC \|
	INTR_MPSAFE, NULL, *ti_edma3_intrs[i].handler,
	sc, &sc->ih_cookie[i]);
	if (err) {
	device_printf(dev, "could not setup %s\n",
	ti_edma3_intrs[i].description);
	return (err);
	}
	}

	return (0);
	}

	static device_method_t ti_edma3_methods[] = {
	DEVMETHOD(device_probe, ti_edma3_probe),
	DEVMETHOD(device_attach, ti_edma3_attach),
	{0, 0},
	};

	static driver_t ti_edma3_driver = {
	"ti_edma3",
	ti_edma3_methods,
	sizeof(struct ti_edma3_softc),
	};
	static devclass_t ti_edma3_devclass;

	DRIVER_MODULE(ti_edma3, simplebus, ti_edma3_driver, ti_edma3_devclass, 0, 0);
	-MODULE_DEPEND(ti_edma3, ti_prcm, 1, 1, 1);
	+MODULE_DEPEND(ti_edma3, ti_sysc, 1, 1, 1);

	static void
	ti_edma3_intr_comp(void *arg)
	{
	printf("%s: unimplemented\n", __func__);
	}

	static void
	ti_edma3_intr_mperr(void *arg)
	{
	printf("%s: unimplemented\n", __func__);
	}

	static void
	ti_edma3_intr_err(void *arg)
	{
	printf("%s: unimplemented\n", __func__);
	}

	void
	ti_edma3_init(unsigned int eqn)
	{
	uint32_t reg;
	int i;

	- /* on AM335x Event queue 0 is always mapped to Transfer Controller 0,
	- * event queue 1 to TC2, etc. So we are asking PRCM to power on specific
	- * TC based on what event queue we need to initialize */
	- ti_prcm_clk_enable(EDMA_TPTC0_CLK + eqn);
	-
	/* Clear Event Missed Regs */
	ti_edma3_cc_wr_4(TI_EDMA3CC_EMCR, 0xFFFFFFFF);
	ti_edma3_cc_wr_4(TI_EDMA3CC_EMCRH, 0xFFFFFFFF);
	ti_edma3_cc_wr_4(TI_EDMA3CC_QEMCR, 0xFFFFFFFF);

	/* Clear Error Reg */
	ti_edma3_cc_wr_4(TI_EDMA3CC_CCERRCLR, 0xFFFFFFFF);

	/* Enable DMA channels 0-63 */
	ti_edma3_cc_wr_4(TI_EDMA3CC_DRAE(0), 0xFFFFFFFF);
	ti_edma3_cc_wr_4(TI_EDMA3CC_DRAEH(0), 0xFFFFFFFF);

	for (i = 0; i < 64; i++) {
	ti_edma3_cc_wr_4(TI_EDMA3CC_DCHMAP(i), i<<5);
	}

	/* Initialize the DMA Queue Number Registers */
	for (i = 0; i < TI_EDMA3_NUM_DMA_CHS; i++) {
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_DMAQNUM(i>>3));
	reg &= TI_EDMA3CC_DMAQNUM_CLR(i);
	reg \|= TI_EDMA3CC_DMAQNUM_SET(i, eqn);
	ti_edma3_cc_wr_4(TI_EDMA3CC_DMAQNUM(i>>3), reg);
	}

	/* Enable the QDMA Region access for all channels */
	ti_edma3_cc_wr_4(TI_EDMA3CC_QRAE(0), (1 << TI_EDMA3_NUM_QDMA_CHS) - 1);

	/Initialize QDMA Queue Number Registers /
	for (i = 0; i < TI_EDMA3_NUM_QDMA_CHS; i++) {
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_QDMAQNUM);
	reg &= TI_EDMA3CC_QDMAQNUM_CLR(i);
	reg \|= TI_EDMA3CC_QDMAQNUM_SET(i, eqn);
	ti_edma3_cc_wr_4(TI_EDMA3CC_QDMAQNUM, reg);
	}
	}

	#ifdef notyet
	int
	ti_edma3_enable_event_intr(unsigned int ch)
	{
	uint32_t reg;

	if (ch >= TI_EDMA3_NUM_DMA_CHS)
	return (EINVAL);

	if (ch < 32) {
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_IESR(0), 1 << ch);
	} else {
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_IESRH(0), 1 << (ch - 32));
	}
	return 0;
	}
	#endif

	int
	ti_edma3_request_dma_ch(unsigned int ch, unsigned int tccn, unsigned int eqn)
	{
	uint32_t reg;

	if (ch >= TI_EDMA3_NUM_DMA_CHS)
	return (EINVAL);

	/* Enable the DMA channel in the DRAE/DRAEH registers */
	if (ch < 32) {
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_DRAE(0));
	reg \|= (0x01 << ch);
	ti_edma3_cc_wr_4(TI_EDMA3CC_DRAE(0), reg);
	} else {
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_DRAEH(0));
	reg \|= (0x01 << (ch - 32));
	ti_edma3_cc_wr_4(TI_EDMA3CC_DRAEH(0), reg);
	}

	/* Associate DMA Channel to Event Queue */
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_DMAQNUM(ch >> 3));
	reg &= TI_EDMA3CC_DMAQNUM_CLR(ch);
	reg \|= TI_EDMA3CC_DMAQNUM_SET((ch), eqn);
	ti_edma3_cc_wr_4(TI_EDMA3CC_DMAQNUM(ch >> 3), reg);

	/* Set TCC in corresponding PaRAM Entry */
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_OPT(ch));
	reg &= TI_EDMA3CC_OPT_TCC_CLR;
	reg \|= TI_EDMA3CC_OPT_TCC_SET(ch);
	ti_edma3_cc_wr_4(TI_EDMA3CC_OPT(ch), reg);

	return 0;
	}

	int
	ti_edma3_request_qdma_ch(unsigned int ch, unsigned int tccn, unsigned int eqn)
	{
	uint32_t reg;

	if (ch >= TI_EDMA3_NUM_DMA_CHS)
	return (EINVAL);

	/* Enable the QDMA channel in the QRAE registers */
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_QRAE(0));
	reg \|= (0x01 << ch);
	ti_edma3_cc_wr_4(TI_EDMA3CC_QRAE(0), reg);

	/* Associate QDMA Channel to Event Queue */
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_QDMAQNUM);
	reg \|= TI_EDMA3CC_QDMAQNUM_SET(ch, eqn);
	ti_edma3_cc_wr_4(TI_EDMA3CC_QDMAQNUM, reg);

	/* Set TCC in corresponding PaRAM Entry */
	reg = ti_edma3_cc_rd_4(TI_EDMA3CC_OPT(ch));
	reg &= TI_EDMA3CC_OPT_TCC_CLR;
	reg \|= TI_EDMA3CC_OPT_TCC_SET(ch);
	ti_edma3_cc_wr_4(TI_EDMA3CC_OPT(ch), reg);

	return 0;
	}

	int
	ti_edma3_enable_transfer_manual(unsigned int ch)
	{
	if (ch >= TI_EDMA3_NUM_DMA_CHS)
	return (EINVAL);

	/* set corresponding bit in ESR/ESRH to set a event */
	if (ch < 32) {
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_ESR(0), 1 << ch);
	} else {
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_ESRH(0), 1 << (ch - 32));
	}

	return 0;
	}

	int
	ti_edma3_enable_transfer_qdma(unsigned int ch)
	{
	if (ch >= TI_EDMA3_NUM_QDMA_CHS)
	return (EINVAL);

	/* set corresponding bit in QEESR to enable QDMA event */
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_QEESR(0), (1 << ch));

	return 0;
	}

	int
	ti_edma3_enable_transfer_event(unsigned int ch)
	{
	if (ch >= TI_EDMA3_NUM_DMA_CHS)
	return (EINVAL);

	/* Clear SECR(H) & EMCR(H) to clean any previous NULL request
	* and set corresponding bit in EESR to enable DMA event */
	if(ch < 32) {
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_SECR(0), (1 << ch));
	ti_edma3_cc_wr_4(TI_EDMA3CC_EMCR, (1 << ch));
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_EESR(0), (1 << ch));
	} else {
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_SECRH(0), 1 << (ch - 32));
	ti_edma3_cc_wr_4(TI_EDMA3CC_EMCRH, 1 << (ch - 32));
	ti_edma3_cc_wr_4(TI_EDMA3CC_S_EESRH(0), 1 << (ch - 32));
	}

	return 0;
	}

	void
	ti_edma3_param_write(unsigned int ch, struct ti_edma3cc_param_set *prs)
	{
	bus_write_region_4(ti_edma3_sc->mem_res[0], TI_EDMA3CC_OPT(ch),
	(uint32_t *) prs, 8);
	}

	void
	ti_edma3_param_read(unsigned int ch, struct ti_edma3cc_param_set *prs)
	{
	bus_read_region_4(ti_edma3_sc->mem_res[0], TI_EDMA3CC_OPT(ch),
	(uint32_t *) prs, 8);
	}
	diff --git a/sys/arm/ti/ti_gpio.c b/sys/arm/ti/ti_gpio.c
	index ccdeeb502be4..8754a4769beb 100644
	--- a/sys/arm/ti/ti_gpio.c
	+++ b/sys/arm/ti/ti_gpio.c
	@@ -1,1087 +1,1141 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2011 Ben Gray <ben.r.gray@gmail.com>.
	* Copyright (c) 2014 Luiz Otavio O Souza <loos@FreeBSD.org>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/**
	* Beware that the OMAP4 datasheet(s) lists GPIO banks 1-6, whereas the code
	* here uses 0-5.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_platform.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>

	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/proc.h>
	#include <sys/rman.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/gpio.h>
	#include <sys/interrupt.h>

	#include <machine/bus.h>
	#include <machine/intr.h>
	#include <machine/resource.h>

	#include <arm/ti/ti_cpuid.h>
	#include <arm/ti/ti_gpio.h>
	#include <arm/ti/ti_scm.h>
	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	+#include <arm/ti/ti_sysc.h>

	#include <dev/gpio/gpiobusvar.h>
	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include "gpio_if.h"
	#include "ti_gpio_if.h"
	#include "pic_if.h"

	#if !defined(SOC_OMAP4) && !defined(SOC_TI_AM335X)
	#error "Unknown SoC"
	#endif

	/* Register definitions */
	#define TI_GPIO_REVISION 0x0000
	#define TI_GPIO_SYSCONFIG 0x0010
	#define TI_GPIO_IRQSTATUS_RAW_0 0x0024
	#define TI_GPIO_IRQSTATUS_RAW_1 0x0028
	#define TI_GPIO_IRQSTATUS_0 0x002C /* writing a 0 has no effect */
	#define TI_GPIO_IRQSTATUS_1 0x0030 /* writing a 0 has no effect */
	#define TI_GPIO_IRQSTATUS_SET_0 0x0034 /* writing a 0 has no effect */
	#define TI_GPIO_IRQSTATUS_SET_1 0x0038 /* writing a 0 has no effect */
	#define TI_GPIO_IRQSTATUS_CLR_0 0x003C /* writing a 0 has no effect */
	#define TI_GPIO_IRQSTATUS_CLR_1 0x0040 /* writing a 0 has no effect */
	#define TI_GPIO_IRQWAKEN_0 0x0044
	#define TI_GPIO_IRQWAKEN_1 0x0048
	#define TI_GPIO_SYSSTATUS 0x0114
	#define TI_GPIO_IRQSTATUS1 0x0118
	#define TI_GPIO_IRQENABLE1 0x011C
	#define TI_GPIO_WAKEUPENABLE 0x0120
	#define TI_GPIO_IRQSTATUS2 0x0128
	#define TI_GPIO_IRQENABLE2 0x012C
	#define TI_GPIO_CTRL 0x0130
	#define TI_GPIO_OE 0x0134
	#define TI_GPIO_DATAIN 0x0138
	#define TI_GPIO_DATAOUT 0x013C
	#define TI_GPIO_LEVELDETECT0 0x0140 /* RW register */
	#define TI_GPIO_LEVELDETECT1 0x0144 /* RW register */
	#define TI_GPIO_RISINGDETECT 0x0148 /* RW register */
	#define TI_GPIO_FALLINGDETECT 0x014C /* RW register */
	#define TI_GPIO_DEBOUNCENABLE 0x0150
	#define TI_GPIO_DEBOUNCINGTIME 0x0154
	#define TI_GPIO_CLEARWKUPENA 0x0180
	#define TI_GPIO_SETWKUENA 0x0184
	#define TI_GPIO_CLEARDATAOUT 0x0190
	#define TI_GPIO_SETDATAOUT 0x0194

	/* Other SoC Specific definitions */
	#define OMAP4_FIRST_GPIO_BANK 1
	#define OMAP4_INTR_PER_BANK 1
	#define OMAP4_GPIO_REV 0x50600801
	#define AM335X_FIRST_GPIO_BANK 0
	#define AM335X_INTR_PER_BANK 2
	#define AM335X_GPIO_REV 0x50600801
	#define PINS_PER_BANK 32
	#define TI_GPIO_MASK(p) (1U << ((p) % PINS_PER_BANK))

	+#define OMAP4_GPIO1_REV 0x00000
	+#define OMAP4_GPIO2_REV 0x55000
	+#define OMAP4_GPIO3_REV 0x57000
	+#define OMAP4_GPIO4_REV 0x59000
	+#define OMAP4_GPIO5_REV 0x5b000
	+#define OMAP4_GPIO6_REV 0x5d000
	+
	+#define AM335X_GPIO0_REV 0x07000
	+#define AM335X_GPIO1_REV 0x4C000
	+#define AM335X_GPIO2_REV 0xAC000
	+#define AM335X_GPIO3_REV 0xAE000
	+
	static int ti_gpio_intr(void *arg);
	static int ti_gpio_detach(device_t);

	static int ti_gpio_pic_attach(struct ti_gpio_softc *sc);
	static int ti_gpio_pic_detach(struct ti_gpio_softc *sc);

	static u_int
	ti_first_gpio_bank(void)
	{
	switch(ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	return (OMAP4_FIRST_GPIO_BANK);
	#endif
	#ifdef SOC_TI_AM335X
	case CHIP_AM335X:
	return (AM335X_FIRST_GPIO_BANK);
	#endif
	}
	return (0);
	}

	static uint32_t
	ti_gpio_rev(void)
	{
	switch(ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	return (OMAP4_GPIO_REV);
	#endif
	#ifdef SOC_TI_AM335X
	case CHIP_AM335X:
	return (AM335X_GPIO_REV);
	#endif
	}
	return (0);
	}

	/**
	* Macros for driver mutex locking
	*/
	#define TI_GPIO_LOCK(_sc) mtx_lock_spin(&(_sc)->sc_mtx)
	#define TI_GPIO_UNLOCK(_sc) mtx_unlock_spin(&(_sc)->sc_mtx)
	#define TI_GPIO_LOCK_INIT(_sc) \
	mtx_init(&_sc->sc_mtx, device_get_nameunit((_sc)->sc_dev), \
	"ti_gpio", MTX_SPIN)
	#define TI_GPIO_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx)
	#define TI_GPIO_ASSERT_LOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED)
	#define TI_GPIO_ASSERT_UNLOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED)

	/**
	* ti_gpio_read_4 - reads a 32-bit value from one of the GPIO registers
	* @sc: GPIO device context
	* @bank: The bank to read from
	* @off: The offset of a register from the GPIO register address range
	*
	*
	* RETURNS:
	* 32-bit value read from the register.
	*/
	static inline uint32_t
	ti_gpio_read_4(struct ti_gpio_softc *sc, bus_size_t off)
	{
	return (bus_read_4(sc->sc_mem_res, off));
	}

	/**
	* ti_gpio_write_4 - writes a 32-bit value to one of the GPIO registers
	* @sc: GPIO device context
	* @bank: The bank to write to
	* @off: The offset of a register from the GPIO register address range
	* @val: The value to write into the register
	*
	* RETURNS:
	* nothing
	*/
	static inline void
	ti_gpio_write_4(struct ti_gpio_softc *sc, bus_size_t off,
	uint32_t val)
	{
	bus_write_4(sc->sc_mem_res, off, val);
	}

	static inline void
	ti_gpio_intr_clr(struct ti_gpio_softc *sc, uint32_t mask)
	{

	/* We clear both set of registers. */
	ti_gpio_write_4(sc, TI_GPIO_IRQSTATUS_CLR_0, mask);
	ti_gpio_write_4(sc, TI_GPIO_IRQSTATUS_CLR_1, mask);
	}

	static inline void
	ti_gpio_intr_set(struct ti_gpio_softc *sc, uint32_t mask)
	{

	/*
	* On OMAP4 we unmask only the MPU interrupt and on AM335x we
	* also activate only the first interrupt.
	*/
	ti_gpio_write_4(sc, TI_GPIO_IRQSTATUS_SET_0, mask);
	}

	static inline void
	ti_gpio_intr_ack(struct ti_gpio_softc *sc, uint32_t mask)
	{

	/*
	* Acknowledge the interrupt on both registers even if we use only
	* the first one.
	*/
	ti_gpio_write_4(sc, TI_GPIO_IRQSTATUS_0, mask);
	ti_gpio_write_4(sc, TI_GPIO_IRQSTATUS_1, mask);
	}

	static inline uint32_t
	ti_gpio_intr_status(struct ti_gpio_softc *sc)
	{
	uint32_t reg;

	/* Get the status from both registers. */
	reg = ti_gpio_read_4(sc, TI_GPIO_IRQSTATUS_0);
	reg \|= ti_gpio_read_4(sc, TI_GPIO_IRQSTATUS_1);

	return (reg);
	}

	static device_t
	ti_gpio_get_bus(device_t dev)
	{
	struct ti_gpio_softc *sc;

	sc = device_get_softc(dev);

	return (sc->sc_busdev);
	}

	/**
	* ti_gpio_pin_max - Returns the maximum number of GPIO pins
	* @dev: gpio device handle
	* @maxpin: pointer to a value that upon return will contain the maximum number
	* of pins in the device.
	*
	*
	* LOCKING:
	* No locking required, returns static data.
	*
	* RETURNS:
	* Returns 0 on success otherwise an error code
	*/
	static int
	ti_gpio_pin_max(device_t dev, int *maxpin)
	{

	*maxpin = PINS_PER_BANK - 1;

	return (0);
	}

	static int
	ti_gpio_valid_pin(struct ti_gpio_softc *sc, int pin)
	{

	if (pin >= sc->sc_maxpin \|\| sc->sc_mem_res == NULL)
	return (EINVAL);

	return (0);
	}

	/**
	* ti_gpio_pin_getcaps - Gets the capabilities of a given pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	* @caps: pointer to a value that upon return will contain the capabilities
	*
	* Currently all pins have the same capability, notably:
	* - GPIO_PIN_INPUT
	* - GPIO_PIN_OUTPUT
	* - GPIO_PIN_PULLUP
	* - GPIO_PIN_PULLDOWN
	* - GPIO_INTR_LEVEL_LOW
	* - GPIO_INTR_LEVEL_HIGH
	* - GPIO_INTR_EDGE_RISING
	* - GPIO_INTR_EDGE_FALLING
	* - GPIO_INTR_EDGE_BOTH
	*
	* LOCKING:
	* No locking required, returns static data.
	*
	* RETURNS:
	* Returns 0 on success otherwise an error code
	*/
	static int
	ti_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps)
	{
	struct ti_gpio_softc *sc;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	*caps = (GPIO_PIN_INPUT \| GPIO_PIN_OUTPUT \| GPIO_PIN_PULLUP \|
	GPIO_PIN_PULLDOWN \| GPIO_INTR_LEVEL_LOW \| GPIO_INTR_LEVEL_HIGH \|
	GPIO_INTR_EDGE_RISING \| GPIO_INTR_EDGE_FALLING \|
	GPIO_INTR_EDGE_BOTH);

	return (0);
	}

	/**
	* ti_gpio_pin_getflags - Gets the current flags of a given pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	* @flags: upon return will contain the current flags of the pin
	*
	* Reads the current flags of a given pin, here we actually read the H/W
	* registers to determine the flags, rather than storing the value in the
	* setflags call.
	*
	* LOCKING:
	* Internally locks the context
	*
	* RETURNS:
	* Returns 0 on success otherwise an error code
	*/
	static int
	ti_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags)
	{
	struct ti_gpio_softc *sc;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	/* Get the current pin state */
	TI_GPIO_LOCK(sc);
	TI_GPIO_GET_FLAGS(dev, pin, flags);
	TI_GPIO_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_gpio_pin_getname - Gets the name of a given pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	* @name: buffer to put the name in
	*
	* The driver simply calls the pins gpio_n, where 'n' is obviously the number
	* of the pin.
	*
	* LOCKING:
	* No locking required, returns static data.
	*
	* RETURNS:
	* Returns 0 on success otherwise an error code
	*/
	static int
	ti_gpio_pin_getname(device_t dev, uint32_t pin, char *name)
	{
	struct ti_gpio_softc *sc;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	/* Set a very simple name */
	snprintf(name, GPIOMAXNAME, "gpio_%u", pin);
	name[GPIOMAXNAME - 1] = '\0';

	return (0);
	}

	/**
	* ti_gpio_pin_setflags - Sets the flags for a given pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	* @flags: the flags to set
	*
	* The flags of the pin correspond to things like input/output mode, pull-ups,
	* pull-downs, etc. This driver doesn't support all flags, only the following:
	* - GPIO_PIN_INPUT
	* - GPIO_PIN_OUTPUT
	* - GPIO_PIN_PULLUP
	* - GPIO_PIN_PULLDOWN
	*
	* LOCKING:
	* Internally locks the context
	*
	* RETURNS:
	* Returns 0 on success otherwise an error code
	*/
	static int
	ti_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags)
	{
	struct ti_gpio_softc *sc;
	uint32_t oe;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	/* Set the GPIO mode and state */
	TI_GPIO_LOCK(sc);
	if (TI_GPIO_SET_FLAGS(dev, pin, flags) != 0) {
	TI_GPIO_UNLOCK(sc);
	return (EINVAL);
	}

	/* If configuring as an output set the "output enable" bit */
	oe = ti_gpio_read_4(sc, TI_GPIO_OE);
	if (flags & GPIO_PIN_INPUT)
	oe \|= TI_GPIO_MASK(pin);
	else
	oe &= ~TI_GPIO_MASK(pin);
	ti_gpio_write_4(sc, TI_GPIO_OE, oe);
	TI_GPIO_UNLOCK(sc);
	-
	+
	return (0);
	}

	/**
	* ti_gpio_pin_set - Sets the current level on a GPIO pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	* @value: non-zero value will drive the pin high, otherwise the pin is
	* driven low.
	*
	*
	* LOCKING:
	* Internally locks the context
	*
	* RETURNS:
	* Returns 0 on success otherwise a error code
	*/
	static int
	ti_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value)
	{
	struct ti_gpio_softc *sc;
	uint32_t reg;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	TI_GPIO_LOCK(sc);
	if (value == GPIO_PIN_LOW)
	reg = TI_GPIO_CLEARDATAOUT;
	else
	reg = TI_GPIO_SETDATAOUT;
	ti_gpio_write_4(sc, reg, TI_GPIO_MASK(pin));
	TI_GPIO_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_gpio_pin_get - Gets the current level on a GPIO pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	* @value: pointer to a value that upond return will contain the pin value
	*
	* The pin must be configured as an input pin beforehand, otherwise this
	* function will fail.
	*
	* LOCKING:
	* Internally locks the context
	*
	* RETURNS:
	* Returns 0 on success otherwise a error code
	*/
	static int
	ti_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *value)
	{
	struct ti_gpio_softc *sc;
	uint32_t oe, reg, val;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	/*
	- * Return data from output latch when set as output and from the
	+ * Return data from output latch when set as output and from the
	* input register otherwise.
	*/
	TI_GPIO_LOCK(sc);
	oe = ti_gpio_read_4(sc, TI_GPIO_OE);
	if (oe & TI_GPIO_MASK(pin))
	reg = TI_GPIO_DATAIN;
	else
	reg = TI_GPIO_DATAOUT;
	val = ti_gpio_read_4(sc, reg);
	*value = (val & TI_GPIO_MASK(pin)) ? 1 : 0;
	TI_GPIO_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_gpio_pin_toggle - Toggles a given GPIO pin
	* @dev: gpio device handle
	* @pin: the number of the pin
	*
	*
	* LOCKING:
	* Internally locks the context
	*
	* RETURNS:
	* Returns 0 on success otherwise a error code
	*/
	static int
	ti_gpio_pin_toggle(device_t dev, uint32_t pin)
	{
	struct ti_gpio_softc *sc;
	uint32_t reg, val;

	sc = device_get_softc(dev);
	if (ti_gpio_valid_pin(sc, pin) != 0)
	return (EINVAL);

	/* Toggle the pin */
	TI_GPIO_LOCK(sc);
	val = ti_gpio_read_4(sc, TI_GPIO_DATAOUT);
	if (val & TI_GPIO_MASK(pin))
	reg = TI_GPIO_CLEARDATAOUT;
	else
	reg = TI_GPIO_SETDATAOUT;
	ti_gpio_write_4(sc, reg, TI_GPIO_MASK(pin));
	TI_GPIO_UNLOCK(sc);

	return (0);
	}

	static int
	ti_gpio_bank_init(device_t dev)
	{
	- int pin;
	+ int pin, err;
	struct ti_gpio_softc *sc;
	uint32_t flags, reg_oe, reg_set, rev;
	- clk_ident_t clk;
	+ uint64_t rev_address;

	sc = device_get_softc(dev);

	/* Enable the interface and functional clocks for the module. */
	- clk = ti_hwmods_get_clock(dev);
	- if (clk == INVALID_CLK_IDENT) {
	- device_printf(dev, "failed to get device id based on ti,hwmods\n");
	+ rev_address = ti_sysc_get_rev_address(device_get_parent(dev));
	+ /* AM335x
	+ * sc->sc_bank used in am335x/am335x_gpio.c and omap4/omap4_gpio.c */
	+ switch(ti_chip()) {
	+#ifdef SOC_OMAP4
	+ case CHIP_OMAP_4:
	+ switch (rev_address) {
	+ case OMAP4_GPIO1_REV:
	+ sc->sc_bank = 0;
	+ break;
	+ case OMAP4_GPIO2_REV:
	+ sc->sc_bank = 1;
	+ break;
	+ case OMAP4_GPIO3_REV:
	+ sc->sc_bank = 2;
	+ break;
	+ case OMAP4_GPIO4_REV:
	+ sc->sc_bank = 3;
	+ break;
	+ case OMAP4_GPIO5_REV:
	+ sc->sc_bank = 4;
	+ break;
	+ case OMAP4_GPIO6_REV:
	+ sc->sc_bank = 5;
	+ break;
	+ }
	+#endif
	+#ifdef SOC_TI_AM335X
	+ case CHIP_AM335X:
	+ switch (rev_address) {
	+ case AM335X_GPIO0_REV:
	+ sc->sc_bank = 0;
	+ break;
	+ case AM335X_GPIO1_REV:
	+ sc->sc_bank = 1;
	+ break;
	+ case AM335X_GPIO2_REV:
	+ sc->sc_bank = 2;
	+ break;
	+ case AM335X_GPIO3_REV:
	+ sc->sc_bank = 3;
	+ break;
	+ }
	+#endif
	+ }
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	+ if (err) {
	+ device_printf(dev, "Failed to enable clock\n");
	return (EINVAL);
	}

	- sc->sc_bank = clk - GPIO1_CLK + ti_first_gpio_bank();
	- ti_prcm_clk_enable(clk);
	-
	/*
	* Read the revision number of the module. TI don't publish the
	* actual revision numbers, so instead the values have been
	* determined by experimentation.
	*/
	- rev = ti_gpio_read_4(sc, TI_GPIO_REVISION);
	+ rev = ti_gpio_read_4(sc,
	+ ti_sysc_get_rev_address_offset_host(device_get_parent(dev)));

	/* Check the revision. */
	if (rev != ti_gpio_rev()) {
	device_printf(dev, "Warning: could not determine the revision "
	"of GPIO module (revision:0x%08x)\n", rev);
	return (EINVAL);
	}

	/* Disable interrupts for all pins. */
	ti_gpio_intr_clr(sc, 0xffffffff);

	/* Init OE register based on pads configuration. */
	reg_oe = 0xffffffff;
	reg_set = 0;
	for (pin = 0; pin < PINS_PER_BANK; pin++) {
	TI_GPIO_GET_FLAGS(dev, pin, &flags);
	if (flags & GPIO_PIN_OUTPUT) {
	reg_oe &= ~(1UL << pin);
	if (flags & GPIO_PIN_PULLUP)
	reg_set \|= (1UL << pin);
	}
	}
	ti_gpio_write_4(sc, TI_GPIO_OE, reg_oe);
	if (reg_set)
	ti_gpio_write_4(sc, TI_GPIO_SETDATAOUT, reg_set);

	return (0);
	}

	/**
	* ti_gpio_attach - attach function for the driver
	* @dev: gpio device handle
	*
	* Allocates and sets up the driver context for all GPIO banks. This function
	* expects the memory ranges and IRQs to already be allocated to the driver.
	*
	* LOCKING:
	* None
	*
	* RETURNS:
	* Always returns 0
	*/
	static int
	ti_gpio_attach(device_t dev)
	{
	struct ti_gpio_softc *sc;
	int err;

	sc = device_get_softc(dev);
	sc->sc_dev = dev;
	TI_GPIO_LOCK_INIT(sc);
	ti_gpio_pin_max(dev, &sc->sc_maxpin);
	sc->sc_maxpin++;

	sc->sc_mem_rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&sc->sc_mem_rid, RF_ACTIVE);
	if (!sc->sc_mem_res) {
	device_printf(dev, "Error: could not allocate mem resources\n");
	ti_gpio_detach(dev);
	return (ENXIO);
	}

	sc->sc_irq_rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
	&sc->sc_irq_rid, RF_ACTIVE);
	if (!sc->sc_irq_res) {
	device_printf(dev, "Error: could not allocate irq resources\n");
	ti_gpio_detach(dev);
	return (ENXIO);
	}

	/*
	* Register our interrupt filter for each of the IRQ resources.
	*/
	if (bus_setup_intr(dev, sc->sc_irq_res,
	INTR_TYPE_MISC \| INTR_MPSAFE, ti_gpio_intr, NULL, sc,
	&sc->sc_irq_hdl) != 0) {
	device_printf(dev,
	"WARNING: unable to register interrupt filter\n");
	ti_gpio_detach(dev);
	return (ENXIO);
	}

	if (ti_gpio_pic_attach(sc) != 0) {
	device_printf(dev, "WARNING: unable to attach PIC\n");
	ti_gpio_detach(dev);
	return (ENXIO);
	}

	/* We need to go through each block and ensure the clocks are running and
	* the module is enabled. It might be better to do this only when the
	* pins are configured which would result in less power used if the GPIO
	- * pins weren't used ...
	+ * pins weren't used ...
	*/
	if (sc->sc_mem_res != NULL) {
	/* Initialize the GPIO module. */
	err = ti_gpio_bank_init(dev);
	if (err != 0) {
	ti_gpio_detach(dev);
	return (err);
	}
	}

	sc->sc_busdev = gpiobus_attach_bus(dev);
	if (sc->sc_busdev == NULL) {
	ti_gpio_detach(dev);
	return (ENXIO);
	}

	return (0);
	}

	/**
	* ti_gpio_detach - detach function for the driver
	* @dev: scm device handle
	*
	* Allocates and sets up the driver context, this simply entails creating a
	* bus mappings for the SCM register set.
	*
	* LOCKING:
	* None
	*
	* RETURNS:
	* Always returns 0
	*/
	static int
	ti_gpio_detach(device_t dev)
	{
	struct ti_gpio_softc *sc = device_get_softc(dev);

	KASSERT(mtx_initialized(&sc->sc_mtx), ("gpio mutex not initialized"));

	/* Disable all interrupts */
	if (sc->sc_mem_res != NULL)
	ti_gpio_intr_clr(sc, 0xffffffff);
	if (sc->sc_busdev != NULL)
	gpiobus_detach_bus(dev);
	if (sc->sc_isrcs != NULL)
	ti_gpio_pic_detach(sc);
	/* Release the memory and IRQ resources. */
	if (sc->sc_irq_hdl) {
	bus_teardown_intr(dev, sc->sc_irq_res,
	sc->sc_irq_hdl);
	}
	if (sc->sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid,
	sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_mem_rid,
	sc->sc_mem_res);
	TI_GPIO_LOCK_DESTROY(sc);

	return (0);
	}

	static inline void
	ti_gpio_rwreg_modify(struct ti_gpio_softc *sc, uint32_t reg, uint32_t mask,
	bool set_bits)
	{
	uint32_t value;

	value = ti_gpio_read_4(sc, reg);
	ti_gpio_write_4(sc, reg, set_bits ? value \| mask : value & ~mask);
	}

	static inline void
	ti_gpio_isrc_mask(struct ti_gpio_softc sc, struct ti_gpio_irqsrc tgi)
	{

	/* Writing a 0 has no effect. */
	ti_gpio_intr_clr(sc, tgi->tgi_mask);
	}

	static inline void
	ti_gpio_isrc_unmask(struct ti_gpio_softc sc, struct ti_gpio_irqsrc tgi)
	{

	/* Writing a 0 has no effect. */
	ti_gpio_intr_set(sc, tgi->tgi_mask);
	}

	static inline void
	ti_gpio_isrc_eoi(struct ti_gpio_softc sc, struct ti_gpio_irqsrc tgi)
	{

	/* Writing a 0 has no effect. */
	ti_gpio_intr_ack(sc, tgi->tgi_mask);
	}

	static inline bool
	ti_gpio_isrc_is_level(struct ti_gpio_irqsrc *tgi)
	{

	return (tgi->tgi_mode == GPIO_INTR_LEVEL_LOW \|\|
	tgi->tgi_mode == GPIO_INTR_LEVEL_HIGH);
	}

	static int
	ti_gpio_intr(void *arg)
	{
	u_int irq;
	uint32_t reg;
	struct ti_gpio_softc *sc;
	struct trapframe *tf;
	struct ti_gpio_irqsrc *tgi;

	sc = (struct ti_gpio_softc *)arg;
	tf = curthread->td_intr_frame;

	reg = ti_gpio_intr_status(sc);
	for (irq = 0; irq < sc->sc_maxpin; irq++) {
	tgi = &sc->sc_isrcs[irq];
	if ((reg & tgi->tgi_mask) == 0)
	continue;
	if (!ti_gpio_isrc_is_level(tgi))
	ti_gpio_isrc_eoi(sc, tgi);
	if (intr_isrc_dispatch(&tgi->tgi_isrc, tf) != 0) {
	ti_gpio_isrc_mask(sc, tgi);
	if (ti_gpio_isrc_is_level(tgi))
	ti_gpio_isrc_eoi(sc, tgi);
	device_printf(sc->sc_dev, "Stray irq %u disabled\n",
	irq);
	}
	}
	return (FILTER_HANDLED);
	}

	static int
	ti_gpio_pic_attach(struct ti_gpio_softc *sc)
	{
	int error;
	uint32_t irq;
	const char *name;

	sc->sc_isrcs = malloc(sizeof(sc->sc_isrcs) sc->sc_maxpin, M_DEVBUF,
	M_WAITOK \| M_ZERO);

	name = device_get_nameunit(sc->sc_dev);
	for (irq = 0; irq < sc->sc_maxpin; irq++) {
	sc->sc_isrcs[irq].tgi_irq = irq;
	sc->sc_isrcs[irq].tgi_mask = TI_GPIO_MASK(irq);
	sc->sc_isrcs[irq].tgi_mode = GPIO_INTR_CONFORM;

	error = intr_isrc_register(&sc->sc_isrcs[irq].tgi_isrc,
	sc->sc_dev, 0, "%s,%u", name, irq);
	if (error != 0)
	return (error); /* XXX deregister ISRCs */
	}
	if (intr_pic_register(sc->sc_dev,
	OF_xref_from_node(ofw_bus_get_node(sc->sc_dev))) == NULL)
	return (ENXIO);

	return (0);
	}

	static int
	ti_gpio_pic_detach(struct ti_gpio_softc *sc)
	{

	/*
	* There has not been established any procedure yet
	* how to detach PIC from living system correctly.
	*/
	device_printf(sc->sc_dev, "%s: not implemented yet\n", __func__);
	return (EBUSY);
	}

	static void
	ti_gpio_pic_config_intr(struct ti_gpio_softc sc, struct ti_gpio_irqsrc tgi,
	uint32_t mode)
	{

	TI_GPIO_LOCK(sc);
	ti_gpio_rwreg_modify(sc, TI_GPIO_RISINGDETECT, tgi->tgi_mask,
	mode == GPIO_INTR_EDGE_RISING \|\| mode == GPIO_INTR_EDGE_BOTH);
	ti_gpio_rwreg_modify(sc, TI_GPIO_FALLINGDETECT, tgi->tgi_mask,
	mode == GPIO_INTR_EDGE_FALLING \|\| mode == GPIO_INTR_EDGE_BOTH);
	ti_gpio_rwreg_modify(sc, TI_GPIO_LEVELDETECT1, tgi->tgi_mask,
	mode == GPIO_INTR_LEVEL_HIGH);
	ti_gpio_rwreg_modify(sc, TI_GPIO_LEVELDETECT0, tgi->tgi_mask,
	mode == GPIO_INTR_LEVEL_LOW);
	tgi->tgi_mode = mode;
	TI_GPIO_UNLOCK(sc);
	}

	static void
	ti_gpio_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
	{
	struct ti_gpio_softc *sc = device_get_softc(dev);
	struct ti_gpio_irqsrc tgi = (struct ti_gpio_irqsrc )isrc;

	ti_gpio_isrc_mask(sc, tgi);
	}

	static void
	ti_gpio_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
	{
	struct ti_gpio_softc *sc = device_get_softc(dev);
	struct ti_gpio_irqsrc tgi = (struct ti_gpio_irqsrc )isrc;

	arm_irq_memory_barrier(tgi->tgi_irq);
	ti_gpio_isrc_unmask(sc, tgi);
	}

	static int
	ti_gpio_pic_map_fdt(struct ti_gpio_softc sc, struct intr_map_data_fdt daf,
	u_int irqp, uint32_t modep)
	{
	uint32_t mode;

	/*
	* The first cell is the interrupt number.
	* The second cell is used to specify flags:
	* bits[3:0] trigger type and level flags:
	* 1 = low-to-high edge triggered.
	* 2 = high-to-low edge triggered.
	* 4 = active high level-sensitive.
	* 8 = active low level-sensitive.
	*/
	if (daf->ncells != 2 \|\| daf->cells[0] >= sc->sc_maxpin)
	return (EINVAL);

	/* Only reasonable modes are supported. */
	if (daf->cells[1] == 1)
	mode = GPIO_INTR_EDGE_RISING;
	else if (daf->cells[1] == 2)
	mode = GPIO_INTR_EDGE_FALLING;
	else if (daf->cells[1] == 3)
	mode = GPIO_INTR_EDGE_BOTH;
	else if (daf->cells[1] == 4)
	mode = GPIO_INTR_LEVEL_HIGH;
	else if (daf->cells[1] == 8)
	mode = GPIO_INTR_LEVEL_LOW;
	else
	return (EINVAL);

	*irqp = daf->cells[0];
	if (modep != NULL)
	*modep = mode;
	return (0);
	}

	static int
	ti_gpio_pic_map_gpio(struct ti_gpio_softc sc, struct intr_map_data_gpio dag,
	u_int irqp, uint32_t modep)
	{
	uint32_t mode;

	if (dag->gpio_pin_num >= sc->sc_maxpin)
	return (EINVAL);

	mode = dag->gpio_intr_mode;
	if (mode != GPIO_INTR_LEVEL_LOW && mode != GPIO_INTR_LEVEL_HIGH &&
	mode != GPIO_INTR_EDGE_RISING && mode != GPIO_INTR_EDGE_FALLING &&
	mode != GPIO_INTR_EDGE_BOTH)
	return (EINVAL);

	*irqp = dag->gpio_pin_num;
	if (modep != NULL)
	*modep = mode;
	return (0);
	}

	static int
	ti_gpio_pic_map(struct ti_gpio_softc sc, struct intr_map_data data,
	u_int irqp, uint32_t modep)
	{

	switch (data->type) {
	case INTR_MAP_DATA_FDT:
	return (ti_gpio_pic_map_fdt(sc,
	(struct intr_map_data_fdt *)data, irqp, modep));
	case INTR_MAP_DATA_GPIO:
	return (ti_gpio_pic_map_gpio(sc,
	(struct intr_map_data_gpio *)data, irqp, modep));
	default:
	return (ENOTSUP);
	}
	}

	static int
	ti_gpio_pic_map_intr(device_t dev, struct intr_map_data *data,
	struct intr_irqsrc **isrcp)
	{
	int error;
	u_int irq;
	struct ti_gpio_softc *sc = device_get_softc(dev);

	error = ti_gpio_pic_map(sc, data, &irq, NULL);
	if (error == 0)
	*isrcp = &sc->sc_isrcs[irq].tgi_isrc;
	return (error);
	}

	static void
	ti_gpio_pic_post_filter(device_t dev, struct intr_irqsrc *isrc)
	{
	struct ti_gpio_softc *sc = device_get_softc(dev);
	struct ti_gpio_irqsrc tgi = (struct ti_gpio_irqsrc )isrc;

	if (ti_gpio_isrc_is_level(tgi))
	ti_gpio_isrc_eoi(sc, tgi);
	}

	static void
	ti_gpio_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
	{

	ti_gpio_pic_enable_intr(dev, isrc);
	}

	static void
	ti_gpio_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
	{
	struct ti_gpio_softc *sc = device_get_softc(dev);
	struct ti_gpio_irqsrc tgi = (struct ti_gpio_irqsrc )isrc;

	ti_gpio_isrc_mask(sc, tgi);
	if (ti_gpio_isrc_is_level(tgi))
	ti_gpio_isrc_eoi(sc, tgi);
	}

	static int
	ti_gpio_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
	struct resource res, struct intr_map_data data)
	{
	u_int irq;
	uint32_t mode;
	struct ti_gpio_softc *sc;
	struct ti_gpio_irqsrc *tgi;

	if (data == NULL)
	return (ENOTSUP);

	sc = device_get_softc(dev);
	tgi = (struct ti_gpio_irqsrc *)isrc;

	/* Get and check config for an interrupt. */
	if (ti_gpio_pic_map(sc, data, &irq, &mode) != 0 \|\| tgi->tgi_irq != irq)
	return (EINVAL);

	/*
	* If this is a setup for another handler,
	* only check that its configuration match.
	*/
	if (isrc->isrc_handlers != 0)
	return (tgi->tgi_mode == mode ? 0 : EINVAL);

	ti_gpio_pic_config_intr(sc, tgi, mode);
	return (0);
	}

	static int
	ti_gpio_pic_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
	struct resource res, struct intr_map_data data)
	{
	struct ti_gpio_softc *sc = device_get_softc(dev);
	struct ti_gpio_irqsrc tgi = (struct ti_gpio_irqsrc )isrc;

	if (isrc->isrc_handlers == 0)
	ti_gpio_pic_config_intr(sc, tgi, GPIO_INTR_CONFORM);
	return (0);
	}

	static phandle_t
	ti_gpio_get_node(device_t bus, device_t dev)
	{

	/* We only have one child, the GPIO bus, which needs our own node. */
	return (ofw_bus_get_node(bus));
	}

	static device_method_t ti_gpio_methods[] = {
	DEVMETHOD(device_attach, ti_gpio_attach),
	DEVMETHOD(device_detach, ti_gpio_detach),

	/* GPIO protocol */
	DEVMETHOD(gpio_get_bus, ti_gpio_get_bus),
	DEVMETHOD(gpio_pin_max, ti_gpio_pin_max),
	DEVMETHOD(gpio_pin_getname, ti_gpio_pin_getname),
	DEVMETHOD(gpio_pin_getflags, ti_gpio_pin_getflags),
	DEVMETHOD(gpio_pin_getcaps, ti_gpio_pin_getcaps),
	DEVMETHOD(gpio_pin_setflags, ti_gpio_pin_setflags),
	DEVMETHOD(gpio_pin_get, ti_gpio_pin_get),
	DEVMETHOD(gpio_pin_set, ti_gpio_pin_set),
	DEVMETHOD(gpio_pin_toggle, ti_gpio_pin_toggle),

	/* Interrupt controller interface */
	DEVMETHOD(pic_disable_intr, ti_gpio_pic_disable_intr),
	DEVMETHOD(pic_enable_intr, ti_gpio_pic_enable_intr),
	DEVMETHOD(pic_map_intr, ti_gpio_pic_map_intr),
	DEVMETHOD(pic_setup_intr, ti_gpio_pic_setup_intr),
	DEVMETHOD(pic_teardown_intr, ti_gpio_pic_teardown_intr),
	DEVMETHOD(pic_post_filter, ti_gpio_pic_post_filter),
	DEVMETHOD(pic_post_ithread, ti_gpio_pic_post_ithread),
	DEVMETHOD(pic_pre_ithread, ti_gpio_pic_pre_ithread),

	/* ofw_bus interface */
	DEVMETHOD(ofw_bus_get_node, ti_gpio_get_node),

	{0, 0},
	};

	driver_t ti_gpio_driver = {
	"gpio",
	ti_gpio_methods,
	sizeof(struct ti_gpio_softc),
	};
	diff --git a/sys/arm/ti/ti_hwmods.c b/sys/arm/ti/ti_hwmods.c
	deleted file mode 100644
	index 97c8f99c828a..000000000000
	--- a/sys/arm/ti/ti_hwmods.c
	+++ /dev/null
	@@ -1,214 +0,0 @@
	-/*-
	- * Copyright (c) 2015 Oleksandr Tymoshenko <gonzo@freebsd.org>
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	- * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
	- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	- * POSSIBILITY OF SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/bus.h>
	-#include <sys/kernel.h>
	-
	-#include <dev/ofw/openfirm.h>
	-#include <dev/ofw/ofw_bus.h>
	-#include <dev/ofw/ofw_bus_subr.h>
	-
	-#include <machine/bus.h>
	-#include <machine/fdt.h>
	-
	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	-
	-struct hwmod {
	- const char *name;
	- int clock_id;
	-};
	-
	-struct hwmod ti_hwmods[] = {
	- {"i2c1", I2C1_CLK},
	- {"i2c2", I2C2_CLK},
	- {"i2c3", I2C3_CLK},
	- {"i2c4", I2C4_CLK},
	- {"i2c5", I2C5_CLK},
	-
	- {"gpio1", GPIO1_CLK},
	- {"gpio2", GPIO2_CLK},
	- {"gpio3", GPIO3_CLK},
	- {"gpio4", GPIO4_CLK},
	- {"gpio5", GPIO5_CLK},
	- {"gpio6", GPIO6_CLK},
	- {"gpio7", GPIO7_CLK},
	-
	- {"mmc1", MMC1_CLK},
	- {"mmc2", MMC2_CLK},
	- {"mmc3", MMC3_CLK},
	- {"mmc4", MMC4_CLK},
	- {"mmc5", MMC5_CLK},
	- {"mmc6", MMC6_CLK},
	-
	- {"epwmss0", PWMSS0_CLK},
	- {"epwmss1", PWMSS1_CLK},
	- {"epwmss2", PWMSS2_CLK},
	-
	- {"spi0", SPI0_CLK},
	- {"spi1", SPI1_CLK},
	-
	- {"timer1", TIMER1_CLK},
	- {"timer2", TIMER2_CLK},
	- {"timer3", TIMER3_CLK},
	- {"timer4", TIMER4_CLK},
	- {"timer5", TIMER5_CLK},
	- {"timer6", TIMER6_CLK},
	- {"timer7", TIMER7_CLK},
	-
	- {"uart1", UART1_CLK},
	- {"uart2", UART2_CLK},
	- {"uart3", UART3_CLK},
	- {"uart4", UART4_CLK},
	- {"uart5", UART5_CLK},
	- {"uart6", UART6_CLK},
	- {"uart7", UART7_CLK},
	-
	- {NULL, 0}
	-};
	-
	-static inline int
	-ti_get_hwmods_prop(phandle_t node, void **name)
	-{
	- int len;
	-
	- if ((len = OF_getprop_alloc(node, "ti,hwmods", name)) > 0)
	- return (len);
	- return (OF_getprop_alloc(OF_parent(node), "ti,hwmods", name));
	-}
	-
	-clk_ident_t
	-ti_hwmods_get_clock(device_t dev)
	-{
	- phandle_t node;
	- int len, l;
	- char *name;
	- char *buf;
	- int clk;
	- struct hwmod *hw;
	-
	- if ((node = ofw_bus_get_node(dev)) == 0)
	- return (INVALID_CLK_IDENT);
	-
	- if ((len = ti_get_hwmods_prop(node, (void **)&name)) <= 0)
	- return (INVALID_CLK_IDENT);
	-
	- buf = name;
	-
	- clk = INVALID_CLK_IDENT;
	- while ((len > 0) && (clk == INVALID_CLK_IDENT)) {
	- for (hw = ti_hwmods; hw->name != NULL; ++hw) {
	- if (strcmp(hw->name, name) == 0) {
	- clk = hw->clock_id;
	- break;
	- }
	- }
	-
	- /* Slide to the next sub-string. */
	- l = strlen(name) + 1;
	- name += l;
	- len -= l;
	- }
	-
	- if (len > 0)
	- device_printf(dev, "WARNING: more than one ti,hwmod \n");
	-
	- OF_prop_free(buf);
	- return (clk);
	-}
	-
	-int ti_hwmods_contains(device_t dev, const char *hwmod)
	-{
	- phandle_t node;
	- int len, l;
	- char *name;
	- char *buf;
	- int result;
	-
	- if ((node = ofw_bus_get_node(dev)) == 0)
	- return (0);
	-
	- if ((len = ti_get_hwmods_prop(node, (void **)&name)) <= 0)
	- return (0);
	-
	- buf = name;
	-
	- result = 0;
	- while (len > 0) {
	- if (strcmp(name, hwmod) == 0) {
	- result = 1;
	- break;
	- }
	-
	- /* Slide to the next sub-string. */
	- l = strlen(name) + 1;
	- name += l;
	- len -= l;
	- }
	-
	- OF_prop_free(buf);
	-
	- return (result);
	-}
	-
	-int
	-ti_hwmods_get_unit(device_t dev, const char *hwmod)
	-{
	- phandle_t node;
	- int l, len, hwmodlen, result;
	- char *name;
	- char *buf;
	-
	- if ((node = ofw_bus_get_node(dev)) == 0)
	- return (0);
	-
	- if ((len = ti_get_hwmods_prop(node, (void **)&name)) <= 0)
	- return (0);
	-
	- buf = name;
	- hwmodlen = strlen(hwmod);
	- result = 0;
	- while (len > 0) {
	- if (strncmp(name, hwmod, hwmodlen) == 0) {
	- result = (int)strtoul(name + hwmodlen, NULL, 10);
	- break;
	- }
	- /* Slide to the next sub-string. */
	- l = strlen(name) + 1;
	- name += l;
	- len -= l;
	- }
	-
	- OF_prop_free(buf);
	- return (result);
	-}
	diff --git a/sys/arm/ti/ti_hwmods.h b/sys/arm/ti/ti_hwmods.h
	deleted file mode 100644
	index c236cc0d0db0..000000000000
	--- a/sys/arm/ti/ti_hwmods.h
	+++ /dev/null
	@@ -1,37 +0,0 @@
	-/*-
	- * Copyright (c) 2015 Oleksandr Tymoshenko <gonzo@freebsd.org>
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	- * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
	- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	- * POSSIBILITY OF SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-#ifndef _TI_HWMODS_H_
	-#define _TI_HWMODS_H_
	-
	-clk_ident_t ti_hwmods_get_clock(device_t dev);
	-int ti_hwmods_contains(device_t dev, const char *hwmod);
	-
	-/* Returns the N from "hwmodN" in the ti,hwmods property; 0 on failure. */
	-int ti_hwmods_get_unit(device_t dev, const char *hwmod);
	-
	-#endif /* _TI_HWMODS_H_ */
	diff --git a/sys/arm/ti/ti_i2c.c b/sys/arm/ti/ti_i2c.c
	index 4a9313d6f501..526cb45481f0 100644
	--- a/sys/arm/ti/ti_i2c.c
	+++ b/sys/arm/ti/ti_i2c.c
	@@ -1,990 +1,978 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2011 Ben Gray <ben.r.gray@gmail.com>.
	* Copyright (c) 2014 Luiz Otavio O Souza <loos@freebsd.org>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/**
	* Driver for the I2C module on the TI SoC.
	*
	* This driver is heavily based on the TWI driver for the AT91 (at91_twi.c).
	*
	* CAUTION: The I2Ci registers are limited to 16 bit and 8 bit data accesses,
	* 32 bit data access is not allowed and can corrupt register content.
	*
	* This driver currently doesn't use DMA for the transfer, although I hope to
	* incorporate that sometime in the future. The idea being that for transaction
	* larger than a certain size the DMA engine is used, for anything less the
	* normal interrupt/fifo driven option is used.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/mbuf.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	#include <machine/bus.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <arm/ti/ti_cpuid.h>
	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_i2c.h>

	#include <dev/iicbus/iiconf.h>
	#include <dev/iicbus/iicbus.h>

	#include "iicbus_if.h"

	/**
	* I2C device driver context, a pointer to this is stored in the device
	* driver structure.
	*/
	struct ti_i2c_softc
	{
	device_t sc_dev;
	- clk_ident_t clk_id;
	struct resource* sc_irq_res;
	struct resource* sc_mem_res;
	device_t sc_iicbus;

	void* sc_irq_h;

	struct mtx sc_mtx;

	struct iic_msg* sc_buffer;
	int sc_bus_inuse;
	int sc_buffer_pos;
	int sc_error;
	int sc_fifo_trsh;
	int sc_timeout;

	uint16_t sc_con_reg;
	uint16_t sc_rev;
	};

	struct ti_i2c_clock_config
	{
	u_int frequency; /* Bus frequency in Hz */
	uint8_t psc; /* Fast/Standard mode prescale divider */
	uint8_t scll; /* Fast/Standard mode SCL low time */
	uint8_t sclh; /* Fast/Standard mode SCL high time */
	uint8_t hsscll; /* High Speed mode SCL low time */
	uint8_t hssclh; /* High Speed mode SCL high time */
	};

	#if defined(SOC_OMAP4)
	/*
	* OMAP4 i2c bus clock is 96MHz / ((psc + 1) * (scll + 7 + sclh + 5)).
	* The prescaler values for 100KHz and 400KHz modes come from the table in the
	* OMAP4 TRM. The table doesn't list 1MHz; these values should give that speed.
	*/
	static struct ti_i2c_clock_config ti_omap4_i2c_clock_configs[] = {
	{ 100000, 23, 13, 15, 0, 0},
	{ 400000, 9, 5, 7, 0, 0},
	{ 1000000, 3, 5, 7, 0, 0},
	/* { 3200000, 1, 113, 115, 7, 10}, - HS mode */
	{ 0 /* Table terminator */ }
	};
	#endif

	#if defined(SOC_TI_AM335X)
	/*
	* AM335x i2c bus clock is 48MHZ / ((psc + 1) * (scll + 7 + sclh + 5))
	* In all cases we prescale the clock to 24MHz as recommended in the manual.
	*/
	static struct ti_i2c_clock_config ti_am335x_i2c_clock_configs[] = {
	{ 100000, 1, 111, 117, 0, 0},
	{ 400000, 1, 23, 25, 0, 0},
	{ 1000000, 1, 5, 7, 0, 0},
	{ 0 /* Table terminator */ }
	};
	#endif

	/**
	* Locking macros used throughout the driver
	*/
	#define TI_I2C_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx)
	#define TI_I2C_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx)
	#define TI_I2C_LOCK_INIT(_sc) \
	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \
	"ti_i2c", MTX_DEF)
	#define TI_I2C_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx)
	#define TI_I2C_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED)
	#define TI_I2C_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED)

	#ifdef DEBUG
	#define ti_i2c_dbg(_sc, fmt, args...) \
	device_printf((_sc)->sc_dev, fmt, ##args)
	#else
	#define ti_i2c_dbg(_sc, fmt, args...)
	#endif

	/**
	* ti_i2c_read_2 - reads a 16-bit value from one of the I2C registers
	* @sc: I2C device context
	* @off: the byte offset within the register bank to read from.
	*
	*
	* LOCKING:
	* No locking required
	*
	* RETURNS:
	* 16-bit value read from the register.
	*/
	static inline uint16_t
	ti_i2c_read_2(struct ti_i2c_softc *sc, bus_size_t off)
	{

	return (bus_read_2(sc->sc_mem_res, off));
	}

	/**
	* ti_i2c_write_2 - writes a 16-bit value to one of the I2C registers
	* @sc: I2C device context
	* @off: the byte offset within the register bank to read from.
	* @val: the value to write into the register
	*
	* LOCKING:
	* No locking required
	*
	* RETURNS:
	* 16-bit value read from the register.
	*/
	static inline void
	ti_i2c_write_2(struct ti_i2c_softc *sc, bus_size_t off, uint16_t val)
	{

	bus_write_2(sc->sc_mem_res, off, val);
	}

	static int
	ti_i2c_transfer_intr(struct ti_i2c_softc* sc, uint16_t status)
	{
	int amount, done, i;

	done = 0;
	amount = 0;
	/* Check for the error conditions. */
	if (status & I2C_STAT_NACK) {
	/* No ACK from slave. */
	ti_i2c_dbg(sc, "NACK\n");
	ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_NACK);
	sc->sc_error = ENXIO;
	} else if (status & I2C_STAT_AL) {
	/* Arbitration lost. */
	ti_i2c_dbg(sc, "Arbitration lost\n");
	ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_AL);
	sc->sc_error = ENXIO;
	}

	/* Check if we have finished. */
	if (status & I2C_STAT_ARDY) {
	/* Register access ready - transaction complete basically. */
	ti_i2c_dbg(sc, "ARDY transaction complete\n");
	if (sc->sc_error != 0 && sc->sc_buffer->flags & IIC_M_NOSTOP) {
	ti_i2c_write_2(sc, I2C_REG_CON,
	sc->sc_con_reg \| I2C_CON_STP);
	}
	ti_i2c_write_2(sc, I2C_REG_STATUS,
	I2C_STAT_ARDY \| I2C_STAT_RDR \| I2C_STAT_RRDY \|
	I2C_STAT_XDR \| I2C_STAT_XRDY);
	return (1);
	}

	if (sc->sc_buffer->flags & IIC_M_RD) {
	/* Read some data. */
	if (status & I2C_STAT_RDR) {
	/*
	* Receive draining interrupt - last data received.
	* The set FIFO threshold won't be reached to trigger
	* RRDY.
	*/
	ti_i2c_dbg(sc, "Receive draining interrupt\n");

	/*
	* Drain the FIFO. Read the pending data in the FIFO.
	*/
	amount = sc->sc_buffer->len - sc->sc_buffer_pos;
	} else if (status & I2C_STAT_RRDY) {
	/*
	* Receive data ready interrupt - FIFO has reached the
	* set threshold.
	*/
	ti_i2c_dbg(sc, "Receive data ready interrupt\n");

	amount = min(sc->sc_fifo_trsh,
	sc->sc_buffer->len - sc->sc_buffer_pos);
	}

	/* Read the bytes from the fifo. */
	for (i = 0; i < amount; i++)
	sc->sc_buffer->buf[sc->sc_buffer_pos++] =
	(uint8_t)(ti_i2c_read_2(sc, I2C_REG_DATA) & 0xff);

	if (status & I2C_STAT_RDR)
	ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_RDR);
	if (status & I2C_STAT_RRDY)
	ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_RRDY);

	} else {
	/* Write some data. */
	if (status & I2C_STAT_XDR) {
	/*
	* Transmit draining interrupt - FIFO level is below
	* the set threshold and the amount of data still to
	* be transferred won't reach the set FIFO threshold.
	*/
	ti_i2c_dbg(sc, "Transmit draining interrupt\n");

	/*
	* Drain the TX data. Write the pending data in the
	* FIFO.
	*/
	amount = sc->sc_buffer->len - sc->sc_buffer_pos;
	} else if (status & I2C_STAT_XRDY) {
	/*
	* Transmit data ready interrupt - the FIFO level
	* is below the set threshold.
	*/
	ti_i2c_dbg(sc, "Transmit data ready interrupt\n");

	amount = min(sc->sc_fifo_trsh,
	sc->sc_buffer->len - sc->sc_buffer_pos);
	}

	/* Write the bytes from the fifo. */
	for (i = 0; i < amount; i++)
	ti_i2c_write_2(sc, I2C_REG_DATA,
	sc->sc_buffer->buf[sc->sc_buffer_pos++]);

	if (status & I2C_STAT_XDR)
	ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_XDR);
	if (status & I2C_STAT_XRDY)
	ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_XRDY);
	}

	return (done);
	}

	/**
	* ti_i2c_intr - interrupt handler for the I2C module
	* @dev: i2c device handle
	*
	*
	*
	* LOCKING:
	* Called from timer context
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	static void
	ti_i2c_intr(void *arg)
	{
	int done;
	struct ti_i2c_softc *sc;
	uint16_t events, status;

	sc = (struct ti_i2c_softc *)arg;

	TI_I2C_LOCK(sc);

	status = ti_i2c_read_2(sc, I2C_REG_STATUS);
	if (status == 0) {
	TI_I2C_UNLOCK(sc);
	return;
	}

	/* Save enabled interrupts. */
	events = ti_i2c_read_2(sc, I2C_REG_IRQENABLE_SET);

	/* We only care about enabled interrupts. */
	status &= events;

	done = 0;

	if (sc->sc_buffer != NULL)
	done = ti_i2c_transfer_intr(sc, status);
	else {
	ti_i2c_dbg(sc, "Transfer interrupt without buffer\n");
	sc->sc_error = EINVAL;
	done = 1;
	}

	if (done)
	/* Wakeup the process that started the transaction. */
	wakeup(sc);

	TI_I2C_UNLOCK(sc);
	}

	/**
	* ti_i2c_transfer - called to perform the transfer
	* @dev: i2c device handle
	* @msgs: the messages to send/receive
	* @nmsgs: the number of messages in the msgs array
	*
	*
	* LOCKING:
	* Internally locked
	*
	* RETURNS:
	* 0 on function succeeded
	* EINVAL if invalid message is passed as an arg
	*/
	static int
	ti_i2c_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs)
	{
	int err, i, repstart, timeout;
	struct ti_i2c_softc *sc;
	uint16_t reg;

	sc = device_get_softc(dev);
	TI_I2C_LOCK(sc);

	/* If the controller is busy wait until it is available. */
	while (sc->sc_bus_inuse == 1)
	mtx_sleep(sc, &sc->sc_mtx, 0, "i2cbuswait", 0);

	/* Now we have control over the I2C controller. */
	sc->sc_bus_inuse = 1;

	err = 0;
	repstart = 0;
	for (i = 0; i < nmsgs; i++) {

	sc->sc_buffer = &msgs[i];
	sc->sc_buffer_pos = 0;
	sc->sc_error = 0;

	/* Zero byte transfers aren't allowed. */
	if (sc->sc_buffer == NULL \|\| sc->sc_buffer->buf == NULL \|\|
	sc->sc_buffer->len == 0) {
	err = EINVAL;
	break;
	}

	/* Check if the i2c bus is free. */
	if (repstart == 0) {
	/*
	* On repeated start we send the START condition while
	* the bus _is_ busy.
	*/
	timeout = 0;
	while (ti_i2c_read_2(sc, I2C_REG_STATUS_RAW) & I2C_STAT_BB) {
	if (timeout++ > 100) {
	err = EBUSY;
	goto out;
	}
	DELAY(1000);
	}
	timeout = 0;
	} else
	repstart = 0;

	if (sc->sc_buffer->flags & IIC_M_NOSTOP)
	repstart = 1;

	/* Set the slave address. */
	ti_i2c_write_2(sc, I2C_REG_SA, msgs[i].slave >> 1);

	/* Write the data length. */
	ti_i2c_write_2(sc, I2C_REG_CNT, sc->sc_buffer->len);

	/* Clear the RX and the TX FIFO. */
	reg = ti_i2c_read_2(sc, I2C_REG_BUF);
	reg \|= I2C_BUF_RXFIFO_CLR \| I2C_BUF_TXFIFO_CLR;
	ti_i2c_write_2(sc, I2C_REG_BUF, reg);

	reg = sc->sc_con_reg \| I2C_CON_STT;
	if (repstart == 0)
	reg \|= I2C_CON_STP;
	if ((sc->sc_buffer->flags & IIC_M_RD) == 0)
	reg \|= I2C_CON_TRX;
	ti_i2c_write_2(sc, I2C_REG_CON, reg);

	/* Wait for an event. */
	err = mtx_sleep(sc, &sc->sc_mtx, 0, "i2ciowait", sc->sc_timeout);
	if (err == 0)
	err = sc->sc_error;

	if (err)
	break;
	}

	out:
	if (timeout == 0) {
	while (ti_i2c_read_2(sc, I2C_REG_STATUS_RAW) & I2C_STAT_BB) {
	if (timeout++ > 100)
	break;
	DELAY(1000);
	}
	}
	/* Put the controller in master mode again. */
	if ((ti_i2c_read_2(sc, I2C_REG_CON) & I2C_CON_MST) == 0)
	ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg);

	sc->sc_buffer = NULL;
	sc->sc_bus_inuse = 0;

	/* Wake up the processes that are waiting for the bus. */
	wakeup(sc);

	TI_I2C_UNLOCK(sc);

	return (err);
	}

	static int
	ti_i2c_reset(struct ti_i2c_softc *sc, u_char speed)
	{
	int timeout;
	struct ti_i2c_clock_config *clkcfg;
	u_int busfreq;
	uint16_t fifo_trsh, reg, scll, sclh;

	switch (ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	clkcfg = ti_omap4_i2c_clock_configs;
	break;
	#endif
	#ifdef SOC_TI_AM335X
	case CHIP_AM335X:
	clkcfg = ti_am335x_i2c_clock_configs;
	break;
	#endif
	default:
	panic("Unknown TI SoC, unable to reset the i2c");
	}

	/*
	* If we haven't attached the bus yet, just init at the default slow
	* speed. This lets us get the hardware initialized enough to attach
	* the bus which is where the real speed configuration is handled. After
	* the bus is attached, get the configured speed from it. Search the
	* configuration table for the best speed we can do that doesn't exceed
	* the requested speed.
	*/
	if (sc->sc_iicbus == NULL)
	busfreq = 100000;
	else
	busfreq = IICBUS_GET_FREQUENCY(sc->sc_iicbus, speed);
	for (;;) {
	if (clkcfg[1].frequency == 0 \|\| clkcfg[1].frequency > busfreq)
	break;
	clkcfg++;
	}

	/*
	* 23.1.4.3 - HS I2C Software Reset
	* From OMAP4 TRM at page 4068.
	*
	* 1. Ensure that the module is disabled.
	*/
	sc->sc_con_reg = 0;
	ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg);

	/* 2. Issue a softreset to the controller. */
	bus_write_2(sc->sc_mem_res, I2C_REG_SYSC, I2C_REG_SYSC_SRST);

	/*
	* 3. Enable the module.
	* The I2Ci.I2C_SYSS[0] RDONE bit is asserted only after the module
	* is enabled by setting the I2Ci.I2C_CON[15] I2C_EN bit to 1.
	*/
	ti_i2c_write_2(sc, I2C_REG_CON, I2C_CON_I2C_EN);

	/* 4. Wait for the software reset to complete. */
	timeout = 0;
	while ((ti_i2c_read_2(sc, I2C_REG_SYSS) & I2C_SYSS_RDONE) == 0) {
	if (timeout++ > 100)
	return (EBUSY);
	DELAY(100);
	}

	/*
	* Disable the I2C controller once again, now that the reset has
	* finished.
	*/
	ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg);

	/*
	* The following sequence is taken from the OMAP4 TRM at page 4077.
	*
	* 1. Enable the functional and interface clocks (see Section
	* 23.1.5.1.1.1.1). Done at ti_i2c_activate().
	*
	* 2. Program the prescaler to obtain an approximately 12MHz internal
	* sampling clock (I2Ci_INTERNAL_CLK) by programming the
	* corresponding value in the I2Ci.I2C_PSC[3:0] PSC field.
	* This value depends on the frequency of the functional clock
	* (I2Ci_FCLK). Because this frequency is 96MHz, the
	* I2Ci.I2C_PSC[7:0] PSC field value is 0x7.
	*/
	ti_i2c_write_2(sc, I2C_REG_PSC, clkcfg->psc);

	/*
	* 3. Program the I2Ci.I2C_SCLL[7:0] SCLL and I2Ci.I2C_SCLH[7:0] SCLH
	* bit fields to obtain a bit rate of 100 Kbps, 400 Kbps or 1Mbps.
	* These values depend on the internal sampling clock frequency
	* (see Table 23-8).
	*/
	scll = clkcfg->scll & I2C_SCLL_MASK;
	sclh = clkcfg->sclh & I2C_SCLH_MASK;

	/*
	* 4. (Optional) Program the I2Ci.I2C_SCLL[15:8] HSSCLL and
	* I2Ci.I2C_SCLH[15:8] HSSCLH fields to obtain a bit rate of
	* 400K bps or 3.4M bps (for the second phase of HS mode). These
	* values depend on the internal sampling clock frequency (see
	* Table 23-8).
	*
	* 5. (Optional) If a bit rate of 3.4M bps is used and the bus line
	* capacitance exceeds 45 pF, (see Section 18.4.8, PAD Functional
	* Multiplexing and Configuration).
	*/
	switch (ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	if ((clkcfg->hsscll + clkcfg->hssclh) > 0) {
	scll \|= clkcfg->hsscll << I2C_HSSCLL_SHIFT;
	sclh \|= clkcfg->hssclh << I2C_HSSCLH_SHIFT;
	sc->sc_con_reg \|= I2C_CON_OPMODE_HS;
	}
	break;
	#endif
	}

	/* Write the selected bit rate. */
	ti_i2c_write_2(sc, I2C_REG_SCLL, scll);
	ti_i2c_write_2(sc, I2C_REG_SCLH, sclh);

	/*
	* 6. Configure the Own Address of the I2C controller by storing it in
	* the I2Ci.I2C_OA0 register. Up to four Own Addresses can be
	* programmed in the I2Ci.I2C_OAi registers (where i = 0, 1, 2, 3)
	* for each I2C controller.
	*
	* Note: For a 10-bit address, set the corresponding expand Own Address
	* bit in the I2Ci.I2C_CON register.
	*
	* Driver currently always in single master mode so ignore this step.
	*/

	/*
	* 7. Set the TX threshold (in transmitter mode) and the RX threshold
	* (in receiver mode) by setting the I2Ci.I2C_BUF[5:0]XTRSH field to
	* (TX threshold - 1) and the I2Ci.I2C_BUF[13:8]RTRSH field to (RX
	* threshold - 1), where the TX and RX thresholds are greater than
	* or equal to 1.
	*
	* The threshold is set to 5 for now.
	*/
	fifo_trsh = (sc->sc_fifo_trsh - 1) & I2C_BUF_TRSH_MASK;
	reg = fifo_trsh \| (fifo_trsh << I2C_BUF_RXTRSH_SHIFT);
	ti_i2c_write_2(sc, I2C_REG_BUF, reg);

	/*
	* 8. Take the I2C controller out of reset by setting the
	* I2Ci.I2C_CON[15] I2C_EN bit to 1.
	*
	* 23.1.5.1.1.1.2 - Initialize the I2C Controller
	*
	* To initialize the I2C controller, perform the following steps:
	*
	* 1. Configure the I2Ci.I2C_CON register:
	* . For master or slave mode, set the I2Ci.I2C_CON[10] MST bit
	* (0: slave, 1: master).
	* . For transmitter or receiver mode, set the I2Ci.I2C_CON[9] TRX
	* bit (0: receiver, 1: transmitter).
	*/

	/* Enable the I2C controller in master mode. */
	sc->sc_con_reg \|= I2C_CON_I2C_EN \| I2C_CON_MST;
	ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg);

	/*
	* 2. If using an interrupt to transmit/receive data, set the
	* corresponding bit in the I2Ci.I2C_IE register (the I2Ci.I2C_IE[4]
	* XRDY_IE bit for the transmit interrupt, the I2Ci.I2C_IE[3] RRDY
	* bit for the receive interrupt).
	*/

	/* Set the interrupts we want to be notified. */
	reg = I2C_IE_XDR \| /* Transmit draining interrupt. */
	I2C_IE_XRDY \| /* Transmit Data Ready interrupt. */
	I2C_IE_RDR \| /* Receive draining interrupt. */
	I2C_IE_RRDY \| /* Receive Data Ready interrupt. */
	I2C_IE_ARDY \| /* Register Access Ready interrupt. */
	I2C_IE_NACK \| /* No Acknowledgment interrupt. */
	I2C_IE_AL; /* Arbitration lost interrupt. */

	/* Enable the interrupts. */
	ti_i2c_write_2(sc, I2C_REG_IRQENABLE_SET, reg);

	/*
	* 3. If using DMA to receive/transmit data, set to 1 the corresponding
	* bit in the I2Ci.I2C_BUF register (the I2Ci.I2C_BUF[15] RDMA_EN
	* bit for the receive DMA channel, the I2Ci.I2C_BUF[7] XDMA_EN bit
	* for the transmit DMA channel).
	*
	* Not using DMA for now, so ignore this.
	*/

	return (0);
	}

	static int
	ti_i2c_iicbus_reset(device_t dev, u_char speed, u_char addr, u_char *oldaddr)
	{
	struct ti_i2c_softc *sc;
	int err;

	sc = device_get_softc(dev);
	TI_I2C_LOCK(sc);
	err = ti_i2c_reset(sc, speed);
	TI_I2C_UNLOCK(sc);
	if (err)
	return (err);

	return (IIC_ENOADDR);
	}

	static int
	ti_i2c_activate(device_t dev)
	{
	int err;
	struct ti_i2c_softc *sc;

	sc = (struct ti_i2c_softc*)device_get_softc(dev);

	/*
	* 1. Enable the functional and interface clocks (see Section
	* 23.1.5.1.1.1.1).
	*/
	- err = ti_prcm_clk_enable(sc->clk_id);
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	if (err)
	return (err);

	return (ti_i2c_reset(sc, IIC_UNKNOWN));
	}

	/**
	* ti_i2c_deactivate - deactivates the controller and releases resources
	* @dev: i2c device handle
	*
	*
	*
	* LOCKING:
	* Assumed called in an atomic context.
	*
	* RETURNS:
	* nothing
	*/
	static void
	ti_i2c_deactivate(device_t dev)
	{
	struct ti_i2c_softc *sc = device_get_softc(dev);

	/* Disable the controller - cancel all transactions. */
	ti_i2c_write_2(sc, I2C_REG_IRQENABLE_CLR, 0xffff);
	ti_i2c_write_2(sc, I2C_REG_STATUS, 0xffff);
	ti_i2c_write_2(sc, I2C_REG_CON, 0);

	/* Release the interrupt handler. */
	if (sc->sc_irq_h != NULL) {
	bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_irq_h);
	sc->sc_irq_h = NULL;
	}

	/* Unmap the I2C controller registers. */
	if (sc->sc_mem_res != NULL) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	sc->sc_mem_res = NULL;
	}

	/* Release the IRQ resource. */
	if (sc->sc_irq_res != NULL) {
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	sc->sc_irq_res = NULL;
	}

	/* Finally disable the functional and interface clocks. */
	- ti_prcm_clk_disable(sc->clk_id);
	+ ti_sysc_clock_disable(device_get_parent(dev));
	}

	static int
	ti_i2c_sysctl_clk(SYSCTL_HANDLER_ARGS)
	{
	int clk, psc, sclh, scll;
	struct ti_i2c_softc *sc;

	sc = arg1;

	TI_I2C_LOCK(sc);
	/* Get the system prescaler value. */
	psc = (int)ti_i2c_read_2(sc, I2C_REG_PSC) + 1;

	/* Get the bitrate. */
	scll = (int)ti_i2c_read_2(sc, I2C_REG_SCLL) & I2C_SCLL_MASK;
	sclh = (int)ti_i2c_read_2(sc, I2C_REG_SCLH) & I2C_SCLH_MASK;

	clk = I2C_CLK / psc / (scll + 7 + sclh + 5);
	TI_I2C_UNLOCK(sc);

	return (sysctl_handle_int(oidp, &clk, 0, req));
	}

	static int
	ti_i2c_sysctl_timeout(SYSCTL_HANDLER_ARGS)
	{
	struct ti_i2c_softc *sc;
	unsigned int val;
	int err;

	sc = arg1;

	/*
	* MTX_DEF lock can't be held while doing uimove in
	* sysctl_handle_int
	*/
	TI_I2C_LOCK(sc);
	val = sc->sc_timeout;
	TI_I2C_UNLOCK(sc);

	err = sysctl_handle_int(oidp, &val, 0, req);
	/* Write request? */
	if ((err == 0) && (req->newptr != NULL)) {
	TI_I2C_LOCK(sc);
	sc->sc_timeout = val;
	TI_I2C_UNLOCK(sc);
	}

	return (err);
	}

	static int
	ti_i2c_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);
	if (!ofw_bus_is_compatible(dev, "ti,omap4-i2c"))
	return (ENXIO);
	device_set_desc(dev, "TI I2C Controller");

	return (0);
	}

	static int
	ti_i2c_attach(device_t dev)
	{
	int err, rid;
	- phandle_t node;
	struct ti_i2c_softc *sc;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid_list *tree;
	uint16_t fifosz;

	sc = device_get_softc(dev);
	sc->sc_dev = dev;

	- /* Get the i2c device id from FDT. */
	- node = ofw_bus_get_node(dev);
	- /* i2c ti,hwmods bindings is special: it start with index 1 */
	- sc->clk_id = ti_hwmods_get_clock(dev);
	- if (sc->clk_id == INVALID_CLK_IDENT) {
	- device_printf(dev, "failed to get device id using ti,hwmod\n");
	- return (ENXIO);
	- }
	-
	/* Get the memory resource for the register mapping. */
	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (sc->sc_mem_res == NULL) {
	device_printf(dev, "Cannot map registers.\n");
	return (ENXIO);
	}

	/* Allocate our IRQ resource. */
	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE \| RF_SHAREABLE);
	if (sc->sc_irq_res == NULL) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "Cannot allocate interrupt.\n");
	return (ENXIO);
	}

	TI_I2C_LOCK_INIT(sc);

	/* First of all, we _must_ activate the H/W. */
	err = ti_i2c_activate(dev);
	if (err) {
	device_printf(dev, "ti_i2c_activate failed\n");
	goto out;
	}

	/* Read the version number of the I2C module */
	sc->sc_rev = ti_i2c_read_2(sc, I2C_REG_REVNB_HI) & 0xff;

	/* Get the fifo size. */
	fifosz = ti_i2c_read_2(sc, I2C_REG_BUFSTAT);
	fifosz >>= I2C_BUFSTAT_FIFODEPTH_SHIFT;
	fifosz &= I2C_BUFSTAT_FIFODEPTH_MASK;

	device_printf(dev, "I2C revision %d.%d FIFO size: %d bytes\n",
	sc->sc_rev >> 4, sc->sc_rev & 0xf, 8 << fifosz);

	/* Set the FIFO threshold to 5 for now. */
	sc->sc_fifo_trsh = 5;

	/* Set I2C bus timeout */
	sc->sc_timeout = 5*hz;

	ctx = device_get_sysctl_ctx(dev);
	tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
	SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "i2c_clock",
	CTLFLAG_RD \| CTLTYPE_UINT \| CTLFLAG_MPSAFE, sc, 0,
	ti_i2c_sysctl_clk, "IU", "I2C bus clock");

	SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "i2c_timeout",
	CTLFLAG_RW \| CTLTYPE_UINT \| CTLFLAG_MPSAFE, sc, 0,
	ti_i2c_sysctl_timeout, "IU", "I2C bus timeout (in ticks)");

	/* Activate the interrupt. */
	err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC \| INTR_MPSAFE,
	NULL, ti_i2c_intr, sc, &sc->sc_irq_h);
	if (err)
	goto out;

	/* Attach the iicbus. */
	if ((sc->sc_iicbus = device_add_child(dev, "iicbus", -1)) == NULL) {
	device_printf(dev, "could not allocate iicbus instance\n");
	err = ENXIO;
	goto out;
	}

	/* Probe and attach the iicbus when interrupts are available. */
	err = bus_delayed_attach_children(dev);

	out:
	if (err) {
	ti_i2c_deactivate(dev);
	TI_I2C_LOCK_DESTROY(sc);
	}

	return (err);
	}

	static int
	ti_i2c_detach(device_t dev)
	{
	struct ti_i2c_softc *sc;
	int rv;

	sc = device_get_softc(dev);

	if ((rv = bus_generic_detach(dev)) != 0) {
	device_printf(dev, "cannot detach child devices\n");
	return (rv);
	}

	if (sc->sc_iicbus &&
	(rv = device_delete_child(dev, sc->sc_iicbus)) != 0)
	return (rv);

	ti_i2c_deactivate(dev);
	TI_I2C_LOCK_DESTROY(sc);

	return (0);
	}

	static phandle_t
	ti_i2c_get_node(device_t bus, device_t dev)
	{

	/* Share controller node with iibus device. */
	return (ofw_bus_get_node(bus));
	}

	static device_method_t ti_i2c_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ti_i2c_probe),
	DEVMETHOD(device_attach, ti_i2c_attach),
	DEVMETHOD(device_detach, ti_i2c_detach),

	/* Bus interface */
	DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
	DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
	DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource),
	DEVMETHOD(bus_release_resource, bus_generic_release_resource),
	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
	DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
	DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
	DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),

	/* OFW methods */
	DEVMETHOD(ofw_bus_get_node, ti_i2c_get_node),

	/* iicbus interface */
	DEVMETHOD(iicbus_callback, iicbus_null_callback),
	DEVMETHOD(iicbus_reset, ti_i2c_iicbus_reset),
	DEVMETHOD(iicbus_transfer, ti_i2c_transfer),

	DEVMETHOD_END
	};

	static driver_t ti_i2c_driver = {
	"iichb",
	ti_i2c_methods,
	sizeof(struct ti_i2c_softc),
	};

	static devclass_t ti_i2c_devclass;

	DRIVER_MODULE(ti_iic, simplebus, ti_i2c_driver, ti_i2c_devclass, 0, 0);
	DRIVER_MODULE(iicbus, ti_iic, iicbus_driver, iicbus_devclass, 0, 0);

	-MODULE_DEPEND(ti_iic, ti_prcm, 1, 1, 1);
	+MODULE_DEPEND(ti_iic, ti_sysc, 1, 1, 1);
	MODULE_DEPEND(ti_iic, iicbus, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_mbox.c b/sys/arm/ti/ti_mbox.c
	index ea8454f96f7a..f77f2d9eafbf 100644
	--- a/sys/arm/ti/ti_mbox.c
	+++ b/sys/arm/ti/ti_mbox.c
	@@ -1,265 +1,268 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 Rui Paulo <rpaulo@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
	* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/rman.h>
	#include <sys/timeet.h>
	#include <sys/timetc.h>
	#include <sys/watchdog.h>
	#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <machine/frame.h>
	#include <machine/intr.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <machine/bus.h>

	#include <arm/ti/ti_mbox.h>
	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>

	#include "mbox_if.h"

	#ifdef DEBUG
	#define DPRINTF(fmt, ...) do { \
	printf("%s: ", __func__); \
	printf(fmt, __VA_ARGS__); \
	} while (0)
	#else
	#define DPRINTF(fmt, ...)
	#endif

	static device_probe_t ti_mbox_probe;
	static device_attach_t ti_mbox_attach;
	static device_detach_t ti_mbox_detach;
	static void ti_mbox_intr(void *);
	static int ti_mbox_read(device_t, int, uint32_t *);
	static int ti_mbox_write(device_t, int, uint32_t);

	struct ti_mbox_softc {
	struct mtx sc_mtx;
	struct resource *sc_mem_res;
	struct resource *sc_irq_res;
	void *sc_intr;
	bus_space_tag_t sc_bt;
	bus_space_handle_t sc_bh;
	};

	#define TI_MBOX_LOCK(sc) mtx_lock(&(sc)->sc_mtx)
	#define TI_MBOX_UNLOCK(sc) mtx_unlock(&(sc)->sc_mtx)

	static device_method_t ti_mbox_methods[] = {
	DEVMETHOD(device_probe, ti_mbox_probe),
	DEVMETHOD(device_attach, ti_mbox_attach),
	DEVMETHOD(device_detach, ti_mbox_detach),

	DEVMETHOD(mbox_read, ti_mbox_read),
	DEVMETHOD(mbox_write, ti_mbox_write),

	DEVMETHOD_END
	};

	static driver_t ti_mbox_driver = {
	"ti_mbox",
	ti_mbox_methods,
	sizeof(struct ti_mbox_softc)
	};

	static devclass_t ti_mbox_devclass;

	DRIVER_MODULE(ti_mbox, simplebus, ti_mbox_driver, ti_mbox_devclass, 0, 0);
	+MODULE_DEPEND(ti_mbox, ti_sysc, 1, 1, 1);

	static __inline uint32_t
	ti_mbox_reg_read(struct ti_mbox_softc *sc, uint16_t reg)
	{
	return (bus_space_read_4(sc->sc_bt, sc->sc_bh, reg));
	}

	static __inline void
	ti_mbox_reg_write(struct ti_mbox_softc *sc, uint16_t reg, uint32_t val)
	{
	bus_space_write_4(sc->sc_bt, sc->sc_bh, reg, val);
	}

	static int
	ti_mbox_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_is_compatible(dev, "ti,omap4-mailbox")) {
	device_set_desc(dev, "TI System Mailbox");
	return (BUS_PROBE_DEFAULT);
	}

	return (ENXIO);
	}

	static int
	ti_mbox_attach(device_t dev)
	{
	struct ti_mbox_softc *sc;
	int rid, delay, chan;
	uint32_t rev, sysconfig;

	- if (ti_prcm_clk_enable(MAILBOX0_CLK) != 0) {
	+ if (ti_sysc_clock_enable(device_get_parent(dev)) != 0) {
	device_printf(dev, "could not enable MBOX clock\n");
	return (ENXIO);
	}
	+
	sc = device_get_softc(dev);
	rid = 0;
	mtx_init(&sc->sc_mtx, "TI mbox", NULL, MTX_DEF);
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (sc->sc_mem_res == NULL) {
	device_printf(dev, "could not allocate memory resource\n");
	return (ENXIO);
	}
	sc->sc_bt = rman_get_bustag(sc->sc_mem_res);
	sc->sc_bh = rman_get_bushandle(sc->sc_mem_res);
	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE);
	if (sc->sc_irq_res == NULL) {
	device_printf(dev, "could not allocate interrupt resource\n");
	ti_mbox_detach(dev);
	return (ENXIO);
	}
	if (bus_setup_intr(dev, sc->sc_irq_res, INTR_MPSAFE \| INTR_TYPE_MISC,
	NULL, ti_mbox_intr, sc, &sc->sc_intr) != 0) {
	device_printf(dev, "unable to setup the interrupt handler\n");
	ti_mbox_detach(dev);
	return (ENXIO);
	}
	/*
	* Reset the controller.
	*/
	sysconfig = ti_mbox_reg_read(sc, TI_MBOX_SYSCONFIG);
	DPRINTF("initial sysconfig %d\n", sysconfig);
	sysconfig \|= TI_MBOX_SYSCONFIG_SOFTRST;
	delay = 100;
	while (ti_mbox_reg_read(sc, TI_MBOX_SYSCONFIG) &
	TI_MBOX_SYSCONFIG_SOFTRST) {
	delay--;
	DELAY(10);
	}
	if (delay == 0) {
	device_printf(dev, "controller reset failed\n");
	ti_mbox_detach(dev);
	return (ENXIO);
	}
	/*
	* Enable smart idle mode.
	*/
	ti_mbox_reg_write(sc, TI_MBOX_SYSCONFIG,
	ti_mbox_reg_read(sc, TI_MBOX_SYSCONFIG) \| TI_MBOX_SYSCONFIG_SMARTIDLE);
	- rev = ti_mbox_reg_read(sc, TI_MBOX_REVISION);
	+ rev = ti_mbox_reg_read(sc,
	+ ti_sysc_get_rev_address_offset_host(device_get_parent(dev)));
	DPRINTF("rev %d\n", rev);
	device_printf(dev, "revision %d.%d\n", (rev >> 8) & 0x4, rev & 0x40);
	/*
	* Enable message interrupts.
	*/
	for (chan = 0; chan < 8; chan++)
	ti_mbox_reg_write(sc, TI_MBOX_IRQENABLE_SET(chan), 1);

	return (0);
	}

	static int
	ti_mbox_detach(device_t dev)
	{
	struct ti_mbox_softc *sc;

	sc = device_get_softc(dev);
	if (sc->sc_intr)
	bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intr);
	if (sc->sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->sc_irq_res),
	sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_mem_res),
	sc->sc_mem_res);

	return (0);
	}

	static void
	ti_mbox_intr(void *arg)
	{
	struct ti_mbox_softc *sc;

	sc = arg;
	DPRINTF("interrupt %p", sc);
	}

	static int
	ti_mbox_read(device_t dev, int chan, uint32_t *data)
	{
	struct ti_mbox_softc *sc;

	if (chan < 0 \|\| chan > 7)
	return (EINVAL);
	sc = device_get_softc(dev);

	return (ti_mbox_reg_read(sc, TI_MBOX_MESSAGE(chan)));
	}

	static int
	ti_mbox_write(device_t dev, int chan, uint32_t data)
	{
	int limit = 500;
	struct ti_mbox_softc *sc;

	if (chan < 0 \|\| chan > 7)
	return (EINVAL);
	sc = device_get_softc(dev);
	TI_MBOX_LOCK(sc);
	/* XXX implement interrupt method */
	while (ti_mbox_reg_read(sc, TI_MBOX_FIFOSTATUS(chan)) == 1 &&
	limit--) {
	DELAY(10);
	}
	if (limit == 0) {
	device_printf(dev, "FIFOSTAUS%d stuck\n", chan);
	TI_MBOX_UNLOCK(sc);
	return (EAGAIN);
	}
	ti_mbox_reg_write(sc, TI_MBOX_MESSAGE(chan), data);

	return (0);
	}
	diff --git a/sys/arm/ti/ti_sysc.c b/sys/arm/ti/ti_omap4_cm.c
	similarity index 66%
	copy from sys/arm/ti/ti_sysc.c
	copy to sys/arm/ti/ti_omap4_cm.c
	index d428dd44a1ab..c9545a612e1f 100644
	--- a/sys/arm/ti/ti_sysc.c
	+++ b/sys/arm/ti/ti_omap4_cm.c
	@@ -1,128 +1,151 @@
	/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	* Copyright (c) 2019 Emmanuel Vadot <manu@FreeBSD.org>
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	+ * Based on sys/arm/ti/ti_sysc.c
	+ *
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/fbio.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/rman.h>
	#include <sys/resource.h>
	#include <machine/bus.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/pmap.h>

	#include <dev/fdt/simplebus.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	+#include <arm/ti/ti_omap4_cm.h>
	+
	static struct ofw_compat_data compat_data[] = {
	- { "ti,sysc", 1 },
	+ { "ti,omap4-cm", 1 },
	{ NULL, 0 }
	};

	-struct ti_sysc_softc {
	+struct ti_omap4_cm_softc {
	struct simplebus_softc sc;
	device_t dev;
	};

	-static int ti_sysc_probe(device_t dev);
	-static int ti_sysc_attach(device_t dev);
	-static int ti_sysc_detach(device_t dev);
	+uint64_t
	+ti_omap4_cm_get_simplebus_base_host(device_t dev) {
	+ struct ti_omap4_cm_softc *sc;
	+
	+ sc = device_get_softc(dev);
	+ if (sc->sc.nranges == 0)
	+ return (0);
	+
	+ return (sc->sc.ranges[0].host);
	+}
	+
	+static int ti_omap4_cm_probe(device_t dev);
	+static int ti_omap4_cm_attach(device_t dev);
	+static int ti_omap4_cm_detach(device_t dev);

	static int
	-ti_sysc_probe(device_t dev)
	+ti_omap4_cm_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	- device_set_desc(dev, "TI SYSC Interconnect");
	+ device_set_desc(dev, "TI OMAP4-CM");
	if (!bootverbose)
	device_quiet(dev);

	return (BUS_PROBE_DEFAULT);
	}

	static int
	-ti_sysc_attach(device_t dev)
	+ti_omap4_cm_attach(device_t dev)
	{
	- struct ti_sysc_softc *sc;
	+ struct ti_omap4_cm_softc *sc;
	device_t cdev;
	phandle_t node, child;

	sc = device_get_softc(dev);
	sc->dev = dev;
	node = ofw_bus_get_node(dev);

	simplebus_init(dev, node);
	if (simplebus_fill_ranges(node, &sc->sc) < 0) {
	device_printf(dev, "could not get ranges\n");
	return (ENXIO);
	}

	+ bus_generic_probe(sc->dev);
	+
	for (child = OF_child(node); child > 0; child = OF_peer(child)) {
	cdev = simplebus_add_device(dev, child, 0, NULL, -1, NULL);
	if (cdev != NULL)
	device_probe_and_attach(cdev);
	}

	return (bus_generic_attach(dev));
	}

	static int
	-ti_sysc_detach(device_t dev)
	+ti_omap4_cm_detach(device_t dev)
	{
	-
	return (EBUSY);
	}

	-static device_method_t ti_sysc_methods[] = {
	+static device_method_t ti_omap4_cm_methods[] = {
	/* Device interface */
	- DEVMETHOD(device_probe, ti_sysc_probe),
	- DEVMETHOD(device_attach, ti_sysc_attach),
	- DEVMETHOD(device_detach, ti_sysc_detach),
	+ DEVMETHOD(device_probe, ti_omap4_cm_probe),
	+ DEVMETHOD(device_attach, ti_omap4_cm_attach),
	+ DEVMETHOD(device_detach, ti_omap4_cm_detach),

	DEVMETHOD_END
	};

	-DEFINE_CLASS_1(ti_sysc, ti_sysc_driver, ti_sysc_methods,
	- sizeof(struct ti_sysc_softc), simplebus_driver);
	+DEFINE_CLASS_1(ti_omap4_cm, ti_omap4_cm_driver, ti_omap4_cm_methods,
	+ sizeof(struct ti_omap4_cm_softc), simplebus_driver);
	+
	+static devclass_t ti_omap4_cm_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_omap4_cm, simplebus, ti_omap4_cm_driver,
	+ti_omap4_cm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);

	-static devclass_t ti_sysc_devclass;
	+EARLY_DRIVER_MODULE(ti_omap4_cm, ofwbus, ti_omap4_cm_driver,
	+ti_omap4_cm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);

	-EARLY_DRIVER_MODULE(ti_sysc, simplebus, ti_sysc_driver,
	-ti_sysc_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
	diff --git a/sys/arm/ti/ti_omap4_cm.h b/sys/arm/ti/ti_omap4_cm.h
	new file mode 100644
	index 000000000000..4da56520cc97
	--- /dev/null
	+++ b/sys/arm/ti/ti_omap4_cm.h
	@@ -0,0 +1,34 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+#ifndef __TI_OMAP4_CM__
	+#define __TI_OMAP4_CM__
	+
	+uint64_t ti_omap4_cm_get_simplebus_base_host(device_t dev);
	+
	+#endif /* __TI_OMAP4_CM__ */
	diff --git a/sys/arm/ti/ti_pinmux.c b/sys/arm/ti/ti_pinmux.c
	index b532d9124ce6..da4b8a85526c 100644
	--- a/sys/arm/ti/ti_pinmux.c
	+++ b/sys/arm/ti/ti_pinmux.c
	@@ -1,461 +1,463 @@
	/*
	* Copyright (c) 2010
	* Ben Gray <ben.r.gray@gmail.com>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. All advertising materials mentioning features or use of this software
	* must display the following acknowledgement:
	* This product includes software developed by Ben Gray.
	* 4. The name of the company nor the name of the author may be used to
	* endorse or promote products derived from this software without specific
	* prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
	* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
	* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	/**
	* Exposes pinmux module to pinctrl-compatible interface
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/resource.h>
	#include <sys/rman.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>

	#include <machine/bus.h>
	#include <machine/resource.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>
	#include <dev/fdt/fdt_pinctrl.h>

	#include <arm/ti/omap4/omap4_scm_padconf.h>
	#include <arm/ti/am335x/am335x_scm_padconf.h>
	#include <arm/ti/ti_cpuid.h>
	#include "ti_pinmux.h"

	struct pincfg {
	uint32_t reg;
	uint32_t conf;
	};

	static struct resource_spec ti_pinmux_res_spec[] = {
	{ SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Control memory window */
	{ -1, 0 }
	};

	static struct ti_pinmux_softc *ti_pinmux_sc;

	#define ti_pinmux_read_2(sc, reg) \
	bus_space_read_2((sc)->sc_bst, (sc)->sc_bsh, (reg))
	#define ti_pinmux_write_2(sc, reg, val) \
	bus_space_write_2((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
	#define ti_pinmux_read_4(sc, reg) \
	bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg))
	#define ti_pinmux_write_4(sc, reg, val) \
	bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))


	/**
	* ti_padconf_devmap - Array of pins, should be defined one per SoC
	*
	* This array is typically defined in one of the targeted *_scm_pinumx.c
	* files and is specific to the given SoC platform. Each entry in the array
	* corresponds to an individual pin.
	*/
	static const struct ti_pinmux_device *ti_pinmux_dev;


	/**
	* ti_pinmux_padconf_from_name - searches the list of pads and returns entry
	* with matching ball name.
	* @ballname: the name of the ball
	*
	* RETURNS:
	* A pointer to the matching padconf or NULL if the ball wasn't found.
	*/
	static const struct ti_pinmux_padconf*
	ti_pinmux_padconf_from_name(const char *ballname)
	{
	const struct ti_pinmux_padconf *padconf;

	padconf = ti_pinmux_dev->padconf;
	while (padconf->ballname != NULL) {
	if (strcmp(ballname, padconf->ballname) == 0)
	return(padconf);
	padconf++;
	}

	return (NULL);
	}

	/**
	* ti_pinmux_padconf_set_internal - sets the muxmode and state for a pad/pin
	* @padconf: pointer to the pad structure
	* @muxmode: the name of the mode to use for the pin, i.e. "uart1_rx"
	* @state: the state to put the pad/pin in, i.e. PADCONF_PIN_???
	*
	*
	* LOCKING:
	* Internally locks it's own context.
	*
	* RETURNS:
	* 0 on success.
	* EINVAL if pin requested is outside valid range or already in use.
	*/
	static int
	ti_pinmux_padconf_set_internal(struct ti_pinmux_softc *sc,
	const struct ti_pinmux_padconf *padconf,
	const char *muxmode, unsigned int state)
	{
	unsigned int mode;
	uint16_t reg_val;

	/* populate the new value for the PADCONF register */
	reg_val = (uint16_t)(state & ti_pinmux_dev->padconf_sate_mask);

	/* find the new mode requested */
	for (mode = 0; mode < 8; mode++) {
	if ((padconf->muxmodes[mode] != NULL) &&
	(strcmp(padconf->muxmodes[mode], muxmode) == 0)) {
	break;
	}
	}

	/* couldn't find the mux mode */
	if (mode >= 8) {
	printf("Invalid mode \"%s\"\n", muxmode);
	return (EINVAL);
	}

	/* set the mux mode */
	reg_val \|= (uint16_t)(mode & ti_pinmux_dev->padconf_muxmode_mask);

	if (bootverbose)
	device_printf(sc->sc_dev, "setting internal %x for %s\n",
	reg_val, muxmode);
	/* write the register value (16-bit writes) */
	ti_pinmux_write_2(sc, padconf->reg_off, reg_val);

	return (0);
	}

	/**
	* ti_pinmux_padconf_set - sets the muxmode and state for a pad/pin
	* @padname: the name of the pad, i.e. "c12"
	* @muxmode: the name of the mode to use for the pin, i.e. "uart1_rx"
	* @state: the state to put the pad/pin in, i.e. PADCONF_PIN_???
	*
	*
	* LOCKING:
	* Internally locks it's own context.
	*
	* RETURNS:
	* 0 on success.
	* EINVAL if pin requested is outside valid range or already in use.
	*/
	int
	ti_pinmux_padconf_set(const char padname, const char muxmode, unsigned int state)
	{
	const struct ti_pinmux_padconf *padconf;

	if (!ti_pinmux_sc)
	return (ENXIO);

	/* find the pin in the devmap */
	padconf = ti_pinmux_padconf_from_name(padname);
	if (padconf == NULL)
	return (EINVAL);

	return (ti_pinmux_padconf_set_internal(ti_pinmux_sc, padconf, muxmode, state));
	}

	/**
	* ti_pinmux_padconf_get - gets the muxmode and state for a pad/pin
	* @padname: the name of the pad, i.e. "c12"
	* @muxmode: upon return will contain the name of the muxmode of the pin
	* @state: upon return will contain the state of the pad/pin
	*
	*
	* LOCKING:
	* Internally locks it's own context.
	*
	* RETURNS:
	* 0 on success.
	* EINVAL if pin requested is outside valid range or already in use.
	*/
	int
	ti_pinmux_padconf_get(const char padname, const char *muxmode,
	unsigned int *state)
	{
	const struct ti_pinmux_padconf *padconf;
	uint16_t reg_val;

	if (!ti_pinmux_sc)
	return (ENXIO);

	/* find the pin in the devmap */
	padconf = ti_pinmux_padconf_from_name(padname);
	if (padconf == NULL)
	return (EINVAL);

	/* read the register value (16-bit reads) */
	reg_val = ti_pinmux_read_2(ti_pinmux_sc, padconf->reg_off);

	/* save the state */
	if (state)
	*state = (reg_val & ti_pinmux_dev->padconf_sate_mask);

	/* save the mode */
	if (muxmode)
	*muxmode = padconf->muxmodes[(reg_val & ti_pinmux_dev->padconf_muxmode_mask)];

	return (0);
	}

	/**
	* ti_pinmux_padconf_set_gpiomode - converts a pad to GPIO mode.
	* @gpio: the GPIO pin number (0-195)
	* @state: the state to put the pad/pin in, i.e. PADCONF_PIN_???
	*
	*
	*
	* LOCKING:
	* Internally locks it's own context.
	*
	* RETURNS:
	* 0 on success.
	* EINVAL if pin requested is outside valid range or already in use.
	*/
	int
	ti_pinmux_padconf_set_gpiomode(uint32_t gpio, unsigned int state)
	{
	const struct ti_pinmux_padconf *padconf;
	uint16_t reg_val;

	if (!ti_pinmux_sc)
	return (ENXIO);

	/* find the gpio pin in the padconf array */
	padconf = ti_pinmux_dev->padconf;
	while (padconf->ballname != NULL) {
	if (padconf->gpio_pin == gpio)
	break;
	padconf++;
	}
	if (padconf->ballname == NULL)
	return (EINVAL);

	/* populate the new value for the PADCONF register */
	reg_val = (uint16_t)(state & ti_pinmux_dev->padconf_sate_mask);

	/* set the mux mode */
	reg_val \|= (uint16_t)(padconf->gpio_mode & ti_pinmux_dev->padconf_muxmode_mask);

	/* write the register value (16-bit writes) */
	ti_pinmux_write_2(ti_pinmux_sc, padconf->reg_off, reg_val);

	return (0);
	}

	/**
	* ti_pinmux_padconf_get_gpiomode - gets the current GPIO mode of the pin
	* @gpio: the GPIO pin number (0-195)
	* @state: upon return will contain the state
	*
	*
	*
	* LOCKING:
	* Internally locks it's own context.
	*
	* RETURNS:
	* 0 on success.
	* EINVAL if pin requested is outside valid range or not configured as GPIO.
	*/
	int
	ti_pinmux_padconf_get_gpiomode(uint32_t gpio, unsigned int *state)
	{
	const struct ti_pinmux_padconf *padconf;
	uint16_t reg_val;

	if (!ti_pinmux_sc)
	return (ENXIO);

	/* find the gpio pin in the padconf array */
	padconf = ti_pinmux_dev->padconf;
	while (padconf->ballname != NULL) {
	if (padconf->gpio_pin == gpio)
	break;
	padconf++;
	}
	if (padconf->ballname == NULL)
	return (EINVAL);

	/* read the current register settings */
	reg_val = ti_pinmux_read_2(ti_pinmux_sc, padconf->reg_off);

	/* check to make sure the pins is configured as GPIO in the first state */
	if ((reg_val & ti_pinmux_dev->padconf_muxmode_mask) != padconf->gpio_mode)
	return (EINVAL);

	/* read and store the reset of the state, i.e. pull-up, pull-down, etc */
	if (state)
	*state = (reg_val & ti_pinmux_dev->padconf_sate_mask);

	return (0);
	}

	static int
	ti_pinmux_configure_pins(device_t dev, phandle_t cfgxref)
	{
	struct pincfg cfgtuples, cfg;
	phandle_t cfgnode;
	int i, ntuples;
	static struct ti_pinmux_softc *sc;

	sc = device_get_softc(dev);
	cfgnode = OF_node_from_xref(cfgxref);
	ntuples = OF_getencprop_alloc_multi(cfgnode, "pinctrl-single,pins",
	sizeof(cfgtuples), (void *)&cfgtuples);

	if (ntuples < 0)
	return (ENOENT);

	if (ntuples == 0)
	return (0); /* Empty property is not an error. */

	for (i = 0, cfg = cfgtuples; i < ntuples; i++, cfg++) {
	if (bootverbose) {
	char name[32];
	OF_getprop(cfgnode, "name", &name, sizeof(name));
	printf("%16s: muxreg 0x%04x muxval 0x%02x\n",
	name, cfg->reg, cfg->conf);
	}

	/* write the register value (16-bit writes) */
	ti_pinmux_write_2(sc, cfg->reg, cfg->conf);
	}

	OF_prop_free(cfgtuples);

	return (0);
	}

	/*
	* Device part of OMAP SCM driver
	*/

	static int
	ti_pinmux_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "pinctrl-single"))
	return (ENXIO);

	if (ti_pinmux_sc) {
	printf("%s: multiple pinctrl modules in device tree data, ignoring\n",
	__func__);
	return (EEXIST);
	}
	switch (ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	ti_pinmux_dev = &omap4_pinmux_dev;
	break;
	#endif
	#ifdef SOC_TI_AM335X
	case CHIP_AM335X:
	ti_pinmux_dev = &ti_am335x_pinmux_dev;
	break;
	#endif
	default:
	printf("Unknown CPU in pinmux\n");
	return (ENXIO);
	}


	device_set_desc(dev, "TI Pinmux Module");
	return (BUS_PROBE_DEFAULT);
	}

	/**
	* ti_pinmux_attach - attaches the pinmux to the simplebus
	* @dev: new device
	*
	* RETURNS
	* Zero on success or ENXIO if an error occuried.
	*/
	static int
	ti_pinmux_attach(device_t dev)
	{
	struct ti_pinmux_softc *sc = device_get_softc(dev);

	#if 0
	if (ti_pinmux_sc)
	return (ENXIO);
	#endif

	sc->sc_dev = dev;

	if (bus_alloc_resources(dev, ti_pinmux_res_spec, sc->sc_res)) {
	device_printf(dev, "could not allocate resources\n");
	return (ENXIO);
	}

	sc->sc_bst = rman_get_bustag(sc->sc_res[0]);
	sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]);

	if (ti_pinmux_sc == NULL)
	ti_pinmux_sc = sc;

	fdt_pinctrl_register(dev, "pinctrl-single,pins");
	fdt_pinctrl_configure_tree(dev);

	return (0);
	}

	static device_method_t ti_pinmux_methods[] = {
	DEVMETHOD(device_probe, ti_pinmux_probe),
	DEVMETHOD(device_attach, ti_pinmux_attach),

	/* fdt_pinctrl interface */
	DEVMETHOD(fdt_pinctrl_configure, ti_pinmux_configure_pins),
	{ 0, 0 }
	};

	static driver_t ti_pinmux_driver = {
	"ti_pinmux",
	ti_pinmux_methods,
	sizeof(struct ti_pinmux_softc),
	};

	static devclass_t ti_pinmux_devclass;

	DRIVER_MODULE(ti_pinmux, simplebus, ti_pinmux_driver, ti_pinmux_devclass, 0, 0);
	+MODULE_VERSION(ti_pinmux, 1);
	+MODULE_DEPEND(ti_pinmux, ti_scm, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_prcm.c b/sys/arm/ti/ti_prcm.c
	index 3b6bbabe6910..ff85e724ada4 100644
	--- a/sys/arm/ti/ti_prcm.c
	+++ b/sys/arm/ti/ti_prcm.c
	@@ -1,357 +1,345 @@
	/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	- * Copyright (c) 2010
	- * Ben Gray <ben.r.gray@gmail.com>.
	+ * Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	* All rights reserved.
	*
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Ben Gray.
	- * 4. The name of the company nor the name of the author may be used to
	- * endorse or promote products derived from this software without specific
	- * prior written permission.
	*
	- * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	- * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
	- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
	- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	- */
	-
	-/**
	- * Power, Reset and Clock Management Module
	- *
	- * This is a very simple driver wrapper around the PRCM set of registers in
	- * the OMAP3 chip. It allows you to turn on and off things like the functional
	- * and interface clocks to the various on-chip modules.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	*
	+ * $FreeBSD$
	*/
	+
	+/* Based on sys/arm/ti/am335x/am335x_prcm.c */
	+
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	+#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	-#include <sys/bus.h>
	-#include <sys/resource.h>
	+#include <sys/malloc.h>
	#include <sys/rman.h>
	-#include <sys/lock.h>
	-#include <sys/mutex.h>
	-
	+#include <sys/timeet.h>
	+#include <sys/timetc.h>
	+#include <sys/watchdog.h>
	#include <machine/bus.h>
	-#include <machine/resource.h>
	+#include <machine/cpu.h>
	#include <machine/intr.h>

	#include <arm/ti/ti_cpuid.h>
	#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/tivar.h>

	-/**
	- * ti_*_clk_devmap - Array of clock devices, should be defined one per SoC
	- *
	- * This array is typically defined in one of the targeted *_prcm_clk.c
	- * files and is specific to the given SoC platform. Each entry in the array
	- * corresponds to an individual clock device.
	- */
	-extern struct ti_clock_dev ti_omap4_clk_devmap[];
	-extern struct ti_clock_dev ti_am335x_clk_devmap[];
	+#include <dev/fdt/simplebus.h>

	-/**
	- * ti_prcm_clk_dev - returns a pointer to the clock device with given id
	- * @clk: the ID of the clock device to get
	- *
	- * Simply iterates through the clk_devmap global array and returns a pointer
	- * to the clock device if found.
	- *
	- * LOCKING:
	- * None
	- *
	- * RETURNS:
	- * The pointer to the clock device on success, on failure NULL is returned.
	- */
	-static struct ti_clock_dev *
	-ti_prcm_clk_dev(clk_ident_t clk)
	+#include <dev/ofw/openfirm.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include "clkdev_if.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+struct ti_prcm_softc {
	+ struct simplebus_softc sc_simplebus;
	+ device_t dev;
	+ struct resource * mem_res;
	+ bus_space_tag_t bst;
	+ bus_space_handle_t bsh;
	+ int attach_done;
	+ struct mtx mtx;
	+};
	+
	+static struct ti_prcm_softc *ti_prcm_sc = NULL;
	+static void omap4_prcm_reset(void);
	+static void am335x_prcm_reset(void);
	+
	+#define TI_AM3_PRCM 18
	+#define TI_AM4_PRCM 17
	+#define TI_OMAP2_PRCM 16
	+#define TI_OMAP3_PRM 15
	+#define TI_OMAP3_CM 14
	+#define TI_OMAP4_CM1 13
	+#define TI_OMAP4_PRM 12
	+#define TI_OMAP4_CM2 11
	+#define TI_OMAP4_SCRM 10
	+#define TI_OMAP5_PRM 9
	+#define TI_OMAP5_CM_CORE_AON 8
	+#define TI_OMAP5_SCRM 7
	+#define TI_OMAP5_CM_CORE 6
	+#define TI_DRA7_PRM 5
	+#define TI_DRA7_CM_CORE_AON 4
	+#define TI_DRA7_CM_CORE 3
	+#define TI_DM814_PRCM 2
	+#define TI_DM816_PRCM 1
	+#define TI_PRCM_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,am3-prcm", TI_AM3_PRCM },
	+ { "ti,am4-prcm", TI_AM4_PRCM },
	+ { "ti,omap2-prcm", TI_OMAP2_PRCM },
	+ { "ti,omap3-prm", TI_OMAP3_PRM },
	+ { "ti,omap3-cm", TI_OMAP3_CM },
	+ { "ti,omap4-cm1", TI_OMAP4_CM1 },
	+ { "ti,omap4-prm", TI_OMAP4_PRM },
	+ { "ti,omap4-cm2", TI_OMAP4_CM2 },
	+ { "ti,omap4-scrm", TI_OMAP4_SCRM },
	+ { "ti,omap5-prm", TI_OMAP5_PRM },
	+ { "ti,omap5-cm-core-aon", TI_OMAP5_CM_CORE_AON },
	+ { "ti,omap5-scrm", TI_OMAP5_SCRM },
	+ { "ti,omap5-cm-core", TI_OMAP5_CM_CORE },
	+ { "ti,dra7-prm", TI_DRA7_PRM },
	+ { "ti,dra7-cm-core-aon", TI_DRA7_CM_CORE_AON },
	+ { "ti,dra7-cm-core", TI_DRA7_CM_CORE },
	+ { "ti,dm814-prcm", TI_DM814_PRCM },
	+ { "ti,dm816-prcm", TI_DM816_PRCM },
	+ { NULL, TI_PRCM_END}
	+};
	+
	+static int
	+ti_prcm_probe(device_t dev)
	{
	- struct ti_clock_dev *clk_dev;
	-
	- /* Find the clock within the devmap - it's a bit inefficent having a for
	- * loop for this, but this function should only called when a driver is
	- * being activated so IMHO not a big issue.
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) {
	+ return (ENXIO);
	+ }
	+
	+ device_set_desc(dev, "TI Power and Clock Management");
	+ return(BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+ti_prcm_attach(device_t dev)
	+{
	+ struct ti_prcm_softc *sc;
	+ phandle_t node, child;
	+ int rid;
	+
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+
	+ node = ofw_bus_get_node(sc->dev);
	+ simplebus_init(sc->dev, node);
	+
	+ if (simplebus_fill_ranges(node, &sc->sc_simplebus) < 0) {
	+ device_printf(sc->dev, "could not get ranges\n");
	+ return (ENXIO);
	+ }
	+ if (sc->sc_simplebus.nranges == 0) {
	+ device_printf(sc->dev, "nranges == 0\n");
	+ return (ENXIO);
	+ }
	+
	+ sc->mem_res = bus_alloc_resource(sc->dev, SYS_RES_MEMORY, &rid,
	+ sc->sc_simplebus.ranges[0].host,
	+ (sc->sc_simplebus.ranges[0].host +
	+ sc->sc_simplebus.ranges[0].size - 1),
	+ sc->sc_simplebus.ranges[0].size,
	+ RF_ACTIVE \| RF_SHAREABLE);
	+
	+ if (sc->mem_res == NULL) {
	+ return (ENXIO);
	+ }
	+
	+ sc->bst = rman_get_bustag(sc->mem_res);
	+ sc->bsh = rman_get_bushandle(sc->mem_res);
	+
	+ mtx_init(&sc->mtx, device_get_nameunit(dev), NULL, MTX_DEF);
	+
	+ /* Fixme: for xxx_prcm_reset functions.
	+ * Get rid of global variables?
	*/
	- clk_dev = NULL;
	+ ti_prcm_sc = sc;
	+
	switch(ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	- clk_dev = &(ti_omap4_clk_devmap[0]);
	+ ti_cpu_reset = omap4_prcm_reset;
	break;
	#endif
	#ifdef SOC_TI_AM335X
	case CHIP_AM335X:
	- clk_dev = &(ti_am335x_clk_devmap[0]);
	+ ti_cpu_reset = am335x_prcm_reset;
	break;
	#endif
	}
	- if (clk_dev == NULL)
	- panic("No clock devmap found");
	- while (clk_dev->id != INVALID_CLK_IDENT) {
	- if (clk_dev->id == clk) {
	- return (clk_dev);
	- }
	- clk_dev++;
	+
	+ bus_generic_probe(sc->dev);
	+ for (child = OF_child(node); child != 0; child = OF_peer(child)) {
	+ simplebus_add_device(dev, child, 0, NULL, -1, NULL);
	}

	- /* Sanity check we managed to find the clock */
	- printf("ti_prcm: Failed to find clock device (%d)\n", clk);
	- return (NULL);
	+ return (bus_generic_attach(sc->dev));
	}

	-/**
	- * ti_prcm_clk_valid - enables a clock for a particular module
	- * @clk: identifier for the module to enable, see ti_prcm.h for a list
	- * of possible modules.
	- * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK.
	- *
	- * This function can enable either a functional or interface clock.
	- *
	- * The real work done to enable the clock is really done in the callback
	- * function associated with the clock, this function is simply a wrapper
	- * around that.
	- *
	- * LOCKING:
	- * Internally locks the driver context.
	- *
	- * RETURNS:
	- * Returns 0 on success or positive error code on failure.
	- */
	int
	-ti_prcm_clk_valid(clk_ident_t clk)
	+ti_prcm_write_4(device_t dev, bus_addr_t addr, uint32_t val)
	{
	- int ret = 0;
	+ struct ti_prcm_softc *sc;

	- if (ti_prcm_clk_dev(clk) == NULL)
	- ret = EINVAL;
	-
	- return (ret);
	+ sc = device_get_softc(dev);
	+ DPRINTF(sc->dev, "offset=%lx write %x\n", addr, val);
	+ bus_space_write_4(sc->bst, sc->bsh, addr, val);
	+ return (0);
	}
	+int
	+ti_prcm_read_4(device_t dev, bus_addr_t addr, uint32_t *val)
	+{
	+ struct ti_prcm_softc *sc;

	+ sc = device_get_softc(dev);
	+
	+ *val = bus_space_read_4(sc->bst, sc->bsh, addr);
	+ DPRINTF(sc->dev, "offset=%lx Read %x\n", addr, *val);
	+ return (0);
	+}

	-/**
	- * ti_prcm_clk_enable - enables a clock for a particular module
	- * @clk: identifier for the module to enable, see ti_prcm.h for a list
	- * of possible modules.
	- * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK.
	- *
	- * This function can enable either a functional or interface clock.
	- *
	- * The real work done to enable the clock is really done in the callback
	- * function associated with the clock, this function is simply a wrapper
	- * around that.
	- *
	- * LOCKING:
	- * Internally locks the driver context.
	- *
	- * RETURNS:
	- * Returns 0 on success or positive error code on failure.
	- */
	int
	-ti_prcm_clk_enable(clk_ident_t clk)
	+ti_prcm_modify_4(device_t dev, bus_addr_t addr, uint32_t clr, uint32_t set)
	{
	- struct ti_clock_dev *clk_dev;
	- int ret;
	-
	- /* Find the clock within the devmap - it's a bit inefficent having a for
	- * loop for this, but this function should only called when a driver is
	- * being activated so IMHO not a big issue.
	- */
	- clk_dev = ti_prcm_clk_dev(clk);
	+ struct ti_prcm_softc *sc;
	+ uint32_t reg;

	- /* Sanity check we managed to find the clock */
	- if (clk_dev == NULL)
	- return (EINVAL);
	+ sc = device_get_softc(dev);

	- /* Activate the clock */
	- if (clk_dev->clk_activate)
	- ret = clk_dev->clk_activate(clk_dev);
	- else
	- ret = EINVAL;
	+ reg = bus_space_read_4(sc->bst, sc->bsh, addr);
	+ reg &= ~clr;
	+ reg \|= set;
	+ bus_space_write_4(sc->bst, sc->bsh, addr, reg);
	+ DPRINTF(sc->dev, "offset=%lx reg: %x (clr %x set %x)\n", addr, reg, clr, set);

	- return (ret);
	+ return (0);
	}

	-
	-/**
	- * ti_prcm_clk_disable - disables a clock for a particular module
	- * @clk: identifier for the module to enable, see ti_prcm.h for a list
	- * of possible modules.
	- * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK.
	- *
	- * This function can enable either a functional or interface clock.
	- *
	- * The real work done to enable the clock is really done in the callback
	- * function associated with the clock, this function is simply a wrapper
	- * around that.
	- *
	- * LOCKING:
	- * Internally locks the driver context.
	- *
	- * RETURNS:
	- * Returns 0 on success or positive error code on failure.
	- */
	-int
	-ti_prcm_clk_disable(clk_ident_t clk)
	+void
	+ti_prcm_device_lock(device_t dev)
	{
	- struct ti_clock_dev *clk_dev;
	- int ret;
	+ struct ti_prcm_softc *sc;

	- /* Find the clock within the devmap - it's a bit inefficent having a for
	- * loop for this, but this function should only called when a driver is
	- * being activated so IMHO not a big issue.
	- */
	- clk_dev = ti_prcm_clk_dev(clk);
	-
	- /* Sanity check we managed to find the clock */
	- if (clk_dev == NULL)
	- return (EINVAL);
	-
	- /* Activate the clock */
	- if (clk_dev->clk_deactivate)
	- ret = clk_dev->clk_deactivate(clk_dev);
	- else
	- ret = EINVAL;
	-
	- return (ret);
	+ sc = device_get_softc(dev);
	+ mtx_lock(&sc->mtx);
	}

	-/**
	- * ti_prcm_clk_set_source - sets the source
	- * @clk: identifier for the module to enable, see ti_prcm.h for a list
	- * of possible modules.
	- * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK.
	- *
	- * This function can enable either a functional or interface clock.
	- *
	- * The real work done to enable the clock is really done in the callback
	- * function associated with the clock, this function is simply a wrapper
	- * around that.
	- *
	- * LOCKING:
	- * Internally locks the driver context.
	- *
	- * RETURNS:
	- * Returns 0 on success or positive error code on failure.
	- */
	-int
	-ti_prcm_clk_set_source(clk_ident_t clk, clk_src_t clksrc)
	+void
	+ti_prcm_device_unlock(device_t dev)
	{
	- struct ti_clock_dev *clk_dev;
	- int ret;
	+ struct ti_prcm_softc *sc;

	- /* Find the clock within the devmap - it's a bit inefficent having a for
	- * loop for this, but this function should only called when a driver is
	- * being activated so IMHO not a big issue.
	- */
	- clk_dev = ti_prcm_clk_dev(clk);
	+ sc = device_get_softc(dev);
	+ mtx_unlock(&sc->mtx);
	+}

	- /* Sanity check we managed to find the clock */
	- if (clk_dev == NULL)
	- return (EINVAL);
	+static device_method_t ti_prcm_methods[] = {
	+ DEVMETHOD(device_probe, ti_prcm_probe),
	+ DEVMETHOD(device_attach, ti_prcm_attach),

	- /* Activate the clock */
	- if (clk_dev->clk_set_source)
	- ret = clk_dev->clk_set_source(clk_dev, clksrc);
	- else
	- ret = EINVAL;
	+ /* clkdev interface */
	+ DEVMETHOD(clkdev_write_4, ti_prcm_write_4),
	+ DEVMETHOD(clkdev_read_4, ti_prcm_read_4),
	+ DEVMETHOD(clkdev_modify_4, ti_prcm_modify_4),
	+ DEVMETHOD(clkdev_device_lock, ti_prcm_device_lock),
	+ DEVMETHOD(clkdev_device_unlock, ti_prcm_device_unlock),

	- return (ret);
	-}
	+ DEVMETHOD_END
	+};

	+DEFINE_CLASS_1(ti_prcm, ti_prcm_driver, ti_prcm_methods,
	+ sizeof(struct ti_prcm_softc), simplebus_driver);

	-/**
	- * ti_prcm_clk_get_source_freq - gets the source clock frequency
	- * @clk: identifier for the module to enable, see ti_prcm.h for a list
	- * of possible modules.
	- * @freq: pointer to an integer that upon return will contain the src freq
	- *
	- * This function returns the frequency of the source clock.
	- *
	- * The real work done to enable the clock is really done in the callback
	- * function associated with the clock, this function is simply a wrapper
	- * around that.
	- *
	- * LOCKING:
	- * Internally locks the driver context.
	- *
	- * RETURNS:
	- * Returns 0 on success or positive error code on failure.
	- */
	-int
	-ti_prcm_clk_get_source_freq(clk_ident_t clk, unsigned int *freq)
	-{
	- struct ti_clock_dev *clk_dev;
	- int ret;
	+static devclass_t ti_prcm_devclass;

	- /* Find the clock within the devmap - it's a bit inefficent having a for
	- * loop for this, but this function should only called when a driver is
	- * being activated so IMHO not a big issue.
	- */
	- clk_dev = ti_prcm_clk_dev(clk);
	+EARLY_DRIVER_MODULE(ti_prcm, ofwbus, ti_prcm_driver,
	+ ti_prcm_devclass, 0, 0, BUS_PASS_BUS);
	+EARLY_DRIVER_MODULE(ti_prcm, simplebus, ti_prcm_driver,
	+ ti_prcm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_prcm, 1);
	+MODULE_DEPEND(ti_prcm, ti_scm, 1, 1, 1);

	- /* Sanity check we managed to find the clock */
	- if (clk_dev == NULL)
	- return (EINVAL);

	- /* Get the source frequency of the clock */
	- if (clk_dev->clk_get_source_freq)
	- ret = clk_dev->clk_get_source_freq(clk_dev, freq);
	- else
	- ret = EINVAL;
	+/* From sys/arm/ti/am335x/am335x_prcm.c
	+ * Copyright (c) 2012 Damjan Marion <dmarion@Freebsd.org>
	+ */
	+#define PRM_DEVICE_OFFSET 0xF00
	+#define AM335x_PRM_RSTCTRL (PRM_DEVICE_OFFSET + 0x00)

	- return (ret);
	+static void
	+am335x_prcm_reset(void)
	+{
	+ ti_prcm_write_4(ti_prcm_sc->dev, AM335x_PRM_RSTCTRL, (1<<1));
	}

	-/**
	- * ti_prcm_clk_set_source_freq - sets the source clock frequency as close to freq as possible
	- * @clk: identifier for the module to enable, see ti_prcm.h for a list
	- * of possible modules.
	- * @freq: requested freq
	+/* FIXME: Is this correct - or should the license part be ontop? */
	+
	+/* From sys/arm/ti/omap4/omap4_prcm_clks.c */
	+/*-
	+ * SPDX-License-Identifier: BSD-3-Clause
	*
	- * LOCKING:
	- * Internally locks the driver context.
	+ * Copyright (c) 2011
	+ * Ben Gray <ben.r.gray@gmail.com>.
	+ * All rights reserved.
	*
	- * RETURNS:
	- * Returns 0 on success or positive error code on failure.
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ * 3. The name of the company nor the name of the author may be used to
	+ * endorse or promote products derived from this software without specific
	+ * prior written permission.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
	+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
	+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/
	-int
	-ti_prcm_clk_set_source_freq(clk_ident_t clk, unsigned int freq)
	-{
	- struct ti_clock_dev *clk_dev;
	- int ret;
	+#define PRM_RSTCTRL 0x1b00
	+#define PRM_RSTCTRL_RESET 0x2

	- clk_dev = ti_prcm_clk_dev(clk);
	-
	- /* Sanity check we managed to find the clock */
	- if (clk_dev == NULL)
	- return (EINVAL);
	-
	- /* Get the source frequency of the clock */
	- if (clk_dev->clk_set_source_freq)
	- ret = clk_dev->clk_set_source_freq(clk_dev, freq);
	- else
	- ret = EINVAL;
	+static void
	+omap4_prcm_reset(void)
	+{
	+ uint32_t reg;

	- return (ret);
	+ ti_prcm_read_4(ti_prcm_sc->dev, PRM_RSTCTRL, &reg);
	+ reg = reg \| PRM_RSTCTRL_RESET;
	+ ti_prcm_write_4(ti_prcm_sc->dev, PRM_RSTCTRL, reg);
	+ ti_prcm_read_4(ti_prcm_sc->dev, PRM_RSTCTRL, &reg);
	}
	diff --git a/sys/arm/ti/ti_prcm.h b/sys/arm/ti/ti_prcm.h
	index 6df39436cb2f..98f8abc15dd7 100644
	--- a/sys/arm/ti/ti_prcm.h
	+++ b/sys/arm/ti/ti_prcm.h
	@@ -1,209 +1,39 @@
	/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	- * Copyright (c) 2010
	- * Ben Gray <ben.r.gray@gmail.com>.
	- * All rights reserved.
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Ben Gray.
	- * 4. The name of the company nor the name of the author may be used to
	- * endorse or promote products derived from this software without specific
	- * prior written permission.
	*
	- * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	- * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
	- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
	- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	+#ifndef __TI_PRCM_H__
	+#define __TI_PRCM_H__

	-/*
	- * Texas Instruments - OMAP3xxx series processors
	- *
	- * Reference:
	- * OMAP35x Applications Processor
	- * Technical Reference Manual
	- * (omap35xx_techref.pdf)
	- */
	-#ifndef _TI_PRCM_H_
	-#define _TI_PRCM_H_
	-
	-typedef enum {
	-
	- INVALID_CLK_IDENT = 0,
	-
	- /* System clocks, typically you can only call ti_prcm_clk_get_source_freq()
	- * on these clocks as they are enabled by default.
	- */
	- SYS_CLK = 1,
	-
	- /* The MPU (ARM) core clock */
	- MPU_CLK = 20,
	-
	- /* MMC modules */
	- MMC1_CLK = 100,
	- MMC2_CLK,
	- MMC3_CLK,
	- MMC4_CLK,
	- MMC5_CLK,
	- MMC6_CLK,
	-
	- /* I2C modules */
	- I2C1_CLK = 200,
	- I2C2_CLK,
	- I2C3_CLK,
	- I2C4_CLK,
	- I2C5_CLK,
	-
	- /* USB module(s) */
	- USBTLL_CLK = 300,
	- USBHSHOST_CLK,
	- USBFSHOST_CLK,
	- USBP1_PHY_CLK,
	- USBP2_PHY_CLK,
	- USBP1_UTMI_CLK,
	- USBP2_UTMI_CLK,
	- USBP1_HSIC_CLK,
	- USBP2_HSIC_CLK,
	-
	- /* UART modules */
	- UART1_CLK = 400,
	- UART2_CLK,
	- UART3_CLK,
	- UART4_CLK,
	- UART5_CLK,
	- UART6_CLK,
	- UART7_CLK,
	- UART8_CLK,
	- UART9_CLK,
	-
	- /* General purpose timer modules */
	- TIMER1_CLK = 500,
	- TIMER2_CLK,
	- TIMER3_CLK,
	- TIMER4_CLK,
	- TIMER5_CLK,
	- TIMER6_CLK,
	- TIMER7_CLK,
	- TIMER8_CLK,
	- TIMER9_CLK,
	- TIMER10_CLK,
	- TIMER11_CLK,
	- TIMER12_CLK,
	-
	- /* McBSP module(s) */
	- MCBSP1_CLK = 600,
	- MCBSP2_CLK,
	- MCBSP3_CLK,
	- MCBSP4_CLK,
	- MCBSP5_CLK,
	-
	- /* General purpose I/O modules */
	- GPIO1_CLK = 700,
	- GPIO2_CLK,
	- GPIO3_CLK,
	- GPIO4_CLK,
	- GPIO5_CLK,
	- GPIO6_CLK,
	- GPIO7_CLK,
	-
	- /* sDMA module */
	- SDMA_CLK = 800,
	-
	- /* CPSW modules */
	- CPSW_CLK = 1000,
	-
	- /* Mentor USB modules */
	- MUSB0_CLK = 1100,
	-
	- /* EDMA module */
	- EDMA_TPCC_CLK = 1200,
	- EDMA_TPTC0_CLK,
	- EDMA_TPTC1_CLK,
	- EDMA_TPTC2_CLK,
	-
	- /* LCD controller module */
	- LCDC_CLK = 1300,
	-
	- /* PWM modules */
	- PWMSS0_CLK = 1400,
	- PWMSS1_CLK,
	- PWMSS2_CLK,
	-
	- /* Mailbox modules */
	- MAILBOX0_CLK = 1500,
	-
	- /* Spinlock modules */
	- SPINLOCK0_CLK = 1600,
	-
	- PRUSS_CLK = 1700,
	-
	- TSC_ADC_CLK = 1800,
	-
	- /* RTC module */
	- RTC_CLK = 1900,
	-
	- /* McSPI */
	- SPI0_CLK = 2000,
	- SPI1_CLK,
	-} clk_ident_t;
	-
	-/*
	- *
	- */
	-typedef enum {
	- SYSCLK_CLK, /* System clock */
	- EXT_CLK,
	-
	- F32KHZ_CLK, /* 32KHz clock */
	- F48MHZ_CLK, /* 48MHz clock */
	- F64MHZ_CLK, /* 64MHz clock */
	- F96MHZ_CLK, /* 96MHz clock */
	-
	-} clk_src_t;
	-
	-struct ti_clock_dev {
	- /* The profile of the timer */
	- clk_ident_t id;
	-
	- /* A bunch of callbacks associated with the clock device */
	- int (clk_activate)(struct ti_clock_dev clkdev);
	- int (clk_deactivate)(struct ti_clock_dev clkdev);
	- int (clk_set_source)(struct ti_clock_dev clkdev,
	- clk_src_t clksrc);
	- int (clk_accessible)(struct ti_clock_dev clkdev);
	- int (clk_set_source_freq)(struct ti_clock_dev clkdev,
	- unsigned int freq);
	- int (clk_get_source_freq)(struct ti_clock_dev clkdev,
	- unsigned int *freq);
	-};
	-
	-int ti_prcm_clk_valid(clk_ident_t clk);
	-int ti_prcm_clk_enable(clk_ident_t clk);
	-int ti_prcm_clk_disable(clk_ident_t clk);
	-int ti_prcm_clk_accessible(clk_ident_t clk);
	-int ti_prcm_clk_disable_autoidle(clk_ident_t clk);
	-int ti_prcm_clk_set_source(clk_ident_t clk, clk_src_t clksrc);
	-int ti_prcm_clk_set_source_freq(clk_ident_t clk, unsigned int freq);
	-int ti_prcm_clk_get_source_freq(clk_ident_t clk, unsigned int *freq);
	-void ti_prcm_reset(void);
	+int ti_prcm_write_4(device_t dev, bus_addr_t addr, uint32_t val);
	+int ti_prcm_read_4(device_t dev, bus_addr_t addr, uint32_t *val);
	+int ti_prcm_modify_4(device_t dev, bus_addr_t addr, uint32_t clr, uint32_t set);
	+void ti_prcm_device_lock(device_t dev);
	+void ti_prcm_device_unlock(device_t dev);

	-#endif /* _TI_PRCM_H_ */
	+#endif
	diff --git a/sys/arm/ti/ti_prm.c b/sys/arm/ti/ti_prm.c
	new file mode 100644
	index 000000000000..4a57fbb8b972
	--- /dev/null
	+++ b/sys/arm/ti/ti_prm.c
	@@ -0,0 +1,210 @@
	+/*-
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+/*
	+ * Power management - simple driver to handle reset and give access to
	+ * memory space region for other drivers through prcm driver.
	+ * Documentation/devicetree/binding/arm/omap/prm-inst.txt
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_prm.h>
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/* relative to prcm address range */
	+#define TI_PRM_PER_RSTCTRL 0xC00
	+
	+struct ti_prm_softc {
	+ device_t dev;
	+ uint8_t type;
	+ bool has_reset;
	+};
	+
	+/* Device */
	+#define TI_OMAP_PRM_INST 10
	+
	+#define TI_AM3_PRM_INST 5
	+#define TI_AM4_PRM_INST 4
	+#define TI_OMAP4_PRM_INST 3
	+#define TI_OMAP5_PRM_INST 2
	+#define TI_DRA7_PRM_INST 1
	+#define TI_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,am3-prm-inst", TI_AM3_PRM_INST },
	+ { "ti,am4-prm-inst", TI_AM4_PRM_INST },
	+ { "ti,omap4-prm-inst", TI_OMAP4_PRM_INST },
	+ { "ti,omap5-prm-inst", TI_OMAP5_PRM_INST },
	+ { "ti,dra7-prm-inst", TI_DRA7_PRM_INST },
	+ { NULL, TI_END }
	+};
	+
	+static struct ofw_compat_data required_data[] = {
	+ { "ti,omap-prm-inst", TI_OMAP_PRM_INST },
	+ { NULL, TI_END }
	+};
	+
	+/* device interface */
	+static int
	+ti_prm_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, required_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI OMAP Power Management");
	+ return(BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+ti_prm_attach(device_t dev)
	+{
	+ struct ti_prm_softc *sc;
	+ phandle_t node;
	+
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+ sc->type = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
	+
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ if (OF_hasprop(node, "#reset-cells")) {
	+ sc->has_reset = true;
	+ } else
	+ sc->has_reset = false;
	+
	+ /* Make device visible for other drivers */
	+ OF_device_register_xref(OF_xref_from_node(node), sc->dev);
	+
	+ return (0);
	+}
	+
	+static int
	+ti_prm_detach(device_t dev) {
	+ return (EBUSY);
	+}
	+
	+int
	+ti_prm_reset(device_t dev)
	+{
	+ struct ti_prm_softc *sc;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+ if (sc->has_reset == false)
	+ return 1;
	+
	+ err = ti_prm_modify_4(dev, TI_PRM_PER_RSTCTRL, 0x2, 0x00);
	+ return (err);
	+}
	+
	+int
	+ti_prm_write_4(device_t dev, bus_addr_t addr, uint32_t val)
	+{
	+ struct ti_prm_softc *sc;
	+ device_t parent;
	+
	+ parent = device_get_parent(dev);
	+ sc = device_get_softc(dev);
	+ DPRINTF(sc->dev, "offset=%lx write %x\n", addr, val);
	+ ti_prcm_device_lock(parent);
	+ ti_prcm_write_4(parent, addr, val);
	+ ti_prcm_device_unlock(parent);
	+ return (0);
	+}
	+
	+int
	+ti_prm_read_4(device_t dev, bus_addr_t addr, uint32_t *val)
	+{
	+ struct ti_prm_softc *sc;
	+ device_t parent;
	+
	+ parent = device_get_parent(dev);
	+ sc = device_get_softc(dev);
	+
	+ ti_prcm_device_lock(parent);
	+ ti_prcm_read_4(parent, addr, val);
	+ ti_prcm_device_unlock(parent);
	+ DPRINTF(sc->dev, "offset=%lx Read %x\n", addr, *val);
	+ return (0);
	+}
	+
	+int
	+ti_prm_modify_4(device_t dev, bus_addr_t addr, uint32_t clr, uint32_t set)
	+{
	+ struct ti_prm_softc *sc;
	+ device_t parent;
	+
	+ parent = device_get_parent(dev);
	+ sc = device_get_softc(dev);
	+
	+ ti_prcm_device_lock(parent);
	+ ti_prcm_modify_4(parent, addr, clr, set);
	+ ti_prcm_device_unlock(parent);
	+ DPRINTF(sc->dev, "offset=%lx (clr %x set %x)\n", addr, clr, set);
	+
	+ return (0);
	+}
	+
	+static device_method_t ti_prm_methods[] = {
	+ DEVMETHOD(device_probe, ti_prm_probe),
	+ DEVMETHOD(device_attach, ti_prm_attach),
	+ DEVMETHOD(device_detach, ti_prm_detach),
	+
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_1(ti_prm, ti_prm_driver, ti_prm_methods,
	+ sizeof(struct ti_prm_softc), simplebus_driver);
	+
	+static devclass_t ti_prm_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_prm, simplebus, ti_prm_driver,
	+ ti_prm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_prm, 1);
	+MODULE_DEPEND(ti_prm, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_prm.h b/sys/arm/ti/ti_prm.h
	new file mode 100644
	index 000000000000..bc3e991088f0
	--- /dev/null
	+++ b/sys/arm/ti/ti_prm.h
	@@ -0,0 +1,38 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+#ifndef __TI_PRM__
	+#define __TI_PRM__
	+
	+int ti_prm_reset(device_t dev);
	+
	+int ti_prm_write_4(device_t dev, bus_addr_t addr, uint32_t val);
	+int ti_prm_read_4(device_t dev, bus_addr_t addr, uint32_t *val);
	+int ti_prm_modify_4(device_t dev, bus_addr_t addr, uint32_t clr, uint32_t set);
	+
	+#endif /* __TI_PRM__ */
	diff --git a/sys/arm/ti/ti_pruss.c b/sys/arm/ti/ti_pruss.c
	index 48c6b17f3ac8..a8dc15ab80b0 100644
	--- a/sys/arm/ti/ti_pruss.c
	+++ b/sys/arm/ti/ti_pruss.c
	@@ -1,772 +1,854 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 Rui Paulo <rpaulo@FreeBSD.org>
	* Copyright (c) 2017 Manuel Stuehn
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
	* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/poll.h>
	#include <sys/time.h>
	#include <sys/uio.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/fcntl.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/types.h>
	#include <sys/sysctl.h>
	#include <sys/event.h>
	#include <sys/selinfo.h>
	#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <machine/frame.h>
	#include <machine/intr.h>
	#include <machine/atomic.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	-#include <arm/ti/ti_prcm.h>
	+#include <dev/extres/clk/clk.h>
	+
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_pruss.h>
	+#include <arm/ti/ti_prm.h>

	#ifdef DEBUG
	#define DPRINTF(fmt, ...) do { \
	printf("%s: ", __func__); \
	printf(fmt, __VA_ARGS__); \
	} while (0)
	#else
	#define DPRINTF(fmt, ...)
	#endif

	static d_open_t ti_pruss_irq_open;
	static d_read_t ti_pruss_irq_read;
	static d_poll_t ti_pruss_irq_poll;

	static device_probe_t ti_pruss_probe;
	static device_attach_t ti_pruss_attach;
	static device_detach_t ti_pruss_detach;
	static void ti_pruss_intr(void *);
	static d_open_t ti_pruss_open;
	static d_mmap_t ti_pruss_mmap;
	static void ti_pruss_irq_kqread_detach(struct knote *);
	static int ti_pruss_irq_kqevent(struct knote *, long);
	static d_kqfilter_t ti_pruss_irq_kqfilter;
	static void ti_pruss_privdtor(void *data);

	#define TI_PRUSS_PRU_IRQS 2
	#define TI_PRUSS_HOST_IRQS 8
	#define TI_PRUSS_IRQS (TI_PRUSS_HOST_IRQS+TI_PRUSS_PRU_IRQS)
	#define TI_PRUSS_EVENTS 64
	#define NOT_SET_STR "NONE"
	#define TI_TS_ARRAY 16

	struct ctl
	{
	size_t cnt;
	size_t idx;
	};

	struct ts_ring_buf
	{
	struct ctl ctl;
	uint64_t ts[TI_TS_ARRAY];
	};

	struct ti_pruss_irqsc
	{
	struct mtx sc_mtx;
	struct cdev *sc_pdev;
	struct selinfo sc_selinfo;
	int8_t channel;
	int8_t last;
	int8_t event;
	bool enable;
	struct ts_ring_buf tstamps;
	};

	static struct cdevsw ti_pruss_cdevirq = {
	.d_version = D_VERSION,
	.d_name = "ti_pruss_irq",
	.d_open = ti_pruss_irq_open,
	.d_read = ti_pruss_irq_read,
	.d_poll = ti_pruss_irq_poll,
	.d_kqfilter = ti_pruss_irq_kqfilter,
	};

	struct ti_pruss_softc {
	struct mtx sc_mtx;
	struct resource *sc_mem_res;
	struct resource *sc_irq_res[TI_PRUSS_HOST_IRQS];
	void *sc_intr[TI_PRUSS_HOST_IRQS];
	struct ti_pruss_irqsc sc_irq_devs[TI_PRUSS_IRQS];
	bus_space_tag_t sc_bt;
	bus_space_handle_t sc_bh;
	struct cdev *sc_pdev;
	struct selinfo sc_selinfo;
	bool sc_glob_irqen;
	};

	static struct cdevsw ti_pruss_cdevsw = {
	.d_version = D_VERSION,
	.d_name = "ti_pruss",
	.d_open = ti_pruss_open,
	.d_mmap = ti_pruss_mmap,
	};

	static device_method_t ti_pruss_methods[] = {
	DEVMETHOD(device_probe, ti_pruss_probe),
	DEVMETHOD(device_attach, ti_pruss_attach),
	DEVMETHOD(device_detach, ti_pruss_detach),

	DEVMETHOD_END
	};

	static driver_t ti_pruss_driver = {
	"ti_pruss",
	ti_pruss_methods,
	sizeof(struct ti_pruss_softc)
	};

	static devclass_t ti_pruss_devclass;

	DRIVER_MODULE(ti_pruss, simplebus, ti_pruss_driver, ti_pruss_devclass, 0, 0);
	-MODULE_DEPEND(ti_pruss, ti_prcm, 1, 1, 1);
	+MODULE_DEPEND(ti_pruss, ti_sysc, 1, 1, 1);
	+MODULE_DEPEND(ti_pruss, ti_prm, 1, 1, 1);

	static struct resource_spec ti_pruss_irq_spec[] = {
	{ SYS_RES_IRQ, 0, RF_ACTIVE },
	{ SYS_RES_IRQ, 1, RF_ACTIVE },
	{ SYS_RES_IRQ, 2, RF_ACTIVE },
	{ SYS_RES_IRQ, 3, RF_ACTIVE },
	{ SYS_RES_IRQ, 4, RF_ACTIVE },
	{ SYS_RES_IRQ, 5, RF_ACTIVE },
	{ SYS_RES_IRQ, 6, RF_ACTIVE },
	{ SYS_RES_IRQ, 7, RF_ACTIVE },
	{ -1, 0, 0 }
	};
	CTASSERT(TI_PRUSS_HOST_IRQS == nitems(ti_pruss_irq_spec) - 1);

	static int
	ti_pruss_irq_open(struct cdev dev, int oflags, int devtype, struct thread td)
	{
	struct ctl* irqs;
	struct ti_pruss_irqsc *sc;
	sc = dev->si_drv1;

	irqs = malloc(sizeof(struct ctl), M_DEVBUF, M_WAITOK);
	if (!irqs)
	return (ENOMEM);

	irqs->cnt = sc->tstamps.ctl.cnt;
	irqs->idx = sc->tstamps.ctl.idx;

	return devfs_set_cdevpriv(irqs, ti_pruss_privdtor);
	}

	static void
	ti_pruss_privdtor(void *data)
	{
	free(data, M_DEVBUF);
	}

	static int
	ti_pruss_irq_poll(struct cdev dev, int events, struct thread td)
	{
	struct ctl* irqs;
	struct ti_pruss_irqsc *sc;
	sc = dev->si_drv1;

	devfs_get_cdevpriv((void**)&irqs);

	if (events & (POLLIN \| POLLRDNORM)) {
	if (sc->tstamps.ctl.cnt != irqs->cnt)
	return events & (POLLIN \| POLLRDNORM);
	else
	selrecord(td, &sc->sc_selinfo);
	}
	return 0;
	}

	static int
	ti_pruss_irq_read(struct cdev cdev, struct uio uio, int ioflag)
	{
	const size_t ts_len = sizeof(uint64_t);
	struct ti_pruss_irqsc* irq;
	struct ctl* priv;
	int error = 0;
	size_t idx;
	ssize_t level;

	irq = cdev->si_drv1;

	if (uio->uio_resid < ts_len)
	return (EINVAL);

	error = devfs_get_cdevpriv((void**)&priv);
	if (error)
	return (error);

	mtx_lock(&irq->sc_mtx);

	if (irq->tstamps.ctl.cnt - priv->cnt > TI_TS_ARRAY)
	{
	priv->cnt = irq->tstamps.ctl.cnt;
	priv->idx = irq->tstamps.ctl.idx;
	mtx_unlock(&irq->sc_mtx);
	return (ENXIO);
	}

	do {
	idx = priv->idx;
	level = irq->tstamps.ctl.idx - idx;
	if (level < 0)
	level += TI_TS_ARRAY;

	if (level == 0) {
	if (ioflag & O_NONBLOCK) {
	mtx_unlock(&irq->sc_mtx);
	return (EWOULDBLOCK);
	}

	error = msleep(irq, &irq->sc_mtx, PCATCH \| PDROP,
	"pruirq", 0);
	if (error)
	return error;

	mtx_lock(&irq->sc_mtx);
	}
	}while(level == 0);

	mtx_unlock(&irq->sc_mtx);

	error = uiomove(&irq->tstamps.ts[idx], ts_len, uio);

	if (++idx == TI_TS_ARRAY)
	idx = 0;
	priv->idx = idx;

	atomic_add_32(&priv->cnt, 1);

	return (error);
	}

	static struct ti_pruss_irq_arg {
	int irq;
	struct ti_pruss_softc *sc;
	} ti_pruss_irq_args[TI_PRUSS_IRQS];

	static __inline uint32_t
	ti_pruss_reg_read(struct ti_pruss_softc *sc, uint32_t reg)
	{
	return (bus_space_read_4(sc->sc_bt, sc->sc_bh, reg));
	}

	static __inline void
	ti_pruss_reg_write(struct ti_pruss_softc *sc, uint32_t reg, uint32_t val)
	{
	bus_space_write_4(sc->sc_bt, sc->sc_bh, reg, val);
	}

	static __inline void
	ti_pruss_interrupts_clear(struct ti_pruss_softc *sc)
	{
	/* disable global interrupt */
	ti_pruss_reg_write(sc, PRUSS_INTC_GER, 0 );

	/* clear all events */
	ti_pruss_reg_write(sc, PRUSS_INTC_SECR0, 0xFFFFFFFF);
	ti_pruss_reg_write(sc, PRUSS_INTC_SECR1, 0xFFFFFFFF);

	/* disable all host interrupts */
	ti_pruss_reg_write(sc, PRUSS_INTC_HIER, 0);
	}

	static __inline int
	ti_pruss_interrupts_enable(struct ti_pruss_softc *sc, int8_t irq, bool enable)
	{
	if (enable && ((sc->sc_irq_devs[irq].channel == -1) \|\|
	(sc->sc_irq_devs[irq].event== -1)))
	{
	device_printf( sc->sc_pdev->si_drv1,
	"Interrupt chain not fully configured, not possible to enable\n" );
	return (EINVAL);
	}

	sc->sc_irq_devs[irq].enable = enable;

	if (sc->sc_irq_devs[irq].sc_pdev) {
	destroy_dev(sc->sc_irq_devs[irq].sc_pdev);
	sc->sc_irq_devs[irq].sc_pdev = NULL;
	}

	if (enable) {
	sc->sc_irq_devs[irq].sc_pdev = make_dev(&ti_pruss_cdevirq, 0, UID_ROOT, GID_WHEEL,
	0600, "pruss%d.irq%d", device_get_unit(sc->sc_pdev->si_drv1), irq);
	sc->sc_irq_devs[irq].sc_pdev->si_drv1 = &sc->sc_irq_devs[irq];

	sc->sc_irq_devs[irq].tstamps.ctl.idx = 0;
	}

	uint32_t reg = enable ? PRUSS_INTC_HIEISR : PRUSS_INTC_HIDISR;
	ti_pruss_reg_write(sc, reg, sc->sc_irq_devs[irq].channel);

	reg = enable ? PRUSS_INTC_EISR : PRUSS_INTC_EICR;
	ti_pruss_reg_write(sc, reg, sc->sc_irq_devs[irq].event );

	return (0);
	}

	static __inline void
	ti_pruss_map_write(struct ti_pruss_softc *sc, uint32_t basereg, uint8_t index, uint8_t content)
	{
	const size_t regadr = basereg + index & ~0x03;
	const size_t bitpos = (index & 0x03) * 8;
	uint32_t rmw = ti_pruss_reg_read(sc, regadr);
	rmw = (rmw & ~( 0xF << bitpos)) \| ( (content & 0xF) << bitpos);
	ti_pruss_reg_write(sc, regadr, rmw);
	}

	static int
	ti_pruss_event_map( SYSCTL_HANDLER_ARGS )
	{
	struct ti_pruss_softc *sc;
	const int8_t irq = arg2;
	int err;
	char event[sizeof(NOT_SET_STR)];

	sc = arg1;

	if(sc->sc_irq_devs[irq].event == -1)
	bcopy(NOT_SET_STR, event, sizeof(event));
	else
	snprintf(event, sizeof(event), "%d", sc->sc_irq_devs[irq].event);

	err = sysctl_handle_string(oidp, event, sizeof(event), req);
	if(err != 0)
	return (err);

	if (req->newptr) { // write event
	if (strcmp(NOT_SET_STR, event) == 0) {
	ti_pruss_interrupts_enable(sc, irq, false);
	sc->sc_irq_devs[irq].event = -1;
	} else {
	if (sc->sc_irq_devs[irq].channel == -1) {
	device_printf( sc->sc_pdev->si_drv1,
	"corresponding channel not configured\n");
	return (ENXIO);
	}

	const int8_t channelnr = sc->sc_irq_devs[irq].channel;
	const int8_t eventnr = strtol( event, NULL, 10 ); // TODO: check if strol is valid
	if (eventnr > TI_PRUSS_EVENTS \|\| eventnr < 0) {
	device_printf( sc->sc_pdev->si_drv1,
	"Event number %d not valid (0 - %d)",
	channelnr, TI_PRUSS_EVENTS -1);
	return (EINVAL);
	}

	sc->sc_irq_devs[irq].channel = channelnr;
	sc->sc_irq_devs[irq].event = eventnr;

	// event[nr] <= channel
	ti_pruss_map_write(sc, PRUSS_INTC_CMR_BASE,
	eventnr, channelnr);
	}
	}
	return (err);
	}

	static int
	ti_pruss_channel_map(SYSCTL_HANDLER_ARGS)
	{
	struct ti_pruss_softc *sc;
	int err;
	char channel[sizeof(NOT_SET_STR)];
	const int8_t irq = arg2;

	sc = arg1;

	if (sc->sc_irq_devs[irq].channel == -1)
	bcopy(NOT_SET_STR, channel, sizeof(channel));
	else
	snprintf(channel, sizeof(channel), "%d", sc->sc_irq_devs[irq].channel);

	err = sysctl_handle_string(oidp, channel, sizeof(channel), req);
	if (err != 0)
	return (err);

	if (req->newptr) { // write event
	if (strcmp(NOT_SET_STR, channel) == 0) {
	ti_pruss_interrupts_enable(sc, irq, false);
	ti_pruss_reg_write(sc, PRUSS_INTC_HIDISR,
	sc->sc_irq_devs[irq].channel);
	sc->sc_irq_devs[irq].channel = -1;
	} else {
	const int8_t channelnr = strtol(channel, NULL, 10); // TODO: check if strol is valid
	if (channelnr > TI_PRUSS_IRQS \|\| channelnr < 0)
	{
	device_printf(sc->sc_pdev->si_drv1,
	"Channel number %d not valid (0 - %d)",
	channelnr, TI_PRUSS_IRQS-1);
	return (EINVAL);
	}

	sc->sc_irq_devs[irq].channel = channelnr;
	sc->sc_irq_devs[irq].last = -1;

	// channel[nr] <= irqnr
	ti_pruss_map_write(sc, PRUSS_INTC_HMR_BASE,
	irq, channelnr);
	}
	}

	return (err);
	}

	static int
	ti_pruss_interrupt_enable(SYSCTL_HANDLER_ARGS)
	{
	struct ti_pruss_softc *sc;
	int err;
	bool irqenable;
	const int8_t irq = arg2;

	sc = arg1;
	irqenable = sc->sc_irq_devs[arg2].enable;

	err = sysctl_handle_bool(oidp, &irqenable, arg2, req);
	if (err != 0)
	return (err);

	if (req->newptr) // write enable
	return ti_pruss_interrupts_enable(sc, irq, irqenable);

	return (err);
	}

	static int
	ti_pruss_global_interrupt_enable(SYSCTL_HANDLER_ARGS)
	{
	struct ti_pruss_softc *sc;
	int err;
	bool glob_irqen;

	sc = arg1;
	glob_irqen = sc->sc_glob_irqen;

	err = sysctl_handle_bool(oidp, &glob_irqen, arg2, req);
	if (err != 0)
	return (err);

	if (req->newptr) {
	sc->sc_glob_irqen = glob_irqen;
	ti_pruss_reg_write(sc, PRUSS_INTC_GER, glob_irqen);
	}

	return (err);
	}
	static int
	ti_pruss_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_is_compatible(dev, "ti,pruss-v1") \|\|
	ofw_bus_is_compatible(dev, "ti,pruss-v2")) {
	device_set_desc(dev, "TI Programmable Realtime Unit Subsystem");
	return (BUS_PROBE_DEFAULT);
	}

	return (ENXIO);
	}

	static int
	ti_pruss_attach(device_t dev)
	{
	struct ti_pruss_softc *sc;
	- int rid, i;
	+ int rid, i, err, ncells;
	+ uint32_t reg;
	+ phandle_t node;
	+ clk_t l3_gclk, pruss_ocp_gclk;
	+ phandle_t ti_prm_ref, *cells;
	+ device_t ti_prm_dev;

	- if (ti_prcm_clk_enable(PRUSS_CLK) != 0) {
	- device_printf(dev, "could not enable PRUSS clock\n");
	+ rid = 0;
	+ sc = device_get_softc(dev);
	+ node = ofw_bus_get_node(device_get_parent(dev));
	+ if (node <= 0) {
	+ device_printf(dev, "Cant get ofw node\n");
	return (ENXIO);
	}
	- sc = device_get_softc(dev);
	- rid = 0;
	+
	+ /*
	+ * Follow activate pattern from sys/arm/ti/am335x/am335x_prcm.c
	+ * by Damjan Marion
	+ */
	+
	+ /* Set MODULEMODE to ENABLE(2) */
	+ /* Wait for MODULEMODE to become ENABLE(2) */
	+ if (ti_sysc_clock_enable(device_get_parent(dev)) != 0) {
	+ device_printf(dev, "Could not enable PRUSS clock\n");
	+ return (ENXIO);
	+ }
	+
	+ /* Set CLKTRCTRL to SW_WKUP(2) */
	+ /* Wait for the 200 MHz OCP clock to become active */
	+ /* Wait for the 200 MHz IEP clock to become active */
	+ /* Wait for the 192 MHz UART clock to become active */
	+ /*
	+ * At the moment there is no reference to CM_PER_PRU_ICSS_CLKSTCTRL@140
	+ * in the devicetree. The register reset state are SW_WKUP(2) as default
	+ * so at the moment ignore setting this register.
	+ */
	+
	+ /* Select L3F as OCP clock */
	+ /* Get the clock and set the parent */
	+ err = clk_get_by_name(dev, "l3_gclk", &l3_gclk);
	+ if (err) {
	+ device_printf(dev, "Cant get l3_gclk err %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ err = clk_get_by_name(dev, "pruss_ocp_gclk@530", &pruss_ocp_gclk);
	+ if (err) {
	+ device_printf(dev, "Cant get pruss_ocp_gclk@530 err %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ err = clk_set_parent_by_clk(pruss_ocp_gclk, l3_gclk);
	+ if (err) {
	+ device_printf(dev,
	+ "Cant set pruss_ocp_gclk parent to l3_gclk err %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ /* Clear the RESET bit */
	+ /* Find the ti_prm */
	+ /* #reset-cells should not been used in this way but... */
	+ err = ofw_bus_parse_xref_list_alloc(node, "resets", "#reset-cells", 0,
	+ &ti_prm_ref, &ncells, &cells);
	+ OF_prop_free(cells);
	+ if (err) {
	+ device_printf(dev,
	+ "Cant fetch \"resets\" reference %x\n", err);
	+ return (ENXIO);
	+ }
	+
	+ ti_prm_dev = OF_device_from_xref(ti_prm_ref);
	+ if (ti_prm_dev == NULL) {
	+ device_printf(dev, "Cant get device from \"resets\"\n");
	+ return (ENXIO);
	+ }
	+
	+ err = ti_prm_reset(ti_prm_dev);
	+ if (err) {
	+ device_printf(dev, "ti_prm_reset failed %d\n", err);
	+ return (ENXIO);
	+ }
	+ /* End of clock activation */
	+
	mtx_init(&sc->sc_mtx, "TI PRUSS", NULL, MTX_DEF);
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (sc->sc_mem_res == NULL) {
	device_printf(dev, "could not allocate memory resource\n");
	return (ENXIO);
	}

	struct sysctl_ctx_list *clist = device_get_sysctl_ctx(dev);
	if (!clist)
	return (EINVAL);

	struct sysctl_oid *poid;
	poid = device_get_sysctl_tree( dev );
	if (!poid)
	return (EINVAL);

	sc->sc_glob_irqen = false;
	struct sysctl_oid *irq_root = SYSCTL_ADD_NODE(clist, SYSCTL_CHILDREN(poid),
	OID_AUTO, "irq", CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"PRUSS Host Interrupts");
	SYSCTL_ADD_PROC(clist, SYSCTL_CHILDREN(poid), OID_AUTO,
	"global_interrupt_enable",
	CTLFLAG_RW \| CTLTYPE_U8 \| CTLFLAG_NEEDGIANT,
	sc, 0, ti_pruss_global_interrupt_enable,
	"CU", "Global interrupt enable");

	sc->sc_bt = rman_get_bustag(sc->sc_mem_res);
	sc->sc_bh = rman_get_bushandle(sc->sc_mem_res);
	if (bus_alloc_resources(dev, ti_pruss_irq_spec, sc->sc_irq_res) != 0) {
	device_printf(dev, "could not allocate interrupt resource\n");
	ti_pruss_detach(dev);
	return (ENXIO);
	}

	ti_pruss_interrupts_clear(sc);

	for (i = 0; i < TI_PRUSS_IRQS; i++) {
	char name[8];
	snprintf(name, sizeof(name), "%d", i);

	struct sysctl_oid *irq_nodes = SYSCTL_ADD_NODE(clist, SYSCTL_CHILDREN(irq_root),
	OID_AUTO, name, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"PRUSS Interrupts");
	SYSCTL_ADD_PROC(clist, SYSCTL_CHILDREN(irq_nodes), OID_AUTO,
	"channel", CTLFLAG_RW \| CTLTYPE_STRING \| CTLFLAG_NEEDGIANT,
	sc, i, ti_pruss_channel_map,
	"A", "Channel attached to this irq");
	SYSCTL_ADD_PROC(clist, SYSCTL_CHILDREN(irq_nodes), OID_AUTO,
	"event", CTLFLAG_RW \| CTLTYPE_STRING \| CTLFLAG_NEEDGIANT,
	sc, i, ti_pruss_event_map,
	"A", "Event attached to this irq");
	SYSCTL_ADD_PROC(clist, SYSCTL_CHILDREN(irq_nodes), OID_AUTO,
	"enable", CTLFLAG_RW \| CTLTYPE_U8 \| CTLFLAG_NEEDGIANT,
	sc, i, ti_pruss_interrupt_enable,
	"CU", "Enable/Disable interrupt");

	sc->sc_irq_devs[i].event = -1;
	sc->sc_irq_devs[i].channel = -1;
	sc->sc_irq_devs[i].tstamps.ctl.idx = 0;

	if (i < TI_PRUSS_HOST_IRQS) {
	ti_pruss_irq_args[i].irq = i;
	ti_pruss_irq_args[i].sc = sc;
	if (bus_setup_intr(dev, sc->sc_irq_res[i],
	INTR_MPSAFE \| INTR_TYPE_MISC,
	NULL, ti_pruss_intr, &ti_pruss_irq_args[i],
	&sc->sc_intr[i]) != 0) {
	device_printf(dev,
	"unable to setup the interrupt handler\n");
	ti_pruss_detach(dev);

	return (ENXIO);
	}
	mtx_init(&sc->sc_irq_devs[i].sc_mtx, "TI PRUSS IRQ", NULL, MTX_DEF);
	knlist_init_mtx(&sc->sc_irq_devs[i].sc_selinfo.si_note, &sc->sc_irq_devs[i].sc_mtx);
	}
	}

	+ reg = ti_pruss_reg_read(sc,
	+ ti_sysc_get_sysc_address_offset_host(device_get_parent(dev)));
	+
	if (ti_pruss_reg_read(sc, PRUSS_AM33XX_INTC) == PRUSS_AM33XX_REV)
	device_printf(dev, "AM33xx PRU-ICSS\n");

	sc->sc_pdev = make_dev(&ti_pruss_cdevsw, 0, UID_ROOT, GID_WHEEL,
	0600, "pruss%d", device_get_unit(dev));
	sc->sc_pdev->si_drv1 = dev;

	/* Acc. to datasheet always write 1 to polarity registers */
	ti_pruss_reg_write(sc, PRUSS_INTC_SIPR0, 0xFFFFFFFF);
	ti_pruss_reg_write(sc, PRUSS_INTC_SIPR1, 0xFFFFFFFF);

	/* Acc. to datasheet always write 0 to event type registers */
	ti_pruss_reg_write(sc, PRUSS_INTC_SITR0, 0);
	ti_pruss_reg_write(sc, PRUSS_INTC_SITR1, 0);

	return (0);
	}

	static int
	ti_pruss_detach(device_t dev)
	{
	struct ti_pruss_softc *sc = device_get_softc(dev);

	ti_pruss_interrupts_clear(sc);

	for (int i = 0; i < TI_PRUSS_HOST_IRQS; i++) {
	ti_pruss_interrupts_enable( sc, i, false );

	if (sc->sc_intr[i])
	bus_teardown_intr(dev, sc->sc_irq_res[i], sc->sc_intr[i]);
	if (sc->sc_irq_res[i])
	bus_release_resource(dev, SYS_RES_IRQ,
	rman_get_rid(sc->sc_irq_res[i]),
	sc->sc_irq_res[i]);
	knlist_clear(&sc->sc_irq_devs[i].sc_selinfo.si_note, 0);
	mtx_lock(&sc->sc_irq_devs[i].sc_mtx);
	if (!knlist_empty(&sc->sc_irq_devs[i].sc_selinfo.si_note))
	printf("IRQ %d KQueue not empty!\n", i );
	mtx_unlock(&sc->sc_irq_devs[i].sc_mtx);
	knlist_destroy(&sc->sc_irq_devs[i].sc_selinfo.si_note);
	mtx_destroy(&sc->sc_irq_devs[i].sc_mtx);
	}

	mtx_destroy(&sc->sc_mtx);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_mem_res),
	sc->sc_mem_res);
	if (sc->sc_pdev)
	destroy_dev(sc->sc_pdev);

	return (0);
	}

	static void
	ti_pruss_intr(void *arg)
	{
	int val;
	struct ti_pruss_irq_arg *iap = arg;
	struct ti_pruss_softc *sc = iap->sc;
	/*
	* Interrupts pr1_host_intr[0:7] are mapped to
	* Host-2 to Host-9 of PRU-ICSS IRQ-controller.
	*/
	const int pru_int = iap->irq + TI_PRUSS_PRU_IRQS;
	const int pru_int_mask = (1 << pru_int);
	const int pru_channel = sc->sc_irq_devs[pru_int].channel;
	const int pru_event = sc->sc_irq_devs[pru_channel].event;

	val = ti_pruss_reg_read(sc, PRUSS_INTC_HIER);
	if (!(val & pru_int_mask))
	return;

	ti_pruss_reg_write(sc, PRUSS_INTC_HIDISR, pru_int);
	ti_pruss_reg_write(sc, PRUSS_INTC_SICR, pru_event);
	ti_pruss_reg_write(sc, PRUSS_INTC_HIEISR, pru_int);

	struct ti_pruss_irqsc* irq = &sc->sc_irq_devs[pru_channel];
	size_t wr = irq->tstamps.ctl.idx;

	struct timespec ts;
	nanouptime(&ts);
	irq->tstamps.ts[wr] = ts.tv_sec * 1000000000 + ts.tv_nsec;

	if (++wr == TI_TS_ARRAY)
	wr = 0;
	atomic_add_32(&irq->tstamps.ctl.cnt, 1);

	irq->tstamps.ctl.idx = wr;

	KNOTE_UNLOCKED(&irq->sc_selinfo.si_note, pru_int);
	wakeup(irq);
	selwakeup(&irq->sc_selinfo);
	}

	static int
	ti_pruss_open(struct cdev *cdev __unused, int oflags __unused,
	int devtype __unused, struct thread *td __unused)
	{
	return (0);
	}

	static int
	ti_pruss_mmap(struct cdev cdev, vm_ooffset_t offset, vm_paddr_t paddr,
	int nprot, vm_memattr_t *memattr)
	{
	device_t dev = cdev->si_drv1;
	struct ti_pruss_softc *sc = device_get_softc(dev);

	if (offset >= rman_get_size(sc->sc_mem_res))
	return (ENOSPC);
	*paddr = rman_get_start(sc->sc_mem_res) + offset;
	*memattr = VM_MEMATTR_UNCACHEABLE;

	return (0);
	}

	static struct filterops ti_pruss_kq_read = {
	.f_isfd = 1,
	.f_detach = ti_pruss_irq_kqread_detach,
	.f_event = ti_pruss_irq_kqevent,
	};

	static void
	ti_pruss_irq_kqread_detach(struct knote *kn)
	{
	struct ti_pruss_irqsc *sc = kn->kn_hook;

	knlist_remove(&sc->sc_selinfo.si_note, kn, 0);
	}

	static int
	ti_pruss_irq_kqevent(struct knote *kn, long hint)
	{
	struct ti_pruss_irqsc* irq_sc;
	int notify;

	irq_sc = kn->kn_hook;

	if (hint > 0)
	kn->kn_data = hint - 2;

	if (hint > 0 \|\| irq_sc->last > 0)
	notify = 1;
	else
	notify = 0;

	irq_sc->last = hint;

	return (notify);
	}

	static int
	ti_pruss_irq_kqfilter(struct cdev cdev, struct knote kn)
	{
	struct ti_pruss_irqsc *sc = cdev->si_drv1;

	switch (kn->kn_filter) {
	case EVFILT_READ:
	kn->kn_hook = sc;
	kn->kn_fop = &ti_pruss_kq_read;
	knlist_add(&sc->sc_selinfo.si_note, kn, 0);
	break;
	default:
	return (EINVAL);
	}

	return (0);
	}
	diff --git a/sys/arm/ti/ti_scm.c b/sys/arm/ti/ti_scm.c
	index fbc87eeb4f92..896a8d09cbf3 100644
	--- a/sys/arm/ti/ti_scm.c
	+++ b/sys/arm/ti/ti_scm.c
	@@ -1,181 +1,162 @@
	/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	+ * Copyright (c) 2019 Emmanuel Vadot <manu@FreeBSD.org>
	*
	- * Copyright (c) 2010
	- * Ben Gray <ben.r.gray@gmail.com>.
	- * All rights reserved.
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Ben Gray.
	- * 4. The name of the company nor the name of the author may be used to
	- * endorse or promote products derived from this software without specific
	- * prior written permission.
	*
	- * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	- * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
	- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
	- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	- */
	-
	-/**
	- * SCM - System Control Module
	- *
	- * Hopefully in the end this module will contain a bunch of utility functions
	- * for configuring and querying the general system control registers, but for
	- * now it only does pin(pad) multiplexing.
	- *
	- * This is different from the GPIO module in that it is used to configure the
	- * pins between modules not just GPIO input/output.
	- *
	- * This file contains the generic top level driver, however it relies on chip
	- * specific settings and therefore expects an array of ti_scm_padconf structs
	- * call ti_padconf_devmap to be located somewhere in the kernel.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	*
	+ * $FreeBSD$
	*/
	+
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	+
	+/* Based on sys/arm/ti/ti_sysc.c */
	+
	#include <sys/param.h>
	#include <sys/systm.h>
	+#include <sys/bus.h>
	+#include <sys/fbio.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	-#include <sys/bus.h>
	-#include <sys/resource.h>
	#include <sys/rman.h>
	-#include <sys/lock.h>
	-#include <sys/mutex.h>
	-
	+#include <sys/resource.h>
	#include <machine/bus.h>
	-#include <machine/resource.h>
	+#include <vm/vm.h>
	+#include <vm/vm_extern.h>
	+#include <vm/vm_kern.h>
	+#include <vm/pmap.h>
	+
	+#include <dev/fdt/simplebus.h>

	-#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>
	-#include <dev/fdt/fdt_pinctrl.h>
	-
	-#include "ti_scm.h"
	-#include "ti_cpuid.h"

	-static struct resource_spec ti_scm_res_spec[] = {
	- { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Control memory window */
	- { -1, 0 }
	+#define TI_AM3_SCM 14
	+#define TI_AM4_SCM 13
	+#define TI_DM814_SCRM 12
	+#define TI_DM816_SCRM 11
	+#define TI_OMAP2_SCM 10
	+#define TI_OMAP3_SCM 9
	+#define TI_OMAP4_SCM_CORE 8
	+#define TI_OMAP4_SCM_PADCONF_CORE 7
	+#define TI_OMAP4_SCM_WKUP 6
	+#define TI_OMAP4_SCM_PADCONF_WKUP 5
	+#define TI_OMAP5_SCM_CORE 4
	+#define TI_OMAP5_SCM_PADCONF_CORE 3
	+#define TI_OMAP5_SCM_WKUP_PAD_CONF 2
	+#define TI_DRA7_SCM_CORE 1
	+#define TI_SCM_END 0
	+
	+static struct ofw_compat_data compat_data[] = {
	+ { "ti,am3-scm", TI_AM3_SCM },
	+ { "ti,am4-scm", TI_AM4_SCM },
	+ { "ti,dm814-scrm", TI_DM814_SCRM },
	+ { "ti,dm816-scrm", TI_DM816_SCRM },
	+ { "ti,omap2-scm", TI_OMAP2_SCM },
	+ { "ti,omap3-scm", TI_OMAP3_SCM },
	+ { "ti,omap4-scm-core", TI_OMAP4_SCM_CORE },
	+ { "ti,omap4-scm-padconf-core", TI_OMAP4_SCM_PADCONF_CORE },
	+ { "ti,omap4-scm-wkup", TI_OMAP4_SCM_WKUP },
	+ { "ti,omap4-scm-padconf-wkup", TI_OMAP4_SCM_PADCONF_WKUP },
	+ { "ti,omap5-scm-core", TI_OMAP5_SCM_CORE },
	+ { "ti,omap5-scm-padconf-core", TI_OMAP5_SCM_PADCONF_CORE },
	+ { "ti,omap5-scm-wkup-pad-conf", TI_OMAP5_SCM_WKUP_PAD_CONF },
	+ { "ti,dra7-scm-core", TI_DRA7_SCM_CORE },
	+ { NULL, TI_SCM_END }
	};

	-static struct ti_scm_softc *ti_scm_sc;
	+struct ti_scm_softc {
	+ struct simplebus_softc sc;
	+ device_t dev;
	+};

	-#define ti_scm_read_4(sc, reg) \
	- bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg))
	-#define ti_scm_write_4(sc, reg, val) \
	- bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
	+static int ti_scm_probe(device_t dev);
	+static int ti_scm_attach(device_t dev);
	+static int ti_scm_detach(device_t dev);

	-/*
	- * Device part of OMAP SCM driver
	- */
	static int
	ti_scm_probe(device_t dev)
	{
	-
	- if (!ti_soc_is_supported())
	- return (ENXIO);
	-
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	- if (!ofw_bus_is_compatible(dev, "syscon"))
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	- if (ti_scm_sc) {
	- return (EEXIST);
	- }
	+ device_set_desc(dev, "TI OMAP Control Module");

	- device_set_desc(dev, "TI Control Module");
	return (BUS_PROBE_DEFAULT);
	}

	-/**
	- * ti_scm_attach - attaches the timer to the simplebus
	- * @dev: new device
	- *
	- * Reserves memory and interrupt resources, stores the softc structure
	- * globally and registers both the timecount and eventtimer objects.
	- *
	- * RETURNS
	- * Zero on success or ENXIO if an error occuried.
	- */
	static int
	ti_scm_attach(device_t dev)
	{
	- struct ti_scm_softc *sc = device_get_softc(dev);
	+ struct ti_scm_softc *sc;
	+ device_t cdev;
	+ phandle_t node, child;

	- sc->sc_dev = dev;
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+ node = ofw_bus_get_node(dev);

	- if (bus_alloc_resources(dev, ti_scm_res_spec, sc->sc_res)) {
	- device_printf(dev, "could not allocate resources\n");
	+ simplebus_init(dev, node);
	+ if (simplebus_fill_ranges(node, &sc->sc) < 0) {
	+ device_printf(dev, "could not get ranges\n");
	return (ENXIO);
	}

	- /* Global timer interface */
	- sc->sc_bst = rman_get_bustag(sc->sc_res[0]);
	- sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]);
	-
	- ti_scm_sc = sc;
	-
	- /* Attach platform extensions, if any. */
	- bus_generic_probe(dev);
	+ for (child = OF_child(node); child > 0; child = OF_peer(child)) {
	+ cdev = simplebus_add_device(dev, child, 0, NULL, -1, NULL);
	+ if (cdev != NULL)
	+ device_probe_and_attach(cdev);
	+ }

	return (bus_generic_attach(dev));
	}

	-int
	-ti_scm_reg_read_4(uint32_t reg, uint32_t *val)
	-{
	- if (!ti_scm_sc)
	- return (ENXIO);
	-
	- *val = ti_scm_read_4(ti_scm_sc, reg);
	- return (0);
	-}
	-
	-int
	-ti_scm_reg_write_4(uint32_t reg, uint32_t val)
	+static int
	+ti_scm_detach(device_t dev)
	{
	- if (!ti_scm_sc)
	- return (ENXIO);
	-
	- ti_scm_write_4(ti_scm_sc, reg, val);
	- return (0);
	+ return (EBUSY);
	}

	-
	static device_method_t ti_scm_methods[] = {
	+ /* Device interface */
	DEVMETHOD(device_probe, ti_scm_probe),
	DEVMETHOD(device_attach, ti_scm_attach),
	+ DEVMETHOD(device_detach, ti_scm_detach),

	- { 0, 0 }
	+ DEVMETHOD_END
	};

	-static driver_t ti_scm_driver = {
	- "ti_scm",
	- ti_scm_methods,
	- sizeof(struct ti_scm_softc),
	-};
	+DEFINE_CLASS_1(ti_scm, ti_scm_driver, ti_scm_methods,
	+ sizeof(struct ti_scm_softc), simplebus_driver);

	static devclass_t ti_scm_devclass;

	-EARLY_DRIVER_MODULE(ti_scm, simplebus, ti_scm_driver, ti_scm_devclass, 0, 0,
	- BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+EARLY_DRIVER_MODULE(ti_scm, simplebus, ti_scm_driver,
	+ ti_scm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
	+MODULE_VERSION(ti_scm, 1);
	+MODULE_DEPEND(ti_scm, ti_sysc, 1, 1, 1);
	+
	diff --git a/sys/arm/ti/ti_scm_syscon.c b/sys/arm/ti/ti_scm_syscon.c
	new file mode 100644
	index 000000000000..2c3fa9345210
	--- /dev/null
	+++ b/sys/arm/ti/ti_scm_syscon.c
	@@ -0,0 +1,294 @@
	+/*-
	+ * Copyright (c) 2019 Emmanuel Vadot <manu@FreeBSD.org>
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+/* Based on sys/arm/ti/ti_sysc.c */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/kernel.h>
	+#include <sys/module.h>
	+#include <sys/bus.h>
	+#include <sys/resource.h>
	+#include <sys/rman.h>
	+#include <sys/lock.h>
	+#include <sys/mutex.h>
	+
	+#include <machine/bus.h>
	+#include <machine/resource.h>
	+
	+#include <dev/fdt/simplebus.h>
	+
	+#include <dev/ofw/openfirm.h>
	+#include <dev/ofw/ofw_bus.h>
	+#include <dev/ofw/ofw_bus_subr.h>
	+
	+#include "syscon_if.h"
	+#include <dev/extres/syscon/syscon.h>
	+#include "clkdev_if.h"
	+
	+#if 0
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+MALLOC_DECLARE(M_SYSCON);
	+
	+struct ti_scm_syscon_softc {
	+ struct simplebus_softc sc_simplebus;
	+ device_t dev;
	+ struct syscon * syscon;
	+ struct resource * res[1];
	+ bus_space_tag_t bst;
	+ bus_space_handle_t bsh;
	+ struct mtx mtx;
	+};
	+
	+static struct resource_spec ti_scm_syscon_res_spec[] = {
	+ { SYS_RES_MEMORY, 0, RF_ACTIVE \| RF_SHAREABLE },
	+ { -1, 0 }
	+};
	+
	+/* Device */
	+static struct ofw_compat_data compat_data[] = {
	+ { "syscon", 1 },
	+ { NULL, 0 }
	+};
	+
	+/* --- dev/extres/syscon syscon_method_t interface --- */
	+static int
	+ti_scm_syscon_write_4(struct syscon *syscon, bus_size_t offset, uint32_t val)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+
	+ sc = device_get_softc(syscon->pdev);
	+ DPRINTF(sc->dev, "offset=%lx write %x\n", offset, val);
	+ mtx_lock(&sc->mtx);
	+ bus_space_write_4(sc->bst, sc->bsh, offset, val);
	+ mtx_unlock(&sc->mtx);
	+ return (0);
	+}
	+
	+static uint32_t
	+ti_scm_syscon_read_4(struct syscon *syscon, bus_size_t offset)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+ uint32_t val;
	+
	+ sc = device_get_softc(syscon->pdev);
	+
	+ mtx_lock(&sc->mtx);
	+ val = bus_space_read_4(sc->bst, sc->bsh, offset);
	+ mtx_unlock(&sc->mtx);
	+ DPRINTF(sc->dev, "offset=%lx Read %x\n", offset, val);
	+ return (val);
	+}
	+static int
	+ti_scm_syscon_modify_4(struct syscon *syscon, bus_size_t offset, uint32_t clr, uint32_t set)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+ uint32_t reg;
	+
	+ sc = device_get_softc(syscon->pdev);
	+
	+ mtx_lock(&sc->mtx);
	+ reg = bus_space_read_4(sc->bst, sc->bsh, offset);
	+ reg &= ~clr;
	+ reg \|= set;
	+ bus_space_write_4(sc->bst, sc->bsh, offset, reg);
	+ mtx_unlock(&sc->mtx);
	+ DPRINTF(sc->dev, "offset=%lx reg: %x (clr %x set %x)\n", offset, reg, clr, set);
	+
	+ return (0);
	+}
	+
	+static syscon_method_t ti_scm_syscon_reg_methods[] = {
	+ SYSCONMETHOD(syscon_read_4, ti_scm_syscon_read_4),
	+ SYSCONMETHOD(syscon_write_4, ti_scm_syscon_write_4),
	+ SYSCONMETHOD(syscon_modify_4, ti_scm_syscon_modify_4),
	+
	+ SYSCONMETHOD_END
	+};
	+
	+DEFINE_CLASS_1(ti_scm_syscon_reg, ti_scm_syscon_reg_class, ti_scm_syscon_reg_methods,
	+ 0, syscon_class);
	+
	+/* device interface */
	+static int
	+ti_scm_syscon_probe(device_t dev)
	+{
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	+ return (ENXIO);
	+
	+ device_set_desc(dev, "TI OMAP Control Module Syscon");
	+ return(BUS_PROBE_DEFAULT);
	+}
	+
	+static int
	+ti_scm_syscon_attach(device_t dev)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+ phandle_t node, child;
	+ int err;
	+
	+ sc = device_get_softc(dev);
	+ sc->dev = dev;
	+
	+ if (bus_alloc_resources(dev, ti_scm_syscon_res_spec, sc->res)) {
	+ device_printf(sc->dev, "Cant allocate resources\n");
	+ return (ENXIO);
	+ }
	+
	+ sc->dev = dev;
	+ sc->bst = rman_get_bustag(sc->res[0]);
	+ sc->bsh = rman_get_bushandle(sc->res[0]);
	+
	+ mtx_init(&sc->mtx, device_get_nameunit(sc->dev), NULL, MTX_DEF);
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ /* dev/extres/syscon interface */
	+ sc->syscon = syscon_create_ofw_node(dev, &ti_scm_syscon_reg_class, node);
	+ if (sc->syscon == NULL) {
	+ device_printf(dev, "Failed to create/register syscon\n");
	+ return (ENXIO);
	+ }
	+
	+ simplebus_init(sc->dev, node);
	+
	+ err = bus_generic_probe(sc->dev);
	+ for (child = OF_child(node); child != 0; child = OF_peer(child)) {
	+ simplebus_add_device(sc->dev, child, 0, NULL, -1, NULL);
	+ }
	+
	+ return (bus_generic_attach(sc->dev));
	+}
	+
	+/* syscon interface */
	+static int
	+ti_scm_syscon_get_handle(device_t dev, struct syscon **syscon)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+
	+ sc = device_get_softc(dev);
	+ *syscon = sc->syscon;
	+ if (*syscon == NULL)
	+ return (ENODEV);
	+ return (0);
	+}
	+
	+/* clkdev interface */
	+static int
	+ti_scm_syscon_clk_write_4(device_t dev, bus_addr_t addr, uint32_t val)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+
	+ sc = device_get_softc(dev);
	+ DPRINTF(sc->dev, "offset=%lx write %x\n", addr, val);
	+ bus_space_write_4(sc->bst, sc->bsh, addr, val);
	+ return (0);
	+}
	+
	+static int
	+ti_scm_syscon_clk_read_4(device_t dev, bus_addr_t addr, uint32_t *val)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+
	+ sc = device_get_softc(dev);
	+
	+ *val = bus_space_read_4(sc->bst, sc->bsh, addr);
	+ DPRINTF(sc->dev, "offset=%lx Read %x\n", addr, *val);
	+ return (0);
	+}
	+
	+static int
	+ti_scm_syscon_clk_modify_4(device_t dev, bus_addr_t addr, uint32_t clr, uint32_t set)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+ uint32_t reg;
	+
	+ sc = device_get_softc(dev);
	+
	+ reg = bus_space_read_4(sc->bst, sc->bsh, addr);
	+ reg &= ~clr;
	+ reg \|= set;
	+ bus_space_write_4(sc->bst, sc->bsh, addr, reg);
	+ DPRINTF(sc->dev, "offset=%lx reg: %x (clr %x set %x)\n", addr, reg, clr, set);
	+
	+ return (0);
	+}
	+
	+static void
	+ti_scm_syscon_clk_device_lock(device_t dev)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+
	+ sc = device_get_softc(dev);
	+ mtx_lock(&sc->mtx);
	+}
	+
	+static void
	+ti_scm_syscon_clk_device_unlock(device_t dev)
	+{
	+ struct ti_scm_syscon_softc *sc;
	+ sc = device_get_softc(dev);
	+ mtx_unlock(&sc->mtx);
	+}
	+
	+static device_method_t ti_scm_syscon_methods[] = {
	+ DEVMETHOD(device_probe, ti_scm_syscon_probe),
	+ DEVMETHOD(device_attach, ti_scm_syscon_attach),
	+
	+ /* syscon interface */
	+ DEVMETHOD(syscon_get_handle, ti_scm_syscon_get_handle),
	+
	+ /* clkdev interface */
	+ DEVMETHOD(clkdev_write_4, ti_scm_syscon_clk_write_4),
	+ DEVMETHOD(clkdev_read_4, ti_scm_syscon_clk_read_4),
	+ DEVMETHOD(clkdev_modify_4, ti_scm_syscon_clk_modify_4),
	+ DEVMETHOD(clkdev_device_lock, ti_scm_syscon_clk_device_lock),
	+ DEVMETHOD(clkdev_device_unlock, ti_scm_syscon_clk_device_unlock),
	+
	+ DEVMETHOD_END
	+};
	+
	+
	+DEFINE_CLASS_1(ti_scm_syscon, ti_scm_syscon_driver, ti_scm_syscon_methods,
	+ sizeof(struct ti_scm_syscon_softc), simplebus_driver);
	+
	+static devclass_t ti_scm_syscon_devclass;
	+
	+EARLY_DRIVER_MODULE(ti_scm_syscon, simplebus, ti_scm_syscon_driver,
	+ ti_scm_syscon_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	+MODULE_VERSION(ti_scm_syscon, 1);
	+MODULE_DEPEND(ti_scm_syscon, ti_scm, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_sdhci.c b/sys/arm/ti/ti_sdhci.c
	index a76453d92588..4d19f4663d2f 100644
	--- a/sys/arm/ti/ti_sdhci.c
	+++ b/sys/arm/ti/ti_sdhci.c
	@@ -1,761 +1,770 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 Ian Lepore <ian@freebsd.org>
	* Copyright (c) 2011 Ben Gray <ben.r.gray@gmail.com>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/gpio.h>
	#include <sys/kernel.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/resource.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	#include <sys/taskqueue.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>

	-#include <machine/bus.h>
	-#include <machine/resource.h>
	-#include <machine/intr.h>
	+#include <arm/ti/ti_cpuid.h>
	+#include <arm/ti/ti_sysc.h>
	+#include "gpio_if.h"

	+#include <dev/extres/clk/clk.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/mmc/bridge.h>
	#include <dev/mmc/mmcreg.h>
	#include <dev/mmc/mmcbrvar.h>

	#include <dev/sdhci/sdhci.h>
	#include <dev/sdhci/sdhci_fdt_gpio.h>
	#include "sdhci_if.h"

	-#include <arm/ti/ti_cpuid.h>
	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	-#include "gpio_if.h"
	+#include <machine/bus.h>
	+#include <machine/resource.h>
	+#include <machine/intr.h>
	+

	#include "opt_mmccam.h"

	struct ti_sdhci_softc {
	device_t dev;
	struct sdhci_fdt_gpio * gpio;
	struct resource * mem_res;
	struct resource * irq_res;
	void * intr_cookie;
	struct sdhci_slot slot;
	- clk_ident_t mmchs_clk_id;
	uint32_t mmchs_reg_off;
	uint32_t sdhci_reg_off;
	- uint32_t baseclk_hz;
	+ uint64_t baseclk_hz;
	uint32_t cmd_and_mode;
	uint32_t sdhci_clkdiv;
	boolean_t disable_highspeed;
	boolean_t force_card_present;
	boolean_t disable_readonly;
	};

	/*
	* Table of supported FDT compat strings.
	*
	* Note that "ti,mmchs" is our own invention, and should be phased out in favor
	* of the documented names.
	*
	* Note that vendor Beaglebone dtsi files use "ti,omap3-hsmmc" for the am335x.
	*/
	static struct ofw_compat_data compat_data[] = {
	{"ti,omap3-hsmmc", 1},
	{"ti,omap4-hsmmc", 1},
	{"ti,mmchs", 1},
	{NULL, 0},
	};

	/*
	* The MMCHS hardware has a few control and status registers at the beginning of
	* the device's memory map, followed by the standard sdhci register block.
	* Different SoCs have the register blocks at different offsets from the
	* beginning of the device. Define some constants to map out the registers we
	* access, and the various per-SoC offsets. The SDHCI_REG_OFFSET is how far
	* beyond the MMCHS block the SDHCI block is found; it's the same on all SoCs.
	*/
	#define OMAP3_MMCHS_REG_OFFSET 0x000
	#define OMAP4_MMCHS_REG_OFFSET 0x100
	#define AM335X_MMCHS_REG_OFFSET 0x100
	#define SDHCI_REG_OFFSET 0x100

	#define MMCHS_SYSCONFIG 0x010
	#define MMCHS_SYSCONFIG_RESET (1 << 1)
	#define MMCHS_SYSSTATUS 0x014
	#define MMCHS_SYSSTATUS_RESETDONE (1 << 0)
	#define MMCHS_CON 0x02C
	#define MMCHS_CON_DW8 (1 << 5)
	#define MMCHS_CON_DVAL_8_4MS (3 << 9)
	#define MMCHS_CON_OD (1 << 0)
	#define MMCHS_SYSCTL 0x12C
	#define MMCHS_SYSCTL_CLKD_MASK 0x3FF
	#define MMCHS_SYSCTL_CLKD_SHIFT 6
	#define MMCHS_SD_CAPA 0x140
	#define MMCHS_SD_CAPA_VS18 (1 << 26)
	#define MMCHS_SD_CAPA_VS30 (1 << 25)
	#define MMCHS_SD_CAPA_VS33 (1 << 24)

	/* Forward declarations, CAM-relataed */
	// static void ti_sdhci_cam_poll(struct cam_sim *);
	// static void ti_sdhci_cam_action(struct cam_sim , union ccb );
	// static int ti_sdhci_cam_settran_settings(struct ti_sdhci_softc sc, union ccb );

	static inline uint32_t
	ti_mmchs_read_4(struct ti_sdhci_softc *sc, bus_size_t off)
	{

	return (bus_read_4(sc->mem_res, off + sc->mmchs_reg_off));
	}

	static inline void
	ti_mmchs_write_4(struct ti_sdhci_softc *sc, bus_size_t off, uint32_t val)
	{

	bus_write_4(sc->mem_res, off + sc->mmchs_reg_off, val);
	}

	static inline uint32_t
	RD4(struct ti_sdhci_softc *sc, bus_size_t off)
	{

	return (bus_read_4(sc->mem_res, off + sc->sdhci_reg_off));
	}

	static inline void
	WR4(struct ti_sdhci_softc *sc, bus_size_t off, uint32_t val)
	{

	bus_write_4(sc->mem_res, off + sc->sdhci_reg_off, val);
	}

	static uint8_t
	ti_sdhci_read_1(device_t dev, struct sdhci_slot *slot, bus_size_t off)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);

	return ((RD4(sc, off & ~3) >> (off & 3) * 8) & 0xff);
	}

	static uint16_t
	ti_sdhci_read_2(device_t dev, struct sdhci_slot *slot, bus_size_t off)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);
	uint32_t clkdiv, val32;

	/*
	* The MMCHS hardware has a non-standard interpretation of the sdclock
	* divisor bits. It uses the same bit positions as SDHCI 3.0 (15..6)
	* but doesn't split them into low:high fields. Instead they're a
	* single number in the range 0..1023 and the number is exactly the
	* clock divisor (with 0 and 1 both meaning divide by 1). The SDHCI
	* driver code expects a v2.0 or v3.0 divisor. The shifting and masking
	* here extracts the MMCHS representation from the hardware word, cleans
	* those bits out, applies the 2N adjustment, and plugs the result into
	* the bit positions for the 2.0 or 3.0 divisor in the returned register
	* value. The ti_sdhci_write_2() routine performs the opposite
	* transformation when the SDHCI driver writes to the register.
	*/
	if (off == SDHCI_CLOCK_CONTROL) {
	val32 = RD4(sc, SDHCI_CLOCK_CONTROL);
	clkdiv = ((val32 >> MMCHS_SYSCTL_CLKD_SHIFT) &
	MMCHS_SYSCTL_CLKD_MASK) / 2;
	val32 &= ~(MMCHS_SYSCTL_CLKD_MASK << MMCHS_SYSCTL_CLKD_SHIFT);
	val32 \|= (clkdiv & SDHCI_DIVIDER_MASK) << SDHCI_DIVIDER_SHIFT;
	if (slot->version >= SDHCI_SPEC_300)
	val32 \|= ((clkdiv >> SDHCI_DIVIDER_MASK_LEN) &
	SDHCI_DIVIDER_HI_MASK) << SDHCI_DIVIDER_HI_SHIFT;
	return (val32 & 0xffff);
	}

	/*
	* Standard 32-bit handling of command and transfer mode.
	*/
	if (off == SDHCI_TRANSFER_MODE) {
	return (sc->cmd_and_mode >> 16);
	} else if (off == SDHCI_COMMAND_FLAGS) {
	return (sc->cmd_and_mode & 0x0000ffff);
	}

	return ((RD4(sc, off & ~3) >> (off & 3) * 8) & 0xffff);
	}

	static uint32_t
	ti_sdhci_read_4(device_t dev, struct sdhci_slot *slot, bus_size_t off)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);
	uint32_t val32;

	val32 = RD4(sc, off);

	/*
	* If we need to disallow highspeed mode due to the OMAP4 erratum, strip
	* that flag from the returned capabilities.
	*/
	if (off == SDHCI_CAPABILITIES && sc->disable_highspeed)
	val32 &= ~SDHCI_CAN_DO_HISPD;

	/*
	* Force the card-present state if necessary.
	*/
	if (off == SDHCI_PRESENT_STATE && sc->force_card_present)
	val32 \|= SDHCI_CARD_PRESENT;

	return (val32);
	}

	static void
	ti_sdhci_read_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint32_t *data, bus_size_t count)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);

	bus_read_multi_4(sc->mem_res, off + sc->sdhci_reg_off, data, count);
	}

	static void
	ti_sdhci_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint8_t val)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);
	uint32_t val32;

	#ifdef MMCCAM
	uint32_t newval32;
	if (off == SDHCI_HOST_CONTROL) {
	val32 = ti_mmchs_read_4(sc, MMCHS_CON);
	newval32 = val32;
	if (val & SDHCI_CTRL_8BITBUS) {
	device_printf(dev, "Custom-enabling 8-bit bus\n");
	newval32 \|= MMCHS_CON_DW8;
	} else {
	device_printf(dev, "Custom-disabling 8-bit bus\n");
	newval32 &= ~MMCHS_CON_DW8;
	}
	if (newval32 != val32)
	ti_mmchs_write_4(sc, MMCHS_CON, newval32);
	}
	#endif
	val32 = RD4(sc, off & ~3);
	val32 &= ~(0xff << (off & 3) * 8);
	val32 \|= (val << (off & 3) * 8);

	WR4(sc, off & ~3, val32);
	}

	static void
	ti_sdhci_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint16_t val)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);
	uint32_t clkdiv, val32;

	/*
	* Translate between the hardware and SDHCI 2.0 or 3.0 representations
	* of the clock divisor. See the comments in ti_sdhci_read_2() for
	* details.
	*/
	if (off == SDHCI_CLOCK_CONTROL) {
	clkdiv = (val >> SDHCI_DIVIDER_SHIFT) & SDHCI_DIVIDER_MASK;
	if (slot->version >= SDHCI_SPEC_300)
	clkdiv \|= ((val >> SDHCI_DIVIDER_HI_SHIFT) &
	SDHCI_DIVIDER_HI_MASK) << SDHCI_DIVIDER_MASK_LEN;
	clkdiv *= 2;
	if (clkdiv > MMCHS_SYSCTL_CLKD_MASK)
	clkdiv = MMCHS_SYSCTL_CLKD_MASK;
	val32 = RD4(sc, SDHCI_CLOCK_CONTROL);
	val32 &= 0xffff0000;
	val32 \|= val & ~(MMCHS_SYSCTL_CLKD_MASK <<
	MMCHS_SYSCTL_CLKD_SHIFT);
	val32 \|= clkdiv << MMCHS_SYSCTL_CLKD_SHIFT;
	WR4(sc, SDHCI_CLOCK_CONTROL, val32);
	return;
	}

	/*
	* Standard 32-bit handling of command and transfer mode.
	*/
	if (off == SDHCI_TRANSFER_MODE) {
	sc->cmd_and_mode = (sc->cmd_and_mode & 0xffff0000) \|
	((uint32_t)val & 0x0000ffff);
	return;
	} else if (off == SDHCI_COMMAND_FLAGS) {
	sc->cmd_and_mode = (sc->cmd_and_mode & 0x0000ffff) \|
	((uint32_t)val << 16);
	WR4(sc, SDHCI_TRANSFER_MODE, sc->cmd_and_mode);
	return;
	}

	val32 = RD4(sc, off & ~3);
	val32 &= ~(0xffff << (off & 3) * 8);
	val32 \|= ((val & 0xffff) << (off & 3) * 8);
	WR4(sc, off & ~3, val32);
	}

	static void
	ti_sdhci_write_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint32_t val)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);

	WR4(sc, off, val);
	}

	static void
	ti_sdhci_write_multi_4(device_t dev, struct sdhci_slot *slot, bus_size_t off,
	uint32_t *data, bus_size_t count)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);

	bus_write_multi_4(sc->mem_res, off + sc->sdhci_reg_off, data, count);
	}

	static void
	ti_sdhci_intr(void *arg)
	{
	struct ti_sdhci_softc *sc = arg;

	sdhci_generic_intr(&sc->slot);
	}

	static int
	ti_sdhci_update_ios(device_t brdev, device_t reqdev)
	{
	struct ti_sdhci_softc *sc = device_get_softc(brdev);
	struct sdhci_slot *slot;
	struct mmc_ios *ios;
	uint32_t val32, newval32;

	slot = device_get_ivars(reqdev);
	ios = &slot->host.ios;

	/*
	* There is an 8-bit-bus bit in the MMCHS control register which, when
	* set, overrides the 1 vs 4 bit setting in the standard SDHCI
	* registers. Set that bit first according to whether an 8-bit bus is
	* requested, then let the standard driver handle everything else.
	*/
	val32 = ti_mmchs_read_4(sc, MMCHS_CON);
	newval32 = val32;

	if (ios->bus_width == bus_width_8)
	newval32 \|= MMCHS_CON_DW8;
	else
	newval32 &= ~MMCHS_CON_DW8;

	if (ios->bus_mode == opendrain)
	newval32 \|= MMCHS_CON_OD;
	else /* if (ios->bus_mode == pushpull) */
	newval32 &= ~MMCHS_CON_OD;

	if (newval32 != val32)
	ti_mmchs_write_4(sc, MMCHS_CON, newval32);

	return (sdhci_generic_update_ios(brdev, reqdev));
	}

	static int
	ti_sdhci_get_ro(device_t brdev, device_t reqdev)
	{
	struct ti_sdhci_softc *sc = device_get_softc(brdev);

	if (sc->disable_readonly)
	return (0);

	return (sdhci_fdt_gpio_get_readonly(sc->gpio));
	}

	static bool
	ti_sdhci_get_card_present(device_t dev, struct sdhci_slot *slot)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);

	return (sdhci_fdt_gpio_get_present(sc->gpio));
	}

	static int
	ti_sdhci_detach(device_t dev)
	{

	/* sdhci_fdt_gpio_teardown(sc->gpio); */

	return (EBUSY);
	}

	-static void
	+static int
	ti_sdhci_hw_init(device_t dev)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);
	uint32_t regval;
	unsigned long timeout;
	+ clk_t mmc_clk;
	+ int err;

	/* Enable the controller and interface/functional clocks */
	- if (ti_prcm_clk_enable(sc->mmchs_clk_id) != 0) {
	+ if (ti_sysc_clock_enable(device_get_parent(dev)) != 0) {
	device_printf(dev, "Error: failed to enable MMC clock\n");
	- return;
	+ return (ENXIO);
	}

	- /* Get the frequency of the source clock */
	- if (ti_prcm_clk_get_source_freq(sc->mmchs_clk_id,
	- &sc->baseclk_hz) != 0) {
	- device_printf(dev, "Error: failed to get source clock freq\n");
	- return;
	+ /* FIXME: Devicetree dosent have any reference to mmc_clk */
	+ err = clk_get_by_name(dev, "mmc_clk", &mmc_clk);
	+ if (err) {
	+ device_printf(dev, "Can not find mmc_clk\n");
	+ return (ENXIO);
	+ }
	+ err = clk_get_freq(mmc_clk, &sc->baseclk_hz);
	+ if (err) {
	+ device_printf(dev, "Cant get mmc_clk frequency\n");
	+ /* AM335x TRM 8.1.6.8 table 8-24 96MHz @ OPP100 */
	+ sc->baseclk_hz = 96000000;
	}

	/* Issue a softreset to the controller */
	ti_mmchs_write_4(sc, MMCHS_SYSCONFIG, MMCHS_SYSCONFIG_RESET);
	timeout = 1000;
	while (!(ti_mmchs_read_4(sc, MMCHS_SYSSTATUS) &
	MMCHS_SYSSTATUS_RESETDONE)) {
	if (--timeout == 0) {
	device_printf(dev,
	"Error: Controller reset operation timed out\n");
	break;
	}
	DELAY(100);
	}

	/*
	* Reset the command and data state machines and also other aspects of
	* the controller such as bus clock and power.
	*
	* If we read the software reset register too fast after writing it we
	* can get back a zero that means the reset hasn't started yet rather
	* than that the reset is complete. Per TI recommendations, work around
	* it by reading until we see the reset bit asserted, then read until
	* it's clear. We also set the SDHCI_QUIRK_WAITFOR_RESET_ASSERTED quirk
	* so that the main sdhci driver uses this same logic in its resets.
	*/
	ti_sdhci_write_1(dev, NULL, SDHCI_SOFTWARE_RESET, SDHCI_RESET_ALL);
	timeout = 10000;
	while ((ti_sdhci_read_1(dev, NULL, SDHCI_SOFTWARE_RESET) &
	SDHCI_RESET_ALL) != SDHCI_RESET_ALL) {
	if (--timeout == 0) {
	break;
	}
	DELAY(1);
	}
	timeout = 10000;
	while ((ti_sdhci_read_1(dev, NULL, SDHCI_SOFTWARE_RESET) &
	SDHCI_RESET_ALL)) {
	if (--timeout == 0) {
	device_printf(dev,
	"Error: Software reset operation timed out\n");
	break;
	}
	DELAY(100);
	}

	/*
	* The attach() routine has examined fdt data and set flags in
	* slot.host.caps to reflect what voltages we can handle. Set those
	* values in the CAPA register. Empirical testing shows that the
	* values in this register can be overwritten at any time, but the
	* manual says that these values should only be set once, "before
	* initialization" whatever that means, and that they survive a reset.
	*/
	regval = ti_mmchs_read_4(sc, MMCHS_SD_CAPA);
	if (sc->slot.host.caps & MMC_OCR_LOW_VOLTAGE)
	regval \|= MMCHS_SD_CAPA_VS18;
	if (sc->slot.host.caps & (MMC_OCR_290_300 \| MMC_OCR_300_310))
	regval \|= MMCHS_SD_CAPA_VS30;
	ti_mmchs_write_4(sc, MMCHS_SD_CAPA, regval);

	/* Set initial host configuration (1-bit, std speed, pwr off). */
	ti_sdhci_write_1(dev, NULL, SDHCI_HOST_CONTROL, 0);
	ti_sdhci_write_1(dev, NULL, SDHCI_POWER_CONTROL, 0);

	/* Set the initial controller configuration. */
	ti_mmchs_write_4(sc, MMCHS_CON, MMCHS_CON_DVAL_8_4MS);
	+
	+ return (0);
	}

	static int
	ti_sdhci_attach(device_t dev)
	{
	struct ti_sdhci_softc *sc = device_get_softc(dev);
	int rid, err;
	pcell_t prop;
	phandle_t node;

	sc->dev = dev;

	/*
	- * Get the MMCHS device id from FDT. If it's not there use the newbus
	- * unit number (which will work as long as the devices are in order and
	- * none are skipped in the fdt). Note that this is a property we made
	- * up and added in freebsd, it doesn't exist in the published bindings.
	+ * Get the MMCHS device id from FDT. Use rev address to identify the unit.
	*/
	node = ofw_bus_get_node(dev);
	- sc->mmchs_clk_id = ti_hwmods_get_clock(dev);
	- if (sc->mmchs_clk_id == INVALID_CLK_IDENT) {
	- device_printf(dev, "failed to get clock based on hwmods property\n");
	- }

	/*
	* The hardware can inherently do dual-voltage (1p8v, 3p0v) on the first
	* device, and only 1p8v on other devices unless an external transceiver
	* is used. The only way we could know about a transceiver is fdt data.
	* Note that we have to do this before calling ti_sdhci_hw_init() so
	* that it can set the right values in the CAPA register.
	*/
	sc->slot.host.caps \|= MMC_OCR_LOW_VOLTAGE;
	- if (sc->mmchs_clk_id == MMC1_CLK \|\| OF_hasprop(node, "ti,dual-volt")) {
	+
	+ if (OF_hasprop(node, "ti,dual-volt")) {
	sc->slot.host.caps \|= MMC_OCR_290_300 \| MMC_OCR_300_310;
	}

	/*
	* Set the offset from the device's memory start to the MMCHS registers.
	* Also for OMAP4 disable high speed mode due to erratum ID i626.
	*/
	switch (ti_chip()) {
	#ifdef SOC_OMAP4
	case CHIP_OMAP_4:
	sc->mmchs_reg_off = OMAP4_MMCHS_REG_OFFSET;
	sc->disable_highspeed = true;
	break;
	#endif
	#ifdef SOC_TI_AM335X
	case CHIP_AM335X:
	sc->mmchs_reg_off = AM335X_MMCHS_REG_OFFSET;
	break;
	#endif
	default:
	panic("Unknown OMAP device\n");
	}

	/*
	* The standard SDHCI registers are at a fixed offset (the same on all
	* SoCs) beyond the MMCHS registers.
	*/
	sc->sdhci_reg_off = sc->mmchs_reg_off + SDHCI_REG_OFFSET;

	/* Resource setup. */
	rid = 0;
	sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (!sc->mem_res) {
	device_printf(dev, "cannot allocate memory window\n");
	err = ENXIO;
	goto fail;
	}

	rid = 0;
	sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE);
	if (!sc->irq_res) {
	device_printf(dev, "cannot allocate interrupt\n");
	err = ENXIO;
	goto fail;
	}

	if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_BIO \| INTR_MPSAFE,
	NULL, ti_sdhci_intr, sc, &sc->intr_cookie)) {
	device_printf(dev, "cannot setup interrupt handler\n");
	err = ENXIO;
	goto fail;
	}

	/*
	* Set up handling of card-detect and write-protect gpio lines.
	*
	* If there is no write protect info in the fdt data, fall back to the
	* historical practice of assuming that the card is writable. This
	* works around bad fdt data from the upstream source. The alternative
	* would be to trust the sdhci controller's PRESENT_STATE register WP
	* bit, but it may say write protect is in effect when it's not if the
	* pinmux setup doesn't route the WP signal into the sdchi block.
	*/
	sc->gpio = sdhci_fdt_gpio_setup(sc->dev, &sc->slot);

	if (!OF_hasprop(node, "wp-gpios") && !OF_hasprop(node, "wp-disable"))
	sc->disable_readonly = true;

	/* Initialise the MMCHS hardware. */
	- ti_sdhci_hw_init(dev);
	+ err = ti_sdhci_hw_init(dev);
	+ if (err != 0) {
	+ /* err should already contain ENXIO from ti_sdhci_hw_init() */
	+ goto fail;
	+ }

	/*
	* The capabilities register can only express base clock frequencies in
	* the range of 0-63MHz for a v2.0 controller. Since our clock runs
	* faster than that, the hardware sets the frequency to zero in the
	* register. When the register contains zero, the sdhci driver expects
	* slot.max_clk to already have the right value in it.
	*/
	sc->slot.max_clk = sc->baseclk_hz;

	/*
	* The MMCHS timeout counter is based on the output sdclock. Tell the
	* sdhci driver to recalculate the timeout clock whenever the output
	* sdclock frequency changes.
	*/
	sc->slot.quirks \|= SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK;

	/*
	* The MMCHS hardware shifts the 136-bit response data (in violation of
	* the spec), so tell the sdhci driver not to do the same in software.
	*/
	sc->slot.quirks \|= SDHCI_QUIRK_DONT_SHIFT_RESPONSE;

	/*
	* Reset bits are broken, have to wait to see the bits asserted
	* before waiting to see them de-asserted.
	*/
	sc->slot.quirks \|= SDHCI_QUIRK_WAITFOR_RESET_ASSERTED;

	/*
	* The controller waits for busy responses.
	*/
	sc->slot.quirks \|= SDHCI_QUIRK_WAIT_WHILE_BUSY;

	/*
	* DMA is not really broken, I just haven't implemented it yet.
	*/
	sc->slot.quirks \|= SDHCI_QUIRK_BROKEN_DMA;

	/*
	* Set up the hardware and go. Note that this sets many of the
	* slot.host.* fields, so we have to do this before overriding any of
	* those values based on fdt data, below.
	*/
	sdhci_init_slot(dev, &sc->slot, 0);

	/*
	* The SDHCI controller doesn't realize it, but we can support 8-bit
	* even though we're not a v3.0 controller. If there's an fdt bus-width
	* property, honor it.
	*/
	if (OF_getencprop(node, "bus-width", &prop, sizeof(prop)) > 0) {
	sc->slot.host.caps &= ~(MMC_CAP_4_BIT_DATA \|
	MMC_CAP_8_BIT_DATA);
	switch (prop) {
	case 8:
	sc->slot.host.caps \|= MMC_CAP_8_BIT_DATA;
	/* FALLTHROUGH */
	case 4:
	sc->slot.host.caps \|= MMC_CAP_4_BIT_DATA;
	break;
	case 1:
	break;
	default:
	device_printf(dev, "Bad bus-width value %u\n", prop);
	break;
	}
	}

	/*
	* If the slot is flagged with the non-removable property, set our flag
	* to always force the SDHCI_CARD_PRESENT bit on.
	*/
	node = ofw_bus_get_node(dev);
	if (OF_hasprop(node, "non-removable"))
	sc->force_card_present = true;

	bus_generic_probe(dev);
	bus_generic_attach(dev);

	sdhci_start_slot(&sc->slot);
	return (0);

	fail:
	if (sc->intr_cookie)
	bus_teardown_intr(dev, sc->irq_res, sc->intr_cookie);
	if (sc->irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->irq_res);
	if (sc->mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->mem_res);

	return (err);
	}

	static int
	ti_sdhci_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) {
	device_set_desc(dev, "TI MMCHS (SDHCI 2.0)");
	return (BUS_PROBE_DEFAULT);
	}

	return (ENXIO);
	}

	static device_method_t ti_sdhci_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ti_sdhci_probe),
	DEVMETHOD(device_attach, ti_sdhci_attach),
	DEVMETHOD(device_detach, ti_sdhci_detach),

	/* Bus interface */
	DEVMETHOD(bus_read_ivar, sdhci_generic_read_ivar),
	DEVMETHOD(bus_write_ivar, sdhci_generic_write_ivar),

	/* MMC bridge interface */
	DEVMETHOD(mmcbr_update_ios, ti_sdhci_update_ios),
	DEVMETHOD(mmcbr_request, sdhci_generic_request),
	DEVMETHOD(mmcbr_get_ro, ti_sdhci_get_ro),
	DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host),
	DEVMETHOD(mmcbr_release_host, sdhci_generic_release_host),

	/* SDHCI registers accessors */
	DEVMETHOD(sdhci_read_1, ti_sdhci_read_1),
	DEVMETHOD(sdhci_read_2, ti_sdhci_read_2),
	DEVMETHOD(sdhci_read_4, ti_sdhci_read_4),
	DEVMETHOD(sdhci_read_multi_4, ti_sdhci_read_multi_4),
	DEVMETHOD(sdhci_write_1, ti_sdhci_write_1),
	DEVMETHOD(sdhci_write_2, ti_sdhci_write_2),
	DEVMETHOD(sdhci_write_4, ti_sdhci_write_4),
	DEVMETHOD(sdhci_write_multi_4, ti_sdhci_write_multi_4),
	DEVMETHOD(sdhci_get_card_present, ti_sdhci_get_card_present),

	DEVMETHOD_END
	};

	static devclass_t ti_sdhci_devclass;

	static driver_t ti_sdhci_driver = {
	"sdhci_ti",
	ti_sdhci_methods,
	sizeof(struct ti_sdhci_softc),
	};

	DRIVER_MODULE(sdhci_ti, simplebus, ti_sdhci_driver, ti_sdhci_devclass, NULL,
	NULL);
	+MODULE_DEPEND(sdhci_ti, ti_sysc, 1, 1, 1);
	SDHCI_DEPEND(sdhci_ti);

	#ifndef MMCCAM
	MMC_DECLARE_BRIDGE(sdhci_ti);
	#endif
	diff --git a/sys/arm/ti/ti_sdma.c b/sys/arm/ti/ti_sdma.c
	index b5c47a109b6f..3df674a33dcd 100644
	--- a/sys/arm/ti/ti_sdma.c
	+++ b/sys/arm/ti/ti_sdma.c
	@@ -1,1251 +1,1252 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2011
	* Ben Gray <ben.r.gray@gmail.com>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/interrupt.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/queue.h>
	#include <sys/taskqueue.h>
	#include <sys/timetc.h>
	#include <machine/bus.h>
	#include <machine/intr.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <arm/ti/ti_cpuid.h>
	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_sdma.h>
	#include <arm/ti/ti_sdmareg.h>

	/**
	* Kernel functions for using the DMA controller
	*
	*
	* DMA TRANSFERS:
	* A DMA transfer block consists of a number of frames (FN). Each frame
	* consists of a number of elements, and each element can have a size of 8, 16,
	* or 32 bits.
	*
	* OMAP44xx and newer chips support linked list (aka scatter gather) transfers,
	* where a linked list of source/destination pairs can be placed in memory
	* for the H/W to process. Earlier chips only allowed you to chain multiple
	* channels together. However currently this linked list feature is not
	* supported by the driver.
	*
	*/

	/**
	* Data structure per DMA channel.
	*
	*
	*/
	struct ti_sdma_channel {

	- /*
	+ /*
	* The configuration registers for the given channel, these are modified
	* by the set functions and only written to the actual registers when a
	* transaction is started.
	*/
	uint32_t reg_csdp;
	uint32_t reg_ccr;
	uint32_t reg_cicr;

	/* Set when one of the configuration registers above change */
	uint32_t need_reg_write;

	/* Callback function used when an interrupt is tripped on the given channel */
	void (callback)(unsigned int ch, uint32_t ch_status, void data);

	/* Callback data passed in the callback ... duh */
	void* callback_data;

	};

	/**
	* DMA driver context, allocated and stored globally, this driver is not
	* intetned to ever be unloaded (see ti_sdma_sc).
	*
	*/
	struct ti_sdma_softc {
	device_t sc_dev;
	struct resource* sc_irq_res;
	struct resource* sc_mem_res;

	- /*
	+ /*
	* I guess in theory we should have a mutex per DMA channel for register
	* modifications. But since we know we are never going to be run on a SMP
	* system, we can use just the single lock for all channels.
	*/
	struct mtx sc_mtx;

	/* Stores the H/W revision read from the registers */
	uint32_t sc_hw_rev;

	- /*
	+ /*
	* Bits in the sc_active_channels data field indicate if the channel has
	* been activated.
	*/
	uint32_t sc_active_channels;

	struct ti_sdma_channel sc_channel[NUM_DMA_CHANNELS];

	};

	static struct ti_sdma_softc *ti_sdma_sc = NULL;

	/**
	* Macros for driver mutex locking
	*/
	#define TI_SDMA_LOCK(_sc) mtx_lock_spin(&(_sc)->sc_mtx)
	#define TI_SDMA_UNLOCK(_sc) mtx_unlock_spin(&(_sc)->sc_mtx)
	#define TI_SDMA_LOCK_INIT(_sc) \
	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \
	"ti_sdma", MTX_SPIN)
	#define TI_SDMA_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx);
	#define TI_SDMA_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED);
	#define TI_SDMA_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);

	/**
	* Function prototypes
	*
	*/
	static void ti_sdma_intr(void *);

	/**
	* ti_sdma_read_4 - reads a 32-bit value from one of the DMA registers
	* @sc: DMA device context
	* @off: The offset of a register from the DMA register address range
	*
	*
	* RETURNS:
	* 32-bit value read from the register.
	*/
	static inline uint32_t
	ti_sdma_read_4(struct ti_sdma_softc *sc, bus_size_t off)
	{
	return bus_read_4(sc->sc_mem_res, off);
	}

	/**
	* ti_sdma_write_4 - writes a 32-bit value to one of the DMA registers
	* @sc: DMA device context
	* @off: The offset of a register from the DMA register address range
	*
	*
	* RETURNS:
	* 32-bit value read from the register.
	*/
	static inline void
	ti_sdma_write_4(struct ti_sdma_softc *sc, bus_size_t off, uint32_t val)
	{
	bus_write_4(sc->sc_mem_res, off, val);
	}

	/**
	* ti_sdma_is_omap3_rev - returns true if H/W is from OMAP3 series
	* @sc: DMA device context
	*
	*/
	static inline int
	ti_sdma_is_omap3_rev(struct ti_sdma_softc *sc)
	{
	return (sc->sc_hw_rev == DMA4_OMAP3_REV);
	}

	/**
	* ti_sdma_is_omap4_rev - returns true if H/W is from OMAP4 series
	* @sc: DMA device context
	*
	*/
	static inline int
	ti_sdma_is_omap4_rev(struct ti_sdma_softc *sc)
	{
	return (sc->sc_hw_rev == DMA4_OMAP4_REV);
	}

	/**
	* ti_sdma_intr - interrupt handler for all 4 DMA IRQs
	* @arg: ignored
	*
	* Called when any of the four DMA IRQs are triggered.
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* nothing
	*/
	static void
	ti_sdma_intr(void *arg)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	uint32_t intr;
	uint32_t csr;
	unsigned int ch, j;
	struct ti_sdma_channel* channel;

	TI_SDMA_LOCK(sc);

	for (j = 0; j < NUM_DMA_IRQS; j++) {

	/* Get the flag interrupts (enabled) */
	intr = ti_sdma_read_4(sc, DMA4_IRQSTATUS_L(j));
	intr &= ti_sdma_read_4(sc, DMA4_IRQENABLE_L(j));
	if (intr == 0x00000000)
	continue;

	/* Loop through checking the status bits */
	for (ch = 0; ch < NUM_DMA_CHANNELS; ch++) {
	if (intr & (1 << ch)) {
	channel = &sc->sc_channel[ch];

	/* Read the CSR regsiter and verify we don't have a spurious IRQ */
	csr = ti_sdma_read_4(sc, DMA4_CSR(ch));
	if (csr == 0) {
	device_printf(sc->sc_dev, "Spurious DMA IRQ for channel "
	"%d\n", ch);
	continue;
	}

	/* Sanity check this channel is active */
	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	device_printf(sc->sc_dev, "IRQ %d for a non-activated "
	"channel %d\n", j, ch);
	continue;
	}

	/* Check the status error codes */
	if (csr & DMA4_CSR_DROP)
	device_printf(sc->sc_dev, "Synchronization event drop "
	"occurred during the transfer on channel %u\n",
	ch);
	if (csr & DMA4_CSR_SECURE_ERR)
	device_printf(sc->sc_dev, "Secure transaction error event "
	"on channel %u\n", ch);
	if (csr & DMA4_CSR_MISALIGNED_ADRS_ERR)
	device_printf(sc->sc_dev, "Misaligned address error event "
	"on channel %u\n", ch);
	if (csr & DMA4_CSR_TRANS_ERR) {
	device_printf(sc->sc_dev, "Transaction error event on "
	"channel %u\n", ch);
	- /*
	+ /*
	* Apparently according to linux code, there is an errata
	* that says the channel is not disabled upon this error.
	* They explicitly disable the channel here .. since I
	* haven't seen the errata, I'm going to ignore for now.
	*/
	}

	/* Clear the status flags for the IRQ */
	ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK);
	ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch));

	/* Call the callback for the given channel */
	if (channel->callback)
	channel->callback(ch, csr, channel->callback_data);
	}
	}
	}

	TI_SDMA_UNLOCK(sc);

	return;
	}

	/**
	* ti_sdma_activate_channel - activates a DMA channel
	* @ch: upon return contains the channel allocated
	* @callback: a callback function to associate with the channel
	* @data: optional data supplied when the callback is called
	*
	* Simply activates a channel be enabling and writing default values to the
	* channel's register set. It doesn't start a transaction, just populates the
	* internal data structures and sets defaults.
	*
	* Note this function doesn't enable interrupts, for that you need to call
	* ti_sdma_enable_channel_irq(). If not using IRQ to detect the end of the
	* transfer, you can use ti_sdma_status_poll() to detect a change in the
	* status.
	*
	* A channel must be activated before any of the other DMA functions can be
	* called on it.
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* 0 on success, otherwise an error code
	*/
	int
	ti_sdma_activate_channel(unsigned int *ch,
	void (callback)(unsigned int ch, uint32_t status, void data),
	void *data)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	struct ti_sdma_channel *channel = NULL;
	uint32_t addr;
	unsigned int i;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	if (ch == NULL)
	return (EINVAL);

	TI_SDMA_LOCK(sc);

	/* Check to see if all channels are in use */
	if (sc->sc_active_channels == 0xffffffff) {
	TI_SDMA_UNLOCK(sc);
	return (ENOMEM);
	}

	/* Find the first non-active channel */
	for (i = 0; i < NUM_DMA_CHANNELS; i++) {
	if (!(sc->sc_active_channels & (0x1 << i))) {
	sc->sc_active_channels \|= (0x1 << i);
	*ch = i;
	break;
	}
	}

	/* Get the channel struct and populate the fields */
	channel = &sc->sc_channel[*ch];

	channel->callback = callback;
	channel->callback_data = data;

	channel->need_reg_write = 1;

	/* Set the default configuration for the DMA channel */
	channel->reg_csdp = DMA4_CSDP_DATA_TYPE(0x2)
	\| DMA4_CSDP_SRC_BURST_MODE(0)
	\| DMA4_CSDP_DST_BURST_MODE(0)
	\| DMA4_CSDP_SRC_ENDIANISM(0)
	\| DMA4_CSDP_DST_ENDIANISM(0)
	\| DMA4_CSDP_WRITE_MODE(0)
	\| DMA4_CSDP_SRC_PACKED(0)
	\| DMA4_CSDP_DST_PACKED(0);

	channel->reg_ccr = DMA4_CCR_DST_ADDRESS_MODE(1)
	\| DMA4_CCR_SRC_ADDRESS_MODE(1)
	\| DMA4_CCR_READ_PRIORITY(0)
	\| DMA4_CCR_WRITE_PRIORITY(0)
	\| DMA4_CCR_SYNC_TRIGGER(0)
	\| DMA4_CCR_FRAME_SYNC(0)
	\| DMA4_CCR_BLOCK_SYNC(0);

	channel->reg_cicr = DMA4_CICR_TRANS_ERR_IE
	\| DMA4_CICR_SECURE_ERR_IE
	\| DMA4_CICR_SUPERVISOR_ERR_IE
	\| DMA4_CICR_MISALIGNED_ADRS_ERR_IE;

	/* Clear all the channel registers, this should abort any transaction */
	for (addr = DMA4_CCR(ch); addr <= DMA4_COLOR(ch); addr += 4)
	ti_sdma_write_4(sc, addr, 0x00000000);

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_deactivate_channel - deactivates a channel
	* @ch: the channel to deactivate
	*
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_deactivate_channel(unsigned int ch)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	unsigned int j;
	unsigned int addr;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	/* First check if the channel is currently active */
	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EBUSY);
	}

	/* Mark the channel as inactive */
	sc->sc_active_channels &= ~(1 << ch);

	/* Disable all DMA interrupts for the channel. */
	ti_sdma_write_4(sc, DMA4_CICR(ch), 0);

	/* Make sure the DMA transfer is stopped. */
	ti_sdma_write_4(sc, DMA4_CCR(ch), 0);

	/* Clear the CSR register and IRQ status register */
	ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK);
	for (j = 0; j < NUM_DMA_IRQS; j++) {
	ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch));
	}

	/* Clear all the channel registers, this should abort any transaction */
	for (addr = DMA4_CCR(ch); addr <= DMA4_COLOR(ch); addr += 4)
	ti_sdma_write_4(sc, addr, 0x00000000);

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_disable_channel_irq - disables IRQ's on the given channel
	* @ch: the channel to disable IRQ's on
	*
	* Disable interrupt generation for the given channel.
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_disable_channel_irq(unsigned int ch)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	uint32_t irq_enable;
	unsigned int j;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	/* Disable all the individual error conditions */
	sc->sc_channel[ch].reg_cicr = 0x0000;
	ti_sdma_write_4(sc, DMA4_CICR(ch), 0x0000);

	/* Disable the channel interrupt enable */
	for (j = 0; j < NUM_DMA_IRQS; j++) {
	irq_enable = ti_sdma_read_4(sc, DMA4_IRQENABLE_L(j));
	irq_enable &= ~(1 << ch);

	ti_sdma_write_4(sc, DMA4_IRQENABLE_L(j), irq_enable);
	}

	/* Indicate the registers need to be rewritten on the next transaction */
	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_sdma_disable_channel_irq - enables IRQ's on the given channel
	* @ch: the channel to enable IRQ's on
	* @flags: bitmask of interrupt types to enable
	*
	* Flags can be a bitmask of the following options:
	* DMA_IRQ_FLAG_DROP
	* DMA_IRQ_FLAG_HALF_FRAME_COMPL
	* DMA_IRQ_FLAG_FRAME_COMPL
	* DMA_IRQ_FLAG_START_LAST_FRAME
	* DMA_IRQ_FLAG_BLOCK_COMPL
	* DMA_IRQ_FLAG_ENDOF_PKT
	* DMA_IRQ_FLAG_DRAIN
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_enable_channel_irq(unsigned int ch, uint32_t flags)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	uint32_t irq_enable;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	/* Always enable the error interrupts if we have interrupts enabled */
	flags \|= DMA4_CICR_TRANS_ERR_IE \| DMA4_CICR_SECURE_ERR_IE \|
	DMA4_CICR_SUPERVISOR_ERR_IE \| DMA4_CICR_MISALIGNED_ADRS_ERR_IE;

	sc->sc_channel[ch].reg_cicr = flags;

	/* Write the values to the register */
	ti_sdma_write_4(sc, DMA4_CICR(ch), flags);

	/* Enable the channel interrupt enable */
	irq_enable = ti_sdma_read_4(sc, DMA4_IRQENABLE_L(0));
	irq_enable \|= (1 << ch);

	ti_sdma_write_4(sc, DMA4_IRQENABLE_L(0), irq_enable);

	/* Indicate the registers need to be rewritten on the next transaction */
	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_sdma_get_channel_status - returns the status of a given channel
	* @ch: the channel number to get the status of
	* @status: upon return will contain the status bitmask, see below for possible
	* values.
	*
	* DMA_STATUS_DROP
	* DMA_STATUS_HALF
	* DMA_STATUS_FRAME
	* DMA_STATUS_LAST
	* DMA_STATUS_BLOCK
	* DMA_STATUS_SYNC
	* DMA_STATUS_PKT
	* DMA_STATUS_TRANS_ERR
	* DMA_STATUS_SECURE_ERR
	* DMA_STATUS_SUPERVISOR_ERR
	* DMA_STATUS_MISALIGNED_ADRS_ERR
	* DMA_STATUS_DRAIN_END
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_get_channel_status(unsigned int ch, uint32_t *status)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	uint32_t csr;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	TI_SDMA_UNLOCK(sc);

	csr = ti_sdma_read_4(sc, DMA4_CSR(ch));

	if (status != NULL)
	*status = csr;

	return (0);
	}

	/**
	* ti_sdma_start_xfer - starts a DMA transfer
	* @ch: the channel number to set the endianness of
	* @src_paddr: the source phsyical address
	* @dst_paddr: the destination phsyical address
	* @frmcnt: the number of frames per block
	* @elmcnt: the number of elements in a frame, an element is either an 8, 16
	* or 32-bit value as defined by ti_sdma_set_xfer_burst()
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_start_xfer(unsigned int ch, unsigned int src_paddr,
	unsigned long dst_paddr,
	unsigned int frmcnt, unsigned int elmcnt)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	struct ti_sdma_channel *channel;
	uint32_t ccr;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	channel = &sc->sc_channel[ch];

	/* a) Write the CSDP register */
	ti_sdma_write_4(sc, DMA4_CSDP(ch),
	channel->reg_csdp \| DMA4_CSDP_WRITE_MODE(1));

	/* b) Set the number of element per frame CEN[23:0] */
	ti_sdma_write_4(sc, DMA4_CEN(ch), elmcnt);

	/* c) Set the number of frame per block CFN[15:0] */
	ti_sdma_write_4(sc, DMA4_CFN(ch), frmcnt);

	/* d) Set the Source/dest start address index CSSA[31:0]/CDSA[31:0] */
	ti_sdma_write_4(sc, DMA4_CSSA(ch), src_paddr);
	ti_sdma_write_4(sc, DMA4_CDSA(ch), dst_paddr);

	/* e) Write the CCR register */
	ti_sdma_write_4(sc, DMA4_CCR(ch), channel->reg_ccr);

	/* f) - Set the source element index increment CSEI[15:0] */
	ti_sdma_write_4(sc, DMA4_CSE(ch), 0x0001);

	/* - Set the source frame index increment CSFI[15:0] */
	ti_sdma_write_4(sc, DMA4_CSF(ch), 0x0001);

	/* - Set the destination element index increment CDEI[15:0]*/
	ti_sdma_write_4(sc, DMA4_CDE(ch), 0x0001);

	/* - Set the destination frame index increment CDFI[31:0] */
	ti_sdma_write_4(sc, DMA4_CDF(ch), 0x0001);

	/* Clear the status register */
	ti_sdma_write_4(sc, DMA4_CSR(ch), 0x1FFE);

	/* Write the start-bit and away we go */
	ccr = ti_sdma_read_4(sc, DMA4_CCR(ch));
	ccr \|= (1 << 7);
	ti_sdma_write_4(sc, DMA4_CCR(ch), ccr);

	/* Clear the reg write flag */
	channel->need_reg_write = 0;

	TI_SDMA_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_sdma_start_xfer_packet - starts a packet DMA transfer
	* @ch: the channel number to use for the transfer
	* @src_paddr: the source physical address
	* @dst_paddr: the destination physical address
	* @frmcnt: the number of frames to transfer
	* @elmcnt: the number of elements in a frame, an element is either an 8, 16
	* or 32-bit value as defined by ti_sdma_set_xfer_burst()
	* @pktsize: the number of elements in each transfer packet
	*
	* The @frmcnt and @elmcnt define the overall number of bytes to transfer,
	* typically @frmcnt is 1 and @elmcnt contains the total number of elements.
	* @pktsize is the size of each individual packet, there might be multiple
	* packets per transfer. i.e. for the following with element size of 32-bits
	*
	* frmcnt = 1, elmcnt = 512, pktsize = 128
	*
	* Total transfer bytes = 1 * 512 = 512 elements or 2048 bytes
	* Packets transferred = 128 / 512 = 4
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_start_xfer_packet(unsigned int ch, unsigned int src_paddr,
	unsigned long dst_paddr, unsigned int frmcnt,
	unsigned int elmcnt, unsigned int pktsize)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	struct ti_sdma_channel *channel;
	uint32_t ccr;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	channel = &sc->sc_channel[ch];

	/* a) Write the CSDP register */
	if (channel->need_reg_write)
	ti_sdma_write_4(sc, DMA4_CSDP(ch),
	channel->reg_csdp \| DMA4_CSDP_WRITE_MODE(1));

	/* b) Set the number of elements to transfer CEN[23:0] */
	ti_sdma_write_4(sc, DMA4_CEN(ch), elmcnt);

	/* c) Set the number of frames to transfer CFN[15:0] */
	ti_sdma_write_4(sc, DMA4_CFN(ch), frmcnt);

	/* d) Set the Source/dest start address index CSSA[31:0]/CDSA[31:0] */
	ti_sdma_write_4(sc, DMA4_CSSA(ch), src_paddr);
	ti_sdma_write_4(sc, DMA4_CDSA(ch), dst_paddr);

	/* e) Write the CCR register */
	ti_sdma_write_4(sc, DMA4_CCR(ch),
	channel->reg_ccr \| DMA4_CCR_PACKET_TRANS);

	/* f) - Set the source element index increment CSEI[15:0] */
	ti_sdma_write_4(sc, DMA4_CSE(ch), 0x0001);

	/* - Set the packet size, this is dependent on the sync source */
	if (channel->reg_ccr & DMA4_CCR_SEL_SRC_DST_SYNC(1))
	ti_sdma_write_4(sc, DMA4_CSF(ch), pktsize);
	else
	ti_sdma_write_4(sc, DMA4_CDF(ch), pktsize);

	/* - Set the destination frame index increment CDFI[31:0] */
	ti_sdma_write_4(sc, DMA4_CDE(ch), 0x0001);

	/* Clear the status register */
	ti_sdma_write_4(sc, DMA4_CSR(ch), 0x1FFE);

	/* Write the start-bit and away we go */
	ccr = ti_sdma_read_4(sc, DMA4_CCR(ch));
	ccr \|= (1 << 7);
	ti_sdma_write_4(sc, DMA4_CCR(ch), ccr);

	/* Clear the reg write flag */
	channel->need_reg_write = 0;

	TI_SDMA_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_sdma_stop_xfer - stops any currently active transfers
	* @ch: the channel number to set the endianness of
	*
	* This function call is effectively a NOP if no transaction is in progress.
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_stop_xfer(unsigned int ch)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	unsigned int j;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	/* Disable all DMA interrupts for the channel. */
	ti_sdma_write_4(sc, DMA4_CICR(ch), 0);

	/* Make sure the DMA transfer is stopped. */
	ti_sdma_write_4(sc, DMA4_CCR(ch), 0);

	/* Clear the CSR register and IRQ status register */
	ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK);
	for (j = 0; j < NUM_DMA_IRQS; j++) {
	ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch));
	}

	/* Configuration registers need to be re-written on the next xfer */
	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return (0);
	}

	/**
	* ti_sdma_set_xfer_endianess - sets the endianness of subsequent transfers
	* @ch: the channel number to set the endianness of
	* @src: the source endianness (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG)
	* @dst: the destination endianness (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG)
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_set_xfer_endianess(unsigned int ch, unsigned int src, unsigned int dst)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_SRC_ENDIANISM(1);
	sc->sc_channel[ch].reg_csdp \|= DMA4_CSDP_SRC_ENDIANISM(src);

	sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DST_ENDIANISM(1);
	sc->sc_channel[ch].reg_csdp \|= DMA4_CSDP_DST_ENDIANISM(dst);

	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_set_xfer_burst - sets the source and destination element size
	* @ch: the channel number to set the burst settings of
	* @src: the source endianness (either DMA_BURST_NONE, DMA_BURST_16, DMA_BURST_32
	* or DMA_BURST_64)
	* @dst: the destination endianness (either DMA_BURST_NONE, DMA_BURST_16,
	* DMA_BURST_32 or DMA_BURST_64)
	*
	* This function sets the size of the elements for all subsequent transfers.
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_set_xfer_burst(unsigned int ch, unsigned int src, unsigned int dst)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_SRC_BURST_MODE(0x3);
	sc->sc_channel[ch].reg_csdp \|= DMA4_CSDP_SRC_BURST_MODE(src);

	sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DST_BURST_MODE(0x3);
	sc->sc_channel[ch].reg_csdp \|= DMA4_CSDP_DST_BURST_MODE(dst);

	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_set_xfer_data_type - driver attach function
	* @ch: the channel number to set the endianness of
	* @type: the xfer data type (either DMA_DATA_8BITS_SCALAR, DMA_DATA_16BITS_SCALAR
	* or DMA_DATA_32BITS_SCALAR)
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_set_xfer_data_type(unsigned int ch, unsigned int type)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DATA_TYPE(0x3);
	sc->sc_channel[ch].reg_csdp \|= DMA4_CSDP_DATA_TYPE(type);

	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_set_callback - driver attach function
	* @dev: dma device handle
	*
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_set_callback(unsigned int ch,
	void (callback)(unsigned int ch, uint32_t status, void data),
	void *data)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	sc->sc_channel[ch].callback = callback;
	sc->sc_channel[ch].callback_data = data;

	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_sync_params - sets channel sync settings
	* @ch: the channel number to set the sync on
	* @trigger: the number of the sync trigger, this depends on what other H/W
	* module is triggering/receiving the DMA transactions
	* @mode: flags describing the sync mode to use, it may have one or more of
	* the following bits set; TI_SDMA_SYNC_FRAME,
	* TI_SDMA_SYNC_BLOCK, TI_SDMA_SYNC_TRIG_ON_SRC.
	*
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_sync_params(unsigned int ch, unsigned int trigger, unsigned int mode)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	uint32_t ccr;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	ccr = sc->sc_channel[ch].reg_ccr;

	ccr &= ~DMA4_CCR_SYNC_TRIGGER(0x7F);
	ccr \|= DMA4_CCR_SYNC_TRIGGER(trigger + 1);

	if (mode & TI_SDMA_SYNC_FRAME)
	ccr \|= DMA4_CCR_FRAME_SYNC(1);
	else
	ccr &= ~DMA4_CCR_FRAME_SYNC(1);

	if (mode & TI_SDMA_SYNC_BLOCK)
	ccr \|= DMA4_CCR_BLOCK_SYNC(1);
	else
	ccr &= ~DMA4_CCR_BLOCK_SYNC(1);

	if (mode & TI_SDMA_SYNC_TRIG_ON_SRC)
	ccr \|= DMA4_CCR_SEL_SRC_DST_SYNC(1);
	else
	ccr &= ~DMA4_CCR_SEL_SRC_DST_SYNC(1);

	sc->sc_channel[ch].reg_ccr = ccr;

	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_set_addr_mode - driver attach function
	* @ch: the channel number to set the endianness of
	* @rd_mode: the xfer source addressing mode (either DMA_ADDR_CONSTANT,
	* DMA_ADDR_POST_INCREMENT, DMA_ADDR_SINGLE_INDEX or
	* DMA_ADDR_DOUBLE_INDEX)
	* @wr_mode: the xfer destination addressing mode (either DMA_ADDR_CONSTANT,
	* DMA_ADDR_POST_INCREMENT, DMA_ADDR_SINGLE_INDEX or
	* DMA_ADDR_DOUBLE_INDEX)
	*
	*
	* LOCKING:
	* DMA registers protected by internal mutex
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	int
	ti_sdma_set_addr_mode(unsigned int ch, unsigned int src_mode,
	unsigned int dst_mode)
	{
	struct ti_sdma_softc *sc = ti_sdma_sc;
	uint32_t ccr;

	/* Sanity check */
	if (sc == NULL)
	return (ENOMEM);

	TI_SDMA_LOCK(sc);

	if ((sc->sc_active_channels & (1 << ch)) == 0) {
	TI_SDMA_UNLOCK(sc);
	return (EINVAL);
	}

	ccr = sc->sc_channel[ch].reg_ccr;

	ccr &= ~DMA4_CCR_SRC_ADDRESS_MODE(0x3);
	ccr \|= DMA4_CCR_SRC_ADDRESS_MODE(src_mode);

	ccr &= ~DMA4_CCR_DST_ADDRESS_MODE(0x3);
	ccr \|= DMA4_CCR_DST_ADDRESS_MODE(dst_mode);

	sc->sc_channel[ch].reg_ccr = ccr;

	sc->sc_channel[ch].need_reg_write = 1;

	TI_SDMA_UNLOCK(sc);

	return 0;
	}

	/**
	* ti_sdma_probe - driver probe function
	* @dev: dma device handle
	*
	*
	*
	* RETURNS:
	* Always returns 0.
	*/
	static int
	ti_sdma_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,omap4430-sdma"))
	return (ENXIO);

	device_set_desc(dev, "TI sDMA Controller");
	return (0);
	}

	/**
	* ti_sdma_attach - driver attach function
	* @dev: dma device handle
	*
	* Initialises memory mapping/pointers to the DMA register set and requests
	* IRQs. This is effectively the setup function for the driver.
	*
	* RETURNS:
	* 0 on success or a negative error code failure.
	*/
	static int
	ti_sdma_attach(device_t dev)
	{
	struct ti_sdma_softc *sc = device_get_softc(dev);
	unsigned int timeout;
	unsigned int i;
	int rid;
	void *ihl;
	int err;

	/* Setup the basics */
	sc->sc_dev = dev;

	/* No channels active at the moment */
	sc->sc_active_channels = 0x00000000;

	/* Mutex to protect the shared data structures */
	TI_SDMA_LOCK_INIT(sc);

	/* Get the memory resource for the register mapping */
	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (sc->sc_mem_res == NULL)
	panic("%s: Cannot map registers", device_get_name(dev));

	/* Enable the interface and functional clocks */
	- ti_prcm_clk_enable(SDMA_CLK);
	+ ti_sysc_clock_enable(device_get_parent(dev));

	/* Read the sDMA revision register and sanity check it's known */
	- sc->sc_hw_rev = ti_sdma_read_4(sc, DMA4_REVISION);
	+ sc->sc_hw_rev = ti_sdma_read_4(sc,
	+ ti_sysc_get_rev_address_offset_host(device_get_parent(dev)));
	device_printf(dev, "sDMA revision %08x\n", sc->sc_hw_rev);

	if (!ti_sdma_is_omap4_rev(sc) && !ti_sdma_is_omap3_rev(sc)) {
	device_printf(sc->sc_dev, "error - unknown sDMA H/W revision\n");
	return (EINVAL);
	}

	/* Disable all interrupts */
	for (i = 0; i < NUM_DMA_IRQS; i++) {
	ti_sdma_write_4(sc, DMA4_IRQENABLE_L(i), 0x00000000);
	}

	/* Soft-reset is only supported on pre-OMAP44xx devices */
	if (ti_sdma_is_omap3_rev(sc)) {

	/* Soft-reset */
	ti_sdma_write_4(sc, DMA4_OCP_SYSCONFIG, 0x0002);

	/* Set the timeout to 100ms*/
	timeout = (hz < 10) ? 1 : ((100 * hz) / 1000);

	/* Wait for DMA reset to complete */
	while ((ti_sdma_read_4(sc, DMA4_SYSSTATUS) & 0x1) == 0x0) {

	/* Sleep for a tick */
	pause("DMARESET", 1);

	if (timeout-- == 0) {
	device_printf(sc->sc_dev, "sDMA reset operation timed out\n");
	return (EINVAL);
	}
	}
	}

	- /*
	+ /*
	* Install interrupt handlers for the for possible interrupts. Any channel
	* can trip one of the four IRQs
	*/
	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE \| RF_SHAREABLE);
	if (sc->sc_irq_res == NULL)
	panic("Unable to setup the dma irq handler.\n");

	err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC \| INTR_MPSAFE,
	NULL, ti_sdma_intr, NULL, &ihl);
	if (err)
	panic("%s: Cannot register IRQ", device_get_name(dev));

	/* Store the DMA structure globally ... this driver should never be unloaded */
	ti_sdma_sc = sc;

	return (0);
	}

	static device_method_t ti_sdma_methods[] = {
	DEVMETHOD(device_probe, ti_sdma_probe),
	DEVMETHOD(device_attach, ti_sdma_attach),
	{0, 0},
	};

	static driver_t ti_sdma_driver = {
	"ti_sdma",
	ti_sdma_methods,
	sizeof(struct ti_sdma_softc),
	};
	static devclass_t ti_sdma_devclass;

	DRIVER_MODULE(ti_sdma, simplebus, ti_sdma_driver, ti_sdma_devclass, 0, 0);
	-MODULE_DEPEND(ti_sdma, ti_prcm, 1, 1, 1);
	+MODULE_DEPEND(ti_sdma, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_spi.c b/sys/arm/ti/ti_spi.c
	index a424f36b8683..19b80605b9b6 100644
	--- a/sys/arm/ti/ti_spi.c
	+++ b/sys/arm/ti/ti_spi.c
	@@ -1,594 +1,582 @@
	/*-
	* Copyright (c) 2016 Rubicon Communications, LLC (Netgate)
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>

	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/rman.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/sysctl.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <machine/intr.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/spibus/spi.h>
	#include <dev/spibus/spibusvar.h>

	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/ti_spireg.h>
	#include <arm/ti/ti_spivar.h>

	#include "spibus_if.h"

	static void ti_spi_intr(void *);
	static int ti_spi_detach(device_t);

	#undef TI_SPI_DEBUG
	#ifdef TI_SPI_DEBUG
	#define IRQSTATUSBITS \
	"\020\1TX0_EMPTY\2TX0_UNDERFLOW\3RX0_FULL\4RX0_OVERFLOW" \
	"\5TX1_EMPTY\6TX1_UNDERFLOW\7RX1_FULL\11TX2_EMPTY" \
	"\12TX1_UNDERFLOW\13RX2_FULL\15TX3_EMPTY\16TX3_UNDERFLOW" \
	"\17RX3_FULL\22EOW"
	#define CONFBITS \
	"\020\1PHA\2POL\7EPOL\17DMAW\20DMAR\21DPE0\22DPE1\23IS" \
	"\24TURBO\25FORCE\30SBE\31SBPOL\34FFEW\35FFER\36CLKG"
	#define STATBITS \
	"\020\1RXS\2TXS\3EOT\4TXFFE\5TXFFF\6RXFFE\7RXFFFF"
	#define MODULCTRLBITS \
	"\020\1SINGLE\2NOSPIEN\3SLAVE\4SYST\10MOA\11FDAA"
	#define CTRLBITS \
	"\020\1ENABLED"

	static void
	ti_spi_printr(device_t dev)
	{
	int clk, conf, ctrl, div, i, j, wl;
	struct ti_spi_softc *sc;
	uint32_t reg;

	sc = device_get_softc(dev);
	reg = TI_SPI_READ(sc, MCSPI_SYSCONFIG);
	device_printf(dev, "SYSCONFIG: %#x\n", reg);
	reg = TI_SPI_READ(sc, MCSPI_SYSSTATUS);
	device_printf(dev, "SYSSTATUS: %#x\n", reg);
	reg = TI_SPI_READ(sc, MCSPI_IRQSTATUS);
	device_printf(dev, "IRQSTATUS: 0x%b\n", reg, IRQSTATUSBITS);
	reg = TI_SPI_READ(sc, MCSPI_IRQENABLE);
	device_printf(dev, "IRQENABLE: 0x%b\n", reg, IRQSTATUSBITS);
	reg = TI_SPI_READ(sc, MCSPI_MODULCTRL);
	device_printf(dev, "MODULCTRL: 0x%b\n", reg, MODULCTRLBITS);
	for (i = 0; i < sc->sc_numcs; i++) {
	ctrl = TI_SPI_READ(sc, MCSPI_CTRL_CH(i));
	conf = TI_SPI_READ(sc, MCSPI_CONF_CH(i));
	device_printf(dev, "CH%dCONF: 0x%b\n", i, conf, CONFBITS);
	if (conf & MCSPI_CONF_CLKG) {
	div = (conf >> MCSPI_CONF_CLK_SHIFT) & MCSPI_CONF_CLK_MSK;
	div \|= ((ctrl >> MCSPI_CTRL_EXTCLK_SHIFT) & MCSPI_CTRL_EXTCLK_MSK) << 4;
	} else {
	div = 1;
	j = (conf >> MCSPI_CONF_CLK_SHIFT) & MCSPI_CONF_CLK_MSK;
	while (j-- > 0)
	div <<= 1;
	}
	clk = TI_SPI_GCLK / div;
	wl = ((conf >> MCSPI_CONF_WL_SHIFT) & MCSPI_CONF_WL_MSK) + 1;
	device_printf(dev, "wordlen: %-2d clock: %d\n", wl, clk);
	reg = TI_SPI_READ(sc, MCSPI_STAT_CH(i));
	device_printf(dev, "CH%dSTAT: 0x%b\n", i, reg, STATBITS);
	device_printf(dev, "CH%dCTRL: 0x%b\n", i, ctrl, CTRLBITS);
	}
	reg = TI_SPI_READ(sc, MCSPI_XFERLEVEL);
	device_printf(dev, "XFERLEVEL: %#x\n", reg);
	}
	#endif

	static void
	ti_spi_set_clock(struct ti_spi_softc *sc, int ch, int freq)
	{
	uint32_t clkdiv, conf, div, extclk, reg;

	clkdiv = TI_SPI_GCLK / freq;
	if (clkdiv > MCSPI_EXTCLK_MSK) {
	extclk = 0;
	clkdiv = 0;
	div = 1;
	while (TI_SPI_GCLK / div > freq && clkdiv <= 0xf) {
	clkdiv++;
	div <<= 1;
	}
	conf = clkdiv << MCSPI_CONF_CLK_SHIFT;
	} else {
	extclk = clkdiv >> 4;
	clkdiv &= MCSPI_CONF_CLK_MSK;
	conf = MCSPI_CONF_CLKG \| clkdiv << MCSPI_CONF_CLK_SHIFT;
	}

	reg = TI_SPI_READ(sc, MCSPI_CTRL_CH(ch));
	reg &= ~(MCSPI_CTRL_EXTCLK_MSK << MCSPI_CTRL_EXTCLK_SHIFT);
	reg \|= extclk << MCSPI_CTRL_EXTCLK_SHIFT;
	TI_SPI_WRITE(sc, MCSPI_CTRL_CH(ch), reg);

	reg = TI_SPI_READ(sc, MCSPI_CONF_CH(ch));
	reg &= ~(MCSPI_CONF_CLKG \| MCSPI_CONF_CLK_MSK << MCSPI_CONF_CLK_SHIFT);
	TI_SPI_WRITE(sc, MCSPI_CONF_CH(ch), reg \| conf);
	}

	static int
	ti_spi_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);
	if (!ofw_bus_is_compatible(dev, "ti,omap4-mcspi"))
	return (ENXIO);

	device_set_desc(dev, "TI McSPI controller");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	ti_spi_attach(device_t dev)
	{
	- int clk_id, err, i, rid, timeout;
	+ int err, i, rid, timeout;
	struct ti_spi_softc *sc;
	uint32_t rev;

	sc = device_get_softc(dev);
	sc->sc_dev = dev;

	- /*
	- * Get the MMCHS device id from FDT. If it's not there use the newbus
	- * unit number (which will work as long as the devices are in order and
	- * none are skipped in the fdt). Note that this is a property we made
	- * up and added in freebsd, it doesn't exist in the published bindings.
	- */
	- clk_id = ti_hwmods_get_clock(dev);
	- if (clk_id == INVALID_CLK_IDENT) {
	- device_printf(dev,
	- "failed to get clock based on hwmods property\n");
	- return (EINVAL);
	- }
	-
	/* Activate the McSPI module. */
	- err = ti_prcm_clk_enable(clk_id);
	+ err = ti_sysc_clock_enable(device_get_parent(dev));
	if (err) {
	device_printf(dev, "Error: failed to activate source clock\n");
	return (err);
	}

	/* Get the number of available channels. */
	if ((OF_getencprop(ofw_bus_get_node(dev), "ti,spi-num-cs",
	&sc->sc_numcs, sizeof(sc->sc_numcs))) <= 0) {
	sc->sc_numcs = 2;
	}

	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (!sc->sc_mem_res) {
	device_printf(dev, "cannot allocate memory window\n");
	return (ENXIO);
	}

	sc->sc_bst = rman_get_bustag(sc->sc_mem_res);
	sc->sc_bsh = rman_get_bushandle(sc->sc_mem_res);

	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	RF_ACTIVE);
	if (!sc->sc_irq_res) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "cannot allocate interrupt\n");
	return (ENXIO);
	}

	/* Hook up our interrupt handler. */
	if (bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC \| INTR_MPSAFE,
	NULL, ti_spi_intr, sc, &sc->sc_intrhand)) {
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);
	device_printf(dev, "cannot setup the interrupt handler\n");
	return (ENXIO);
	}

	mtx_init(&sc->sc_mtx, "ti_spi", NULL, MTX_DEF);

	/* Issue a softreset to the controller */
	TI_SPI_WRITE(sc, MCSPI_SYSCONFIG, MCSPI_SYSCONFIG_SOFTRESET);
	timeout = 1000;
	while (!(TI_SPI_READ(sc, MCSPI_SYSSTATUS) &
	MCSPI_SYSSTATUS_RESETDONE)) {
	if (--timeout == 0) {
	device_printf(dev,
	"Error: Controller reset operation timed out\n");
	ti_spi_detach(dev);
	return (ENXIO);
	}
	DELAY(100);
	}

	/* Print the McSPI module revision. */
	- rev = TI_SPI_READ(sc, MCSPI_REVISION);
	+ rev = TI_SPI_READ(sc,
	+ ti_sysc_get_rev_address_offset_host(device_get_parent(dev)));
	device_printf(dev,
	"scheme: %#x func: %#x rtl: %d rev: %d.%d custom rev: %d\n",
	(rev >> MCSPI_REVISION_SCHEME_SHIFT) & MCSPI_REVISION_SCHEME_MSK,
	(rev >> MCSPI_REVISION_FUNC_SHIFT) & MCSPI_REVISION_FUNC_MSK,
	(rev >> MCSPI_REVISION_RTL_SHIFT) & MCSPI_REVISION_RTL_MSK,
	(rev >> MCSPI_REVISION_MAJOR_SHIFT) & MCSPI_REVISION_MAJOR_MSK,
	(rev >> MCSPI_REVISION_MINOR_SHIFT) & MCSPI_REVISION_MINOR_MSK,
	(rev >> MCSPI_REVISION_CUSTOM_SHIFT) & MCSPI_REVISION_CUSTOM_MSK);

	/* Set Master mode, single channel. */
	TI_SPI_WRITE(sc, MCSPI_MODULCTRL, MCSPI_MODULCTRL_SINGLE);

	/* Clear pending interrupts and disable interrupts. */
	TI_SPI_WRITE(sc, MCSPI_IRQENABLE, 0x0);
	TI_SPI_WRITE(sc, MCSPI_IRQSTATUS, 0xffff);

	for (i = 0; i < sc->sc_numcs; i++) {
	/*
	* Default to SPI mode 0, CS active low, 8 bits word length and
	* 500kHz clock.
	*/
	TI_SPI_WRITE(sc, MCSPI_CONF_CH(i),
	MCSPI_CONF_DPE0 \| MCSPI_CONF_EPOL \|
	(8 - 1) << MCSPI_CONF_WL_SHIFT);
	/* Set initial clock - 500kHz. */
	ti_spi_set_clock(sc, i, 500000);
	}

	#ifdef TI_SPI_DEBUG
	ti_spi_printr(dev);
	#endif

	device_add_child(dev, "spibus", -1);

	return (bus_generic_attach(dev));
	}

	static int
	ti_spi_detach(device_t dev)
	{
	struct ti_spi_softc *sc;

	sc = device_get_softc(dev);

	/* Clear pending interrupts and disable interrupts. */
	TI_SPI_WRITE(sc, MCSPI_IRQENABLE, 0);
	TI_SPI_WRITE(sc, MCSPI_IRQSTATUS, 0xffff);

	/* Reset controller. */
	TI_SPI_WRITE(sc, MCSPI_SYSCONFIG, MCSPI_SYSCONFIG_SOFTRESET);

	bus_generic_detach(dev);

	mtx_destroy(&sc->sc_mtx);
	if (sc->sc_intrhand)
	bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intrhand);
	if (sc->sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res);

	return (0);
	}

	static int
	ti_spi_fill_fifo(struct ti_spi_softc *sc)
	{
	int bytes, timeout;
	struct spi_command *cmd;
	uint32_t written;
	uint8_t *data;

	cmd = sc->sc_cmd;
	bytes = min(sc->sc_len - sc->sc_written, sc->sc_fifolvl);
	while (bytes-- > 0) {
	data = (uint8_t *)cmd->tx_cmd;
	written = sc->sc_written++;
	if (written >= cmd->tx_cmd_sz) {
	data = (uint8_t *)cmd->tx_data;
	written -= cmd->tx_cmd_sz;
	}
	if (sc->sc_fifolvl == 1) {
	/* FIFO disabled. */
	timeout = 1000;
	while (--timeout > 0 && (TI_SPI_READ(sc,
	MCSPI_STAT_CH(sc->sc_cs)) & MCSPI_STAT_TXS) == 0) {
	DELAY(100);
	}
	if (timeout == 0)
	return (-1);
	}
	TI_SPI_WRITE(sc, MCSPI_TX_CH(sc->sc_cs), data[written]);
	}

	return (0);
	}

	static int
	ti_spi_drain_fifo(struct ti_spi_softc *sc)
	{
	int bytes, timeout;
	struct spi_command *cmd;
	uint32_t read;
	uint8_t *data;

	cmd = sc->sc_cmd;
	bytes = min(sc->sc_len - sc->sc_read, sc->sc_fifolvl);
	while (bytes-- > 0) {
	data = (uint8_t *)cmd->rx_cmd;
	read = sc->sc_read++;
	if (read >= cmd->rx_cmd_sz) {
	data = (uint8_t *)cmd->rx_data;
	read -= cmd->rx_cmd_sz;
	}
	if (sc->sc_fifolvl == 1) {
	/* FIFO disabled. */
	timeout = 1000;
	while (--timeout > 0 && (TI_SPI_READ(sc,
	MCSPI_STAT_CH(sc->sc_cs)) & MCSPI_STAT_RXS) == 0) {
	DELAY(100);
	}
	if (timeout == 0)
	return (-1);
	}
	data[read] = TI_SPI_READ(sc, MCSPI_RX_CH(sc->sc_cs));
	}

	return (0);
	}

	static void
	ti_spi_intr(void *arg)
	{
	int eow;
	struct ti_spi_softc *sc;
	uint32_t status;

	eow = 0;
	sc = (struct ti_spi_softc *)arg;
	TI_SPI_LOCK(sc);
	status = TI_SPI_READ(sc, MCSPI_IRQSTATUS);

	/*
	* No new TX_empty or RX_full event will be asserted while the CPU has
	* not performed the number of writes or reads defined by
	* MCSPI_XFERLEVEL[AEL] and MCSPI_XFERLEVEL[AFL]. It is responsibility
	* of CPU perform the right number of writes and reads.
	*/
	if (status & MCSPI_IRQ_TX0_EMPTY)
	ti_spi_fill_fifo(sc);
	if (status & MCSPI_IRQ_RX0_FULL)
	ti_spi_drain_fifo(sc);

	if (status & MCSPI_IRQ_EOW)
	eow = 1;

	/* Clear interrupt status. */
	TI_SPI_WRITE(sc, MCSPI_IRQSTATUS, status);

	/* Check for end of transfer. */
	if (sc->sc_written == sc->sc_len && sc->sc_read == sc->sc_len) {
	sc->sc_flags \|= TI_SPI_DONE;
	wakeup(sc->sc_dev);
	}

	TI_SPI_UNLOCK(sc);
	}

	static int
	ti_spi_pio_transfer(struct ti_spi_softc *sc)
	{

	while (sc->sc_len - sc->sc_written > 0) {
	if (ti_spi_fill_fifo(sc) == -1)
	return (EIO);
	if (ti_spi_drain_fifo(sc) == -1)
	return (EIO);
	}

	return (0);
	}

	static int
	ti_spi_gcd(int a, int b)
	{
	int m;

	while ((m = a % b) != 0) {
	a = b;
	b = m;
	}

	return (b);
	}

	static int
	ti_spi_transfer(device_t dev, device_t child, struct spi_command *cmd)
	{
	int err;
	struct ti_spi_softc *sc;
	uint32_t clockhz, cs, mode, reg;

	sc = device_get_softc(dev);

	KASSERT(cmd->tx_cmd_sz == cmd->rx_cmd_sz,
	("TX/RX command sizes should be equal"));
	KASSERT(cmd->tx_data_sz == cmd->rx_data_sz,
	("TX/RX data sizes should be equal"));

	/* Get the proper chip select for this child. */
	spibus_get_cs(child, &cs);
	spibus_get_clock(child, &clockhz);
	spibus_get_mode(child, &mode);

	cs &= ~SPIBUS_CS_HIGH;

	if (cs > sc->sc_numcs) {
	device_printf(dev, "Invalid chip select %d requested by %s\n",
	cs, device_get_nameunit(child));
	return (EINVAL);
	}

	if (mode > 3)
	{
	device_printf(dev, "Invalid mode %d requested by %s\n", mode,
	device_get_nameunit(child));
	return (EINVAL);
	}

	TI_SPI_LOCK(sc);

	/* If the controller is in use wait until it is available. */
	while (sc->sc_flags & TI_SPI_BUSY)
	mtx_sleep(dev, &sc->sc_mtx, 0, "ti_spi", 0);

	/* Now we have control over SPI controller. */
	sc->sc_flags = TI_SPI_BUSY;

	/* Save the SPI command data. */
	sc->sc_cs = cs;
	sc->sc_cmd = cmd;
	sc->sc_read = 0;
	sc->sc_written = 0;
	sc->sc_len = cmd->tx_cmd_sz + cmd->tx_data_sz;
	sc->sc_fifolvl = ti_spi_gcd(sc->sc_len, TI_SPI_FIFOSZ);
	if (sc->sc_fifolvl < 2 \|\| sc->sc_len > 0xffff)
	sc->sc_fifolvl = 1; /* FIFO disabled. */
	/* Disable FIFO for now. */
	sc->sc_fifolvl = 1;

	/* Set the bus frequency. */
	ti_spi_set_clock(sc, sc->sc_cs, clockhz);

	/* Disable the FIFO. */
	TI_SPI_WRITE(sc, MCSPI_XFERLEVEL, 0);

	/* 8 bits word, d0 miso, d1 mosi, mode 0 and CS active low. */
	reg = TI_SPI_READ(sc, MCSPI_CONF_CH(sc->sc_cs));
	reg &= ~(MCSPI_CONF_FFER \| MCSPI_CONF_FFEW \| MCSPI_CONF_SBPOL \|
	MCSPI_CONF_SBE \| MCSPI_CONF_TURBO \| MCSPI_CONF_IS \|
	MCSPI_CONF_DPE1 \| MCSPI_CONF_DPE0 \| MCSPI_CONF_DMAR \|
	MCSPI_CONF_DMAW \| MCSPI_CONF_EPOL);
	reg \|= MCSPI_CONF_DPE0 \| MCSPI_CONF_EPOL \| MCSPI_CONF_WL8BITS;
	reg \|= mode; /* POL and PHA are the low bits, we can just OR-in mode */
	TI_SPI_WRITE(sc, MCSPI_CONF_CH(sc->sc_cs), reg);

	#if 0
	/* Enable channel interrupts. */
	reg = TI_SPI_READ(sc, MCSPI_IRQENABLE);
	reg \|= 0xf;
	TI_SPI_WRITE(sc, MCSPI_IRQENABLE, reg);
	#endif

	/* Start the transfer. */
	reg = TI_SPI_READ(sc, MCSPI_CTRL_CH(sc->sc_cs));
	TI_SPI_WRITE(sc, MCSPI_CTRL_CH(sc->sc_cs), reg \| MCSPI_CTRL_ENABLE);

	/* Force CS on. */
	reg = TI_SPI_READ(sc, MCSPI_CONF_CH(sc->sc_cs));
	TI_SPI_WRITE(sc, MCSPI_CONF_CH(sc->sc_cs), reg \|= MCSPI_CONF_FORCE);

	err = 0;
	if (sc->sc_fifolvl == 1)
	err = ti_spi_pio_transfer(sc);

	/* Force CS off. */
	reg = TI_SPI_READ(sc, MCSPI_CONF_CH(sc->sc_cs));
	reg &= ~MCSPI_CONF_FORCE;
	TI_SPI_WRITE(sc, MCSPI_CONF_CH(sc->sc_cs), reg);

	/* Disable IRQs. */
	reg = TI_SPI_READ(sc, MCSPI_IRQENABLE);
	reg &= ~0xf;
	TI_SPI_WRITE(sc, MCSPI_IRQENABLE, reg);
	TI_SPI_WRITE(sc, MCSPI_IRQSTATUS, 0xf);

	/* Disable the SPI channel. */
	reg = TI_SPI_READ(sc, MCSPI_CTRL_CH(sc->sc_cs));
	reg &= ~MCSPI_CTRL_ENABLE;
	TI_SPI_WRITE(sc, MCSPI_CTRL_CH(sc->sc_cs), reg);

	/* Disable FIFO. */
	reg = TI_SPI_READ(sc, MCSPI_CONF_CH(sc->sc_cs));
	reg &= ~(MCSPI_CONF_FFER \| MCSPI_CONF_FFEW);
	TI_SPI_WRITE(sc, MCSPI_CONF_CH(sc->sc_cs), reg);

	/* Release the controller and wakeup the next thread waiting for it. */
	sc->sc_flags = 0;
	wakeup_one(dev);
	TI_SPI_UNLOCK(sc);

	return (err);
	}

	static phandle_t
	ti_spi_get_node(device_t bus, device_t dev)
	{

	/* Share controller node with spibus. */
	return (ofw_bus_get_node(bus));
	}

	static device_method_t ti_spi_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ti_spi_probe),
	DEVMETHOD(device_attach, ti_spi_attach),
	DEVMETHOD(device_detach, ti_spi_detach),

	/* SPI interface */
	DEVMETHOD(spibus_transfer, ti_spi_transfer),

	/* ofw_bus interface */
	DEVMETHOD(ofw_bus_get_node, ti_spi_get_node),

	DEVMETHOD_END
	};

	static devclass_t ti_spi_devclass;

	static driver_t ti_spi_driver = {
	"spi",
	ti_spi_methods,
	sizeof(struct ti_spi_softc),
	};

	DRIVER_MODULE(ti_spi, simplebus, ti_spi_driver, ti_spi_devclass, 0, 0);
	+MODULE_DEPEND(ti_spi, ti_sysc, 1, 1, 1);
	diff --git a/sys/arm/ti/ti_sysc.c b/sys/arm/ti/ti_sysc.c
	index d428dd44a1ab..171520643c13 100644
	--- a/sys/arm/ti/ti_sysc.c
	+++ b/sys/arm/ti/ti_sysc.c
	@@ -1,128 +1,622 @@
	/*-
	* Copyright (c) 2019 Emmanuel Vadot <manu@FreeBSD.org>
	*
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/fbio.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	+#include <sys/queue.h>
	#include <sys/rman.h>
	#include <sys/resource.h>
	#include <machine/bus.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/pmap.h>

	#include <dev/fdt/simplebus.h>

	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	+#include <dev/extres/clk/clk.h>
	+
	+#include <arm/ti/ti_sysc.h>
	+#include <arm/ti/clk/clock_common.h>
	+
	+#define DEBUG_SYSC 0
	+
	+#if DEBUG_SYSC
	+#define DPRINTF(dev, msg...) device_printf(dev, msg)
	+#else
	+#define DPRINTF(dev, msg...)
	+#endif
	+
	+/* Documentation/devicetree/bindings/bus/ti-sysc.txt
	+ *
	+ * Documentation/devicetree/clock/clock-bindings.txt
	+ * Defines phandle + optional pair
	+ * Documentation/devicetree/clock/ti-clkctl.txt
	+ */
	+
	+static int ti_sysc_probe(device_t dev);
	+static int ti_sysc_attach(device_t dev);
	+static int ti_sysc_detach(device_t dev);
	+
	+#define TI_SYSC_DRA7_MCAN 15
	+#define TI_SYSC_USB_HOST_FS 14
	+#define TI_SYSC_DRA7_MCASP 13
	+#define TI_SYSC_MCASP 12
	+#define TI_SYSC_OMAP_AES 11
	+#define TI_SYSC_OMAP3_SHAM 10
	+#define TI_SYSC_OMAP4_SR 9
	+#define TI_SYSC_OMAP3630_SR 8
	+#define TI_SYSC_OMAP3430_SR 7
	+#define TI_SYSC_OMAP4_TIMER 6
	+#define TI_SYSC_OMAP2_TIMER 5
	+/* Above needs special workarounds */
	+#define TI_SYSC_OMAP4_SIMPLE 4
	+#define TI_SYSC_OMAP4 3
	+#define TI_SYSC_OMAP2 2
	+#define TI_SYSC 1
	+#define TI_SYSC_END 0
	+
	static struct ofw_compat_data compat_data[] = {
	- { "ti,sysc", 1 },
	- { NULL, 0 }
	+ { "ti,sysc-dra7-mcan", TI_SYSC_DRA7_MCAN },
	+ { "ti,sysc-usb-host-fs", TI_SYSC_USB_HOST_FS },
	+ { "ti,sysc-dra7-mcasp", TI_SYSC_DRA7_MCASP },
	+ { "ti,sysc-mcasp", TI_SYSC_MCASP },
	+ { "ti,sysc-omap-aes", TI_SYSC_OMAP_AES },
	+ { "ti,sysc-omap3-sham", TI_SYSC_OMAP3_SHAM },
	+ { "ti,sysc-omap4-sr", TI_SYSC_OMAP4_SR },
	+ { "ti,sysc-omap3630-sr", TI_SYSC_OMAP3630_SR },
	+ { "ti,sysc-omap3430-sr", TI_SYSC_OMAP3430_SR },
	+ { "ti,sysc-omap4-timer", TI_SYSC_OMAP4_TIMER },
	+ { "ti,sysc-omap2-timer", TI_SYSC_OMAP2_TIMER },
	+ /* Above needs special workarounds */
	+ { "ti,sysc-omap4-simple", TI_SYSC_OMAP4_SIMPLE },
	+ { "ti,sysc-omap4", TI_SYSC_OMAP4 },
	+ { "ti,sysc-omap2", TI_SYSC_OMAP2 },
	+ { "ti,sysc", TI_SYSC },
	+ { NULL, TI_SYSC_END }
	+};
	+
	+/* reg-names can be "rev", "sysc" and "syss" */
	+static const char * reg_names[] = { "rev", "sysc", "syss" };
	+#define REG_REV 0
	+#define REG_SYSC 1
	+#define REG_SYSS 2
	+#define REG_MAX 3
	+
	+/* master idle / slave idle mode defined in 8.1.3.2.1 / 8.1.3.2.2 */
	+#include <gnu/dts/include/dt-bindings/bus/ti-sysc.h>
	+#define SYSC_IDLE_MAX 4
	+
	+struct sysc_reg {
	+ uint64_t address;
	+ uint64_t size;
	+};
	+
	+struct clk_list {
	+ TAILQ_ENTRY(clk_list) next;
	+ clk_t clk;
	};

	struct ti_sysc_softc {
	struct simplebus_softc sc;
	+ bool attach_done;
	+
	device_t dev;
	+ int device_type;
	+
	+ struct sysc_reg reg[REG_MAX];
	+ /* Offset from host base address */
	+ uint64_t offset_reg[REG_MAX];
	+
	+ uint32_t ti_sysc_mask;
	+ int32_t ti_sysc_midle[SYSC_IDLE_MAX];
	+ int32_t ti_sysc_sidle[SYSC_IDLE_MAX];
	+ uint32_t ti_sysc_delay_us;
	+ uint32_t ti_syss_mask;
	+
	+ int num_clocks;
	+ TAILQ_HEAD(, clk_list) clk_list;
	+
	+ /* deprecated ti_hwmods */
	+ bool ti_no_reset_on_init;
	+ bool ti_no_idle_on_init;
	+ bool ti_no_idle;
	};

	-static int ti_sysc_probe(device_t dev);
	-static int ti_sysc_attach(device_t dev);
	-static int ti_sysc_detach(device_t dev);
	+/*
	+ * All sysc seems to have a reg["rev"] register.
	+ * Lets use that for identification of which module the driver are connected to.
	+ */
	+uint64_t
	+ti_sysc_get_rev_address(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ return (sc->reg[REG_REV].address);
	+}
	+
	+uint64_t
	+ti_sysc_get_rev_address_offset_host(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ return (sc->offset_reg[REG_REV]);
	+}
	+
	+uint64_t
	+ti_sysc_get_sysc_address(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ return (sc->reg[REG_SYSC].address);
	+}
	+
	+uint64_t
	+ti_sysc_get_sysc_address_offset_host(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ return (sc->offset_reg[REG_SYSC]);
	+}
	+
	+uint64_t
	+ti_sysc_get_syss_address(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ return (sc->reg[REG_SYSS].address);
	+}
	+
	+uint64_t
	+ti_sysc_get_syss_address_offset_host(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ return (sc->offset_reg[REG_SYSS]);
	+}
	+
	+/*
	+ * Due no memory region is assigned the sysc driver the children needs to
	+ * handle the practical read/writes to the registers.
	+ * Check if sysc has reset bit.
	+ */
	+uint32_t
	+ti_sysc_get_soft_reset_bit(device_t dev) {
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+ switch (sc->device_type) {
	+ case TI_SYSC_OMAP4_TIMER:
	+ case TI_SYSC_OMAP4_SIMPLE:
	+ case TI_SYSC_OMAP4:
	+ if (sc->ti_sysc_mask & SYSC_OMAP4_SOFTRESET) {
	+ return (SYSC_OMAP4_SOFTRESET);
	+ }
	+ break;
	+
	+ case TI_SYSC_OMAP2_TIMER:
	+ case TI_SYSC_OMAP2:
	+ case TI_SYSC:
	+ if (sc->ti_sysc_mask & SYSC_OMAP2_SOFTRESET) {
	+ return (SYSC_OMAP2_SOFTRESET);
	+ }
	+ break;
	+ default:
	+ break;
	+ }
	+
	+ return (0);
	+}
	+
	+int
	+ti_sysc_clock_enable(device_t dev) {
	+ struct clk_list clkp, clkp_tmp;
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+ int err;
	+
	+ TAILQ_FOREACH_SAFE(clkp, &sc->clk_list, next, clkp_tmp) {
	+ err = clk_enable(clkp->clk);
	+
	+ if (err) {
	+ DPRINTF(sc->dev, "clk_enable %s failed %d\n",
	+ clk_get_name(clkp->clk), err);
	+ break;
	+ }
	+ }
	+ return (err);
	+}
	+
	+int
	+ti_sysc_clock_disable(device_t dev) {
	+ struct clk_list clkp, clkp_tmp;
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+ int err = 0;
	+
	+ TAILQ_FOREACH_SAFE(clkp, &sc->clk_list, next, clkp_tmp) {
	+ err = clk_disable(clkp->clk);
	+
	+ if (err) {
	+ DPRINTF(sc->dev, "clk_enable %s failed %d\n",
	+ clk_get_name(clkp->clk), err);
	+ break;
	+ }
	+ }
	+ return (err);
	+}

	+static int
	+parse_regfields(struct ti_sysc_softc *sc) {
	+ phandle_t node;
	+ uint32_t parent_address_cells;
	+ uint32_t parent_size_cells;
	+ cell_t *reg;
	+ ssize_t nreg;
	+ int err, k, reg_i, prop_idx;
	+ uint32_t idx;
	+
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ /* Get parents address and size properties */
	+ err = OF_searchencprop(OF_parent(node), "#address-cells",
	+ &parent_address_cells, sizeof(parent_address_cells));
	+ if (err == -1)
	+ return (ENXIO);
	+ if (!(parent_address_cells == 1 \|\| parent_address_cells == 2)) {
	+ DPRINTF(sc->dev, "Expect parent #address-cells=[1\|\|2]\n");
	+ return (ENXIO);
	+ }
	+
	+ err = OF_searchencprop(OF_parent(node), "#size-cells",
	+ &parent_size_cells, sizeof(parent_size_cells));
	+ if (err == -1)
	+ return (ENXIO);
	+
	+ if (!(parent_size_cells == 1 \|\| parent_size_cells == 2)) {
	+ DPRINTF(sc->dev, "Expect parent #size-cells = [1\|\|2]\n");
	+ return (ENXIO);
	+ }
	+
	+ /* Grab the content of reg properties */
	+ nreg = OF_getproplen(node, "reg");
	+ reg = malloc(nreg, M_DEVBUF, M_WAITOK);
	+ OF_getencprop(node, "reg", reg, nreg);
	+
	+ /* Make sure address & size are 0 */
	+ for (idx = 0; idx < REG_MAX; idx++) {
	+ sc->reg[idx].address = 0;
	+ sc->reg[idx].size = 0;
	+ }
	+
	+ /* Loop through reg-names and figure out which reg-name corresponds to
	+ * index populate the values into the reg array.
	+ */
	+ for (idx = 0, reg_i = 0; idx < REG_MAX && reg_i < nreg; idx++) {
	+ err = ofw_bus_find_string_index(node, "reg-names",
	+ reg_names[idx], &prop_idx);
	+ if (err != 0)
	+ continue;
	+
	+ for (k = 0; k < parent_address_cells; k++) {
	+ sc->reg[prop_idx].address <<= 32;
	+ sc->reg[prop_idx].address \|= reg[reg_i++];
	+ }
	+
	+ for (k = 0; k < parent_size_cells; k++) {
	+ sc->reg[prop_idx].size <<= 32;
	+ sc->reg[prop_idx].size \|= reg[reg_i++];
	+ }
	+
	+ if (sc->sc.nranges == 0)
	+ sc->offset_reg[prop_idx] = sc->reg[prop_idx].address;
	+ else
	+ sc->offset_reg[prop_idx] = sc->reg[prop_idx].address -
	+ sc->sc.ranges[REG_REV].host;
	+
	+ DPRINTF(sc->dev, "reg[%s] adress %#jx size %#jx\n",
	+ reg_names[idx],
	+ sc->reg[prop_idx].address,
	+ sc->reg[prop_idx].size);
	+ }
	+ free(reg, M_DEVBUF);
	+ return (0);
	+}
	+
	+static void
	+parse_idle(struct ti_sysc_softc sc, const char name, uint32_t *idle) {
	+ phandle_t node;
	+ cell_t value[SYSC_IDLE_MAX];
	+ int len, no, i;
	+
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ if (!OF_hasprop(node, name)) {
	+ return;
	+ }
	+
	+ len = OF_getproplen(node, name);
	+ no = len / sizeof(cell_t);
	+ if (no >= SYSC_IDLE_MAX) {
	+ DPRINTF(sc->dev, "Limit %s\n", name);
	+ no = SYSC_IDLE_MAX-1;
	+ len = no * sizeof(cell_t);
	+ }
	+
	+ OF_getencprop(node, name, value, len);
	+ for (i = 0; i < no; i++) {
	+ idle[i] = value[i];
	+#if DEBUG_SYSC
	+ DPRINTF(sc->dev, "%s[%d] = %d ",
	+ name, i, value[i]);
	+ switch(value[i]) {
	+ case SYSC_IDLE_FORCE:
	+ DPRINTF(sc->dev, "SYSC_IDLE_FORCE\n");
	+ break;
	+ case SYSC_IDLE_NO:
	+ DPRINTF(sc->dev, "SYSC_IDLE_NO\n");
	+ break;
	+ case SYSC_IDLE_SMART:
	+ DPRINTF(sc->dev, "SYSC_IDLE_SMART\n");
	+ break;
	+ case SYSC_IDLE_SMART_WKUP:
	+ DPRINTF(sc->dev, "SYSC_IDLE_SMART_WKUP\n");
	+ break;
	+ }
	+#endif
	+ }
	+ for ( ; i < SYSC_IDLE_MAX; i++)
	+ idle[i] = -1;
	+}
	+
	+static int
	+ti_sysc_attach_clocks(struct ti_sysc_softc *sc) {
	+ clk_t *clk;
	+ struct clk_list *clkp;
	+ int index, err;
	+ phandle_t cnode;
	+
	+ clk = malloc(sc->num_clocks*sizeof(clk_t), M_DEVBUF, M_WAITOK \| M_ZERO);
	+
	+ cnode = ofw_bus_get_node(sc->dev);
	+
	+ /* Check if all clocks can be found */
	+ for (index = 0; index < sc->num_clocks; index++) {
	+ err = clk_get_by_ofw_index(sc->dev, 0, index, &clk[index]);
	+
	+ if (err != 0) {
	+ free(clk, M_DEVBUF);
	+ return (1);
	+ }
	+ }
	+
	+ /* All clocks are found, add to list */
	+ for (index = 0; index < sc->num_clocks; index++) {
	+ clkp = malloc(sizeof(*clkp), M_DEVBUF, M_WAITOK \| M_ZERO);
	+ clkp->clk = clk[index];
	+ TAILQ_INSERT_TAIL(&sc->clk_list, clkp, next);
	+ }
	+
	+ /* Release the clk array */
	+ free(clk, M_DEVBUF);
	+ return (0);
	+}
	+
	+static int
	+ti_sysc_simplebus_attach_child(device_t dev) {
	+ device_t cdev;
	+ phandle_t node, child;
	+ struct ti_sysc_softc *sc = device_get_softc(dev);
	+
	+ node = ofw_bus_get_node(sc->dev);
	+
	+ for (child = OF_child(node); child > 0; child = OF_peer(child)) {
	+ cdev = simplebus_add_device(sc->dev, child, 0, NULL, -1, NULL);
	+ if (cdev != NULL)
	+ device_probe_and_attach(cdev);
	+ }
	+ return (0);
	+}
	+
	+/* Device interface */
	static int
	ti_sysc_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
	return (ENXIO);

	device_set_desc(dev, "TI SYSC Interconnect");
	- if (!bootverbose)
	- device_quiet(dev);

	return (BUS_PROBE_DEFAULT);
	}

	static int
	ti_sysc_attach(device_t dev)
	{
	struct ti_sysc_softc *sc;
	- device_t cdev;
	- phandle_t node, child;
	+ phandle_t node;
	+ int err;
	+ cell_t value;

	sc = device_get_softc(dev);
	sc->dev = dev;
	- node = ofw_bus_get_node(dev);
	+ sc->device_type = ofw_bus_search_compatible(dev, compat_data)->ocd_data;

	- simplebus_init(dev, node);
	+ node = ofw_bus_get_node(sc->dev);
	+ /* ranges - use simplebus */
	+ simplebus_init(sc->dev, node);
	if (simplebus_fill_ranges(node, &sc->sc) < 0) {
	- device_printf(dev, "could not get ranges\n");
	+ DPRINTF(sc->dev, "could not get ranges\n");
	return (ENXIO);
	}

	- for (child = OF_child(node); child > 0; child = OF_peer(child)) {
	- cdev = simplebus_add_device(dev, child, 0, NULL, -1, NULL);
	- if (cdev != NULL)
	- device_probe_and_attach(cdev);
	+ if (sc->sc.nranges == 0) {
	+ DPRINTF(sc->dev, "nranges == 0\n");
	+ return (ENXIO);
	+ }
	+
	+ /* Required field reg & reg-names - assume at least "rev" exists */
	+ err = parse_regfields(sc);
	+ if (err) {
	+ DPRINTF(sc->dev, "parse_regfields failed %d\n", err);
	+ return (ENXIO);
	+ }
	+
	+ /* Optional */
	+ if (OF_hasprop(node, "ti,sysc-mask")) {
	+ OF_getencprop(node, "ti,sysc-mask", &value, sizeof(cell_t));
	+ sc->ti_sysc_mask = value;
	+ }
	+ if (OF_hasprop(node, "ti,syss-mask")) {
	+ OF_getencprop(node, "ti,syss-mask", &value, sizeof(cell_t));
	+ sc->ti_syss_mask = value;
	+ }
	+ if (OF_hasprop(node, "ti,sysc-delay-us")) {
	+ OF_getencprop(node, "ti,sysc-delay-us", &value, sizeof(cell_t));
	+ sc->ti_sysc_delay_us = value;
	+ }
	+
	+ DPRINTF(sc->dev, "sysc_mask %x syss_mask %x delay_us %x\n",
	+ sc->ti_sysc_mask, sc->ti_syss_mask, sc->ti_sysc_delay_us);
	+
	+ parse_idle(sc, "ti,sysc-midle", sc->ti_sysc_midle);
	+ parse_idle(sc, "ti,sysc-sidle", sc->ti_sysc_sidle);
	+
	+ if (OF_hasprop(node, "ti,no-reset-on-init"))
	+ sc->ti_no_reset_on_init = true;
	+ else
	+ sc->ti_no_reset_on_init = false;
	+
	+ if (OF_hasprop(node, "ti,no-idle-on-init"))
	+ sc->ti_no_idle_on_init = true;
	+ else
	+ sc->ti_no_idle_on_init = false;
	+
	+ if (OF_hasprop(node, "ti,no-idle"))
	+ sc->ti_no_idle = true;
	+ else
	+ sc->ti_no_idle = false;
	+
	+ DPRINTF(sc->dev,
	+ "no-reset-on-init %d, no-idle-on-init %d, no-idle %d\n",
	+ sc->ti_no_reset_on_init,
	+ sc->ti_no_idle_on_init,
	+ sc->ti_no_idle);
	+
	+ if (OF_hasprop(node, "clocks")) {
	+ struct clock_cell_info cell_info;
	+ read_clock_cells(sc->dev, &cell_info);
	+ free(cell_info.clock_cells, M_DEVBUF);
	+ free(cell_info.clock_cells_ncells, M_DEVBUF);
	+
	+ sc->num_clocks = cell_info.num_real_clocks;
	+ TAILQ_INIT(&sc->clk_list);
	+
	+ err = ti_sysc_attach_clocks(sc);
	+ if (err) {
	+ DPRINTF(sc->dev, "Failed to attach clocks\n");
	+ return (bus_generic_attach(sc->dev));
	+ }
	+ }
	+
	+ err = ti_sysc_simplebus_attach_child(sc->dev);
	+ if (err) {
	+ DPRINTF(sc->dev, "ti_sysc_simplebus_attach_child %d\n",
	+ err);
	+ return (err);
	}

	- return (bus_generic_attach(dev));
	+ sc->attach_done = true;
	+
	+ return (bus_generic_attach(sc->dev));
	}

	static int
	ti_sysc_detach(device_t dev)
	{
	-
	return (EBUSY);
	}

	+/* Bus interface */
	+static void
	+ti_sysc_new_pass(device_t dev)
	+{
	+ struct ti_sysc_softc *sc;
	+ int err;
	+ phandle_t node;
	+
	+ sc = device_get_softc(dev);
	+
	+ if (sc->attach_done) {
	+ bus_generic_new_pass(sc->dev);
	+ return;
	+ }
	+
	+ node = ofw_bus_get_node(sc->dev);
	+ if (OF_hasprop(node, "clocks")) {
	+ err = ti_sysc_attach_clocks(sc);
	+ if (err) {
	+ DPRINTF(sc->dev, "Failed to attach clocks\n");
	+ return;
	+ }
	+ }
	+
	+ err = ti_sysc_simplebus_attach_child(sc->dev);
	+ if (err) {
	+ DPRINTF(sc->dev,
	+ "ti_sysc_simplebus_attach_child failed %d\n", err);
	+ return;
	+ }
	+ sc->attach_done = true;
	+
	+ bus_generic_attach(sc->dev);
	+}
	+
	static device_method_t ti_sysc_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ti_sysc_probe),
	DEVMETHOD(device_attach, ti_sysc_attach),
	DEVMETHOD(device_detach, ti_sysc_detach),

	+ /* Bus interface */
	+ DEVMETHOD(bus_new_pass, ti_sysc_new_pass),
	+
	DEVMETHOD_END
	};

	DEFINE_CLASS_1(ti_sysc, ti_sysc_driver, ti_sysc_methods,
	- sizeof(struct ti_sysc_softc), simplebus_driver);
	+ sizeof(struct ti_sysc_softc), simplebus_driver);

	static devclass_t ti_sysc_devclass;

	EARLY_DRIVER_MODULE(ti_sysc, simplebus, ti_sysc_driver,
	-ti_sysc_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
	+ ti_sysc_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_FIRST);
	diff --git a/sys/arm/ti/ti_sysc.h b/sys/arm/ti/ti_sysc.h
	new file mode 100644
	index 000000000000..b74222f05772
	--- /dev/null
	+++ b/sys/arm/ti/ti_sysc.h
	@@ -0,0 +1,43 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Oskar Holmlund <oskar.holmlund@ohdata.se>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
	+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
	+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+#ifndef __TI_SYSC__
	+#define __TI_SYSC__
	+
	+uint64_t ti_sysc_get_rev_address(device_t dev);
	+uint64_t ti_sysc_get_rev_address_offset_host(device_t dev);
	+uint64_t ti_sysc_get_sysc_address(device_t dev);
	+uint64_t ti_sysc_get_sysc_address_offset_host(device_t dev);
	+uint64_t ti_sysc_get_syss_address(device_t dev);
	+uint64_t ti_sysc_get_syss_address_offset_host(device_t dev);
	+int ti_sysc_clock_enable(device_t dev);
	+int ti_sysc_clock_disable(device_t dev);
	+
	+uint32_t ti_sysc_get_soft_reset_bit(device_t dev);
	+
	+#endif /* __TI_SYSC__ */
	diff --git a/sys/arm/ti/ti_wdt.c b/sys/arm/ti/ti_wdt.c
	index 539e4d93950f..29ae41eac531 100644
	--- a/sys/arm/ti/ti_wdt.c
	+++ b/sys/arm/ti/ti_wdt.c
	@@ -1,276 +1,276 @@
	/*-
	* Copyright (c) 2014 Rui Paulo <rpaulo@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
	* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
	* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/eventhandler.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/rman.h>
	#include <sys/event.h>
	#include <sys/selinfo.h>
	#include <sys/watchdog.h>
	#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <machine/frame.h>
	#include <machine/intr.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <machine/bus.h>

	-#include <arm/ti/ti_prcm.h>
	#include <arm/ti/ti_wdt.h>

	#ifdef DEBUG
	#define DPRINTF(fmt, ...) do { \
	printf("%s: ", __func__); \
	printf(fmt, __VA_ARGS__); \
	} while (0)
	#else
	#define DPRINTF(fmt, ...)
	#endif

	static device_probe_t ti_wdt_probe;
	static device_attach_t ti_wdt_attach;
	static device_detach_t ti_wdt_detach;
	static void ti_wdt_intr(void *);
	static void ti_wdt_event(void , unsigned int, int );

	struct ti_wdt_softc {
	struct resource *sc_mem_res;
	struct resource *sc_irq_res;
	void *sc_intr;
	bus_space_tag_t sc_bt;
	bus_space_handle_t sc_bh;
	eventhandler_tag sc_ev_tag;
	};

	static device_method_t ti_wdt_methods[] = {
	DEVMETHOD(device_probe, ti_wdt_probe),
	DEVMETHOD(device_attach, ti_wdt_attach),
	DEVMETHOD(device_detach, ti_wdt_detach),

	DEVMETHOD_END
	};

	static driver_t ti_wdt_driver = {
	"ti_wdt",
	ti_wdt_methods,
	sizeof(struct ti_wdt_softc)
	};

	static devclass_t ti_wdt_devclass;

	DRIVER_MODULE(ti_wdt, simplebus, ti_wdt_driver, ti_wdt_devclass, 0, 0);
	+MODULE_DEPEND(ti_wdt, ti_sysc, 1, 1, 1);

	static __inline uint32_t
	ti_wdt_reg_read(struct ti_wdt_softc *sc, uint32_t reg)
	{

	return (bus_space_read_4(sc->sc_bt, sc->sc_bh, reg));
	}

	static __inline void
	ti_wdt_reg_write(struct ti_wdt_softc *sc, uint32_t reg, uint32_t val)
	{

	bus_space_write_4(sc->sc_bt, sc->sc_bh, reg, val);
	}

	/*
	* Wait for the write to a specific synchronised register to complete.
	*/
	static __inline void
	ti_wdt_reg_wait(struct ti_wdt_softc *sc, uint32_t bit)
	{

	while (ti_wdt_reg_read(sc, TI_WDT_WWPS) & bit)
	DELAY(10);
	}

	static __inline void
	ti_wdt_disable(struct ti_wdt_softc *sc)
	{

	DPRINTF("disabling watchdog %p\n", sc);
	ti_wdt_reg_write(sc, TI_WDT_WSPR, 0xAAAA);
	ti_wdt_reg_wait(sc, TI_W_PEND_WSPR);
	ti_wdt_reg_write(sc, TI_WDT_WSPR, 0x5555);
	ti_wdt_reg_wait(sc, TI_W_PEND_WSPR);
	}

	static __inline void
	ti_wdt_enable(struct ti_wdt_softc *sc)
	{

	DPRINTF("enabling watchdog %p\n", sc);
	ti_wdt_reg_write(sc, TI_WDT_WSPR, 0xBBBB);
	ti_wdt_reg_wait(sc, TI_W_PEND_WSPR);
	ti_wdt_reg_write(sc, TI_WDT_WSPR, 0x4444);
	ti_wdt_reg_wait(sc, TI_W_PEND_WSPR);
	}

	static int
	ti_wdt_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);
	if (ofw_bus_is_compatible(dev, "ti,omap3-wdt")) {
	device_set_desc(dev, "TI Watchdog Timer");
	return (BUS_PROBE_DEFAULT);
	}

	return (ENXIO);
	}

	static int
	ti_wdt_attach(device_t dev)
	{
	struct ti_wdt_softc *sc;
	int rid;

	sc = device_get_softc(dev);
	rid = 0;
	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	RF_ACTIVE);
	if (sc->sc_mem_res == NULL) {
	device_printf(dev, "could not allocate memory resource\n");
	return (ENXIO);
	}
	sc->sc_bt = rman_get_bustag(sc->sc_mem_res);
	sc->sc_bh = rman_get_bushandle(sc->sc_mem_res);
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE);
	if (sc->sc_irq_res == NULL) {
	device_printf(dev, "could not allocate interrupt resource\n");
	ti_wdt_detach(dev);
	return (ENXIO);
	}
	if (bus_setup_intr(dev, sc->sc_irq_res, INTR_MPSAFE \| INTR_TYPE_MISC,
	NULL, ti_wdt_intr, sc, &sc->sc_intr) != 0) {
	device_printf(dev,
	"unable to setup the interrupt handler\n");
	ti_wdt_detach(dev);
	return (ENXIO);
	}
	/* Reset, enable interrupts and stop the watchdog. */
	ti_wdt_reg_write(sc, TI_WDT_WDSC,
	ti_wdt_reg_read(sc, TI_WDT_WDSC) \| TI_WDSC_SR);
	while (ti_wdt_reg_read(sc, TI_WDT_WDSC) & TI_WDSC_SR)
	DELAY(10);
	ti_wdt_reg_write(sc, TI_WDT_WIRQENSET, TI_IRQ_EN_OVF \| TI_IRQ_EN_DLY);
	ti_wdt_disable(sc);
	if (bootverbose)
	device_printf(dev, "revision: 0x%x\n",
	ti_wdt_reg_read(sc, TI_WDT_WIDR));
	sc->sc_ev_tag = EVENTHANDLER_REGISTER(watchdog_list, ti_wdt_event, sc,
	0);

	return (0);
	}

	static int
	ti_wdt_detach(device_t dev)
	{
	struct ti_wdt_softc *sc;

	sc = device_get_softc(dev);
	if (sc->sc_ev_tag)
	EVENTHANDLER_DEREGISTER(watchdog_list, sc->sc_ev_tag);
	if (sc->sc_intr)
	bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intr);
	if (sc->sc_irq_res)
	bus_release_resource(dev, SYS_RES_IRQ,
	rman_get_rid(sc->sc_irq_res), sc->sc_irq_res);
	if (sc->sc_mem_res)
	bus_release_resource(dev, SYS_RES_MEMORY,
	rman_get_rid(sc->sc_mem_res), sc->sc_mem_res);

	return (0);
	}

	static void
	ti_wdt_intr(void *arg)
	{
	struct ti_wdt_softc *sc;

	sc = arg;
	DPRINTF("interrupt %p", sc);
	ti_wdt_reg_write(sc, TI_WDT_WIRQSTAT, TI_IRQ_EV_OVF \| TI_IRQ_EV_DLY);
	/* TODO: handle interrupt */
	}

	static void
	ti_wdt_event(void arg, unsigned int cmd, int error)
	{
	struct ti_wdt_softc *sc;
	uint8_t s;
	uint32_t wldr;
	uint32_t ptv;

	sc = arg;
	ti_wdt_disable(sc);
	if (cmd == WD_TO_NEVER) {
	*error = 0;
	return;
	}
	DPRINTF("cmd 0x%x\n", cmd);
	cmd &= WD_INTERVAL;
	if (cmd < WD_TO_1SEC) {
	*error = EINVAL;
	return;
	}
	s = 1 << (cmd - WD_TO_1SEC);
	DPRINTF("seconds %u\n", s);
	/*
	* Leave the pre-scaler with its default values:
	* PTV = 0 == 2**0 == 1
	* PRE = 1 (enabled)
	*
	* Compute the load register value assuming a 32kHz clock.
	* See OVF_Rate in the WDT section of the AM335x TRM.
	*/
	ptv = 0;
	wldr = 0xffffffff - (s * (32768 / (1 << ptv))) + 1;
	DPRINTF("wldr 0x%x\n", wldr);
	ti_wdt_reg_write(sc, TI_WDT_WLDR, wldr);
	/*
	* Trigger a timer reload.
	*/
	ti_wdt_reg_write(sc, TI_WDT_WTGR,
	ti_wdt_reg_read(sc, TI_WDT_WTGR) + 1);
	ti_wdt_reg_wait(sc, TI_W_PEND_WTGR);
	ti_wdt_enable(sc);
	*error = 0;
	}
	diff --git a/sys/arm/ti/usb/omap_ehci.c b/sys/arm/ti/usb/omap_ehci.c
	index c14a483b7175..adc2c122f054 100644
	--- a/sys/arm/ti/usb/omap_ehci.c
	+++ b/sys/arm/ti/usb/omap_ehci.c
	@@ -1,472 +1,471 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2011
	* Ben Gray <ben.r.gray@gmail.com>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/rman.h>
	#include <sys/module.h>
	#include <sys/proc.h>
	#include <sys/condvar.h>

	#include <dev/fdt/simplebus.h>
	#include <dev/fdt/fdt_common.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/usb/usb.h>
	#include <dev/usb/usbdi.h>

	#include <dev/usb/usb_core.h>
	#include <dev/usb/usb_busdma.h>
	#include <dev/usb/usb_process.h>
	#include <dev/usb/usb_util.h>

	#include <dev/usb/usb_controller.h>
	#include <dev/usb/usb_bus.h>
	#include <dev/usb/controller/ehci.h>
	#include <dev/usb/controller/ehcireg.h>

	#include <machine/bus.h>

	-#include <arm/ti/ti_prcm.h>
	#include <arm/ti/usb/omap_usb.h>

	#include <arm/ti/omap4/pandaboard/pandaboard.h>

	/* EHCI */
	#define OMAP_USBHOST_HCCAPBASE 0x0000
	#define OMAP_USBHOST_HCSPARAMS 0x0004
	#define OMAP_USBHOST_HCCPARAMS 0x0008
	#define OMAP_USBHOST_USBCMD 0x0010
	#define OMAP_USBHOST_USBSTS 0x0014
	#define OMAP_USBHOST_USBINTR 0x0018
	#define OMAP_USBHOST_FRINDEX 0x001C
	#define OMAP_USBHOST_CTRLDSSEGMENT 0x0020
	#define OMAP_USBHOST_PERIODICLISTBASE 0x0024
	#define OMAP_USBHOST_ASYNCLISTADDR 0x0028
	#define OMAP_USBHOST_CONFIGFLAG 0x0050
	#define OMAP_USBHOST_PORTSC(i) (0x0054 + (0x04 * (i)))
	#define OMAP_USBHOST_INSNREG00 0x0090
	#define OMAP_USBHOST_INSNREG01 0x0094
	#define OMAP_USBHOST_INSNREG02 0x0098
	#define OMAP_USBHOST_INSNREG03 0x009C
	#define OMAP_USBHOST_INSNREG04 0x00A0
	#define OMAP_USBHOST_INSNREG05_UTMI 0x00A4
	#define OMAP_USBHOST_INSNREG05_ULPI 0x00A4
	#define OMAP_USBHOST_INSNREG06 0x00A8
	#define OMAP_USBHOST_INSNREG07 0x00AC
	#define OMAP_USBHOST_INSNREG08 0x00B0

	#define OMAP_USBHOST_INSNREG04_DISABLE_UNSUSPEND (1 << 5)

	#define OMAP_USBHOST_INSNREG05_ULPI_CONTROL_SHIFT 31
	#define OMAP_USBHOST_INSNREG05_ULPI_PORTSEL_SHIFT 24
	#define OMAP_USBHOST_INSNREG05_ULPI_OPSEL_SHIFT 22
	#define OMAP_USBHOST_INSNREG05_ULPI_REGADD_SHIFT 16
	#define OMAP_USBHOST_INSNREG05_ULPI_EXTREGADD_SHIFT 8
	#define OMAP_USBHOST_INSNREG05_ULPI_WRDATA_SHIFT 0

	#define ULPI_FUNC_CTRL_RESET (1 << 5)

	/-------------------------------------------------------------------------/

	/*
	* Macros for Set and Clear
	* See ULPI 1.1 specification to find the registers with Set and Clear offsets
	*/
	#define ULPI_SET(a) (a + 1)
	#define ULPI_CLR(a) (a + 2)

	/-------------------------------------------------------------------------/

	/*
	* Register Map
	*/
	#define ULPI_VENDOR_ID_LOW 0x00
	#define ULPI_VENDOR_ID_HIGH 0x01
	#define ULPI_PRODUCT_ID_LOW 0x02
	#define ULPI_PRODUCT_ID_HIGH 0x03
	#define ULPI_FUNC_CTRL 0x04
	#define ULPI_IFC_CTRL 0x07
	#define ULPI_OTG_CTRL 0x0a
	#define ULPI_USB_INT_EN_RISE 0x0d
	#define ULPI_USB_INT_EN_FALL 0x10
	#define ULPI_USB_INT_STS 0x13
	#define ULPI_USB_INT_LATCH 0x14
	#define ULPI_DEBUG 0x15
	#define ULPI_SCRATCH 0x16

	#define OMAP_EHCI_HC_DEVSTR "TI OMAP USB 2.0 controller"

	struct omap_ehci_softc {
	ehci_softc_t base; /* storage for EHCI code */
	device_t sc_dev;
	};

	static device_attach_t omap_ehci_attach;
	static device_detach_t omap_ehci_detach;

	/**
	* omap_ehci_read_4 - read a 32-bit value from the EHCI registers
	* omap_ehci_write_4 - write a 32-bit value from the EHCI registers
	* @sc: omap ehci device context
	* @off: byte offset within the register set to read from
	* @val: the value to write into the register
	*
	*
	* LOCKING:
	* None
	*
	* RETURNS:
	* nothing in case of write function, if read function returns the value read.
	*/
	static inline uint32_t
	omap_ehci_read_4(struct omap_ehci_softc *sc, bus_size_t off)
	{
	return (bus_read_4(sc->base.sc_io_res, off));
	}

	static inline void
	omap_ehci_write_4(struct omap_ehci_softc *sc, bus_size_t off, uint32_t val)
	{
	bus_write_4(sc->base.sc_io_res, off, val);
	}

	/**
	* omap_ehci_soft_phy_reset - resets the phy using the reset command
	* @isc: omap ehci device context
	* @port: port to send the reset over
	*
	*
	* LOCKING:
	* none
	*
	* RETURNS:
	* nothing
	*/
	static void
	omap_ehci_soft_phy_reset(struct omap_ehci_softc *isc, unsigned int port)
	{
	unsigned long timeout = (hz < 10) ? 1 : ((100 * hz) / 1000);
	uint32_t reg;

	reg = ULPI_FUNC_CTRL_RESET
	/* FUNCTION_CTRL_SET register */
	\| (ULPI_SET(ULPI_FUNC_CTRL) << OMAP_USBHOST_INSNREG05_ULPI_REGADD_SHIFT)
	/* Write */
	\| (2 << OMAP_USBHOST_INSNREG05_ULPI_OPSEL_SHIFT)
	/* PORTn */
	\| ((port + 1) << OMAP_USBHOST_INSNREG05_ULPI_PORTSEL_SHIFT)
	/* start ULPI access*/
	\| (1 << OMAP_USBHOST_INSNREG05_ULPI_CONTROL_SHIFT);

	omap_ehci_write_4(isc, OMAP_USBHOST_INSNREG05_ULPI, reg);

	/* Wait for ULPI access completion */
	while ((omap_ehci_read_4(isc, OMAP_USBHOST_INSNREG05_ULPI)
	& (1 << OMAP_USBHOST_INSNREG05_ULPI_CONTROL_SHIFT))) {

	/* Sleep for a tick */
	pause("USBPHY_RESET", 1);

	if (timeout-- == 0) {
	device_printf(isc->sc_dev, "PHY reset operation timed out\n");
	break;
	}
	}
	}

	/**
	* omap_ehci_init - initialises the USB host EHCI controller
	* @isc: omap ehci device context
	*
	* This initialisation routine is quite heavily based on the work done by the
	* OMAP Linux team (for which I thank them very much). The init sequence is
	* almost identical, diverging only for the FreeBSD specifics.
	*
	* LOCKING:
	* none
	*
	* RETURNS:
	* 0 on success, a negative error code on failure.
	*/
	static int
	omap_ehci_init(struct omap_ehci_softc *isc)
	{
	uint32_t reg = 0;
	int i;
	device_t uhh_dev;

	uhh_dev = device_get_parent(isc->sc_dev);
	device_printf(isc->sc_dev, "Starting TI EHCI USB Controller\n");

	/* Set the interrupt threshold control, it controls the maximum rate at
	* which the host controller issues interrupts. We set it to 1 microframe
	* at startup - the default is 8 mircoframes (equates to 1ms).
	*/
	reg = omap_ehci_read_4(isc, OMAP_USBHOST_USBCMD);
	reg &= 0xff00ffff;
	reg \|= (1 << 16);
	omap_ehci_write_4(isc, OMAP_USBHOST_USBCMD, reg);

	/* Soft reset the PHY using PHY reset command over ULPI */
	for (i = 0; i < OMAP_HS_USB_PORTS; i++) {
	if (omap_usb_port_mode(uhh_dev, i) == EHCI_HCD_OMAP_MODE_PHY)
	omap_ehci_soft_phy_reset(isc, i);

	}

	return(0);
	}

	/**
	* omap_ehci_probe - starts the given command
	* @dev:
	*
	* Effectively boilerplate EHCI resume code.
	*
	* LOCKING:
	* Caller should be holding the OMAP3_MMC lock.
	*
	* RETURNS:
	* EH_HANDLED or EH_NOT_HANDLED
	*/
	static int
	omap_ehci_probe(device_t dev)
	{
	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,ehci-omap"))
	return (ENXIO);

	device_set_desc(dev, OMAP_EHCI_HC_DEVSTR);

	return (BUS_PROBE_DEFAULT);
	}

	/**
	* omap_ehci_attach - driver entry point, sets up the ECHI controller/driver
	* @dev: the new device handle
	*
	* Sets up bus spaces, interrupt handles, etc for the EHCI controller. It also
	* parses the resource hints and calls omap_ehci_init() to initialise the
	* H/W.
	*
	* LOCKING:
	* none
	*
	* RETURNS:
	* 0 on success or a positive error code on failure.
	*/
	static int
	omap_ehci_attach(device_t dev)
	{
	struct omap_ehci_softc *isc = device_get_softc(dev);
	ehci_softc_t *sc = &isc->base;
	#ifdef SOC_OMAP4
	phandle_t root;
	#endif
	int err;
	int rid;

	#ifdef SOC_OMAP4
	/*
	* If we're running a Pandaboard, run Pandaboard-specific
	* init code.
	*/
	root = OF_finddevice("/");
	if (ofw_bus_node_is_compatible(root, "ti,omap4-panda"))
	pandaboard_usb_hub_init();
	#endif

	/* initialise some bus fields */
	sc->sc_bus.parent = dev;
	sc->sc_bus.devices = sc->sc_devices;
	sc->sc_bus.devices_max = EHCI_MAX_DEVICES;
	sc->sc_bus.dma_bits = 32;

	sprintf(sc->sc_vendor, "Texas Instruments");

	/* save the device */
	isc->sc_dev = dev;

	/* get all DMA memory */
	if (usb_bus_mem_alloc_all(&sc->sc_bus, USB_GET_DMA_TAG(dev),
	&ehci_iterate_hw_softc)) {
	return (ENOMEM);
	}

	/* Allocate resource for the EHCI register set */
	rid = 0;
	sc->sc_io_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (!sc->sc_io_res) {
	device_printf(dev, "Error: Could not map EHCI memory\n");
	goto error;
	}
	/* Request an interrupt resource */
	rid = 0;
	sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE);
	if (sc->sc_irq_res == NULL) {
	device_printf(dev, "Error: could not allocate irq\n");
	goto error;
	}

	/* Add this device as a child of the USBus device */
	sc->sc_bus.bdev = device_add_child(dev, "usbus", -1);
	if (!sc->sc_bus.bdev) {
	device_printf(dev, "Error: could not add USB device\n");
	goto error;
	}

	device_set_ivars(sc->sc_bus.bdev, &sc->sc_bus);
	device_set_desc(sc->sc_bus.bdev, OMAP_EHCI_HC_DEVSTR);

	/* Initialise the ECHI registers */
	err = omap_ehci_init(isc);
	if (err) {
	device_printf(dev, "Error: could not setup OMAP EHCI, %d\n", err);
	goto error;
	}

	/* Set the tag and size of the register set in the EHCI context */
	sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res);
	sc->sc_io_tag = rman_get_bustag(sc->sc_io_res);
	sc->sc_io_size = rman_get_size(sc->sc_io_res);

	/* Setup the interrupt */
	err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO \| INTR_MPSAFE,
	NULL, (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl);
	if (err) {
	device_printf(dev, "Error: could not setup irq, %d\n", err);
	sc->sc_intr_hdl = NULL;
	goto error;
	}

	/* Finally we are ready to kick off the ECHI host controller */
	err = ehci_init(sc);
	if (err == 0) {
	err = device_probe_and_attach(sc->sc_bus.bdev);
	}
	if (err) {
	device_printf(dev, "Error: USB init failed err=%d\n", err);
	goto error;
	}

	return (0);

	error:
	omap_ehci_detach(dev);
	return (ENXIO);
	}

	/**
	* omap_ehci_detach - detach the device and cleanup the driver
	* @dev: device handle
	*
	* Clean-up routine where everything initialised in omap_ehci_attach is
	* freed and cleaned up. This function calls omap_ehci_fini() to shutdown
	* the on-chip module.
	*
	* LOCKING:
	* none
	*
	* RETURNS:
	* Always returns 0 (success).
	*/
	static int
	omap_ehci_detach(device_t dev)
	{
	struct omap_ehci_softc *isc = device_get_softc(dev);
	ehci_softc_t *sc = &isc->base;
	int err;

	/* during module unload there are lots of children leftover */
	device_delete_children(dev);

	/*
	* disable interrupts that might have been switched on in ehci_init
	*/
	if (sc->sc_io_res) {
	EWRITE4(sc, EHCI_USBINTR, 0);
	}

	if (sc->sc_irq_res && sc->sc_intr_hdl) {
	/*
	* only call ehci_detach() after ehci_init()
	*/
	ehci_detach(sc);

	err = bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_intr_hdl);
	if (err)
	device_printf(dev, "Error: could not tear down irq, %d\n", err);
	sc->sc_intr_hdl = NULL;
	}

	/* Free the resources stored in the base EHCI handler */
	if (sc->sc_irq_res) {
	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res);
	sc->sc_irq_res = NULL;
	}
	if (sc->sc_io_res) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_io_res);
	sc->sc_io_res = NULL;
	}

	return (0);
	}

	static device_method_t ehci_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, omap_ehci_probe),
	DEVMETHOD(device_attach, omap_ehci_attach),
	DEVMETHOD(device_detach, omap_ehci_detach),

	DEVMETHOD(device_suspend, bus_generic_suspend),
	DEVMETHOD(device_resume, bus_generic_resume),
	DEVMETHOD(device_shutdown, bus_generic_shutdown),

	/* Bus interface */
	DEVMETHOD(bus_print_child, bus_generic_print_child),

	{0, 0}
	};

	static driver_t ehci_driver = {
	"ehci",
	ehci_methods,
	sizeof(struct omap_ehci_softc),
	};

	static devclass_t ehci_devclass;

	DRIVER_MODULE(omap_ehci, omap_uhh, ehci_driver, ehci_devclass, 0, 0);
	diff --git a/sys/arm/ti/usb/omap_host.c b/sys/arm/ti/usb/omap_host.c
	index 304e80d33df8..736ccf17262e 100644
	--- a/sys/arm/ti/usb/omap_host.c
	+++ b/sys/arm/ti/usb/omap_host.c
	@@ -1,465 +1,468 @@
	/*-
	* Copyright (c) 2015 Oleksandr Tymoshenko <gonzo@freebsd.org>
	* Copyright (c) 2011 Ben Gray <ben.r.gray@gmail.com>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/rman.h>
	#include <sys/module.h>

	#include <dev/fdt/simplebus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <machine/bus.h>

	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/usb/omap_usb.h>

	/*
	* USB Host Module
	*/

	/* UHH */
	#define OMAP_USBHOST_UHH_REVISION 0x0000
	#define OMAP_USBHOST_UHH_SYSCONFIG 0x0010
	#define OMAP_USBHOST_UHH_SYSSTATUS 0x0014
	#define OMAP_USBHOST_UHH_HOSTCONFIG 0x0040
	#define OMAP_USBHOST_UHH_DEBUG_CSR 0x0044

	/* UHH Register Set */
	#define UHH_SYSCONFIG_MIDLEMODE_MASK (3UL << 12)
	#define UHH_SYSCONFIG_MIDLEMODE_SMARTSTANDBY (2UL << 12)
	#define UHH_SYSCONFIG_MIDLEMODE_NOSTANDBY (1UL << 12)
	#define UHH_SYSCONFIG_MIDLEMODE_FORCESTANDBY (0UL << 12)
	#define UHH_SYSCONFIG_CLOCKACTIVITY (1UL << 8)
	#define UHH_SYSCONFIG_SIDLEMODE_MASK (3UL << 3)
	#define UHH_SYSCONFIG_SIDLEMODE_SMARTIDLE (2UL << 3)
	#define UHH_SYSCONFIG_SIDLEMODE_NOIDLE (1UL << 3)
	#define UHH_SYSCONFIG_SIDLEMODE_FORCEIDLE (0UL << 3)
	#define UHH_SYSCONFIG_ENAWAKEUP (1UL << 2)
	#define UHH_SYSCONFIG_SOFTRESET (1UL << 1)
	#define UHH_SYSCONFIG_AUTOIDLE (1UL << 0)

	#define UHH_HOSTCONFIG_APP_START_CLK (1UL << 31)
	#define UHH_HOSTCONFIG_P3_CONNECT_STATUS (1UL << 10)
	#define UHH_HOSTCONFIG_P2_CONNECT_STATUS (1UL << 9)
	#define UHH_HOSTCONFIG_P1_CONNECT_STATUS (1UL << 8)
	#define UHH_HOSTCONFIG_ENA_INCR_ALIGN (1UL << 5)
	#define UHH_HOSTCONFIG_ENA_INCR16 (1UL << 4)
	#define UHH_HOSTCONFIG_ENA_INCR8 (1UL << 3)
	#define UHH_HOSTCONFIG_ENA_INCR4 (1UL << 2)
	#define UHH_HOSTCONFIG_AUTOPPD_ON_OVERCUR_EN (1UL << 1)
	#define UHH_HOSTCONFIG_P1_ULPI_BYPASS (1UL << 0)

	/* The following are on rev2 (OMAP44xx) of the EHCI only */
	#define UHH_SYSCONFIG_IDLEMODE_MASK (3UL << 2)
	#define UHH_SYSCONFIG_IDLEMODE_NOIDLE (1UL << 2)
	#define UHH_SYSCONFIG_STANDBYMODE_MASK (3UL << 4)
	#define UHH_SYSCONFIG_STANDBYMODE_NOSTDBY (1UL << 4)

	#define UHH_HOSTCONFIG_P1_MODE_MASK (3UL << 16)
	#define UHH_HOSTCONFIG_P1_MODE_ULPI_PHY (0UL << 16)
	#define UHH_HOSTCONFIG_P1_MODE_UTMI_PHY (1UL << 16)
	#define UHH_HOSTCONFIG_P1_MODE_HSIC (3UL << 16)
	#define UHH_HOSTCONFIG_P2_MODE_MASK (3UL << 18)
	#define UHH_HOSTCONFIG_P2_MODE_ULPI_PHY (0UL << 18)
	#define UHH_HOSTCONFIG_P2_MODE_UTMI_PHY (1UL << 18)
	#define UHH_HOSTCONFIG_P2_MODE_HSIC (3UL << 18)

	/*
	* Values of UHH_REVISION - Note: these are not given in the TRM but taken
	* from the linux OMAP EHCI driver (thanks guys). It has been verified on
	* a Panda and Beagle board.
	*/
	#define OMAP_UHH_REV1 0x00000010 /* OMAP3 */
	#define OMAP_UHH_REV2 0x50700100 /* OMAP4 */

	struct omap_uhh_softc {
	struct simplebus_softc simplebus_sc;
	device_t sc_dev;

	/* UHH register set */
	struct resource* uhh_mem_res;

	/* The revision of the HS USB HOST read from UHH_REVISION */
	uint32_t uhh_rev;

	/* The following details are provided by conf hints */
	int port_mode[3];
	};

	static device_attach_t omap_uhh_attach;
	static device_detach_t omap_uhh_detach;

	static inline uint32_t
	omap_uhh_read_4(struct omap_uhh_softc *sc, bus_size_t off)
	{
	return bus_read_4(sc->uhh_mem_res, off);
	}

	static inline void
	omap_uhh_write_4(struct omap_uhh_softc *sc, bus_size_t off, uint32_t val)
	{
	bus_write_4(sc->uhh_mem_res, off, val);
	}

	static int
	omap_uhh_init(struct omap_uhh_softc *isc)
	{
	uint8_t tll_ch_mask;
	uint32_t reg;
	int i;

	/* Enable Clocks for high speed USBHOST */
	- ti_prcm_clk_enable(USBHSHOST_CLK);
	+ ti_sysc_clock_enable(device_get_parent(isc->sc_dev));

	/* Read the UHH revision */
	isc->uhh_rev = omap_uhh_read_4(isc, OMAP_USBHOST_UHH_REVISION);
	device_printf(isc->sc_dev, "UHH revision 0x%08x\n", isc->uhh_rev);

	+ /* FIXME */
	+#if 0
	if (isc->uhh_rev == OMAP_UHH_REV2) {
	/* For OMAP44xx devices you have to enable the per-port clocks:
	* PHY_MODE - External ULPI clock
	* TTL_MODE - Internal UTMI clock
	* HSIC_MODE - Internal 480Mhz and 60Mhz clocks
	*/
	switch(isc->port_mode[0]) {
	case EHCI_HCD_OMAP_MODE_UNKNOWN:
	break;
	case EHCI_HCD_OMAP_MODE_PHY:
	if (ti_prcm_clk_set_source(USBP1_PHY_CLK, EXT_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock source for port 0\n");
	if (ti_prcm_clk_enable(USBP1_PHY_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock USBP1_PHY_CLK source for port 0\n");
	break;
	case EHCI_HCD_OMAP_MODE_TLL:
	if (ti_prcm_clk_enable(USBP1_UTMI_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock USBP1_PHY_CLK source for port 0\n");
	break;
	case EHCI_HCD_OMAP_MODE_HSIC:
	if (ti_prcm_clk_enable(USBP1_HSIC_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock USBP1_PHY_CLK source for port 0\n");
	break;
	default:
	device_printf(isc->sc_dev, "unknown port mode %d for port 0\n", isc->port_mode[0]);
	}
	switch(isc->port_mode[1]) {
	case EHCI_HCD_OMAP_MODE_UNKNOWN:
	break;
	case EHCI_HCD_OMAP_MODE_PHY:
	if (ti_prcm_clk_set_source(USBP2_PHY_CLK, EXT_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock source for port 0\n");
	if (ti_prcm_clk_enable(USBP2_PHY_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock USBP2_PHY_CLK source for port 1\n");
	break;
	case EHCI_HCD_OMAP_MODE_TLL:
	if (ti_prcm_clk_enable(USBP2_UTMI_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock USBP2_UTMI_CLK source for port 1\n");
	break;
	case EHCI_HCD_OMAP_MODE_HSIC:
	if (ti_prcm_clk_enable(USBP2_HSIC_CLK))
	device_printf(isc->sc_dev,
	"failed to set clock USBP2_HSIC_CLK source for port 1\n");
	break;
	default:
	device_printf(isc->sc_dev, "unknown port mode %d for port 1\n", isc->port_mode[1]);
	}
	}
	+#endif

	/* Put UHH in SmartIdle/SmartStandby mode */
	reg = omap_uhh_read_4(isc, OMAP_USBHOST_UHH_SYSCONFIG);
	if (isc->uhh_rev == OMAP_UHH_REV1) {
	reg &= ~(UHH_SYSCONFIG_SIDLEMODE_MASK \|
	UHH_SYSCONFIG_MIDLEMODE_MASK);
	reg \|= (UHH_SYSCONFIG_ENAWAKEUP \|
	UHH_SYSCONFIG_AUTOIDLE \|
	UHH_SYSCONFIG_CLOCKACTIVITY \|
	UHH_SYSCONFIG_SIDLEMODE_SMARTIDLE \|
	UHH_SYSCONFIG_MIDLEMODE_SMARTSTANDBY);
	} else if (isc->uhh_rev == OMAP_UHH_REV2) {
	reg &= ~UHH_SYSCONFIG_IDLEMODE_MASK;
	reg \|= UHH_SYSCONFIG_IDLEMODE_NOIDLE;
	reg &= ~UHH_SYSCONFIG_STANDBYMODE_MASK;
	reg \|= UHH_SYSCONFIG_STANDBYMODE_NOSTDBY;
	}
	omap_uhh_write_4(isc, OMAP_USBHOST_UHH_SYSCONFIG, reg);
	device_printf(isc->sc_dev, "OMAP_UHH_SYSCONFIG: 0x%08x\n", reg);

	reg = omap_uhh_read_4(isc, OMAP_USBHOST_UHH_HOSTCONFIG);

	/* Setup ULPI bypass and burst configurations */
	reg \|= (UHH_HOSTCONFIG_ENA_INCR4 \|
	UHH_HOSTCONFIG_ENA_INCR8 \|
	UHH_HOSTCONFIG_ENA_INCR16);
	reg &= ~UHH_HOSTCONFIG_ENA_INCR_ALIGN;

	if (isc->uhh_rev == OMAP_UHH_REV1) {
	if (isc->port_mode[0] == EHCI_HCD_OMAP_MODE_UNKNOWN)
	reg &= ~UHH_HOSTCONFIG_P1_CONNECT_STATUS;
	if (isc->port_mode[1] == EHCI_HCD_OMAP_MODE_UNKNOWN)
	reg &= ~UHH_HOSTCONFIG_P2_CONNECT_STATUS;
	if (isc->port_mode[2] == EHCI_HCD_OMAP_MODE_UNKNOWN)
	reg &= ~UHH_HOSTCONFIG_P3_CONNECT_STATUS;

	/* Bypass the TLL module for PHY mode operation */
	if ((isc->port_mode[0] == EHCI_HCD_OMAP_MODE_PHY) \|\|
	(isc->port_mode[1] == EHCI_HCD_OMAP_MODE_PHY) \|\|
	(isc->port_mode[2] == EHCI_HCD_OMAP_MODE_PHY))
	reg &= ~UHH_HOSTCONFIG_P1_ULPI_BYPASS;
	else
	reg \|= UHH_HOSTCONFIG_P1_ULPI_BYPASS;

	} else if (isc->uhh_rev == OMAP_UHH_REV2) {
	reg \|= UHH_HOSTCONFIG_APP_START_CLK;

	/* Clear port mode fields for PHY mode*/
	reg &= ~UHH_HOSTCONFIG_P1_MODE_MASK;
	reg &= ~UHH_HOSTCONFIG_P2_MODE_MASK;

	if (isc->port_mode[0] == EHCI_HCD_OMAP_MODE_TLL)
	reg \|= UHH_HOSTCONFIG_P1_MODE_UTMI_PHY;
	else if (isc->port_mode[0] == EHCI_HCD_OMAP_MODE_HSIC)
	reg \|= UHH_HOSTCONFIG_P1_MODE_HSIC;

	if (isc->port_mode[1] == EHCI_HCD_OMAP_MODE_TLL)
	reg \|= UHH_HOSTCONFIG_P2_MODE_UTMI_PHY;
	else if (isc->port_mode[1] == EHCI_HCD_OMAP_MODE_HSIC)
	reg \|= UHH_HOSTCONFIG_P2_MODE_HSIC;
	}

	omap_uhh_write_4(isc, OMAP_USBHOST_UHH_HOSTCONFIG, reg);
	device_printf(isc->sc_dev, "UHH setup done, uhh_hostconfig=0x%08x\n", reg);


	/* I found the code and comments in the Linux EHCI driver - thanks guys :)
	*
	* "An undocumented "feature" in the OMAP3 EHCI controller, causes suspended
	* ports to be taken out of suspend when the USBCMD.Run/Stop bit is cleared
	* (for example when we do omap_uhh_bus_suspend). This breaks suspend-resume if
	* the root-hub is allowed to suspend. Writing 1 to this undocumented
	* register bit disables this feature and restores normal behavior."
	*/
	#if 0
	omap_uhh_write_4(isc, OMAP_USBHOST_INSNREG04,
	OMAP_USBHOST_INSNREG04_DISABLE_UNSUSPEND);
	#endif
	tll_ch_mask = 0;
	for (i = 0; i < OMAP_HS_USB_PORTS; i++) {
	if (isc->port_mode[i] == EHCI_HCD_OMAP_MODE_TLL)
	tll_ch_mask \|= (1 << i);
	}
	if (tll_ch_mask)
	omap_tll_utmi_enable(tll_ch_mask);

	return(0);
	}

	/**
	* omap_uhh_fini - shutdown the EHCI controller
	* @isc: omap ehci device context
	*
	*
	*
	* LOCKING:
	* none
	*
	* RETURNS:
	* 0 on success, a negative error code on failure.
	*/
	static void
	omap_uhh_fini(struct omap_uhh_softc *isc)
	{
	unsigned long timeout;

	device_printf(isc->sc_dev, "Stopping TI EHCI USB Controller\n");

	/* Set the timeout */
	if (hz < 10)
	timeout = 1;
	else
	timeout = (100 * hz) / 1000;

	/* Reset the UHH, OHCI and EHCI modules */
	omap_uhh_write_4(isc, OMAP_USBHOST_UHH_SYSCONFIG, 0x0002);
	while ((omap_uhh_read_4(isc, OMAP_USBHOST_UHH_SYSSTATUS) & 0x07) == 0x00) {
	/* Sleep for a tick */
	pause("USBRESET", 1);

	if (timeout-- == 0) {
	device_printf(isc->sc_dev, "operation timed out\n");
	break;
	}
	}

	/* Disable functional and interface clocks for the TLL and HOST modules */
	- ti_prcm_clk_disable(USBHSHOST_CLK);
	+ ti_sysc_clock_disable(device_get_parent(isc->sc_dev));

	device_printf(isc->sc_dev, "Clock to USB host has been disabled\n");
	}

	int
	omap_usb_port_mode(device_t dev, int port)
	{
	struct omap_uhh_softc *isc;

	isc = device_get_softc(dev);
	if ((port < 0) \|\| (port >= OMAP_HS_USB_PORTS))
	return (-1);

	return isc->port_mode[port];
	}

	static int
	omap_uhh_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,usbhs-host"))
	return (ENXIO);

	device_set_desc(dev, "TI OMAP USB 2.0 Host module");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	omap_uhh_attach(device_t dev)
	{
	struct omap_uhh_softc *isc = device_get_softc(dev);
	int err;
	int rid;
	int i;
	phandle_t node;
	char propname[16];
	char *mode;

	/* save the device */
	isc->sc_dev = dev;

	/* Allocate resource for the UHH register set */
	rid = 0;
	isc->uhh_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (!isc->uhh_mem_res) {
	device_printf(dev, "Error: Could not map UHH memory\n");
	goto error;
	}

	node = ofw_bus_get_node(dev);

	if (node == -1)
	goto error;

	/* Get port modes from FDT */
	for (i = 0; i < OMAP_HS_USB_PORTS; i++) {
	isc->port_mode[i] = EHCI_HCD_OMAP_MODE_UNKNOWN;
	snprintf(propname, sizeof(propname),
	"port%d-mode", i+1);

	if (OF_getprop_alloc(node, propname, (void**)&mode) <= 0)
	continue;
	if (strcmp(mode, "ehci-phy") == 0)
	isc->port_mode[i] = EHCI_HCD_OMAP_MODE_PHY;
	else if (strcmp(mode, "ehci-tll") == 0)
	isc->port_mode[i] = EHCI_HCD_OMAP_MODE_TLL;
	else if (strcmp(mode, "ehci-hsic") == 0)
	isc->port_mode[i] = EHCI_HCD_OMAP_MODE_HSIC;
	}

	/* Initialise the ECHI registers */
	err = omap_uhh_init(isc);
	if (err) {
	device_printf(dev, "Error: could not setup OMAP EHCI, %d\n", err);
	goto error;
	}

	simplebus_init(dev, node);

	/*
	* Allow devices to identify.
	*/
	bus_generic_probe(dev);

	/*
	* Now walk the OFW tree and attach top-level devices.
	*/
	for (node = OF_child(node); node > 0; node = OF_peer(node))
	simplebus_add_device(dev, node, 0, NULL, -1, NULL);
	return (bus_generic_attach(dev));

	error:
	omap_uhh_detach(dev);
	return (ENXIO);
	}

	static int
	omap_uhh_detach(device_t dev)
	{
	struct omap_uhh_softc *isc = device_get_softc(dev);

	/* during module unload there are lots of children leftover */
	device_delete_children(dev);

	if (isc->uhh_mem_res) {
	bus_release_resource(dev, SYS_RES_MEMORY, 0, isc->uhh_mem_res);
	isc->uhh_mem_res = NULL;
	}

	omap_uhh_fini(isc);

	return (0);
	}

	static device_method_t omap_uhh_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, omap_uhh_probe),
	DEVMETHOD(device_attach, omap_uhh_attach),
	DEVMETHOD(device_detach, omap_uhh_detach),

	DEVMETHOD(device_suspend, bus_generic_suspend),
	DEVMETHOD(device_resume, bus_generic_resume),
	DEVMETHOD(device_shutdown, bus_generic_shutdown),

	DEVMETHOD_END
	};

	DEFINE_CLASS_1(omap_uhh, omap_uhh_driver, omap_uhh_methods,
	sizeof(struct omap_uhh_softc), simplebus_driver);
	static devclass_t omap_uhh_devclass;
	DRIVER_MODULE(omap_uhh, simplebus, omap_uhh_driver, omap_uhh_devclass, 0, 0);
	diff --git a/sys/arm/ti/usb/omap_tll.c b/sys/arm/ti/usb/omap_tll.c
	index eb3e246a61d6..c5383e3d52d3 100644
	--- a/sys/arm/ti/usb/omap_tll.c
	+++ b/sys/arm/ti/usb/omap_tll.c
	@@ -1,363 +1,363 @@
	/*-
	* Copyright (c) 2011
	* Ben Gray <ben.r.gray@gmail.com>.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/rman.h>
	#include <sys/module.h>

	#include <dev/ofw/ofw_bus_subr.h>

	#include <machine/bus.h>

	-#include <arm/ti/ti_prcm.h>
	+#include <arm/ti/ti_sysc.h>
	#include <arm/ti/usb/omap_usb.h>

	/*
	* USB TLL Module
	*/
	#define OMAP_USBTLL_REVISION 0x0000
	#define OMAP_USBTLL_SYSCONFIG 0x0010
	#define OMAP_USBTLL_SYSSTATUS 0x0014
	#define OMAP_USBTLL_IRQSTATUS 0x0018
	#define OMAP_USBTLL_IRQENABLE 0x001C
	#define OMAP_USBTLL_TLL_SHARED_CONF 0x0030
	#define OMAP_USBTLL_TLL_CHANNEL_CONF(i) (0x0040 + (0x04 * (i)))
	#define OMAP_USBTLL_SAR_CNTX(i) (0x0400 + (0x04 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_ID_LO(i) (0x0800 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_ID_HI(i) (0x0801 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_PRODUCT_ID_LO(i) (0x0802 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_PRODUCT_ID_HI(i) (0x0803 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_FUNCTION_CTRL(i) (0x0804 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_FUNCTION_CTRL_SET(i) (0x0805 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_FUNCTION_CTRL_CLR(i) (0x0806 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_INTERFACE_CTRL(i) (0x0807 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_INTERFACE_CTRL_SET(i) (0x0808 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_INTERFACE_CTRL_CLR(i) (0x0809 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_OTG_CTRL(i) (0x080A + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_OTG_CTRL_SET(i) (0x080B + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_OTG_CTRL_CLR(i) (0x080C + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_EN_RISE(i) (0x080D + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_EN_RISE_SET(i) (0x080E + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_EN_RISE_CLR(i) (0x080F + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_EN_FALL(i) (0x0810 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_EN_FALL_SET(i) (0x0811 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_EN_FALL_CLR(i) (0x0812 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_STATUS(i) (0x0813 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_LATCH(i) (0x0814 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_DEBUG(i) (0x0815 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_SCRATCH_REGISTER(i) (0x0816 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_SCRATCH_REGISTER_SET(i) (0x0817 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_SCRATCH_REGISTER_CLR(i) (0x0818 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_EXTENDED_SET_ACCESS(i) (0x082F + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VCONTROL_EN(i) (0x0830 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VCONTROL_EN_SET(i) (0x0831 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VCONTROL_EN_CLR(i) (0x0832 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VCONTROL_STATUS(i) (0x0833 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VCONTROL_LATCH(i) (0x0834 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VSTATUS(i) (0x0835 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VSTATUS_SET(i) (0x0836 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_UTMI_VSTATUS_CLR(i) (0x0837 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_USB_INT_LATCH_NOCLR(i) (0x0838 + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_INT_EN(i) (0x083B + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_INT_EN_SET(i) (0x083C + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_INT_EN_CLR(i) (0x083D + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_INT_STATUS(i) (0x083E + (0x100 * (i)))
	#define OMAP_USBTLL_ULPI_VENDOR_INT_LATCH(i) (0x083F + (0x100 * (i)))

	/* TLL Register Set */
	#define TLL_SYSCONFIG_CACTIVITY (1UL << 8)
	#define TLL_SYSCONFIG_SIDLE_SMART_IDLE (2UL << 3)
	#define TLL_SYSCONFIG_SIDLE_NO_IDLE (1UL << 3)
	#define TLL_SYSCONFIG_SIDLE_FORCED_IDLE (0UL << 3)
	#define TLL_SYSCONFIG_ENAWAKEUP (1UL << 2)
	#define TLL_SYSCONFIG_SOFTRESET (1UL << 1)
	#define TLL_SYSCONFIG_AUTOIDLE (1UL << 0)

	#define TLL_SYSSTATUS_RESETDONE (1UL << 0)

	#define TLL_SHARED_CONF_USB_90D_DDR_EN (1UL << 6)
	#define TLL_SHARED_CONF_USB_180D_SDR_EN (1UL << 5)
	#define TLL_SHARED_CONF_USB_DIVRATIO_MASK (7UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_128 (7UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_64 (6UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_32 (5UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_16 (4UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_8 (3UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_4 (2UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_2 (1UL << 2)
	#define TLL_SHARED_CONF_USB_DIVRATIO_1 (0UL << 2)
	#define TLL_SHARED_CONF_FCLK_REQ (1UL << 1)
	#define TLL_SHARED_CONF_FCLK_IS_ON (1UL << 0)

	#define TLL_CHANNEL_CONF_DRVVBUS (1UL << 16)
	#define TLL_CHANNEL_CONF_CHRGVBUS (1UL << 15)
	#define TLL_CHANNEL_CONF_ULPINOBITSTUFF (1UL << 11)
	#define TLL_CHANNEL_CONF_ULPIAUTOIDLE (1UL << 10)
	#define TLL_CHANNEL_CONF_UTMIAUTOIDLE (1UL << 9)
	#define TLL_CHANNEL_CONF_ULPIDDRMODE (1UL << 8)
	#define TLL_CHANNEL_CONF_ULPIOUTCLKMODE (1UL << 7)
	#define TLL_CHANNEL_CONF_TLLFULLSPEED (1UL << 6)
	#define TLL_CHANNEL_CONF_TLLCONNECT (1UL << 5)
	#define TLL_CHANNEL_CONF_TLLATTACH (1UL << 4)
	#define TLL_CHANNEL_CONF_UTMIISADEV (1UL << 3)
	#define TLL_CHANNEL_CONF_CHANEN (1UL << 0)

	struct omap_tll_softc {
	device_t sc_dev;

	/* TLL register set */
	struct resource* tll_mem_res;
	int tll_mem_rid;
	};

	static struct omap_tll_softc *omap_tll_sc;

	static int omap_tll_attach(device_t dev);
	static int omap_tll_detach(device_t dev);

	static inline uint32_t
	omap_tll_read_4(struct omap_tll_softc *sc, bus_size_t off)
	{
	return bus_read_4(sc->tll_mem_res, off);
	}

	static inline void
	omap_tll_write_4(struct omap_tll_softc *sc, bus_size_t off, uint32_t val)
	{
	bus_write_4(sc->tll_mem_res, off, val);
	}

	void
	omap_tll_utmi_enable(unsigned int en_mask)
	{
	struct omap_tll_softc *sc;
	unsigned int i;
	uint32_t reg;

	sc = omap_tll_sc;
	if (sc == NULL)
	return;

	/* There are 3 TLL channels, one per USB controller so set them all up the
	* same, SDR mode, bit stuffing and no autoidle.
	*/
	for (i=0; i<3; i++) {
	reg = omap_tll_read_4(sc, OMAP_USBTLL_TLL_CHANNEL_CONF(i));

	reg &= ~(TLL_CHANNEL_CONF_UTMIAUTOIDLE
	\| TLL_CHANNEL_CONF_ULPINOBITSTUFF
	\| TLL_CHANNEL_CONF_ULPIDDRMODE);

	omap_tll_write_4(sc, OMAP_USBTLL_TLL_CHANNEL_CONF(i), reg);
	}

	/* Program the common TLL register */
	reg = omap_tll_read_4(sc, OMAP_USBTLL_TLL_SHARED_CONF);

	reg &= ~( TLL_SHARED_CONF_USB_90D_DDR_EN
	\| TLL_SHARED_CONF_USB_DIVRATIO_MASK);
	reg \|= ( TLL_SHARED_CONF_FCLK_IS_ON
	\| TLL_SHARED_CONF_USB_DIVRATIO_2
	\| TLL_SHARED_CONF_USB_180D_SDR_EN);

	omap_tll_write_4(sc, OMAP_USBTLL_TLL_SHARED_CONF, reg);

	/* Enable channels now */
	for (i = 0; i < 3; i++) {
	reg = omap_tll_read_4(sc, OMAP_USBTLL_TLL_CHANNEL_CONF(i));

	/* Enable only the reg that is needed */
	if ((en_mask & (1 << i)) == 0)
	continue;

	reg \|= TLL_CHANNEL_CONF_CHANEN;
	omap_tll_write_4(sc, OMAP_USBTLL_TLL_CHANNEL_CONF(i), reg);
	}
	}

	static int
	omap_tll_init(struct omap_tll_softc *sc)
	{
	unsigned long timeout;
	int ret = 0;

	/* Enable the USB TLL */
	- ti_prcm_clk_enable(USBTLL_CLK);
	+ ti_sysc_clock_enable(device_get_parent(sc->sc_dev));

	/* Perform TLL soft reset, and wait until reset is complete */
	omap_tll_write_4(sc, OMAP_USBTLL_SYSCONFIG, TLL_SYSCONFIG_SOFTRESET);

	/* Set the timeout to 100ms*/
	timeout = (hz < 10) ? 1 : ((100 * hz) / 1000);

	/* Wait for TLL reset to complete */
	while ((omap_tll_read_4(sc, OMAP_USBTLL_SYSSTATUS) &
	TLL_SYSSTATUS_RESETDONE) == 0x00) {

	/* Sleep for a tick */
	pause("USBRESET", 1);

	if (timeout-- == 0) {
	device_printf(sc->sc_dev, "TLL reset operation timed out\n");
	ret = EINVAL;
	goto err_sys_status;
	}
	}

	/* CLOCKACTIVITY = 1 : OCP-derived internal clocks ON during idle
	* SIDLEMODE = 2 : Smart-idle mode. Sidleack asserted after Idlereq
	* assertion when no more activity on the USB.
	* ENAWAKEUP = 1 : Wakeup generation enabled
	*/
	omap_tll_write_4(sc, OMAP_USBTLL_SYSCONFIG, TLL_SYSCONFIG_ENAWAKEUP \|
	TLL_SYSCONFIG_AUTOIDLE \|
	TLL_SYSCONFIG_SIDLE_SMART_IDLE \|
	TLL_SYSCONFIG_CACTIVITY);

	return(0);

	err_sys_status:
	/* Disable the TLL clocks */
	- ti_prcm_clk_disable(USBTLL_CLK);
	+ ti_sysc_clock_disable(device_get_parent(sc->sc_dev));

	return(ret);
	}

	static void
	omap_tll_disable(struct omap_tll_softc *sc)
	{
	unsigned long timeout;

	timeout = (hz < 10) ? 1 : ((100 * hz) / 1000);

	/* Reset the TLL module */
	omap_tll_write_4(sc, OMAP_USBTLL_SYSCONFIG, 0x0002);
	while ((omap_tll_read_4(sc, OMAP_USBTLL_SYSSTATUS) & (0x01)) == 0x00) {
	/* Sleep for a tick */
	pause("USBRESET", 1);

	if (timeout-- == 0) {
	device_printf(sc->sc_dev, "operation timed out\n");
	break;
	}
	}

	/* Disable functional and interface clocks for the TLL and HOST modules */
	- ti_prcm_clk_disable(USBTLL_CLK);
	+ ti_sysc_clock_disable(device_get_parent(sc->sc_dev));
	}

	static int
	omap_tll_probe(device_t dev)
	{

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (!ofw_bus_is_compatible(dev, "ti,usbhs-tll"))
	return (ENXIO);

	device_set_desc(dev, "TI OMAP USB 2.0 TLL module");

	return (BUS_PROBE_DEFAULT);
	}

	static int
	omap_tll_attach(device_t dev)
	{
	struct omap_tll_softc *sc;

	sc = device_get_softc(dev);
	/* save the device */
	sc->sc_dev = dev;

	/* Allocate resource for the TLL register set */
	sc->tll_mem_rid = 0;
	sc->tll_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&sc->tll_mem_rid, RF_ACTIVE);
	if (!sc->tll_mem_res) {
	device_printf(dev, "Error: Could not map TLL memory\n");
	goto error;
	}

	omap_tll_init(sc);

	omap_tll_sc = sc;

	return (0);

	error:
	omap_tll_detach(dev);
	return (ENXIO);
	}

	static int
	omap_tll_detach(device_t dev)
	{
	struct omap_tll_softc *sc;

	sc = device_get_softc(dev);
	omap_tll_disable(sc);

	/* Release the other register set memory maps */
	if (sc->tll_mem_res) {
	bus_release_resource(dev, SYS_RES_MEMORY,
	sc->tll_mem_rid, sc->tll_mem_res);
	sc->tll_mem_res = NULL;
	}

	omap_tll_sc = NULL;

	return (0);
	}

	static device_method_t omap_tll_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, omap_tll_probe),
	DEVMETHOD(device_attach, omap_tll_attach),
	DEVMETHOD(device_detach, omap_tll_detach),
	DEVMETHOD(device_suspend, bus_generic_suspend),
	DEVMETHOD(device_resume, bus_generic_resume),
	DEVMETHOD(device_shutdown, bus_generic_shutdown),

	{0, 0}
	};

	static driver_t omap_tll_driver = {
	"omap_tll",
	omap_tll_methods,
	sizeof(struct omap_tll_softc),
	};

	static devclass_t omap_tll_devclass;

	DRIVER_MODULE(omap_tll, simplebus, omap_tll_driver, omap_tll_devclass, 0, 0);
	diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
	index 1667d5993998..423adac08208 100644
	--- a/sys/arm64/arm64/machdep.c
	+++ b/sys/arm64/arm64/machdep.c
	@@ -1,1372 +1,1374 @@
	/*-
	* Copyright (c) 2014 Andrew Turner
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/

	#include "opt_acpi.h"
	#include "opt_platform.h"
	#include "opt_ddb.h"

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/buf.h>
	#include <sys/bus.h>
	#include <sys/cons.h>
	#include <sys/cpu.h>
	#include <sys/csan.h>
	#include <sys/devmap.h>
	#include <sys/efi.h>
	#include <sys/exec.h>
	#include <sys/imgact.h>
	#include <sys/kdb.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/limits.h>
	#include <sys/linker.h>
	#include <sys/msgbuf.h>
	#include <sys/pcpu.h>
	#include <sys/physmem.h>
	#include <sys/proc.h>
	#include <sys/ptrace.h>
	#include <sys/reboot.h>
	#include <sys/rwlock.h>
	#include <sys/sched.h>
	#include <sys/signalvar.h>
	#include <sys/syscallsubr.h>
	#include <sys/sysent.h>
	#include <sys/sysproto.h>
	#include <sys/ucontext.h>
	#include <sys/vdso.h>
	#include <sys/vmmeter.h>

	#include <vm/vm.h>
	#include <vm/vm_param.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_phys.h>
	#include <vm/pmap.h>
	#include <vm/vm_map.h>
	#include <vm/vm_pager.h>

	#include <machine/armreg.h>
	#include <machine/cpu.h>
	#include <machine/debug_monitor.h>
	#include <machine/kdb.h>
	#include <machine/machdep.h>
	#include <machine/metadata.h>
	#include <machine/md_var.h>
	#include <machine/pcb.h>
	#include <machine/reg.h>
	#include <machine/undefined.h>
	#include <machine/vmparam.h>

	#ifdef VFP
	#include <machine/vfp.h>
	#endif

	#ifdef DEV_ACPI
	#include <contrib/dev/acpica/include/acpi.h>
	#include <machine/acpica_machdep.h>
	#endif

	#ifdef FDT
	#include <dev/fdt/fdt_common.h>
	#include <dev/ofw/openfirm.h>
	#endif

	static void get_fpcontext(struct thread td, mcontext_t mcp);
	static void set_fpcontext(struct thread td, mcontext_t mcp);

	enum arm64_bus arm64_bus_method = ARM64_BUS_NONE;

	struct pcpu __pcpu[MAXCPU];

	static struct trapframe proc0_tf;

	int early_boot = 1;
	int cold = 1;
	static int boot_el;

	struct kva_md_info kmi;

	int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */
	int has_pan;

	/*
	* Physical address of the EFI System Table. Stashed from the metadata hints
	* passed into the kernel and used by the EFI code to call runtime services.
	*/
	vm_paddr_t efi_systbl_phys;
	static struct efi_map_header *efihdr;

	/* pagezero_* implementations are provided in support.S */
	void pagezero_simple(void *);
	void pagezero_cache(void *);

	/* pagezero_simple is default pagezero */
	void (pagezero)(void p) = pagezero_simple;

	+int (*apei_nmi)(void);
	+
	static void
	pan_setup(void)
	{
	uint64_t id_aa64mfr1;

	id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
	if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE)
	has_pan = 1;
	}

	void
	pan_enable(void)
	{

	/*
	* The LLVM integrated assembler doesn't understand the PAN
	* PSTATE field. Because of this we need to manually create
	* the instruction in an asm block. This is equivalent to:
	* msr pan, #1
	*
	* This sets the PAN bit, stopping the kernel from accessing
	* memory when userspace can also access it unless the kernel
	* uses the userspace load/store instructions.
	*/
	if (has_pan) {
	WRITE_SPECIALREG(sctlr_el1,
	READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN);
	__asm __volatile(".inst 0xd500409f \| (0x1 << 8)");
	}
	}

	bool
	has_hyp(void)
	{

	return (boot_el == 2);
	}

	static void
	cpu_startup(void *dummy)
	{
	vm_paddr_t size;
	int i;

	printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem),
	ptoa((uintmax_t)realmem) / 1024 / 1024);

	if (bootverbose) {
	printf("Physical memory chunk(s):\n");
	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
	size = phys_avail[i + 1] - phys_avail[i];
	printf("%#016jx - %#016jx, %ju bytes (%ju pages)\n",
	(uintmax_t)phys_avail[i],
	(uintmax_t)phys_avail[i + 1] - 1,
	(uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
	}
	}

	printf("avail memory = %ju (%ju MB)\n",
	ptoa((uintmax_t)vm_free_count()),
	ptoa((uintmax_t)vm_free_count()) / 1024 / 1024);

	undef_init();
	install_cpu_errata();

	vm_ksubmap_init(&kmi);
	bufinit();
	vm_pager_bufferinit();
	}

	SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);

	static void
	late_ifunc_resolve(void *dummy __unused)
	{
	link_elf_late_ireloc();
	}
	SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL);

	int
	cpu_idle_wakeup(int cpu)
	{

	return (0);
	}

	int
	fill_regs(struct thread td, struct reg regs)
	{
	struct trapframe *frame;

	frame = td->td_frame;
	regs->sp = frame->tf_sp;
	regs->lr = frame->tf_lr;
	regs->elr = frame->tf_elr;
	regs->spsr = frame->tf_spsr;

	memcpy(regs->x, frame->tf_x, sizeof(regs->x));

	#ifdef COMPAT_FREEBSD32
	/*
	* We may be called here for a 32bits process, if we're using a
	* 64bits debugger. If so, put PC and SPSR where it expects it.
	*/
	if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
	regs->x[15] = frame->tf_elr;
	regs->x[16] = frame->tf_spsr;
	}
	#endif
	return (0);
	}

	int
	set_regs(struct thread td, struct reg regs)
	{
	struct trapframe *frame;

	frame = td->td_frame;
	frame->tf_sp = regs->sp;
	frame->tf_lr = regs->lr;
	frame->tf_elr = regs->elr;
	frame->tf_spsr &= ~PSR_FLAGS;
	frame->tf_spsr \|= regs->spsr & PSR_FLAGS;

	memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x));

	#ifdef COMPAT_FREEBSD32
	if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
	/*
	* We may be called for a 32bits process if we're using
	* a 64bits debugger. If so, get PC and SPSR from where
	* it put it.
	*/
	frame->tf_elr = regs->x[15];
	frame->tf_spsr = regs->x[16] & PSR_FLAGS;
	}
	#endif
	return (0);
	}

	int
	fill_fpregs(struct thread td, struct fpreg regs)
	{
	#ifdef VFP
	struct pcb *pcb;

	pcb = td->td_pcb;
	if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
	/*
	* If we have just been running VFP instructions we will
	* need to save the state to memcpy it below.
	*/
	if (td == curthread)
	vfp_save_state(td, pcb);

	KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
	("Called fill_fpregs while the kernel is using the VFP"));
	memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs,
	sizeof(regs->fp_q));
	regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr;
	regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr;
	} else
	#endif
	memset(regs, 0, sizeof(*regs));
	return (0);
	}

	int
	set_fpregs(struct thread td, struct fpreg regs)
	{
	#ifdef VFP
	struct pcb *pcb;

	pcb = td->td_pcb;
	KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate,
	("Called set_fpregs while the kernel is using the VFP"));
	memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q));
	pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr;
	pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr;
	#endif
	return (0);
	}

	int
	fill_dbregs(struct thread td, struct dbreg regs)
	{
	struct debug_monitor_state *monitor;
	int count, i;
	uint8_t debug_ver, nbkpts;

	memset(regs, 0, sizeof(*regs));

	extract_user_id_field(ID_AA64DFR0_EL1, ID_AA64DFR0_DebugVer_SHIFT,
	&debug_ver);
	extract_user_id_field(ID_AA64DFR0_EL1, ID_AA64DFR0_BRPs_SHIFT,
	&nbkpts);

	/*
	* The BRPs field contains the number of breakpoints - 1. Armv8-A
	* allows the hardware to provide 2-16 breakpoints so this won't
	* overflow an 8 bit value.
	*/
	count = nbkpts + 1;

	regs->db_info = debug_ver;
	regs->db_info <<= 8;
	regs->db_info \|= count;

	monitor = &td->td_pcb->pcb_dbg_regs;
	if ((monitor->dbg_flags & DBGMON_ENABLED) != 0) {
	for (i = 0; i < count; i++) {
	regs->db_regs[i].dbr_addr = monitor->dbg_bvr[i];
	regs->db_regs[i].dbr_ctrl = monitor->dbg_bcr[i];
	}
	}

	return (0);
	}

	int
	set_dbregs(struct thread td, struct dbreg regs)
	{
	struct debug_monitor_state *monitor;
	int count;
	int i;

	monitor = &td->td_pcb->pcb_dbg_regs;
	count = 0;
	monitor->dbg_enable_count = 0;
	for (i = 0; i < DBG_BRP_MAX; i++) {
	/* TODO: Check these values */
	monitor->dbg_bvr[i] = regs->db_regs[i].dbr_addr;
	monitor->dbg_bcr[i] = regs->db_regs[i].dbr_ctrl;
	if ((monitor->dbg_bcr[i] & 1) != 0)
	monitor->dbg_enable_count++;
	}
	if (monitor->dbg_enable_count > 0)
	monitor->dbg_flags \|= DBGMON_ENABLED;

	return (0);
	}

	#ifdef COMPAT_FREEBSD32
	int
	fill_regs32(struct thread td, struct reg32 regs)
	{
	int i;
	struct trapframe *tf;

	tf = td->td_frame;
	for (i = 0; i < 13; i++)
	regs->r[i] = tf->tf_x[i];
	/* For arm32, SP is r13 and LR is r14 */
	regs->r_sp = tf->tf_x[13];
	regs->r_lr = tf->tf_x[14];
	regs->r_pc = tf->tf_elr;
	regs->r_cpsr = tf->tf_spsr;

	return (0);
	}

	int
	set_regs32(struct thread td, struct reg32 regs)
	{
	int i;
	struct trapframe *tf;

	tf = td->td_frame;
	for (i = 0; i < 13; i++)
	tf->tf_x[i] = regs->r[i];
	/* For arm 32, SP is r13 an LR is r14 */
	tf->tf_x[13] = regs->r_sp;
	tf->tf_x[14] = regs->r_lr;
	tf->tf_elr = regs->r_pc;
	tf->tf_spsr = regs->r_cpsr;


	return (0);
	}

	int
	fill_fpregs32(struct thread td, struct fpreg32 regs)
	{

	printf("ARM64TODO: fill_fpregs32");
	return (EDOOFUS);
	}

	int
	set_fpregs32(struct thread td, struct fpreg32 regs)
	{

	printf("ARM64TODO: set_fpregs32");
	return (EDOOFUS);
	}

	int
	fill_dbregs32(struct thread td, struct dbreg32 regs)
	{

	printf("ARM64TODO: fill_dbregs32");
	return (EDOOFUS);
	}

	int
	set_dbregs32(struct thread td, struct dbreg32 regs)
	{

	printf("ARM64TODO: set_dbregs32");
	return (EDOOFUS);
	}
	#endif

	int
	ptrace_set_pc(struct thread *td, u_long addr)
	{

	td->td_frame->tf_elr = addr;
	return (0);
	}

	int
	ptrace_single_step(struct thread *td)
	{

	td->td_frame->tf_spsr \|= PSR_SS;
	td->td_pcb->pcb_flags \|= PCB_SINGLE_STEP;
	return (0);
	}

	int
	ptrace_clear_single_step(struct thread *td)
	{

	td->td_frame->tf_spsr &= ~PSR_SS;
	td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP;
	return (0);
	}

	void
	exec_setregs(struct thread td, struct image_params imgp, uintptr_t stack)
	{
	struct trapframe *tf = td->td_frame;

	memset(tf, 0, sizeof(struct trapframe));

	tf->tf_x[0] = stack;
	tf->tf_sp = STACKALIGN(stack);
	tf->tf_lr = imgp->entry_addr;
	tf->tf_elr = imgp->entry_addr;
	}

	/* Sanity check these are the same size, they will be memcpy'd to and fro */
	CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
	sizeof((struct gpregs *)0)->gp_x);
	CTASSERT(sizeof(((struct trapframe *)0)->tf_x) ==
	sizeof((struct reg *)0)->x);

	int
	get_mcontext(struct thread td, mcontext_t mcp, int clear_ret)
	{
	struct trapframe *tf = td->td_frame;

	if (clear_ret & GET_MC_CLEAR_RET) {
	mcp->mc_gpregs.gp_x[0] = 0;
	mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C;
	} else {
	mcp->mc_gpregs.gp_x[0] = tf->tf_x[0];
	mcp->mc_gpregs.gp_spsr = tf->tf_spsr;
	}

	memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1],
	sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1));

	mcp->mc_gpregs.gp_sp = tf->tf_sp;
	mcp->mc_gpregs.gp_lr = tf->tf_lr;
	mcp->mc_gpregs.gp_elr = tf->tf_elr;
	get_fpcontext(td, mcp);

	return (0);
	}

	int
	set_mcontext(struct thread td, mcontext_t mcp)
	{
	struct trapframe *tf = td->td_frame;
	uint32_t spsr;

	spsr = mcp->mc_gpregs.gp_spsr;
	if ((spsr & PSR_M_MASK) != PSR_M_EL0t \|\|
	(spsr & PSR_AARCH32) != 0 \|\|
	(spsr & PSR_DAIF) != (td->td_frame->tf_spsr & PSR_DAIF))
	return (EINVAL);

	memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x));

	tf->tf_sp = mcp->mc_gpregs.gp_sp;
	tf->tf_lr = mcp->mc_gpregs.gp_lr;
	tf->tf_elr = mcp->mc_gpregs.gp_elr;
	tf->tf_spsr = mcp->mc_gpregs.gp_spsr;
	set_fpcontext(td, mcp);

	return (0);
	}

	static void
	get_fpcontext(struct thread td, mcontext_t mcp)
	{
	#ifdef VFP
	struct pcb *curpcb;

	critical_enter();

	curpcb = curthread->td_pcb;

	if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) {
	/*
	* If we have just been running VFP instructions we will
	* need to save the state to memcpy it below.
	*/
	vfp_save_state(td, curpcb);

	KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
	("Called get_fpcontext while the kernel is using the VFP"));
	KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0,
	("Non-userspace FPU flags set in get_fpcontext"));
	memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs,
	sizeof(mcp->mc_fpregs));
	mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr;
	mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr;
	mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags;
	mcp->mc_flags \|= _MC_FP_VALID;
	}

	critical_exit();
	#endif
	}

	static void
	set_fpcontext(struct thread td, mcontext_t mcp)
	{
	#ifdef VFP
	struct pcb *curpcb;

	critical_enter();

	if ((mcp->mc_flags & _MC_FP_VALID) != 0) {
	curpcb = curthread->td_pcb;

	/*
	* Discard any vfp state for the current thread, we
	* are about to override it.
	*/
	vfp_discard(td);

	KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate,
	("Called set_fpcontext while the kernel is using the VFP"));
	memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q,
	sizeof(mcp->mc_fpregs));
	curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr;
	curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr;
	curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK;
	}

	critical_exit();
	#endif
	}

	void
	cpu_idle(int busy)
	{

	spinlock_enter();
	if (!busy)
	cpu_idleclock();
	if (!sched_runnable())
	__asm __volatile(
	"dsb sy \n"
	"wfi \n");
	if (!busy)
	cpu_activeclock();
	spinlock_exit();
	}

	void
	cpu_halt(void)
	{

	/* We should have shutdown by now, if not enter a low power sleep */
	intr_disable();
	while (1) {
	__asm __volatile("wfi");
	}
	}

	/*
	* Flush the D-cache for non-DMA I/O so that the I-cache can
	* be made coherent later.
	*/
	void
	cpu_flush_dcache(void *ptr, size_t len)
	{

	/* ARM64TODO TBD */
	}

	/* Get current clock frequency for the given CPU ID. */
	int
	cpu_est_clockrate(int cpu_id, uint64_t *rate)
	{
	struct pcpu *pc;

	pc = pcpu_find(cpu_id);
	if (pc == NULL \|\| rate == NULL)
	return (EINVAL);

	if (pc->pc_clock == 0)
	return (EOPNOTSUPP);

	*rate = pc->pc_clock;
	return (0);
	}

	void
	cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
	{

	pcpu->pc_acpi_id = 0xffffffff;
	}

	void
	spinlock_enter(void)
	{
	struct thread *td;
	register_t daif;

	td = curthread;
	if (td->td_md.md_spinlock_count == 0) {
	daif = intr_disable();
	td->td_md.md_spinlock_count = 1;
	td->td_md.md_saved_daif = daif;
	critical_enter();
	} else
	td->td_md.md_spinlock_count++;
	}

	void
	spinlock_exit(void)
	{
	struct thread *td;
	register_t daif;

	td = curthread;
	daif = td->td_md.md_saved_daif;
	td->td_md.md_spinlock_count--;
	if (td->td_md.md_spinlock_count == 0) {
	critical_exit();
	intr_restore(daif);
	}
	}

	#ifndef _SYS_SYSPROTO_H_
	struct sigreturn_args {
	ucontext_t *ucp;
	};
	#endif

	int
	sys_sigreturn(struct thread td, struct sigreturn_args uap)
	{
	ucontext_t uc;
	int error;

	if (copyin(uap->sigcntxp, &uc, sizeof(uc)))
	return (EFAULT);

	error = set_mcontext(td, &uc.uc_mcontext);
	if (error != 0)
	return (error);

	/* Restore signal mask. */
	kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);

	return (EJUSTRETURN);
	}

	/*
	* Construct a PCB from a trapframe. This is called from kdb_trap() where
	* we want to start a backtrace from the function that caused us to enter
	* the debugger. We have the context in the trapframe, but base the trace
	* on the PCB. The PCB doesn't have to be perfect, as long as it contains
	* enough for a backtrace.
	*/
	void
	makectx(struct trapframe tf, struct pcb pcb)
	{
	int i;

	for (i = 0; i < PCB_LR; i++)
	pcb->pcb_x[i] = tf->tf_x[i];

	pcb->pcb_x[PCB_LR] = tf->tf_lr;
	pcb->pcb_pc = tf->tf_elr;
	pcb->pcb_sp = tf->tf_sp;
	}

	void
	sendsig(sig_t catcher, ksiginfo_t ksi, sigset_t mask)
	{
	struct thread *td;
	struct proc *p;
	struct trapframe *tf;
	struct sigframe *fp, frame;
	struct sigacts *psp;
	struct sysentvec *sysent;
	int onstack, sig;

	td = curthread;
	p = td->td_proc;
	PROC_LOCK_ASSERT(p, MA_OWNED);

	sig = ksi->ksi_signo;
	psp = p->p_sigacts;
	mtx_assert(&psp->ps_mtx, MA_OWNED);

	tf = td->td_frame;
	onstack = sigonstack(tf->tf_sp);

	CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm,
	catcher, sig);

	/* Allocate and validate space for the signal handler context. */
	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack &&
	SIGISMEMBER(psp->ps_sigonstack, sig)) {
	fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp +
	td->td_sigstk.ss_size);
	#if defined(COMPAT_43)
	td->td_sigstk.ss_flags \|= SS_ONSTACK;
	#endif
	} else {
	fp = (struct sigframe *)td->td_frame->tf_sp;
	}

	/* Make room, keeping the stack aligned */
	fp--;
	fp = (struct sigframe *)STACKALIGN(fp);

	/* Fill in the frame to copy out */
	bzero(&frame, sizeof(frame));
	get_mcontext(td, &frame.sf_uc.uc_mcontext, 0);
	frame.sf_si = ksi->ksi_info;
	frame.sf_uc.uc_sigmask = *mask;
	frame.sf_uc.uc_stack = td->td_sigstk;
	frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) != 0 ?
	(onstack ? SS_ONSTACK : 0) : SS_DISABLE;
	mtx_unlock(&psp->ps_mtx);
	PROC_UNLOCK(td->td_proc);

	/* Copy the sigframe out to the user's stack. */
	if (copyout(&frame, fp, sizeof(*fp)) != 0) {
	/* Process has trashed its stack. Kill it. */
	CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp);
	PROC_LOCK(p);
	sigexit(td, SIGILL);
	}

	tf->tf_x[0]= sig;
	tf->tf_x[1] = (register_t)&fp->sf_si;
	tf->tf_x[2] = (register_t)&fp->sf_uc;

	tf->tf_elr = (register_t)catcher;
	tf->tf_sp = (register_t)fp;
	sysent = p->p_sysent;
	if (sysent->sv_sigcode_base != 0)
	tf->tf_lr = (register_t)sysent->sv_sigcode_base;
	else
	tf->tf_lr = (register_t)(sysent->sv_psstrings -
	*(sysent->sv_szsigcode));

	CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr,
	tf->tf_sp);

	PROC_LOCK(p);
	mtx_lock(&psp->ps_mtx);
	}

	static void
	init_proc0(vm_offset_t kstack)
	{
	struct pcpu *pcpup = &__pcpu[0];

	proc_linkup0(&proc0, &thread0);
	thread0.td_kstack = kstack;
	thread0.td_kstack_pages = KSTACK_PAGES;
	thread0.td_pcb = (struct pcb *)(thread0.td_kstack +
	thread0.td_kstack_pages * PAGE_SIZE) - 1;
	thread0.td_pcb->pcb_fpflags = 0;
	thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate;
	thread0.td_pcb->pcb_vfpcpu = UINT_MAX;
	thread0.td_frame = &proc0_tf;
	pcpup->pc_curpcb = thread0.td_pcb;
	}

	typedef struct {
	uint32_t type;
	uint64_t phys_start;
	uint64_t virt_start;
	uint64_t num_pages;
	uint64_t attr;
	} EFI_MEMORY_DESCRIPTOR;

	typedef void (efi_map_entry_cb)(struct efi_md );

	static void
	foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb)
	{
	struct efi_md map, p;
	size_t efisz;
	int ndesc, i;

	/*
	* Memory map data provided by UEFI via the GetMemoryMap
	* Boot Services API.
	*/
	efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
	map = (struct efi_md )((uint8_t )efihdr + efisz);

	if (efihdr->descriptor_size == 0)
	return;
	ndesc = efihdr->memory_size / efihdr->descriptor_size;

	for (i = 0, p = map; i < ndesc; i++,
	p = efi_next_descriptor(p, efihdr->descriptor_size)) {
	cb(p);
	}
	}

	static void
	exclude_efi_map_entry(struct efi_md *p)
	{

	switch (p->md_type) {
	case EFI_MD_TYPE_CODE:
	case EFI_MD_TYPE_DATA:
	case EFI_MD_TYPE_BS_CODE:
	case EFI_MD_TYPE_BS_DATA:
	case EFI_MD_TYPE_FREE:
	/*
	* We're allowed to use any entry with these types.
	*/
	break;
	default:
	physmem_exclude_region(p->md_phys, p->md_pages * PAGE_SIZE,
	EXFLAG_NOALLOC);
	}
	}

	static void
	exclude_efi_map_entries(struct efi_map_header *efihdr)
	{

	foreach_efi_map_entry(efihdr, exclude_efi_map_entry);
	}

	static void
	add_efi_map_entry(struct efi_md *p)
	{

	switch (p->md_type) {
	case EFI_MD_TYPE_RT_DATA:
	/*
	* Runtime data will be excluded after the DMAP
	* region is created to stop it from being added
	* to phys_avail.
	*/
	case EFI_MD_TYPE_CODE:
	case EFI_MD_TYPE_DATA:
	case EFI_MD_TYPE_BS_CODE:
	case EFI_MD_TYPE_BS_DATA:
	case EFI_MD_TYPE_FREE:
	/*
	* We're allowed to use any entry with these types.
	*/
	physmem_hardware_region(p->md_phys,
	p->md_pages * PAGE_SIZE);
	break;
	}
	}

	static void
	add_efi_map_entries(struct efi_map_header *efihdr)
	{

	foreach_efi_map_entry(efihdr, add_efi_map_entry);
	}

	static void
	print_efi_map_entry(struct efi_md *p)
	{
	const char *type;
	static const char *types[] = {
	"Reserved",
	"LoaderCode",
	"LoaderData",
	"BootServicesCode",
	"BootServicesData",
	"RuntimeServicesCode",
	"RuntimeServicesData",
	"ConventionalMemory",
	"UnusableMemory",
	"ACPIReclaimMemory",
	"ACPIMemoryNVS",
	"MemoryMappedIO",
	"MemoryMappedIOPortSpace",
	"PalCode",
	"PersistentMemory"
	};

	if (p->md_type < nitems(types))
	type = types[p->md_type];
	else
	type = "<INVALID>";
	printf("%23s %012lx %12p %08lx ", type, p->md_phys,
	p->md_virt, p->md_pages);
	if (p->md_attr & EFI_MD_ATTR_UC)
	printf("UC ");
	if (p->md_attr & EFI_MD_ATTR_WC)
	printf("WC ");
	if (p->md_attr & EFI_MD_ATTR_WT)
	printf("WT ");
	if (p->md_attr & EFI_MD_ATTR_WB)
	printf("WB ");
	if (p->md_attr & EFI_MD_ATTR_UCE)
	printf("UCE ");
	if (p->md_attr & EFI_MD_ATTR_WP)
	printf("WP ");
	if (p->md_attr & EFI_MD_ATTR_RP)
	printf("RP ");
	if (p->md_attr & EFI_MD_ATTR_XP)
	printf("XP ");
	if (p->md_attr & EFI_MD_ATTR_NV)
	printf("NV ");
	if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
	printf("MORE_RELIABLE ");
	if (p->md_attr & EFI_MD_ATTR_RO)
	printf("RO ");
	if (p->md_attr & EFI_MD_ATTR_RT)
	printf("RUNTIME");
	printf("\n");
	}

	static void
	print_efi_map_entries(struct efi_map_header *efihdr)
	{

	printf("%23s %12s %12s %8s %4s\n",
	"Type", "Physical", "Virtual", "#Pages", "Attr");
	foreach_efi_map_entry(efihdr, print_efi_map_entry);
	}

	#ifdef FDT
	static void
	try_load_dtb(caddr_t kmdp)
	{
	vm_offset_t dtbp;

	dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
	#if defined(FDT_DTB_STATIC)
	/*
	* In case the device tree blob was not retrieved (from metadata) try
	* to use the statically embedded one.
	*/
	if (dtbp == 0)
	dtbp = (vm_offset_t)&fdt_static_dtb;
	#endif

	if (dtbp == (vm_offset_t)NULL) {
	printf("ERROR loading DTB\n");
	return;
	}

	if (OF_install(OFW_FDT, 0) == FALSE)
	panic("Cannot install FDT");

	if (OF_init((void *)dtbp) != 0)
	panic("OF_init failed with the found device tree");

	parse_fdt_bootargs();
	}
	#endif

	static bool
	bus_probe(void)
	{
	bool has_acpi, has_fdt;
	char order, env;

	has_acpi = has_fdt = false;

	#ifdef FDT
	has_fdt = (OF_peer(0) != 0);
	#endif
	#ifdef DEV_ACPI
	has_acpi = (acpi_find_table(ACPI_SIG_SPCR) != 0);
	#endif

	env = kern_getenv("kern.cfg.order");
	if (env != NULL) {
	order = env;
	while (order != NULL) {
	if (has_acpi &&
	strncmp(order, "acpi", 4) == 0 &&
	(order[4] == ',' \|\| order[4] == '\0')) {
	arm64_bus_method = ARM64_BUS_ACPI;
	break;
	}
	if (has_fdt &&
	strncmp(order, "fdt", 3) == 0 &&
	(order[3] == ',' \|\| order[3] == '\0')) {
	arm64_bus_method = ARM64_BUS_FDT;
	break;
	}
	order = strchr(order, ',');
	}
	freeenv(env);

	/* If we set the bus method it is valid */
	if (arm64_bus_method != ARM64_BUS_NONE)
	return (true);
	}
	/* If no order or an invalid order was set use the default */
	if (arm64_bus_method == ARM64_BUS_NONE) {
	if (has_fdt)
	arm64_bus_method = ARM64_BUS_FDT;
	else if (has_acpi)
	arm64_bus_method = ARM64_BUS_ACPI;
	}

	/*
	* If no option was set the default is valid, otherwise we are
	* setting one to get cninit() working, then calling panic to tell
	* the user about the invalid bus setup.
	*/
	return (env == NULL);
	}

	static void
	cache_setup(void)
	{
	int dczva_line_shift;
	uint32_t dczid_el0;

	identify_cache(READ_SPECIALREG(ctr_el0));

	dczid_el0 = READ_SPECIALREG(dczid_el0);

	/* Check if dc zva is not prohibited */
	if (dczid_el0 & DCZID_DZP)
	dczva_line_size = 0;
	else {
	/* Same as with above calculations */
	dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
	dczva_line_size = sizeof(int) << dczva_line_shift;

	/* Change pagezero function */
	pagezero = pagezero_cache;
	}
	}

	int
	memory_mapping_mode(vm_paddr_t pa)
	{
	struct efi_md map, p;
	size_t efisz;
	int ndesc, i;

	if (efihdr == NULL)
	return (VM_MEMATTR_WRITE_BACK);

	/*
	* Memory map data provided by UEFI via the GetMemoryMap
	* Boot Services API.
	*/
	efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
	map = (struct efi_md )((uint8_t )efihdr + efisz);

	if (efihdr->descriptor_size == 0)
	return (VM_MEMATTR_WRITE_BACK);
	ndesc = efihdr->memory_size / efihdr->descriptor_size;

	for (i = 0, p = map; i < ndesc; i++,
	p = efi_next_descriptor(p, efihdr->descriptor_size)) {
	if (pa < p->md_phys \|\|
	pa >= p->md_phys + p->md_pages * EFI_PAGE_SIZE)
	continue;
	if (p->md_type == EFI_MD_TYPE_IOMEM \|\|
	p->md_type == EFI_MD_TYPE_IOPORT)
	return (VM_MEMATTR_DEVICE);
	else if ((p->md_attr & EFI_MD_ATTR_WB) != 0 \|\|
	p->md_type == EFI_MD_TYPE_RECLAIM)
	return (VM_MEMATTR_WRITE_BACK);
	else if ((p->md_attr & EFI_MD_ATTR_WT) != 0)
	return (VM_MEMATTR_WRITE_THROUGH);
	else if ((p->md_attr & EFI_MD_ATTR_WC) != 0)
	return (VM_MEMATTR_WRITE_COMBINING);
	break;
	}

	return (VM_MEMATTR_DEVICE);
	}

	void
	initarm(struct arm64_bootparams *abp)
	{
	struct efi_fb *efifb;
	struct pcpu *pcpup;
	char *env;
	#ifdef FDT
	struct mem_region mem_regions[FDT_MEM_REGIONS];
	int mem_regions_sz;
	#endif
	vm_offset_t lastaddr;
	caddr_t kmdp;
	bool valid;

	boot_el = abp->boot_el;

	/* Parse loader or FDT boot parametes. Determine last used address. */
	lastaddr = parse_boot_param(abp);

	/* Find the kernel address */
	kmdp = preload_search_by_type("elf kernel");
	if (kmdp == NULL)
	kmdp = preload_search_by_type("elf64 kernel");

	identify_cpu(0);
	update_special_regs(0);

	link_elf_ireloc(kmdp);
	try_load_dtb(kmdp);

	efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t);

	/* Load the physical memory ranges */
	efihdr = (struct efi_map_header *)preload_search_info(kmdp,
	MODINFO_METADATA \| MODINFOMD_EFI_MAP);
	if (efihdr != NULL)
	add_efi_map_entries(efihdr);
	#ifdef FDT
	else {
	/* Grab physical memory regions information from device tree. */
	if (fdt_get_mem_regions(mem_regions, &mem_regions_sz,
	NULL) != 0)
	panic("Cannot get physical memory regions");
	physmem_hardware_regions(mem_regions, mem_regions_sz);
	}
	if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0)
	physmem_exclude_regions(mem_regions, mem_regions_sz,
	EXFLAG_NODUMP \| EXFLAG_NOALLOC);
	#endif

	/* Exclude the EFI framebuffer from our view of physical memory. */
	efifb = (struct efi_fb *)preload_search_info(kmdp,
	MODINFO_METADATA \| MODINFOMD_EFI_FB);
	if (efifb != NULL)
	physmem_exclude_region(efifb->fb_addr, efifb->fb_size,
	EXFLAG_NOALLOC);

	/* Set the pcpu data, this is needed by pmap_bootstrap */
	pcpup = &__pcpu[0];
	pcpu_init(pcpup, 0, sizeof(struct pcpu));

	/*
	* Set the pcpu pointer with a backup in tpidr_el1 to be
	* loaded when entering the kernel from userland.
	*/
	__asm __volatile(
	"mov x18, %0 \n"
	"msr tpidr_el1, %0" :: "r"(pcpup));

	PCPU_SET(curthread, &thread0);
	PCPU_SET(midr, get_midr());

	/* Do basic tuning, hz etc */
	init_param1();

	cache_setup();
	pan_setup();

	/* Bootstrap enough of pmap to enter the kernel proper */
	pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt,
	KERNBASE - abp->kern_delta, lastaddr - KERNBASE);
	/* Exclude entries neexed in teh DMAP region, but not phys_avail */
	if (efihdr != NULL)
	exclude_efi_map_entries(efihdr);
	physmem_init_kernel_globals();

	devmap_bootstrap(0, NULL);

	valid = bus_probe();

	cninit();

	if (!valid)
	panic("Invalid bus configuration: %s",
	kern_getenv("kern.cfg.order"));

	init_proc0(abp->kern_stack);
	msgbufinit(msgbufp, msgbufsize);
	mutex_init();
	init_param2(physmem);

	dbg_init();
	kdb_init();
	pan_enable();

	kcsan_cpu_init(0);

	env = kern_getenv("kernelname");
	if (env != NULL)
	strlcpy(kernelname, env, sizeof(kernelname));

	if (boothowto & RB_VERBOSE) {
	print_efi_map_entries(efihdr);
	physmem_print_tables();
	}

	early_boot = 0;
	}

	void
	dbg_init(void)
	{

	/* Clear OS lock */
	WRITE_SPECIALREG(oslar_el1, 0);

	/* This permits DDB to use debug registers for watchpoints. */
	dbg_monitor_init();

	/* TODO: Eventually will need to initialize debug registers here. */
	}

	#ifdef DDB
	#include <ddb/ddb.h>

	DB_SHOW_COMMAND(specialregs, db_show_spregs)
	{
	#define PRINT_REG(reg) \
	db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg))

	PRINT_REG(actlr_el1);
	PRINT_REG(afsr0_el1);
	PRINT_REG(afsr1_el1);
	PRINT_REG(aidr_el1);
	PRINT_REG(amair_el1);
	PRINT_REG(ccsidr_el1);
	PRINT_REG(clidr_el1);
	PRINT_REG(contextidr_el1);
	PRINT_REG(cpacr_el1);
	PRINT_REG(csselr_el1);
	PRINT_REG(ctr_el0);
	PRINT_REG(currentel);
	PRINT_REG(daif);
	PRINT_REG(dczid_el0);
	PRINT_REG(elr_el1);
	PRINT_REG(esr_el1);
	PRINT_REG(far_el1);
	#if 0
	/* ARM64TODO: Enable VFP before reading floating-point registers */
	PRINT_REG(fpcr);
	PRINT_REG(fpsr);
	#endif
	PRINT_REG(id_aa64afr0_el1);
	PRINT_REG(id_aa64afr1_el1);
	PRINT_REG(id_aa64dfr0_el1);
	PRINT_REG(id_aa64dfr1_el1);
	PRINT_REG(id_aa64isar0_el1);
	PRINT_REG(id_aa64isar1_el1);
	PRINT_REG(id_aa64pfr0_el1);
	PRINT_REG(id_aa64pfr1_el1);
	PRINT_REG(id_afr0_el1);
	PRINT_REG(id_dfr0_el1);
	PRINT_REG(id_isar0_el1);
	PRINT_REG(id_isar1_el1);
	PRINT_REG(id_isar2_el1);
	PRINT_REG(id_isar3_el1);
	PRINT_REG(id_isar4_el1);
	PRINT_REG(id_isar5_el1);
	PRINT_REG(id_mmfr0_el1);
	PRINT_REG(id_mmfr1_el1);
	PRINT_REG(id_mmfr2_el1);
	PRINT_REG(id_mmfr3_el1);
	#if 0
	/* Missing from llvm */
	PRINT_REG(id_mmfr4_el1);
	#endif
	PRINT_REG(id_pfr0_el1);
	PRINT_REG(id_pfr1_el1);
	PRINT_REG(isr_el1);
	PRINT_REG(mair_el1);
	PRINT_REG(midr_el1);
	PRINT_REG(mpidr_el1);
	PRINT_REG(mvfr0_el1);
	PRINT_REG(mvfr1_el1);
	PRINT_REG(mvfr2_el1);
	PRINT_REG(revidr_el1);
	PRINT_REG(sctlr_el1);
	PRINT_REG(sp_el0);
	PRINT_REG(spsel);
	PRINT_REG(spsr_el1);
	PRINT_REG(tcr_el1);
	PRINT_REG(tpidr_el0);
	PRINT_REG(tpidr_el1);
	PRINT_REG(tpidrro_el0);
	PRINT_REG(ttbr0_el1);
	PRINT_REG(ttbr1_el1);
	PRINT_REG(vbar_el1);
	#undef PRINT_REG
	}

	DB_SHOW_COMMAND(vtop, db_show_vtop)
	{
	uint64_t phys;

	if (have_addr) {
	phys = arm64_address_translate_s1e1r(addr);
	db_printf("EL1 physical address reg (read): 0x%016lx\n", phys);
	phys = arm64_address_translate_s1e1w(addr);
	db_printf("EL1 physical address reg (write): 0x%016lx\n", phys);
	phys = arm64_address_translate_s1e0r(addr);
	db_printf("EL0 physical address reg (read): 0x%016lx\n", phys);
	phys = arm64_address_translate_s1e0w(addr);
	db_printf("EL0 physical address reg (write): 0x%016lx\n", phys);
	} else
	db_printf("show vtop <virt_addr>\n");
	}
	#endif
	diff --git a/sys/arm64/include/acpica_machdep.h b/sys/arm64/include/acpica_machdep.h
	index 282c79f5eaec..7f8139f9134b 100644
	--- a/sys/arm64/include/acpica_machdep.h
	+++ b/sys/arm64/include/acpica_machdep.h
	@@ -1,62 +1,64 @@
	/*-
	* Copyright (c) 2002 Mitsuru IWASAKI
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	/******************************************************************************
	*
	* Name: acpica_machdep.h - arch-specific defines, etc.
	* $Revision$
	*
	*****************************************************************************/

	#ifndef __ACPICA_MACHDEP_H__
	#define __ACPICA_MACHDEP_H__


	#ifdef _KERNEL

	#include <machine/_bus.h>

	/* Only use the reduced hardware model */
	#define ACPI_REDUCED_HARDWARE 1

	/* Section 5.2.10.1: global lock acquire/release functions */
	int acpi_acquire_global_lock(volatile uint32_t *);
	int acpi_release_global_lock(volatile uint32_t *);

	void acpi_map_table(vm_paddr_t pa, const char sig);
	void acpi_unmap_table(void *table);
	vm_paddr_t acpi_find_table(const char *sig);

	struct acpi_generic_address;

	int acpi_map_addr(struct acpi_generic_address , bus_space_tag_t ,
	bus_space_handle_t *, bus_size_t);

	+extern int (*apei_nmi)(void);
	+
	#endif /* _KERNEL */

	#endif /* __ACPICA_MACHDEP_H__ */
	diff --git a/sys/conf/files b/sys/conf/files
	index 8ed21c37d214..cbb29fa45663 100644
	--- a/sys/conf/files
	+++ b/sys/conf/files
	@@ -1,4995 +1,4994 @@
	# $FreeBSD$
	#
	# The long compile-with and dependency lines are required because of
	# limitations in config: backslash-newline doesn't work in strings, and
	# dependency lines other than the first are silently ignored.
	#
	acpi_quirks.h optional acpi \
	dependency "$S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \
	compile-with "${AWK} -f $S/tools/acpi_quirks2h.awk $S/dev/acpica/acpi_quirks" \
	no-obj no-implicit-rule before-depend \
	clean "acpi_quirks.h"
	bhnd_nvram_map.h optional bhnd \
	dependency "$S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/tools/nvram_map_gen.awk $S/dev/bhnd/nvram/nvram_map" \
	compile-with "sh $S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/nvram/nvram_map -h" \
	no-obj no-implicit-rule before-depend \
	clean "bhnd_nvram_map.h"
	bhnd_nvram_map_data.h optional bhnd \
	dependency "$S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/tools/nvram_map_gen.awk $S/dev/bhnd/nvram/nvram_map" \
	compile-with "sh $S/dev/bhnd/tools/nvram_map_gen.sh $S/dev/bhnd/nvram/nvram_map -d" \
	no-obj no-implicit-rule before-depend \
	clean "bhnd_nvram_map_data.h"
	fdt_static_dtb.h optional fdt fdt_dtb_static \
	compile-with "sh -c 'MACHINE=${MACHINE} $S/tools/fdt/make_dtbh.sh ${FDT_DTS_FILE} ${.CURDIR}'" \
	dependency "${FDT_DTS_FILE:T:R}.dtb" \
	no-obj no-implicit-rule before-depend \
	clean "fdt_static_dtb.h"
	feeder_eq_gen.h optional sound \
	dependency "$S/tools/sound/feeder_eq_mkfilter.awk" \
	compile-with "${AWK} -f $S/tools/sound/feeder_eq_mkfilter.awk -- ${FEEDER_EQ_PRESETS} > feeder_eq_gen.h" \
	no-obj no-implicit-rule before-depend \
	clean "feeder_eq_gen.h"
	feeder_rate_gen.h optional sound \
	dependency "$S/tools/sound/feeder_rate_mkfilter.awk" \
	compile-with "${AWK} -f $S/tools/sound/feeder_rate_mkfilter.awk -- ${FEEDER_RATE_PRESETS} > feeder_rate_gen.h" \
	no-obj no-implicit-rule before-depend \
	clean "feeder_rate_gen.h"
	font.h optional sc_dflt_font \
	compile-with "uudecode < ${SRCTOP}/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'u_char dflt_font_16[16256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < ${SRCTOP}/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'u_char dflt_font_14[14256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < ${SRCTOP}/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \
	no-obj no-implicit-rule before-depend \
	clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8"
	snd_fxdiv_gen.h optional sound \
	dependency "$S/tools/sound/snd_fxdiv_gen.awk" \
	compile-with "${AWK} -f $S/tools/sound/snd_fxdiv_gen.awk -- > snd_fxdiv_gen.h" \
	no-obj no-implicit-rule before-depend \
	clean "snd_fxdiv_gen.h"
	miidevs.h optional miibus \| mii \
	dependency "$S/tools/miidevs2h.awk $S/dev/mii/miidevs" \
	compile-with "${AWK} -f $S/tools/miidevs2h.awk $S/dev/mii/miidevs" \
	no-obj no-implicit-rule before-depend \
	clean "miidevs.h"
	pccarddevs.h standard \
	dependency "$S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \
	compile-with "${AWK} -f $S/tools/pccarddevs2h.awk $S/dev/pccard/pccarddevs" \
	no-obj no-implicit-rule before-depend \
	clean "pccarddevs.h"
	kbdmuxmap.h optional kbdmux_dflt_keymap \
	compile-with "${KEYMAP} -L ${KBDMUX_DFLT_KEYMAP} \| ${KEYMAP_FIX} > ${.TARGET}" \
	no-obj no-implicit-rule before-depend \
	clean "kbdmuxmap.h"
	teken_state.h optional sc \| vt \
	dependency "$S/teken/gensequences $S/teken/sequences" \
	compile-with "${AWK} -f $S/teken/gensequences $S/teken/sequences > teken_state.h" \
	no-obj no-implicit-rule before-depend \
	clean "teken_state.h"
	ukbdmap.h optional ukbd_dflt_keymap \
	compile-with "${KEYMAP} -L ${UKBD_DFLT_KEYMAP} \| ${KEYMAP_FIX} > ${.TARGET}" \
	no-obj no-implicit-rule before-depend \
	clean "ukbdmap.h"
	usbdevs.h optional usb \
	dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \
	compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -h" \
	no-obj no-implicit-rule before-depend \
	clean "usbdevs.h"
	usbdevs_data.h optional usb \
	dependency "$S/tools/usbdevs2h.awk $S/dev/usb/usbdevs" \
	compile-with "${AWK} -f $S/tools/usbdevs2h.awk $S/dev/usb/usbdevs -d" \
	no-obj no-implicit-rule before-depend \
	clean "usbdevs_data.h"
	sdiodevs.h optional mmccam \
	dependency "$S/tools/sdiodevs2h.awk $S/dev/sdio/sdiodevs" \
	compile-with "${AWK} -f $S/tools/sdiodevs2h.awk $S/dev/sdio/sdiodevs -h" \
	no-obj no-implicit-rule before-depend \
	clean "sdiodevs.h"
	sdiodevs_data.h optional mmccam \
	dependency "$S/tools/sdiodevs2h.awk $S/dev/sdio/sdiodevs" \
	compile-with "${AWK} -f $S/tools/sdiodevs2h.awk $S/dev/sdio/sdiodevs -d" \
	no-obj no-implicit-rule before-depend \
	clean "sdiodevs_data.h"
	cam/cam.c optional scbus
	cam/cam_compat.c optional scbus
	cam/cam_iosched.c optional scbus
	cam/cam_periph.c optional scbus
	cam/cam_queue.c optional scbus
	cam/cam_sim.c optional scbus
	cam/cam_xpt.c optional scbus
	cam/ata/ata_all.c optional scbus
	cam/ata/ata_xpt.c optional scbus
	cam/ata/ata_pmp.c optional scbus
	cam/nvme/nvme_all.c optional scbus
	cam/nvme/nvme_da.c optional nda \| da
	cam/nvme/nvme_xpt.c optional scbus
	cam/scsi/scsi_xpt.c optional scbus
	cam/scsi/scsi_all.c optional scbus
	cam/scsi/scsi_cd.c optional cd
	cam/scsi/scsi_ch.c optional ch
	cam/ata/ata_da.c optional ada \| da
	cam/ctl/ctl.c optional ctl
	cam/ctl/ctl_backend.c optional ctl
	cam/ctl/ctl_backend_block.c optional ctl
	cam/ctl/ctl_backend_ramdisk.c optional ctl
	cam/ctl/ctl_cmd_table.c optional ctl
	cam/ctl/ctl_frontend.c optional ctl
	cam/ctl/ctl_frontend_cam_sim.c optional ctl
	cam/ctl/ctl_frontend_ioctl.c optional ctl
	cam/ctl/ctl_frontend_iscsi.c optional ctl cfiscsi
	cam/ctl/ctl_ha.c optional ctl
	cam/ctl/ctl_scsi_all.c optional ctl
	cam/ctl/ctl_tpc.c optional ctl
	cam/ctl/ctl_tpc_local.c optional ctl
	cam/ctl/ctl_error.c optional ctl
	cam/ctl/ctl_util.c optional ctl
	cam/ctl/scsi_ctl.c optional ctl
	cam/mmc/mmc_xpt.c optional scbus mmccam
	cam/mmc/mmc_da.c optional scbus mmccam da
	cam/scsi/scsi_da.c optional da
	cam/scsi/scsi_pass.c optional pass
	cam/scsi/scsi_pt.c optional pt
	cam/scsi/scsi_sa.c optional sa
	cam/scsi/scsi_enc.c optional ses
	cam/scsi/scsi_enc_ses.c optional ses
	cam/scsi/scsi_enc_safte.c optional ses
	cam/scsi/scsi_sg.c optional sg
	cam/scsi/scsi_targ_bh.c optional targbh
	cam/scsi/scsi_target.c optional targ
	cam/scsi/smp_all.c optional scbus
	# shared between zfs and dtrace
	cddl/compat/opensolaris/kern/opensolaris.c optional zfs \| dtrace compile-with "${CDDL_C}"
	cddl/compat/opensolaris/kern/opensolaris_cmn_err.c optional zfs \| dtrace compile-with "${CDDL_C}"
	cddl/compat/opensolaris/kern/opensolaris_kmem.c optional zfs \| dtrace compile-with "${CDDL_C}"
	cddl/compat/opensolaris/kern/opensolaris_misc.c optional zfs \| dtrace compile-with "${CDDL_C}"
	cddl/compat/opensolaris/kern/opensolaris_proc.c optional zfs \| dtrace compile-with "${CDDL_C}"
	cddl/compat/opensolaris/kern/opensolaris_sunddi.c optional zfs \| dtrace compile-with "${CDDL_C}"
	cddl/compat/opensolaris/kern/opensolaris_taskq.c optional zfs \| dtrace compile-with "${CDDL_C}"
	# zfs specific
	cddl/compat/opensolaris/kern/opensolaris_acl.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_dtrace.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_kobj.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_kstat.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_lookup.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_policy.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_string.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_sysevent.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_uio.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_vfs.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_vm.c optional zfs compile-with "${ZFS_C}"
	cddl/compat/opensolaris/kern/opensolaris_zone.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/acl/acl_common.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/avl/avl.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/lz4/lz4.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/nvpair/opensolaris_fnvpair.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair_alloc_fixed.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/unicode/u8_textprep.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfeature_common.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfs_comutil.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfs_deleg.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfs_fletcher.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfs_ioctl_compat.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfs_namecheck.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zfs_prop.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zpool_prop.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/common/zfs/zprop_common.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/vnode.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/abd.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/aggsum.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/blkptr.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/bplist.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/bpobj.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/bptree.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/bqueue.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/cityhash.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf_stats.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_diff.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_object.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_tx.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_zfetch.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c optional zfs compile-with "${ZFS_C}" \
	warning "kernel contains CDDL licensed ZFS filesystem"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_bookmark.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deadlist.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_deleg.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_destroy.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_userhold.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_synctask.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/gzip.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lzjb.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/mmp.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/multilist.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/rrwlock.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/sa.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/sha256.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/skein_zfs.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/spa_checkpoint.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/spa_errlog.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/spa_history.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/space_map.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/space_reftree.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/trim_map.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/uberblock.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/unique.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_cache.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_indirect.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_indirect_births.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_indirect_mapping.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_initialize.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_removal.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_root.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zap_leaf.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zap_micro.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zcp.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_get.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_global.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_iter.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_synctask.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_acl.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_byteswap.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ctldir.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_debug.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fm.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_fuid.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_log.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_onexit.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_replay.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_rlock.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_sa.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zio_checksum.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zio_inject.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zle.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zthr.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/os/callb.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/os/fm.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/os/list.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/zmod/zmod.c optional zfs compile-with "${ZFS_C}"
	# zfs lua support
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lapi.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lauxlib.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lbaselib.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lbitlib.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lcode.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lcompat.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lcorolib.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lctype.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ldebug.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ldo.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ldump.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lfunc.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lgc.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/llex.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lmem.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lobject.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lopcodes.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lparser.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lstate.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lstring.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lstrlib.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ltable.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ltablib.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/ltm.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lundump.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lvm.c optional zfs compile-with "${ZFS_C}"
	cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lzio.c optional zfs compile-with "${ZFS_C}"
	# dtrace specific
	cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c optional dtrace compile-with "${DTRACE_C}" \
	warning "kernel contains CDDL licensed DTRACE"
	cddl/contrib/opensolaris/uts/common/dtrace/dtrace_xoroshiro128_plus.c optional dtrace compile-with "${DTRACE_C}"
	cddl/dev/dtmalloc/dtmalloc.c optional dtmalloc \| dtraceall compile-with "${CDDL_C}"
	cddl/dev/profile/profile.c optional dtrace_profile \| dtraceall compile-with "${CDDL_C}"
	cddl/dev/sdt/sdt.c optional dtrace_sdt \| dtraceall compile-with "${CDDL_C}"
	cddl/dev/fbt/fbt.c optional dtrace_fbt \| dtraceall compile-with "${FBT_C}"
	cddl/dev/systrace/systrace.c optional dtrace_systrace \| dtraceall compile-with "${CDDL_C}"
	cddl/dev/prototype.c optional dtrace_prototype \| dtraceall compile-with "${CDDL_C}"
	fs/nfsclient/nfs_clkdtrace.c optional dtnfscl nfscl \| dtraceall nfscl compile-with "${CDDL_C}"
	compat/cloudabi/cloudabi_clock.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_errno.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_fd.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_file.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_futex.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_mem.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_proc.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_random.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_sock.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_thread.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi/cloudabi_vdso.c optional compat_cloudabi32 \| compat_cloudabi64
	compat/cloudabi32/cloudabi32_fd.c optional compat_cloudabi32
	compat/cloudabi32/cloudabi32_module.c optional compat_cloudabi32
	compat/cloudabi32/cloudabi32_poll.c optional compat_cloudabi32
	compat/cloudabi32/cloudabi32_sock.c optional compat_cloudabi32
	compat/cloudabi32/cloudabi32_syscalls.c optional compat_cloudabi32
	compat/cloudabi32/cloudabi32_sysent.c optional compat_cloudabi32
	compat/cloudabi32/cloudabi32_thread.c optional compat_cloudabi32
	compat/cloudabi64/cloudabi64_fd.c optional compat_cloudabi64
	compat/cloudabi64/cloudabi64_module.c optional compat_cloudabi64
	compat/cloudabi64/cloudabi64_poll.c optional compat_cloudabi64
	compat/cloudabi64/cloudabi64_sock.c optional compat_cloudabi64
	compat/cloudabi64/cloudabi64_syscalls.c optional compat_cloudabi64
	compat/cloudabi64/cloudabi64_sysent.c optional compat_cloudabi64
	compat/cloudabi64/cloudabi64_thread.c optional compat_cloudabi64
	compat/freebsd32/freebsd32_capability.c optional compat_freebsd32
	compat/freebsd32/freebsd32_ioctl.c optional compat_freebsd32
	compat/freebsd32/freebsd32_misc.c optional compat_freebsd32
	compat/freebsd32/freebsd32_syscalls.c optional compat_freebsd32
	compat/freebsd32/freebsd32_sysent.c optional compat_freebsd32
	contrib/ck/src/ck_array.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_barrier_centralized.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_barrier_combining.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_barrier_dissemination.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_barrier_mcs.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_barrier_tournament.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_epoch.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_hp.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_hs.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_ht.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/ck/src/ck_rhs.c standard compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	contrib/dev/acpica/common/ahids.c optional acpi acpi_debug
	contrib/dev/acpica/common/ahuuids.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbcmds.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbconvert.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbdisply.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbexec.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbhistry.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbinput.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbmethod.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbnames.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbobject.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbstats.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbtest.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbutils.c optional acpi acpi_debug
	contrib/dev/acpica/components/debugger/dbxface.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmbuffer.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmcstyle.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmdeferred.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmnames.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmopcode.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmresrc.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmresrcl.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmresrcl2.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmresrcs.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmutils.c optional acpi acpi_debug
	contrib/dev/acpica/components/disassembler/dmwalk.c optional acpi acpi_debug
	contrib/dev/acpica/components/dispatcher/dsargs.c optional acpi
	contrib/dev/acpica/components/dispatcher/dscontrol.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsdebug.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsfield.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsinit.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsmethod.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsmthdat.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsobject.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsopcode.c optional acpi
	contrib/dev/acpica/components/dispatcher/dspkginit.c optional acpi
	contrib/dev/acpica/components/dispatcher/dsutils.c optional acpi
	contrib/dev/acpica/components/dispatcher/dswexec.c optional acpi
	contrib/dev/acpica/components/dispatcher/dswload.c optional acpi
	contrib/dev/acpica/components/dispatcher/dswload2.c optional acpi
	contrib/dev/acpica/components/dispatcher/dswscope.c optional acpi
	contrib/dev/acpica/components/dispatcher/dswstate.c optional acpi
	contrib/dev/acpica/components/events/evevent.c optional acpi
	contrib/dev/acpica/components/events/evglock.c optional acpi
	contrib/dev/acpica/components/events/evgpe.c optional acpi
	contrib/dev/acpica/components/events/evgpeblk.c optional acpi
	contrib/dev/acpica/components/events/evgpeinit.c optional acpi
	contrib/dev/acpica/components/events/evgpeutil.c optional acpi
	contrib/dev/acpica/components/events/evhandler.c optional acpi
	contrib/dev/acpica/components/events/evmisc.c optional acpi
	contrib/dev/acpica/components/events/evregion.c optional acpi
	contrib/dev/acpica/components/events/evrgnini.c optional acpi
	contrib/dev/acpica/components/events/evsci.c optional acpi
	contrib/dev/acpica/components/events/evxface.c optional acpi
	contrib/dev/acpica/components/events/evxfevnt.c optional acpi
	contrib/dev/acpica/components/events/evxfgpe.c optional acpi
	contrib/dev/acpica/components/events/evxfregn.c optional acpi
	contrib/dev/acpica/components/executer/exconcat.c optional acpi
	contrib/dev/acpica/components/executer/exconfig.c optional acpi
	contrib/dev/acpica/components/executer/exconvrt.c optional acpi
	contrib/dev/acpica/components/executer/excreate.c optional acpi
	contrib/dev/acpica/components/executer/exdebug.c optional acpi
	contrib/dev/acpica/components/executer/exdump.c optional acpi
	contrib/dev/acpica/components/executer/exfield.c optional acpi
	contrib/dev/acpica/components/executer/exfldio.c optional acpi
	contrib/dev/acpica/components/executer/exmisc.c optional acpi
	contrib/dev/acpica/components/executer/exmutex.c optional acpi
	contrib/dev/acpica/components/executer/exnames.c optional acpi
	contrib/dev/acpica/components/executer/exoparg1.c optional acpi
	contrib/dev/acpica/components/executer/exoparg2.c optional acpi
	contrib/dev/acpica/components/executer/exoparg3.c optional acpi
	contrib/dev/acpica/components/executer/exoparg6.c optional acpi
	contrib/dev/acpica/components/executer/exprep.c optional acpi
	contrib/dev/acpica/components/executer/exregion.c optional acpi
	contrib/dev/acpica/components/executer/exresnte.c optional acpi
	contrib/dev/acpica/components/executer/exresolv.c optional acpi
	contrib/dev/acpica/components/executer/exresop.c optional acpi
	contrib/dev/acpica/components/executer/exserial.c optional acpi
	contrib/dev/acpica/components/executer/exstore.c optional acpi
	contrib/dev/acpica/components/executer/exstoren.c optional acpi
	contrib/dev/acpica/components/executer/exstorob.c optional acpi
	contrib/dev/acpica/components/executer/exsystem.c optional acpi
	contrib/dev/acpica/components/executer/extrace.c optional acpi
	contrib/dev/acpica/components/executer/exutils.c optional acpi
	contrib/dev/acpica/components/hardware/hwacpi.c optional acpi
	contrib/dev/acpica/components/hardware/hwesleep.c optional acpi
	contrib/dev/acpica/components/hardware/hwgpe.c optional acpi
	contrib/dev/acpica/components/hardware/hwpci.c optional acpi
	contrib/dev/acpica/components/hardware/hwregs.c optional acpi
	contrib/dev/acpica/components/hardware/hwsleep.c optional acpi
	contrib/dev/acpica/components/hardware/hwtimer.c optional acpi
	contrib/dev/acpica/components/hardware/hwvalid.c optional acpi
	contrib/dev/acpica/components/hardware/hwxface.c optional acpi
	contrib/dev/acpica/components/hardware/hwxfsleep.c optional acpi
	contrib/dev/acpica/components/namespace/nsaccess.c optional acpi
	contrib/dev/acpica/components/namespace/nsalloc.c optional acpi
	contrib/dev/acpica/components/namespace/nsarguments.c optional acpi
	contrib/dev/acpica/components/namespace/nsconvert.c optional acpi
	contrib/dev/acpica/components/namespace/nsdump.c optional acpi
	contrib/dev/acpica/components/namespace/nseval.c optional acpi
	contrib/dev/acpica/components/namespace/nsinit.c optional acpi
	contrib/dev/acpica/components/namespace/nsload.c optional acpi
	contrib/dev/acpica/components/namespace/nsnames.c optional acpi
	contrib/dev/acpica/components/namespace/nsobject.c optional acpi
	contrib/dev/acpica/components/namespace/nsparse.c optional acpi
	contrib/dev/acpica/components/namespace/nspredef.c optional acpi
	contrib/dev/acpica/components/namespace/nsprepkg.c optional acpi
	contrib/dev/acpica/components/namespace/nsrepair.c optional acpi
	contrib/dev/acpica/components/namespace/nsrepair2.c optional acpi
	contrib/dev/acpica/components/namespace/nssearch.c optional acpi
	contrib/dev/acpica/components/namespace/nsutils.c optional acpi
	contrib/dev/acpica/components/namespace/nswalk.c optional acpi
	contrib/dev/acpica/components/namespace/nsxfeval.c optional acpi
	contrib/dev/acpica/components/namespace/nsxfname.c optional acpi
	contrib/dev/acpica/components/namespace/nsxfobj.c optional acpi
	contrib/dev/acpica/components/parser/psargs.c optional acpi
	contrib/dev/acpica/components/parser/psloop.c optional acpi
	contrib/dev/acpica/components/parser/psobject.c optional acpi
	contrib/dev/acpica/components/parser/psopcode.c optional acpi
	contrib/dev/acpica/components/parser/psopinfo.c optional acpi
	contrib/dev/acpica/components/parser/psparse.c optional acpi
	contrib/dev/acpica/components/parser/psscope.c optional acpi
	contrib/dev/acpica/components/parser/pstree.c optional acpi
	contrib/dev/acpica/components/parser/psutils.c optional acpi
	contrib/dev/acpica/components/parser/pswalk.c optional acpi
	contrib/dev/acpica/components/parser/psxface.c optional acpi
	contrib/dev/acpica/components/resources/rsaddr.c optional acpi
	contrib/dev/acpica/components/resources/rscalc.c optional acpi
	contrib/dev/acpica/components/resources/rscreate.c optional acpi
	contrib/dev/acpica/components/resources/rsdump.c optional acpi acpi_debug
	contrib/dev/acpica/components/resources/rsdumpinfo.c optional acpi
	contrib/dev/acpica/components/resources/rsinfo.c optional acpi
	contrib/dev/acpica/components/resources/rsio.c optional acpi
	contrib/dev/acpica/components/resources/rsirq.c optional acpi
	contrib/dev/acpica/components/resources/rslist.c optional acpi
	contrib/dev/acpica/components/resources/rsmemory.c optional acpi
	contrib/dev/acpica/components/resources/rsmisc.c optional acpi
	contrib/dev/acpica/components/resources/rsserial.c optional acpi
	contrib/dev/acpica/components/resources/rsutils.c optional acpi
	contrib/dev/acpica/components/resources/rsxface.c optional acpi
	contrib/dev/acpica/components/tables/tbdata.c optional acpi
	contrib/dev/acpica/components/tables/tbfadt.c optional acpi
	contrib/dev/acpica/components/tables/tbfind.c optional acpi
	contrib/dev/acpica/components/tables/tbinstal.c optional acpi
	contrib/dev/acpica/components/tables/tbprint.c optional acpi
	contrib/dev/acpica/components/tables/tbutils.c optional acpi
	contrib/dev/acpica/components/tables/tbxface.c optional acpi
	contrib/dev/acpica/components/tables/tbxfload.c optional acpi
	contrib/dev/acpica/components/tables/tbxfroot.c optional acpi
	contrib/dev/acpica/components/utilities/utaddress.c optional acpi
	contrib/dev/acpica/components/utilities/utalloc.c optional acpi
	contrib/dev/acpica/components/utilities/utascii.c optional acpi
	contrib/dev/acpica/components/utilities/utbuffer.c optional acpi
	contrib/dev/acpica/components/utilities/utcache.c optional acpi
	contrib/dev/acpica/components/utilities/utcopy.c optional acpi
	contrib/dev/acpica/components/utilities/utdebug.c optional acpi
	contrib/dev/acpica/components/utilities/utdecode.c optional acpi
	contrib/dev/acpica/components/utilities/utdelete.c optional acpi
	contrib/dev/acpica/components/utilities/uterror.c optional acpi
	contrib/dev/acpica/components/utilities/uteval.c optional acpi
	contrib/dev/acpica/components/utilities/utexcep.c optional acpi
	contrib/dev/acpica/components/utilities/utglobal.c optional acpi
	contrib/dev/acpica/components/utilities/uthex.c optional acpi
	contrib/dev/acpica/components/utilities/utids.c optional acpi
	contrib/dev/acpica/components/utilities/utinit.c optional acpi
	contrib/dev/acpica/components/utilities/utlock.c optional acpi
	contrib/dev/acpica/components/utilities/utmath.c optional acpi
	contrib/dev/acpica/components/utilities/utmisc.c optional acpi
	contrib/dev/acpica/components/utilities/utmutex.c optional acpi
	contrib/dev/acpica/components/utilities/utnonansi.c optional acpi
	contrib/dev/acpica/components/utilities/utobject.c optional acpi
	contrib/dev/acpica/components/utilities/utosi.c optional acpi
	contrib/dev/acpica/components/utilities/utownerid.c optional acpi
	contrib/dev/acpica/components/utilities/utpredef.c optional acpi
	contrib/dev/acpica/components/utilities/utresdecode.c optional acpi acpi_debug
	contrib/dev/acpica/components/utilities/utresrc.c optional acpi
	contrib/dev/acpica/components/utilities/utstate.c optional acpi
	contrib/dev/acpica/components/utilities/utstring.c optional acpi
	contrib/dev/acpica/components/utilities/utstrsuppt.c optional acpi
	contrib/dev/acpica/components/utilities/utstrtoul64.c optional acpi
	contrib/dev/acpica/components/utilities/utuuid.c optional acpi acpi_debug
	contrib/dev/acpica/components/utilities/utxface.c optional acpi
	contrib/dev/acpica/components/utilities/utxferror.c optional acpi
	contrib/dev/acpica/components/utilities/utxfinit.c optional acpi
	contrib/dev/acpica/os_specific/service_layers/osgendbg.c optional acpi acpi_debug
	contrib/ipfilter/netinet/fil.c optional ipfilter inet \
	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_auth.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_fil_freebsd.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_frag.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_log.c optional ipfilter inet \
	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_nat.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_proxy.c optional ipfilter inet \
	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_state.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_lookup.c optional ipfilter inet \
	compile-with "${NORMAL_C} ${NO_WSELF_ASSIGN} -Wno-unused -Wno-error -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_pool.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_htable.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter ${NO_WTAUTOLOGICAL_POINTER_COMPARE}"
	contrib/ipfilter/netinet/ip_sync.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/mlfk_ipl.c optional ipfilter inet \
	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_nat6.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_rules.c optional ipfilter inet \
	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_scan.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/ip_dstlist.c optional ipfilter inet \
	compile-with "${NORMAL_C} -Wno-unused -I$S/contrib/ipfilter"
	contrib/ipfilter/netinet/radix_ipf.c optional ipfilter inet \
	compile-with "${NORMAL_C} -I$S/contrib/ipfilter"
	contrib/libfdt/fdt.c optional fdt
	contrib/libfdt/fdt_ro.c optional fdt
	contrib/libfdt/fdt_rw.c optional fdt
	contrib/libfdt/fdt_strerror.c optional fdt
	contrib/libfdt/fdt_sw.c optional fdt
	contrib/libfdt/fdt_wip.c optional fdt
	contrib/libnv/cnvlist.c standard
	contrib/libnv/dnvlist.c standard
	contrib/libnv/nvlist.c standard
	contrib/libnv/bsd_nvpair.c standard
	contrib/ngatm/netnatm/api/cc_conn.c optional ngatm_ccatm \
	compile-with "${NORMAL_C_NOWERROR} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/api/cc_data.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/api/cc_dump.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/api/cc_port.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/api/cc_sig.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/api/cc_user.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/api/unisap.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/misc/straddr.c optional ngatm_atmbase \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/misc/unimsg_common.c optional ngatm_atmbase \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/msg/traffic.c optional ngatm_atmbase \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/msg/uni_ie.c optional ngatm_atmbase \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/msg/uni_msg.c optional ngatm_atmbase \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/saal/saal_sscfu.c optional ngatm_sscfu \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/saal/saal_sscop.c optional ngatm_sscop \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_call.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_coord.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_party.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_print.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_reset.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_uni.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_unimsgcpy.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	contrib/ngatm/netnatm/sig/sig_verify.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	# xz
	dev/xz/xz_mod.c optional xz \
	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
	contrib/xz-embedded/linux/lib/xz/xz_crc32.c optional xz \
	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
	contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c optional xz \
	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
	contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c optional xz \
	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
	contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c optional xz \
	compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/"
	# Zstd
	contrib/zstd/lib/freebsd/zstd_kmalloc.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/common/zstd_common.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/common/fse_decompress.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/common/entropy_common.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/common/error_private.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/common/xxhash.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_compress.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_compress_literals.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_compress_sequences.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_compress_superblock.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/fse_compress.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/hist.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/huf_compress.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_double_fast.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_fast.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_lazy.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_ldm.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/compress/zstd_opt.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/decompress/zstd_ddict.c optional zstdio compile-with ${ZSTD_C}
	contrib/zstd/lib/decompress/zstd_decompress.c optional zstdio compile-with ${ZSTD_C}
	# See comment in sys/conf/kern.pre.mk
	contrib/zstd/lib/decompress/zstd_decompress_block.c optional zstdio \
	compile-with "${ZSTD_C} ${ZSTD_DECOMPRESS_BLOCK_FLAGS}"
	contrib/zstd/lib/decompress/huf_decompress.c optional zstdio compile-with ${ZSTD_C}
	# Blake 2
	contrib/libb2/blake2b-ref.c optional crypto \| ipsec \| ipsec_support \
	compile-with "${NORMAL_C} -I$S/crypto/blake2 -Wno-cast-qual -DSUFFIX=_ref -Wno-unused-function"
	contrib/libb2/blake2s-ref.c optional crypto \| ipsec \| ipsec_support \
	compile-with "${NORMAL_C} -I$S/crypto/blake2 -Wno-cast-qual -DSUFFIX=_ref -Wno-unused-function"
	crypto/blake2/blake2-sw.c optional crypto \| ipsec \| ipsec_support \
	compile-with "${NORMAL_C} -I$S/crypto/blake2 -Wno-cast-qual"
	crypto/camellia/camellia.c optional crypto \| ipsec \| ipsec_support
	crypto/camellia/camellia-api.c optional crypto \| ipsec \| ipsec_support
	crypto/chacha20/chacha.c standard
	crypto/chacha20/chacha-sw.c optional crypto \| ipsec \| ipsec_support
	crypto/des/des_ecb.c optional netsmb
	crypto/des/des_setkey.c optional netsmb
	crypto/rc4/rc4.c optional netgraph_mppc_encryption
	crypto/rijndael/rijndael-alg-fst.c optional crypto \| ekcd \| geom_bde \| \
	ipsec \| ipsec_support \| !random_loadable \| wlan_ccmp
	crypto/rijndael/rijndael-api-fst.c optional ekcd \| geom_bde \| !random_loadable
	crypto/rijndael/rijndael-api.c optional crypto \| ipsec \| ipsec_support \| \
	wlan_ccmp
	crypto/sha1.c optional carp \| crypto \| ether \| ipsec \| \
	ipsec_support \| netgraph_mppc_encryption \| sctp
	crypto/sha2/sha256c.c optional crypto \| ekcd \| geom_bde \| ipsec \| \
	ipsec_support \| !random_loadable \| sctp \| zfs
	crypto/sha2/sha512c.c optional crypto \| geom_bde \| ipsec \| \
	ipsec_support \| zfs
	crypto/skein/skein.c optional crypto \| zfs
	crypto/skein/skein_block.c optional crypto \| zfs
	crypto/siphash/siphash.c optional inet \| inet6
	crypto/siphash/siphash_test.c optional inet \| inet6
	ddb/db_access.c optional ddb
	ddb/db_break.c optional ddb
	ddb/db_capture.c optional ddb
	ddb/db_command.c optional ddb
	ddb/db_examine.c optional ddb
	ddb/db_expr.c optional ddb
	ddb/db_input.c optional ddb
	ddb/db_lex.c optional ddb
	ddb/db_main.c optional ddb
	ddb/db_output.c optional ddb
	ddb/db_print.c optional ddb
	ddb/db_ps.c optional ddb
	ddb/db_run.c optional ddb
	ddb/db_script.c optional ddb
	ddb/db_sym.c optional ddb
	ddb/db_thread.c optional ddb
	ddb/db_textdump.c optional ddb
	ddb/db_variables.c optional ddb
	ddb/db_watch.c optional ddb
	ddb/db_write_cmd.c optional ddb
	dev/aac/aac.c optional aac
	dev/aac/aac_cam.c optional aacp aac
	dev/aac/aac_debug.c optional aac
	dev/aac/aac_disk.c optional aac
	dev/aac/aac_linux.c optional aac compat_linux
	dev/aac/aac_pci.c optional aac pci
	dev/aacraid/aacraid.c optional aacraid
	dev/aacraid/aacraid_cam.c optional aacraid scbus
	dev/aacraid/aacraid_debug.c optional aacraid
	dev/aacraid/aacraid_linux.c optional aacraid compat_linux
	dev/aacraid/aacraid_pci.c optional aacraid pci
	dev/acpi_support/acpi_wmi.c optional acpi_wmi acpi
	dev/acpi_support/acpi_asus.c optional acpi_asus acpi
	dev/acpi_support/acpi_asus_wmi.c optional acpi_asus_wmi acpi
	dev/acpi_support/acpi_fujitsu.c optional acpi_fujitsu acpi
	dev/acpi_support/acpi_hp.c optional acpi_hp acpi
	dev/acpi_support/acpi_ibm.c optional acpi_ibm acpi
	dev/acpi_support/acpi_panasonic.c optional acpi_panasonic acpi
	dev/acpi_support/acpi_sony.c optional acpi_sony acpi
	dev/acpi_support/acpi_toshiba.c optional acpi_toshiba acpi
	dev/acpi_support/atk0110.c optional aibs acpi
	dev/acpica/Osd/OsdDebug.c optional acpi
	dev/acpica/Osd/OsdHardware.c optional acpi
	dev/acpica/Osd/OsdInterrupt.c optional acpi
	dev/acpica/Osd/OsdMemory.c optional acpi
	dev/acpica/Osd/OsdSchedule.c optional acpi
	dev/acpica/Osd/OsdStream.c optional acpi
	dev/acpica/Osd/OsdSynch.c optional acpi
	dev/acpica/Osd/OsdTable.c optional acpi
	dev/acpica/acpi.c optional acpi
	dev/acpica/acpi_acad.c optional acpi
	+dev/acpica/acpi_apei.c optional acpi
	dev/acpica/acpi_battery.c optional acpi
	dev/acpica/acpi_button.c optional acpi
	dev/acpica/acpi_cmbat.c optional acpi
	dev/acpica/acpi_cpu.c optional acpi
	dev/acpica/acpi_ec.c optional acpi
	dev/acpica/acpi_isab.c optional acpi isa
	dev/acpica/acpi_lid.c optional acpi
	dev/acpica/acpi_package.c optional acpi
	dev/acpica/acpi_perf.c optional acpi
	dev/acpica/acpi_powerres.c optional acpi
	dev/acpica/acpi_quirk.c optional acpi
	dev/acpica/acpi_resource.c optional acpi
	dev/acpica/acpi_container.c optional acpi
	dev/acpica/acpi_smbat.c optional acpi
	dev/acpica/acpi_thermal.c optional acpi
	dev/acpica/acpi_throttle.c optional acpi
	dev/acpica/acpi_video.c optional acpi_video acpi
	dev/acpica/acpi_dock.c optional acpi_dock acpi
	dev/adlink/adlink.c optional adlink
	dev/ae/if_ae.c optional ae pci
	dev/age/if_age.c optional age pci
	dev/agp/agp.c optional agp pci
	dev/agp/agp_if.m optional agp pci
	dev/ahci/ahci.c optional ahci
	dev/ahci/ahciem.c optional ahci
	dev/ahci/ahci_pci.c optional ahci pci
	dev/aic7xxx/ahc_isa.c optional ahc isa
	dev/aic7xxx/ahc_pci.c optional ahc pci \
	compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}"
	dev/aic7xxx/ahd_pci.c optional ahd pci \
	compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}"
	dev/aic7xxx/aic7770.c optional ahc
	dev/aic7xxx/aic79xx.c optional ahd pci
	dev/aic7xxx/aic79xx_osm.c optional ahd pci
	dev/aic7xxx/aic79xx_pci.c optional ahd pci
	dev/aic7xxx/aic79xx_reg_print.c optional ahd pci ahd_reg_pretty_print
	dev/aic7xxx/aic7xxx.c optional ahc
	dev/aic7xxx/aic7xxx_93cx6.c optional ahc
	dev/aic7xxx/aic7xxx_osm.c optional ahc
	dev/aic7xxx/aic7xxx_pci.c optional ahc pci
	dev/aic7xxx/aic7xxx_reg_print.c optional ahc ahc_reg_pretty_print
	dev/al_eth/al_eth.c optional al_eth fdt \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	dev/al_eth/al_init_eth_lm.c optional al_eth fdt \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	dev/al_eth/al_init_eth_kr.c optional al_eth fdt \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_iofic.c optional al_iofic \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_serdes_25g.c optional al_serdes \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_serdes_hssp.c optional al_serdes \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_udma_config.c optional al_udma \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_udma_debug.c optional al_udma \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_udma_iofic.c optional al_udma \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_hal_udma_main.c optional al_udma \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/al_serdes.c optional al_serdes \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/eth/al_hal_eth_kr.c optional al_eth \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	contrib/alpine-hal/eth/al_hal_eth_main.c optional al_eth \
	no-depend \
	compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}"
	dev/alc/if_alc.c optional alc pci
	dev/ale/if_ale.c optional ale pci
	dev/alpm/alpm.c optional alpm pci
	dev/altera/avgen/altera_avgen.c optional altera_avgen
	dev/altera/avgen/altera_avgen_fdt.c optional altera_avgen fdt
	dev/altera/avgen/altera_avgen_nexus.c optional altera_avgen
	dev/altera/msgdma/msgdma.c optional altera_msgdma xdma
	dev/altera/sdcard/altera_sdcard.c optional altera_sdcard
	dev/altera/sdcard/altera_sdcard_disk.c optional altera_sdcard
	dev/altera/sdcard/altera_sdcard_io.c optional altera_sdcard
	dev/altera/sdcard/altera_sdcard_fdt.c optional altera_sdcard fdt
	dev/altera/sdcard/altera_sdcard_nexus.c optional altera_sdcard
	dev/altera/softdma/softdma.c optional altera_softdma xdma fdt
	dev/altera/pio/pio.c optional altera_pio
	dev/altera/pio/pio_if.m optional altera_pio
	dev/amdpm/amdpm.c optional amdpm pci \| nfpm pci
	dev/amdsmb/amdsmb.c optional amdsmb pci
	dev/amr/amr.c optional amr
	dev/amr/amr_cam.c optional amrp amr
	dev/amr/amr_disk.c optional amr
	dev/amr/amr_linux.c optional amr compat_linux
	dev/amr/amr_pci.c optional amr pci
	dev/an/if_an.c optional an
	dev/an/if_an_isa.c optional an isa
	dev/an/if_an_pccard.c optional an pccard
	dev/an/if_an_pci.c optional an pci
	#
	dev/ata/ata_if.m optional ata \| atacore
	dev/ata/ata-all.c optional ata \| atacore
	dev/ata/ata-dma.c optional ata \| atacore
	dev/ata/ata-lowlevel.c optional ata \| atacore
	dev/ata/ata-sata.c optional ata \| atacore
	dev/ata/ata-card.c optional ata pccard \| atapccard
	dev/ata/ata-isa.c optional ata isa \| ataisa
	dev/ata/ata-pci.c optional ata pci \| atapci
	dev/ata/chipsets/ata-acard.c optional ata pci \| ataacard
	dev/ata/chipsets/ata-acerlabs.c optional ata pci \| ataacerlabs
	dev/ata/chipsets/ata-amd.c optional ata pci \| ataamd
	dev/ata/chipsets/ata-ati.c optional ata pci \| ataati
	dev/ata/chipsets/ata-cenatek.c optional ata pci \| atacenatek
	dev/ata/chipsets/ata-cypress.c optional ata pci \| atacypress
	dev/ata/chipsets/ata-cyrix.c optional ata pci \| atacyrix
	dev/ata/chipsets/ata-highpoint.c optional ata pci \| atahighpoint
	dev/ata/chipsets/ata-intel.c optional ata pci \| ataintel
	dev/ata/chipsets/ata-ite.c optional ata pci \| ataite
	dev/ata/chipsets/ata-jmicron.c optional ata pci \| atajmicron
	dev/ata/chipsets/ata-marvell.c optional ata pci \| atamarvell
	dev/ata/chipsets/ata-micron.c optional ata pci \| atamicron
	dev/ata/chipsets/ata-national.c optional ata pci \| atanational
	dev/ata/chipsets/ata-netcell.c optional ata pci \| atanetcell
	dev/ata/chipsets/ata-nvidia.c optional ata pci \| atanvidia
	dev/ata/chipsets/ata-promise.c optional ata pci \| atapromise
	dev/ata/chipsets/ata-serverworks.c optional ata pci \| ataserverworks
	dev/ata/chipsets/ata-siliconimage.c optional ata pci \| atasiliconimage \| ataati
	dev/ata/chipsets/ata-sis.c optional ata pci \| atasis
	dev/ata/chipsets/ata-via.c optional ata pci \| atavia
	#
	dev/ath/if_ath_pci.c optional ath_pci pci \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	#
	dev/ath/if_ath_ahb.c optional ath_ahb \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	#
	dev/ath/if_ath.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_alq.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_beacon.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_btcoex.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_btcoex_mci.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_debug.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_descdma.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_keycache.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_ioctl.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_led.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_lna_div.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_tx.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_tx_edma.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_tx_ht.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_tdma.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_sysctl.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_rx.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_rx_edma.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/if_ath_spectral.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ah_osdep.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	#
	dev/ath/ath_hal/ah.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_hal/ah_eeprom_v1.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_hal/ah_eeprom_v3.c optional ath_hal \| ath_ar5211 \| ath_ar5212 \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_hal/ah_eeprom_v14.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_hal/ah_eeprom_v4k.c \
	optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_hal/ah_eeprom_9287.c \
	optional ath_hal \| ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_hal/ah_regdomain.c optional ath \
	compile-with "${NORMAL_C} ${NO_WSHIFT_COUNT_NEGATIVE} ${NO_WSHIFT_COUNT_OVERFLOW} -I$S/dev/ath"
	# ar5210
	dev/ath/ath_hal/ar5210/ar5210_attach.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_beacon.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_interrupts.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_keycache.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_misc.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_phy.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_power.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_recv.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_reset.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5210/ar5210_xmit.c optional ath_hal \| ath_ar5210 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar5211
	dev/ath/ath_hal/ar5211/ar5211_attach.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_beacon.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_interrupts.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_keycache.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_misc.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_phy.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_power.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_recv.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_reset.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5211/ar5211_xmit.c optional ath_hal \| ath_ar5211 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar5212
	dev/ath/ath_hal/ar5212/ar5212_ani.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_attach.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_beacon.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_eeprom.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_gpio.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_interrupts.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_keycache.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_misc.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_phy.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_power.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_recv.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_reset.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_rfgain.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5212_xmit.c \
	optional ath_hal \| ath_ar5212 \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| \
	ath_ar9285 ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar5416 (depends on ar5212)
	dev/ath/ath_hal/ar5416/ar5416_ani.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_attach.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_beacon.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_btcoex.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_cal.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_cal_iq.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_cal_adcgain.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_cal_adcdc.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_eeprom.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_gpio.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_interrupts.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_keycache.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_misc.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_phy.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_power.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_radar.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_recv.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_reset.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_spectral.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar5416_xmit.c \
	optional ath_hal \| ath_ar5416 \| ath_ar9160 \| ath_ar9280 \| ath_ar9285 \| \
	ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar9130 (depends upon ar5416) - also requires AH_SUPPORT_AR9130
	#
	# Since this is an embedded MAC SoC, there's no need to compile it into the
	# default HAL.
	dev/ath/ath_hal/ar9001/ar9130_attach.c optional ath_ar9130 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9001/ar9130_phy.c optional ath_ar9130 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9001/ar9130_eeprom.c optional ath_ar9130 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar9160 (depends on ar5416)
	dev/ath/ath_hal/ar9001/ar9160_attach.c optional ath_hal \| ath_ar9160 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar9280 (depends on ar5416)
	dev/ath/ath_hal/ar9002/ar9280_attach.c optional ath_hal \| ath_ar9280 \| \
	ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9280_olc.c optional ath_hal \| ath_ar9280 \| \
	ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar9285 (depends on ar5416 and ar9280)
	dev/ath/ath_hal/ar9002/ar9285_attach.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9285_btcoex.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9285_reset.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9285_cal.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9285_phy.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9285_diversity.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	# ar9287 (depends on ar5416)
	dev/ath/ath_hal/ar9002/ar9287_attach.c optional ath_hal \| ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9287_reset.c optional ath_hal \| ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9287_cal.c optional ath_hal \| ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9287_olc.c optional ath_hal \| ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"

	# ar9300
	contrib/dev/ath/ath_hal/ar9300/ar9300_ani.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_attach.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_beacon.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_eeprom.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WCONSTANT_CONVERSION}"
	contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_gpio.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_interrupts.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_keycache.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_mci.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_misc.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_paprd.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_phy.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_power.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_radar.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_radio.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_recv.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_recv_ds.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_reset.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal ${NO_WSOMETIMES_UNINITIALIZED} -Wno-unused-function"
	contrib/dev/ath/ath_hal/ar9300/ar9300_stub.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_stub_funcs.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_spectral.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_timer.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_xmit.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"
	contrib/dev/ath/ath_hal/ar9300/ar9300_xmit_ds.c optional ath_hal \| ath_ar9300 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal -I$S/contrib/dev/ath/ath_hal"

	# rf backends
	dev/ath/ath_hal/ar5212/ar2316.c optional ath_rf2316 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar2317.c optional ath_rf2317 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar2413.c optional ath_hal \| ath_rf2413 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar2425.c optional ath_hal \| ath_rf2425 \| ath_rf2417 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5111.c optional ath_hal \| ath_rf5111 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5112.c optional ath_hal \| ath_rf5112 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5212/ar5413.c optional ath_hal \| ath_rf5413 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar5416/ar2133.c optional ath_hal \| ath_ar5416 \| \
	ath_ar9130 \| ath_ar9160 \| ath_ar9280 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9280.c optional ath_hal \| ath_ar9280 \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9285.c optional ath_hal \| ath_ar9285 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
	dev/ath/ath_hal/ar9002/ar9287.c optional ath_hal \| ath_ar9287 \
	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"

	# ath rate control algorithms
	dev/ath/ath_rate/amrr/amrr.c optional ath_rate_amrr \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_rate/onoe/onoe.c optional ath_rate_onoe \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	dev/ath/ath_rate/sample/sample.c optional ath_rate_sample \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	# ath DFS modules
	dev/ath/ath_dfs/null/dfs_null.c optional ath \
	compile-with "${NORMAL_C} -I$S/dev/ath"
	#
	dev/bce/if_bce.c optional bce
	dev/bfe/if_bfe.c optional bfe
	dev/bge/if_bge.c optional bge
	dev/bhnd/bhnd.c optional bhnd
	dev/bhnd/bhnd_erom.c optional bhnd
	dev/bhnd/bhnd_erom_if.m optional bhnd
	dev/bhnd/bhnd_subr.c optional bhnd
	dev/bhnd/bhnd_bus_if.m optional bhnd
	dev/bhnd/bhndb/bhnd_bhndb.c optional bhndb bhnd
	dev/bhnd/bhndb/bhndb.c optional bhndb bhnd
	dev/bhnd/bhndb/bhndb_bus_if.m optional bhndb bhnd
	dev/bhnd/bhndb/bhndb_hwdata.c optional bhndb bhnd
	dev/bhnd/bhndb/bhndb_if.m optional bhndb bhnd
	dev/bhnd/bhndb/bhndb_pci.c optional bhndb_pci bhndb bhnd pci
	dev/bhnd/bhndb/bhndb_pci_hwdata.c optional bhndb_pci bhndb bhnd pci
	dev/bhnd/bhndb/bhndb_pci_sprom.c optional bhndb_pci bhndb bhnd pci
	dev/bhnd/bhndb/bhndb_subr.c optional bhndb bhnd
	dev/bhnd/bcma/bcma.c optional bcma bhnd
	dev/bhnd/bcma/bcma_bhndb.c optional bcma bhnd bhndb
	dev/bhnd/bcma/bcma_erom.c optional bcma bhnd
	dev/bhnd/bcma/bcma_subr.c optional bcma bhnd
	dev/bhnd/cores/chipc/bhnd_chipc_if.m optional bhnd
	dev/bhnd/cores/chipc/bhnd_sprom_chipc.c optional bhnd
	dev/bhnd/cores/chipc/bhnd_pmu_chipc.c optional bhnd
	dev/bhnd/cores/chipc/chipc.c optional bhnd
	dev/bhnd/cores/chipc/chipc_cfi.c optional bhnd cfi
	dev/bhnd/cores/chipc/chipc_gpio.c optional bhnd gpio
	dev/bhnd/cores/chipc/chipc_slicer.c optional bhnd cfi \| bhnd spibus
	dev/bhnd/cores/chipc/chipc_spi.c optional bhnd spibus
	dev/bhnd/cores/chipc/chipc_subr.c optional bhnd
	dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl.c optional bhnd
	dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl_if.m optional bhnd
	dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl_hostb_if.m optional bhnd
	dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl_subr.c optional bhnd
	dev/bhnd/cores/pci/bhnd_pci.c optional bhnd pci
	dev/bhnd/cores/pci/bhnd_pci_hostb.c optional bhndb bhnd pci
	dev/bhnd/cores/pci/bhnd_pcib.c optional bhnd_pcib bhnd pci
	dev/bhnd/cores/pcie2/bhnd_pcie2.c optional bhnd pci
	dev/bhnd/cores/pcie2/bhnd_pcie2_hostb.c optional bhndb bhnd pci
	dev/bhnd/cores/pcie2/bhnd_pcie2b.c optional bhnd_pcie2b bhnd pci
	dev/bhnd/cores/pmu/bhnd_pmu.c optional bhnd
	dev/bhnd/cores/pmu/bhnd_pmu_core.c optional bhnd
	dev/bhnd/cores/pmu/bhnd_pmu_if.m optional bhnd
	dev/bhnd/cores/pmu/bhnd_pmu_subr.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data_bcm.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data_bcmraw.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data_btxt.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data_sprom.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data_sprom_subr.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_data_tlv.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_if.m optional bhnd
	dev/bhnd/nvram/bhnd_nvram_io.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_iobuf.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_ioptr.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_iores.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_plist.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_store.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_store_subr.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_subr.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_value.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_value_fmts.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_value_prf.c optional bhnd
	dev/bhnd/nvram/bhnd_nvram_value_subr.c optional bhnd
	dev/bhnd/nvram/bhnd_sprom.c optional bhnd
	dev/bhnd/siba/siba.c optional siba bhnd
	dev/bhnd/siba/siba_bhndb.c optional siba bhnd bhndb
	dev/bhnd/siba/siba_erom.c optional siba bhnd
	dev/bhnd/siba/siba_subr.c optional siba bhnd
	#
	dev/bnxt/bnxt_hwrm.c optional bnxt iflib pci
	dev/bnxt/bnxt_sysctl.c optional bnxt iflib pci
	dev/bnxt/bnxt_txrx.c optional bnxt iflib pci
	dev/bnxt/if_bnxt.c optional bnxt iflib pci
	dev/bwi/bwimac.c optional bwi
	dev/bwi/bwiphy.c optional bwi
	dev/bwi/bwirf.c optional bwi
	dev/bwi/if_bwi.c optional bwi
	dev/bwi/if_bwi_pci.c optional bwi pci
	dev/bwn/if_bwn.c optional bwn bhnd
	dev/bwn/if_bwn_pci.c optional bwn pci bhnd bhndb bhndb_pci
	dev/bwn/if_bwn_phy_common.c optional bwn bhnd
	dev/bwn/if_bwn_phy_g.c optional bwn bhnd
	dev/bwn/if_bwn_phy_lp.c optional bwn bhnd
	dev/bwn/if_bwn_phy_n.c optional bwn bhnd
	dev/bwn/if_bwn_util.c optional bwn bhnd
	dev/cadence/if_cgem.c optional cgem fdt
	dev/cardbus/cardbus.c optional cardbus
	dev/cardbus/cardbus_cis.c optional cardbus
	dev/cardbus/cardbus_device.c optional cardbus
	dev/cas/if_cas.c optional cas
	dev/cfi/cfi_bus_fdt.c optional cfi fdt
	dev/cfi/cfi_bus_nexus.c optional cfi
	dev/cfi/cfi_core.c optional cfi
	dev/cfi/cfi_dev.c optional cfi
	dev/cfi/cfi_disk.c optional cfid
	dev/chromebook_platform/chromebook_platform.c optional chromebook_platform
	dev/ciss/ciss.c optional ciss
	dev/cmx/cmx.c optional cmx
	dev/cmx/cmx_pccard.c optional cmx pccard
	dev/cpufreq/ichss.c optional cpufreq pci
	dev/cxgb/cxgb_main.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/cxgb_sge.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_mc5.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_vsc7323.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_vsc8211.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_ael1002.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_aq100x.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_mv88e1xxx.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_xgmac.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_t3_hw.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/common/cxgb_tn1010.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/sys/uipc_mvec.c optional cxgb pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgb/cxgb_t3fw.c optional cxgb cxgb_t3fw \
	compile-with "${NORMAL_C} -I$S/dev/cxgb"
	dev/cxgbe/t4_clip.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_filter.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_if.m optional cxgbe pci
	dev/cxgbe/t4_iov.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_mp_ring.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_main.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_netmap.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_sched.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_sge.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_smt.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_l2t.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_tracer.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/t4_vf.c optional cxgbev pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/common/t4_hw.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/common/t4vf_hw.c optional cxgbev pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/crypto/t4_kern_tls.c optional cxgbe pci kern_tls \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/crypto/t4_keyctx.c optional cxgbe pci \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/cudbg/cudbg_common.c optional cxgbe \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/cudbg/cudbg_flash_utils.c optional cxgbe \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/cudbg/cudbg_lib.c optional cxgbe \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/cudbg/cudbg_wtp.c optional cxgbe \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/cudbg/fastlz.c optional cxgbe \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	dev/cxgbe/cudbg/fastlz_api.c optional cxgbe \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	t4fw_cfg.c optional cxgbe \
	compile-with "${AWK} -f $S/tools/fw_stub.awk t4fw_cfg.fw:t4fw_cfg t4fw_cfg_uwire.fw:t4fw_cfg_uwire t4fw.fw:t4fw -mt4fw_cfg -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "t4fw_cfg.c"
	t4fw_cfg.fwo optional cxgbe \
	dependency "t4fw_cfg.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t4fw_cfg.fwo"
	t4fw_cfg.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t4fw_cfg.txt" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t4fw_cfg.fw"
	t4fw_cfg_uwire.fwo optional cxgbe \
	dependency "t4fw_cfg_uwire.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t4fw_cfg_uwire.fwo"
	t4fw_cfg_uwire.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t4fw_cfg_uwire.txt" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t4fw_cfg_uwire.fw"
	t4fw.fwo optional cxgbe \
	dependency "t4fw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t4fw.fwo"
	t4fw.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t4fw-1.24.12.0.bin" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t4fw.fw"
	t5fw_cfg.c optional cxgbe \
	compile-with "${AWK} -f $S/tools/fw_stub.awk t5fw_cfg.fw:t5fw_cfg t5fw_cfg_uwire.fw:t5fw_cfg_uwire t5fw.fw:t5fw -mt5fw_cfg -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "t5fw_cfg.c"
	t5fw_cfg.fwo optional cxgbe \
	dependency "t5fw_cfg.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t5fw_cfg.fwo"
	t5fw_cfg.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t5fw_cfg.txt" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t5fw_cfg.fw"
	t5fw_cfg_uwire.fwo optional cxgbe \
	dependency "t5fw_cfg_uwire.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t5fw_cfg_uwire.fwo"
	t5fw_cfg_uwire.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t5fw_cfg_uwire.txt" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t5fw_cfg_uwire.fw"
	t5fw.fwo optional cxgbe \
	dependency "t5fw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t5fw.fwo"
	t5fw.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t5fw-1.24.12.0.bin" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t5fw.fw"
	t6fw_cfg.c optional cxgbe \
	compile-with "${AWK} -f $S/tools/fw_stub.awk t6fw_cfg.fw:t6fw_cfg t6fw_cfg_uwire.fw:t6fw_cfg_uwire t6fw.fw:t6fw -mt6fw_cfg -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "t6fw_cfg.c"
	t6fw_cfg.fwo optional cxgbe \
	dependency "t6fw_cfg.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t6fw_cfg.fwo"
	t6fw_cfg.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t6fw_cfg.txt" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t6fw_cfg.fw"
	t6fw_cfg_uwire.fwo optional cxgbe \
	dependency "t6fw_cfg_uwire.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t6fw_cfg_uwire.fwo"
	t6fw_cfg_uwire.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t6fw_cfg_uwire.txt" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t6fw_cfg_uwire.fw"
	t6fw.fwo optional cxgbe \
	dependency "t6fw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "t6fw.fwo"
	t6fw.fw optional cxgbe \
	dependency "$S/dev/cxgbe/firmware/t6fw-1.24.12.0.bin" \
	compile-with "${CP} ${.ALLSRC} ${.TARGET}" \
	no-obj no-implicit-rule \
	clean "t6fw.fw"
	dev/cxgbe/crypto/t4_crypto.c optional ccr \
	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
	-dev/cy/cy.c optional cy
	-dev/cy/cy_isa.c optional cy isa
	-dev/cy/cy_pci.c optional cy pci
	dev/cyapa/cyapa.c optional cyapa iicbus
	dev/dc/if_dc.c optional dc pci
	dev/dc/dcphy.c optional dc pci
	dev/dc/pnphy.c optional dc pci
	dev/dcons/dcons.c optional dcons
	dev/dcons/dcons_crom.c optional dcons_crom
	dev/dcons/dcons_os.c optional dcons
	dev/dme/if_dme.c optional dme
	dev/drm2/drm_agpsupport.c optional drm2
	dev/drm2/drm_auth.c optional drm2
	dev/drm2/drm_bufs.c optional drm2
	dev/drm2/drm_buffer.c optional drm2
	dev/drm2/drm_context.c optional drm2
	dev/drm2/drm_crtc.c optional drm2
	dev/drm2/drm_crtc_helper.c optional drm2
	dev/drm2/drm_dma.c optional drm2
	dev/drm2/drm_dp_helper.c optional drm2
	dev/drm2/drm_dp_iic_helper.c optional drm2
	dev/drm2/drm_drv.c optional drm2
	dev/drm2/drm_edid.c optional drm2
	dev/drm2/drm_fb_helper.c optional drm2
	dev/drm2/drm_fops.c optional drm2
	dev/drm2/drm_gem.c optional drm2
	dev/drm2/drm_gem_names.c optional drm2
	dev/drm2/drm_global.c optional drm2
	dev/drm2/drm_hashtab.c optional drm2
	dev/drm2/drm_ioctl.c optional drm2
	dev/drm2/drm_irq.c optional drm2
	dev/drm2/drm_linux_list_sort.c optional drm2
	dev/drm2/drm_lock.c optional drm2
	dev/drm2/drm_memory.c optional drm2
	dev/drm2/drm_mm.c optional drm2
	dev/drm2/drm_modes.c optional drm2
	dev/drm2/drm_pci.c optional drm2
	dev/drm2/drm_platform.c optional drm2
	dev/drm2/drm_scatter.c optional drm2
	dev/drm2/drm_stub.c optional drm2
	dev/drm2/drm_sysctl.c optional drm2
	dev/drm2/drm_vm.c optional drm2
	dev/drm2/drm_os_freebsd.c optional drm2
	dev/drm2/ttm/ttm_agp_backend.c optional drm2
	dev/drm2/ttm/ttm_lock.c optional drm2
	dev/drm2/ttm/ttm_object.c optional drm2
	dev/drm2/ttm/ttm_tt.c optional drm2
	dev/drm2/ttm/ttm_bo_util.c optional drm2
	dev/drm2/ttm/ttm_bo.c optional drm2
	dev/drm2/ttm/ttm_bo_manager.c optional drm2
	dev/drm2/ttm/ttm_execbuf_util.c optional drm2
	dev/drm2/ttm/ttm_memory.c optional drm2
	dev/drm2/ttm/ttm_page_alloc.c optional drm2
	dev/drm2/ttm/ttm_bo_vm.c optional drm2
	dev/efidev/efidev.c optional efirt
	dev/efidev/efirt.c optional efirt
	dev/efidev/efirtc.c optional efirt
	dev/e1000/if_em.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/em_txrx.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/igb_txrx.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_80003es2lan.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_82540.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_82541.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_82542.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_82543.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_82571.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_82575.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_ich8lan.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_i210.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_api.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_mac.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_manage.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_nvm.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_phy.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_vf.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_mbx.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/e1000/e1000_osdep.c optional em \
	compile-with "${NORMAL_C} -I$S/dev/e1000"
	dev/et/if_et.c optional et
	dev/ena/ena.c optional ena \
	compile-with "${NORMAL_C} -I$S/contrib"
	+dev/ena/ena_datapath.c optional ena \
	+ compile-with "${NORMAL_C} -I$S/contrib"
	+dev/ena/ena_netmap.c optional ena \
	+ compile-with "${NORMAL_C} -I$S/contrib"
	dev/ena/ena_sysctl.c optional ena \
	compile-with "${NORMAL_C} -I$S/contrib"
	contrib/ena-com/ena_com.c optional ena
	contrib/ena-com/ena_eth_com.c optional ena
	dev/esp/esp_pci.c optional esp pci
	dev/esp/ncr53c9x.c optional esp
	dev/etherswitch/arswitch/arswitch.c optional arswitch
	dev/etherswitch/arswitch/arswitch_reg.c optional arswitch
	dev/etherswitch/arswitch/arswitch_phy.c optional arswitch
	dev/etherswitch/arswitch/arswitch_8216.c optional arswitch
	dev/etherswitch/arswitch/arswitch_8226.c optional arswitch
	dev/etherswitch/arswitch/arswitch_8316.c optional arswitch
	dev/etherswitch/arswitch/arswitch_8327.c optional arswitch
	dev/etherswitch/arswitch/arswitch_7240.c optional arswitch
	dev/etherswitch/arswitch/arswitch_9340.c optional arswitch
	dev/etherswitch/arswitch/arswitch_vlans.c optional arswitch
	dev/etherswitch/etherswitch.c optional etherswitch
	dev/etherswitch/etherswitch_if.m optional etherswitch
	dev/etherswitch/ip17x/ip17x.c optional ip17x
	dev/etherswitch/ip17x/ip175c.c optional ip17x
	dev/etherswitch/ip17x/ip175d.c optional ip17x
	dev/etherswitch/ip17x/ip17x_phy.c optional ip17x
	dev/etherswitch/ip17x/ip17x_vlans.c optional ip17x
	dev/etherswitch/miiproxy.c optional miiproxy
	dev/etherswitch/rtl8366/rtl8366rb.c optional rtl8366rb
	dev/etherswitch/e6000sw/e6000sw.c optional e6000sw
	dev/etherswitch/e6000sw/e6060sw.c optional e6060sw
	dev/etherswitch/infineon/adm6996fc.c optional adm6996fc
	dev/etherswitch/micrel/ksz8995ma.c optional ksz8995ma
	dev/etherswitch/ukswitch/ukswitch.c optional ukswitch
	dev/evdev/cdev.c optional evdev
	dev/evdev/evdev.c optional evdev
	dev/evdev/evdev_mt.c optional evdev
	dev/evdev/evdev_utils.c optional evdev
	dev/evdev/uinput.c optional evdev uinput
	dev/exca/exca.c optional cbb
	dev/extres/clk/clk.c optional ext_resources clk fdt
	dev/extres/clk/clkdev_if.m optional ext_resources clk fdt
	dev/extres/clk/clknode_if.m optional ext_resources clk fdt
	dev/extres/clk/clk_bus.c optional ext_resources clk fdt
	dev/extres/clk/clk_div.c optional ext_resources clk fdt
	dev/extres/clk/clk_fixed.c optional ext_resources clk fdt
	dev/extres/clk/clk_gate.c optional ext_resources clk fdt
	dev/extres/clk/clk_link.c optional ext_resources clk fdt
	dev/extres/clk/clk_mux.c optional ext_resources clk fdt
	dev/extres/phy/phy.c optional ext_resources phy fdt
	dev/extres/phy/phydev_if.m optional ext_resources phy fdt
	dev/extres/phy/phynode_if.m optional ext_resources phy fdt
	dev/extres/phy/phy_usb.c optional ext_resources phy fdt
	dev/extres/phy/phynode_usb_if.m optional ext_resources phy fdt
	dev/extres/hwreset/hwreset.c optional ext_resources hwreset fdt
	dev/extres/hwreset/hwreset_if.m optional ext_resources hwreset fdt
	dev/extres/nvmem/nvmem.c optional ext_resources nvmem fdt
	dev/extres/nvmem/nvmem_if.m optional ext_resources nvmem fdt
	dev/extres/regulator/regdev_if.m optional ext_resources regulator fdt
	dev/extres/regulator/regnode_if.m optional ext_resources regulator fdt
	dev/extres/regulator/regulator.c optional ext_resources regulator fdt
	dev/extres/regulator/regulator_bus.c optional ext_resources regulator fdt
	dev/extres/regulator/regulator_fixed.c optional ext_resources regulator fdt
	dev/extres/syscon/syscon.c optional ext_resources syscon
	dev/extres/syscon/syscon_generic.c optional ext_resources syscon fdt
	dev/extres/syscon/syscon_if.m optional ext_resources syscon
	dev/extres/syscon/syscon_power.c optional ext_resources syscon syscon_power fdt
	dev/fb/fbd.c optional fbd \| vt
	dev/fb/fb_if.m standard
	dev/fb/splash.c optional sc splash
	dev/fdt/fdt_clock.c optional fdt fdt_clock
	dev/fdt/fdt_clock_if.m optional fdt fdt_clock
	dev/fdt/fdt_common.c optional fdt
	dev/fdt/fdt_pinctrl.c optional fdt fdt_pinctrl
	dev/fdt/fdt_pinctrl_if.m optional fdt fdt_pinctrl
	dev/fdt/fdt_slicer.c optional fdt cfi \| fdt mx25l \| fdt n25q \| fdt at45d
	dev/fdt/fdt_static_dtb.S optional fdt fdt_dtb_static \
	dependency "${FDT_DTS_FILE:T:R}.dtb"
	dev/fdt/simplebus.c optional fdt
	dev/fdt/simple_mfd.c optional syscon fdt
	dev/filemon/filemon.c optional filemon
	dev/firewire/firewire.c optional firewire
	dev/firewire/fwcrom.c optional firewire
	dev/firewire/fwdev.c optional firewire
	dev/firewire/fwdma.c optional firewire
	dev/firewire/fwmem.c optional firewire
	dev/firewire/fwohci.c optional firewire
	dev/firewire/fwohci_pci.c optional firewire pci
	dev/firewire/if_fwe.c optional fwe
	dev/firewire/if_fwip.c optional fwip
	dev/firewire/sbp.c optional sbp
	dev/firewire/sbp_targ.c optional sbp_targ
	dev/flash/at45d.c optional at45d
	dev/flash/cqspi.c optional cqspi fdt xdma
	dev/flash/mx25l.c optional mx25l
	dev/flash/n25q.c optional n25q fdt
	dev/flash/qspi_if.m optional cqspi fdt \| n25q fdt
	dev/fxp/if_fxp.c optional fxp
	dev/fxp/inphy.c optional fxp
	dev/gem/if_gem.c optional gem
	dev/gem/if_gem_pci.c optional gem pci
	dev/goldfish/goldfish_rtc.c optional goldfish_rtc fdt
	dev/gpio/dwgpio/dwgpio.c optional gpio dwgpio fdt
	dev/gpio/dwgpio/dwgpio_bus.c optional gpio dwgpio fdt
	dev/gpio/dwgpio/dwgpio_if.m optional gpio dwgpio fdt
	dev/gpio/gpiobacklight.c optional gpiobacklight fdt
	dev/gpio/gpiokeys.c optional gpiokeys fdt
	dev/gpio/gpiokeys_codes.c optional gpiokeys fdt
	dev/gpio/gpiobus.c optional gpio \
	dependency "gpiobus_if.h"
	dev/gpio/gpioc.c optional gpio \
	dependency "gpio_if.h"
	dev/gpio/gpioiic.c optional gpioiic
	dev/gpio/gpioled.c optional gpioled !fdt
	dev/gpio/gpioled_fdt.c optional gpioled fdt
	dev/gpio/gpiomdio.c optional gpiomdio mii_bitbang
	dev/gpio/gpiopower.c optional gpiopower fdt
	dev/gpio/gpioregulator.c optional gpioregulator fdt ext_resources
	dev/gpio/gpiospi.c optional gpiospi
	dev/gpio/gpioths.c optional gpioths
	dev/gpio/gpio_if.m optional gpio
	dev/gpio/gpiobus_if.m optional gpio
	dev/gpio/gpiopps.c optional gpiopps fdt
	dev/gpio/ofw_gpiobus.c optional fdt gpio
	dev/hifn/hifn7751.c optional hifn
	dev/hme/if_hme.c optional hme
	dev/hme/if_hme_pci.c optional hme pci
	dev/hptiop/hptiop.c optional hptiop scbus
	dev/hwpmc/hwpmc_logging.c optional hwpmc
	dev/hwpmc/hwpmc_mod.c optional hwpmc
	dev/hwpmc/hwpmc_soft.c optional hwpmc
	dev/ichiic/ig4_acpi.c optional ig4 acpi iicbus
	dev/ichiic/ig4_iic.c optional ig4 iicbus
	dev/ichiic/ig4_pci.c optional ig4 pci iicbus
	dev/ichsmb/ichsmb.c optional ichsmb
	dev/ichsmb/ichsmb_pci.c optional ichsmb pci
	dev/ida/ida.c optional ida
	dev/ida/ida_disk.c optional ida
	dev/ida/ida_pci.c optional ida pci
	dev/iicbus/acpi_iicbus.c optional acpi iicbus
	dev/iicbus/ad7418.c optional ad7418
	dev/iicbus/ads111x.c optional ads111x
	dev/iicbus/ds1307.c optional ds1307
	dev/iicbus/ds13rtc.c optional ds13rtc \| ds133x \| ds1374
	dev/iicbus/ds1672.c optional ds1672
	dev/iicbus/ds3231.c optional ds3231
	dev/iicbus/syr827.c optional syr827 ext_resources fdt
	dev/iicbus/icee.c optional icee
	dev/iicbus/if_ic.c optional ic
	dev/iicbus/iic.c optional iic
	dev/iicbus/iic_recover_bus.c optional iicbus
	dev/iicbus/iicbb.c optional iicbb
	dev/iicbus/iicbb_if.m optional iicbb
	dev/iicbus/iicbus.c optional iicbus
	dev/iicbus/iicbus_if.m optional iicbus
	dev/iicbus/iiconf.c optional iicbus
	dev/iicbus/iicsmb.c optional iicsmb \
	dependency "iicbus_if.h"
	dev/iicbus/iicoc.c optional iicoc
	dev/iicbus/iicoc_fdt.c optional iicoc ext_resources fdt
	dev/iicbus/iicoc_pci.c optional iicoc pci
	dev/iicbus/isl12xx.c optional isl12xx
	dev/iicbus/lm75.c optional lm75
	dev/iicbus/mux/iicmux.c optional iicmux
	dev/iicbus/mux/iicmux_if.m optional iicmux
	dev/iicbus/mux/iic_gpiomux.c optional iic_gpiomux fdt
	dev/iicbus/mux/ltc430x.c optional ltc430x
	dev/iicbus/nxprtc.c optional nxprtc \| pcf8563
	dev/iicbus/ofw_iicbus.c optional fdt iicbus
	dev/iicbus/rtc8583.c optional rtc8583
	dev/iicbus/rtc/rx8803.c optional rx8803 iicbus fdt
	dev/iicbus/s35390a.c optional s35390a
	dev/iicbus/sy8106a.c optional sy8106a ext_resources fdt
	dev/iicbus/gpio/tca6416.c optional tca6416 fdt
	dev/iir/iir.c optional iir
	dev/iir/iir_ctrl.c optional iir
	dev/iir/iir_pci.c optional iir pci
	dev/intpm/intpm.c optional intpm pci
	# XXX Work around clang warning, until maintainer approves fix.
	dev/ips/ips.c optional ips \
	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
	dev/ips/ips_commands.c optional ips
	dev/ips/ips_disk.c optional ips
	dev/ips/ips_ioctl.c optional ips
	dev/ips/ips_pci.c optional ips pci
	dev/ipw/if_ipw.c optional ipw
	ipwbssfw.c optional ipwbssfw \| ipwfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_bss.fw:ipw_bss:130 -lintel_ipw -mipw_bss -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "ipwbssfw.c"
	ipw_bss.fwo optional ipwbssfw \| ipwfw \
	dependency "ipw_bss.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "ipw_bss.fwo"
	ipw_bss.fw optional ipwbssfw \| ipwfw \
	dependency "$S/contrib/dev/ipw/ipw2100-1.3.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "ipw_bss.fw"
	ipwibssfw.c optional ipwibssfw \| ipwfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_ibss.fw:ipw_ibss:130 -lintel_ipw -mipw_ibss -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "ipwibssfw.c"
	ipw_ibss.fwo optional ipwibssfw \| ipwfw \
	dependency "ipw_ibss.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "ipw_ibss.fwo"
	ipw_ibss.fw optional ipwibssfw \| ipwfw \
	dependency "$S/contrib/dev/ipw/ipw2100-1.3-i.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "ipw_ibss.fw"
	ipwmonitorfw.c optional ipwmonitorfw \| ipwfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk ipw_monitor.fw:ipw_monitor:130 -lintel_ipw -mipw_monitor -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "ipwmonitorfw.c"
	ipw_monitor.fwo optional ipwmonitorfw \| ipwfw \
	dependency "ipw_monitor.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "ipw_monitor.fwo"
	ipw_monitor.fw optional ipwmonitorfw \| ipwfw \
	dependency "$S/contrib/dev/ipw/ipw2100-1.3-p.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "ipw_monitor.fw"
	dev/iscsi/icl.c optional iscsi
	dev/iscsi/icl_conn_if.m optional cfiscsi \| iscsi
	dev/iscsi/icl_soft.c optional iscsi
	dev/iscsi/icl_soft_proxy.c optional iscsi
	dev/iscsi/iscsi.c optional iscsi scbus
	dev/iscsi_initiator/iscsi.c optional iscsi_initiator scbus
	dev/iscsi_initiator/iscsi_subr.c optional iscsi_initiator scbus
	dev/iscsi_initiator/isc_cam.c optional iscsi_initiator scbus
	dev/iscsi_initiator/isc_soc.c optional iscsi_initiator scbus
	dev/iscsi_initiator/isc_sm.c optional iscsi_initiator scbus
	dev/iscsi_initiator/isc_subr.c optional iscsi_initiator scbus
	dev/ismt/ismt.c optional ismt
	dev/isl/isl.c optional isl iicbus
	dev/isp/isp.c optional isp
	dev/isp/isp_freebsd.c optional isp
	dev/isp/isp_library.c optional isp
	dev/isp/isp_pci.c optional isp pci
	dev/isp/isp_target.c optional isp
	dev/ispfw/ispfw.c optional ispfw
	dev/iwi/if_iwi.c optional iwi
	iwibssfw.c optional iwibssfw \| iwifw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_bss.fw:iwi_bss:300 -lintel_iwi -miwi_bss -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwibssfw.c"
	iwi_bss.fwo optional iwibssfw \| iwifw \
	dependency "iwi_bss.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwi_bss.fwo"
	iwi_bss.fw optional iwibssfw \| iwifw \
	dependency "$S/contrib/dev/iwi/ipw2200-bss.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwi_bss.fw"
	iwiibssfw.c optional iwiibssfw \| iwifw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_ibss.fw:iwi_ibss:300 -lintel_iwi -miwi_ibss -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwiibssfw.c"
	iwi_ibss.fwo optional iwiibssfw \| iwifw \
	dependency "iwi_ibss.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwi_ibss.fwo"
	iwi_ibss.fw optional iwiibssfw \| iwifw \
	dependency "$S/contrib/dev/iwi/ipw2200-ibss.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwi_ibss.fw"
	iwimonitorfw.c optional iwimonitorfw \| iwifw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwi_monitor.fw:iwi_monitor:300 -lintel_iwi -miwi_monitor -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwimonitorfw.c"
	iwi_monitor.fwo optional iwimonitorfw \| iwifw \
	dependency "iwi_monitor.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwi_monitor.fwo"
	iwi_monitor.fw optional iwimonitorfw \| iwifw \
	dependency "$S/contrib/dev/iwi/ipw2200-sniffer.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwi_monitor.fw"
	dev/iwm/if_iwm.c optional iwm
	dev/iwm/if_iwm_7000.c optional iwm
	dev/iwm/if_iwm_8000.c optional iwm
	dev/iwm/if_iwm_9000.c optional iwm
	dev/iwm/if_iwm_9260.c optional iwm
	dev/iwm/if_iwm_binding.c optional iwm
	dev/iwm/if_iwm_fw.c optional iwm
	dev/iwm/if_iwm_led.c optional iwm
	dev/iwm/if_iwm_mac_ctxt.c optional iwm
	dev/iwm/if_iwm_notif_wait.c optional iwm
	dev/iwm/if_iwm_pcie_trans.c optional iwm
	dev/iwm/if_iwm_phy_ctxt.c optional iwm
	dev/iwm/if_iwm_phy_db.c optional iwm
	dev/iwm/if_iwm_power.c optional iwm
	dev/iwm/if_iwm_scan.c optional iwm
	dev/iwm/if_iwm_sf.c optional iwm
	dev/iwm/if_iwm_sta.c optional iwm
	dev/iwm/if_iwm_time_event.c optional iwm
	dev/iwm/if_iwm_util.c optional iwm
	iwm3160fw.c optional iwm3160fw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm3160.fw:iwm3160fw -miwm3160fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm3160fw.c"
	iwm3160fw.fwo optional iwm3160fw \| iwmfw \
	dependency "iwm3160.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm3160fw.fwo"
	iwm3160.fw optional iwm3160fw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-3160-17.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm3160.fw"
	iwm3168fw.c optional iwm3168fw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm3168.fw:iwm3168fw -miwm3168fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm3168fw.c"
	iwm3168fw.fwo optional iwm3168fw \| iwmfw \
	dependency "iwm3168.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm3168fw.fwo"
	iwm3168.fw optional iwm3168fw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-3168-22.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm3168.fw"
	iwm7260fw.c optional iwm7260fw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm7260.fw:iwm7260fw -miwm7260fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm7260fw.c"
	iwm7260fw.fwo optional iwm7260fw \| iwmfw \
	dependency "iwm7260.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm7260fw.fwo"
	iwm7260.fw optional iwm7260fw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-7260-17.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm7260.fw"
	iwm7265fw.c optional iwm7265fw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm7265.fw:iwm7265fw -miwm7265fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm7265fw.c"
	iwm7265fw.fwo optional iwm7265fw \| iwmfw \
	dependency "iwm7265.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm7265fw.fwo"
	iwm7265.fw optional iwm7265fw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-7265-17.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm7265.fw"
	iwm7265Dfw.c optional iwm7265Dfw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm7265D.fw:iwm7265Dfw -miwm7265Dfw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm7265Dfw.c"
	iwm7265Dfw.fwo optional iwm7265Dfw \| iwmfw \
	dependency "iwm7265D.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm7265Dfw.fwo"
	iwm7265D.fw optional iwm7265Dfw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-7265D-17.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm7265D.fw"
	iwm8000Cfw.c optional iwm8000Cfw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm8000C.fw:iwm8000Cfw -miwm8000Cfw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm8000Cfw.c"
	iwm8000Cfw.fwo optional iwm8000Cfw \| iwmfw \
	dependency "iwm8000C.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm8000Cfw.fwo"
	iwm8000C.fw optional iwm8000Cfw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-8000C-16.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm8000C.fw"
	iwm8265.fw optional iwm8265fw \| iwmfw \
	dependency "$S/contrib/dev/iwm/iwm-8265-22.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwm8265.fw"
	iwm8265fw.c optional iwm8265fw \| iwmfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwm8265.fw:iwm8265fw -miwm8265fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwm8265fw.c"
	iwm8265fw.fwo optional iwm8265fw \| iwmfw \
	dependency "iwm8265.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwm8265fw.fwo"
	dev/iwn/if_iwn.c optional iwn
	iwn1000fw.c optional iwn1000fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn1000.fw:iwn1000fw -miwn1000fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn1000fw.c"
	iwn1000fw.fwo optional iwn1000fw \| iwnfw \
	dependency "iwn1000.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn1000fw.fwo"
	iwn1000.fw optional iwn1000fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-1000-39.31.5.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn1000.fw"
	iwn100fw.c optional iwn100fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn100.fw:iwn100fw -miwn100fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn100fw.c"
	iwn100fw.fwo optional iwn100fw \| iwnfw \
	dependency "iwn100.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn100fw.fwo"
	iwn100.fw optional iwn100fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-100-39.31.5.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn100.fw"
	iwn105fw.c optional iwn105fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn105.fw:iwn105fw -miwn105fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn105fw.c"
	iwn105fw.fwo optional iwn105fw \| iwnfw \
	dependency "iwn105.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn105fw.fwo"
	iwn105.fw optional iwn105fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-105-6-18.168.6.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn105.fw"
	iwn135fw.c optional iwn135fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn135.fw:iwn135fw -miwn135fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn135fw.c"
	iwn135fw.fwo optional iwn135fw \| iwnfw \
	dependency "iwn135.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn135fw.fwo"
	iwn135.fw optional iwn135fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-135-6-18.168.6.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn135.fw"
	iwn2000fw.c optional iwn2000fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn2000.fw:iwn2000fw -miwn2000fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn2000fw.c"
	iwn2000fw.fwo optional iwn2000fw \| iwnfw \
	dependency "iwn2000.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn2000fw.fwo"
	iwn2000.fw optional iwn2000fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-2000-18.168.6.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn2000.fw"
	iwn2030fw.c optional iwn2030fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn2030.fw:iwn2030fw -miwn2030fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn2030fw.c"
	iwn2030fw.fwo optional iwn2030fw \| iwnfw \
	dependency "iwn2030.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn2030fw.fwo"
	iwn2030.fw optional iwn2030fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwnwifi-2030-18.168.6.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn2030.fw"
	iwn4965fw.c optional iwn4965fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn4965.fw:iwn4965fw -miwn4965fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn4965fw.c"
	iwn4965fw.fwo optional iwn4965fw \| iwnfw \
	dependency "iwn4965.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn4965fw.fwo"
	iwn4965.fw optional iwn4965fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-4965-228.61.2.24.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn4965.fw"
	iwn5000fw.c optional iwn5000fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5000.fw:iwn5000fw -miwn5000fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn5000fw.c"
	iwn5000fw.fwo optional iwn5000fw \| iwnfw \
	dependency "iwn5000.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn5000fw.fwo"
	iwn5000.fw optional iwn5000fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-5000-8.83.5.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn5000.fw"
	iwn5150fw.c optional iwn5150fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn5150.fw:iwn5150fw -miwn5150fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn5150fw.c"
	iwn5150fw.fwo optional iwn5150fw \| iwnfw \
	dependency "iwn5150.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn5150fw.fwo"
	iwn5150.fw optional iwn5150fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-5150-8.24.2.2.fw.uu"\
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn5150.fw"
	iwn6000fw.c optional iwn6000fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000.fw:iwn6000fw -miwn6000fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn6000fw.c"
	iwn6000fw.fwo optional iwn6000fw \| iwnfw \
	dependency "iwn6000.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn6000fw.fwo"
	iwn6000.fw optional iwn6000fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-6000-9.221.4.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn6000.fw"
	iwn6000g2afw.c optional iwn6000g2afw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2a.fw:iwn6000g2afw -miwn6000g2afw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn6000g2afw.c"
	iwn6000g2afw.fwo optional iwn6000g2afw \| iwnfw \
	dependency "iwn6000g2a.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn6000g2afw.fwo"
	iwn6000g2a.fw optional iwn6000g2afw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-6000g2a-18.168.6.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn6000g2a.fw"
	iwn6000g2bfw.c optional iwn6000g2bfw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6000g2b.fw:iwn6000g2bfw -miwn6000g2bfw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn6000g2bfw.c"
	iwn6000g2bfw.fwo optional iwn6000g2bfw \| iwnfw \
	dependency "iwn6000g2b.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn6000g2bfw.fwo"
	iwn6000g2b.fw optional iwn6000g2bfw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-6000g2b-18.168.6.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn6000g2b.fw"
	iwn6050fw.c optional iwn6050fw \| iwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk iwn6050.fw:iwn6050fw -miwn6050fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "iwn6050fw.c"
	iwn6050fw.fwo optional iwn6050fw \| iwnfw \
	dependency "iwn6050.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "iwn6050fw.fwo"
	iwn6050.fw optional iwn6050fw \| iwnfw \
	dependency "$S/contrib/dev/iwn/iwlwifi-6050-41.28.5.1.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "iwn6050.fw"
	dev/ixgbe/if_ix.c optional ix inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe -DSMP"
	dev/ixgbe/if_ixv.c optional ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe -DSMP"
	dev/ixgbe/if_bypass.c optional ix inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/if_fdir.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/if_sriov.c optional ix inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ix_txrx.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_osdep.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_phy.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_api.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_common.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_mbx.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_vf.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_82598.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_82599.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_x540.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_x550.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_dcb.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_dcb_82598.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/ixgbe/ixgbe_dcb_82599.c optional ix inet \| ixv inet \
	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
	dev/jedec_dimm/jedec_dimm.c optional jedec_dimm smbus
	dev/jme/if_jme.c optional jme pci
	dev/kbd/kbd.c optional atkbd \| pckbd \| sc \| ukbd \| vt
	dev/kbdmux/kbdmux.c optional kbdmux
	dev/ksyms/ksyms.c optional ksyms
	dev/le/am7990.c optional le
	dev/le/am79900.c optional le
	dev/le/if_le_pci.c optional le pci
	dev/le/lance.c optional le
	dev/led/led.c standard
	dev/lge/if_lge.c optional lge
	dev/liquidio/base/cn23xx_pf_device.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_console.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_ctrl.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_device.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_droq.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_mem_ops.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_request_manager.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/base/lio_response_manager.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/lio_core.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/lio_ioctl.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/lio_main.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/lio_rss.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/lio_rxtx.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	dev/liquidio/lio_sysctl.c optional lio \
	compile-with "${NORMAL_C} \
	-I$S/dev/liquidio -I$S/dev/liquidio/base -DSMP"
	lio.c optional lio \
	compile-with "${AWK} -f $S/tools/fw_stub.awk lio_23xx_nic.bin.fw:lio_23xx_nic.bin -mlio_23xx_nic.bin -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "lio.c"
	lio_23xx_nic.bin.fw.fwo optional lio \
	dependency "lio_23xx_nic.bin.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "lio_23xx_nic.bin.fw.fwo"
	lio_23xx_nic.bin.fw optional lio \
	dependency "$S/contrib/dev/liquidio/lio_23xx_nic.bin.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "lio_23xx_nic.bin.fw"
	dev/malo/if_malo.c optional malo
	dev/malo/if_malohal.c optional malo
	dev/malo/if_malo_pci.c optional malo pci
	dev/mc146818/mc146818.c optional mc146818
	dev/md/md.c optional md
	dev/mdio/mdio_if.m optional miiproxy \| mdio
	dev/mdio/mdio.c optional miiproxy \| mdio
	dev/mem/memdev.c optional mem
	dev/mem/memutil.c optional mem
	dev/mfi/mfi.c optional mfi
	dev/mfi/mfi_debug.c optional mfi
	dev/mfi/mfi_pci.c optional mfi pci
	dev/mfi/mfi_disk.c optional mfi
	dev/mfi/mfi_syspd.c optional mfi
	dev/mfi/mfi_tbolt.c optional mfi
	dev/mfi/mfi_linux.c optional mfi compat_linux
	dev/mfi/mfi_cam.c optional mfip scbus
	dev/mii/acphy.c optional miibus \| acphy
	dev/mii/amphy.c optional miibus \| amphy
	dev/mii/atphy.c optional miibus \| atphy
	dev/mii/axphy.c optional miibus \| axphy
	dev/mii/bmtphy.c optional miibus \| bmtphy
	dev/mii/brgphy.c optional miibus \| brgphy
	dev/mii/ciphy.c optional miibus \| ciphy
	dev/mii/e1000phy.c optional miibus \| e1000phy
	dev/mii/gentbi.c optional miibus \| gentbi
	dev/mii/icsphy.c optional miibus \| icsphy
	dev/mii/ip1000phy.c optional miibus \| ip1000phy
	dev/mii/jmphy.c optional miibus \| jmphy
	dev/mii/lxtphy.c optional miibus \| lxtphy
	dev/mii/micphy.c optional miibus fdt \| micphy fdt
	dev/mii/mii.c optional miibus \| mii
	dev/mii/mii_bitbang.c optional miibus \| mii_bitbang
	dev/mii/mii_physubr.c optional miibus \| mii
	dev/mii/mii_fdt.c optional miibus fdt \| mii fdt
	dev/mii/miibus_if.m optional miibus \| mii
	dev/mii/mlphy.c optional miibus \| mlphy
	dev/mii/nsgphy.c optional miibus \| nsgphy
	dev/mii/nsphy.c optional miibus \| nsphy
	dev/mii/nsphyter.c optional miibus \| nsphyter
	dev/mii/pnaphy.c optional miibus \| pnaphy
	dev/mii/qsphy.c optional miibus \| qsphy
	dev/mii/rdcphy.c optional miibus \| rdcphy
	dev/mii/rgephy.c optional miibus \| rgephy
	dev/mii/rlphy.c optional miibus \| rlphy
	dev/mii/rlswitch.c optional rlswitch
	dev/mii/smcphy.c optional miibus \| smcphy
	dev/mii/smscphy.c optional miibus \| smscphy
	dev/mii/tdkphy.c optional miibus \| tdkphy
	dev/mii/tlphy.c optional miibus \| tlphy
	dev/mii/truephy.c optional miibus \| truephy
	dev/mii/ukphy.c optional miibus \| mii
	dev/mii/ukphy_subr.c optional miibus \| mii
	dev/mii/vscphy.c optional miibus \| vscphy
	dev/mii/xmphy.c optional miibus \| xmphy
	dev/mk48txx/mk48txx.c optional mk48txx
	dev/mlxfw/mlxfw_fsm.c optional mlxfw \
	compile-with "${MLXFW_C}"
	dev/mlxfw/mlxfw_mfa2.c optional mlxfw \
	compile-with "${MLXFW_C}"
	dev/mlxfw/mlxfw_mfa2_tlv_multi.c optional mlxfw \
	compile-with "${MLXFW_C}"
	dev/mlx/mlx.c optional mlx
	dev/mlx/mlx_disk.c optional mlx
	dev/mlx/mlx_pci.c optional mlx pci
	dev/mly/mly.c optional mly
	dev/mmc/mmc_subr.c optional mmc \| mmcsd !mmccam
	dev/mmc/mmc.c optional mmc !mmccam
	dev/mmc/mmcbr_if.m standard
	dev/mmc/mmcbus_if.m standard
	dev/mmc/mmcsd.c optional mmcsd !mmccam
	dev/mmc/mmc_fdt_helpers.c optional mmc fdt \| mmccam fdt
	dev/mmcnull/mmcnull.c optional mmcnull
	dev/mn/if_mn.c optional mn pci
	dev/mpr/mpr.c optional mpr
	dev/mpr/mpr_config.c optional mpr
	# XXX Work around clang warning, until maintainer approves fix.
	dev/mpr/mpr_mapping.c optional mpr \
	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
	dev/mpr/mpr_pci.c optional mpr pci
	dev/mpr/mpr_sas.c optional mpr \
	compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}"
	dev/mpr/mpr_sas_lsi.c optional mpr
	dev/mpr/mpr_table.c optional mpr
	dev/mpr/mpr_user.c optional mpr
	dev/mps/mps.c optional mps
	dev/mps/mps_config.c optional mps
	# XXX Work around clang warning, until maintainer approves fix.
	dev/mps/mps_mapping.c optional mps \
	compile-with "${NORMAL_C} ${NO_WSOMETIMES_UNINITIALIZED}"
	dev/mps/mps_pci.c optional mps pci
	dev/mps/mps_sas.c optional mps \
	compile-with "${NORMAL_C} ${NO_WUNNEEDED_INTERNAL_DECL}"
	dev/mps/mps_sas_lsi.c optional mps
	dev/mps/mps_table.c optional mps
	dev/mps/mps_user.c optional mps
	dev/mpt/mpt.c optional mpt
	dev/mpt/mpt_cam.c optional mpt
	dev/mpt/mpt_debug.c optional mpt
	dev/mpt/mpt_pci.c optional mpt pci
	dev/mpt/mpt_raid.c optional mpt
	dev/mpt/mpt_user.c optional mpt
	dev/mrsas/mrsas.c optional mrsas
	dev/mrsas/mrsas_cam.c optional mrsas
	dev/mrsas/mrsas_ioctl.c optional mrsas
	dev/mrsas/mrsas_fp.c optional mrsas
	dev/msk/if_msk.c optional msk
	dev/mvs/mvs.c optional mvs
	dev/mvs/mvs_if.m optional mvs
	dev/mvs/mvs_pci.c optional mvs pci
	dev/mwl/if_mwl.c optional mwl
	dev/mwl/if_mwl_pci.c optional mwl pci
	dev/mwl/mwlhal.c optional mwl
	mwlfw.c optional mwlfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk mw88W8363.fw:mw88W8363fw mwlboot.fw:mwlboot -mmwl -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "mwlfw.c"
	mw88W8363.fwo optional mwlfw \
	dependency "mw88W8363.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "mw88W8363.fwo"
	mw88W8363.fw optional mwlfw \
	dependency "$S/contrib/dev/mwl/mw88W8363.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "mw88W8363.fw"
	mwlboot.fwo optional mwlfw \
	dependency "mwlboot.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "mwlboot.fwo"
	mwlboot.fw optional mwlfw \
	dependency "$S/contrib/dev/mwl/mwlboot.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "mwlboot.fw"
	dev/mxge/if_mxge.c optional mxge pci
	dev/mxge/mxge_eth_z8e.c optional mxge pci
	dev/mxge/mxge_ethp_z8e.c optional mxge pci
	dev/mxge/mxge_rss_eth_z8e.c optional mxge pci
	dev/mxge/mxge_rss_ethp_z8e.c optional mxge pci
	dev/my/if_my.c optional my
	dev/netmap/if_ptnet.c optional netmap inet
	dev/netmap/netmap.c optional netmap
	dev/netmap/netmap_bdg.c optional netmap
	dev/netmap/netmap_freebsd.c optional netmap
	dev/netmap/netmap_generic.c optional netmap
	dev/netmap/netmap_kloop.c optional netmap
	dev/netmap/netmap_legacy.c optional netmap
	dev/netmap/netmap_mbq.c optional netmap
	dev/netmap/netmap_mem2.c optional netmap
	dev/netmap/netmap_monitor.c optional netmap
	dev/netmap/netmap_null.c optional netmap
	dev/netmap/netmap_offloadings.c optional netmap
	dev/netmap/netmap_pipe.c optional netmap
	dev/netmap/netmap_vale.c optional netmap
	# compile-with "${NORMAL_C} -Wconversion -Wextra"
	dev/nfsmb/nfsmb.c optional nfsmb pci
	dev/nge/if_nge.c optional nge
	dev/nmdm/nmdm.c optional nmdm
	dev/null/null.c standard
	dev/nvd/nvd.c optional nvd nvme
	dev/nvme/nvme.c optional nvme
	dev/nvme/nvme_ahci.c optional nvme ahci
	dev/nvme/nvme_ctrlr.c optional nvme
	dev/nvme/nvme_ctrlr_cmd.c optional nvme
	dev/nvme/nvme_ns.c optional nvme
	dev/nvme/nvme_ns_cmd.c optional nvme
	dev/nvme/nvme_pci.c optional nvme pci
	dev/nvme/nvme_qpair.c optional nvme
	dev/nvme/nvme_sim.c optional nvme scbus
	dev/nvme/nvme_sysctl.c optional nvme
	dev/nvme/nvme_test.c optional nvme
	dev/nvme/nvme_util.c optional nvme
	dev/oce/oce_hw.c optional oce pci
	dev/oce/oce_if.c optional oce pci
	dev/oce/oce_mbox.c optional oce pci
	dev/oce/oce_queue.c optional oce pci
	dev/oce/oce_sysctl.c optional oce pci
	dev/oce/oce_util.c optional oce pci
	dev/ocs_fc/ocs_pci.c optional ocs_fc pci
	dev/ocs_fc/ocs_ioctl.c optional ocs_fc pci
	dev/ocs_fc/ocs_os.c optional ocs_fc pci
	dev/ocs_fc/ocs_utils.c optional ocs_fc pci
	dev/ocs_fc/ocs_hw.c optional ocs_fc pci
	dev/ocs_fc/ocs_hw_queues.c optional ocs_fc pci
	dev/ocs_fc/sli4.c optional ocs_fc pci
	dev/ocs_fc/ocs_sm.c optional ocs_fc pci
	dev/ocs_fc/ocs_device.c optional ocs_fc pci
	dev/ocs_fc/ocs_xport.c optional ocs_fc pci
	dev/ocs_fc/ocs_domain.c optional ocs_fc pci
	dev/ocs_fc/ocs_sport.c optional ocs_fc pci
	dev/ocs_fc/ocs_els.c optional ocs_fc pci
	dev/ocs_fc/ocs_fabric.c optional ocs_fc pci
	dev/ocs_fc/ocs_io.c optional ocs_fc pci
	dev/ocs_fc/ocs_node.c optional ocs_fc pci
	dev/ocs_fc/ocs_scsi.c optional ocs_fc pci
	dev/ocs_fc/ocs_unsol.c optional ocs_fc pci
	dev/ocs_fc/ocs_ddump.c optional ocs_fc pci
	dev/ocs_fc/ocs_mgmt.c optional ocs_fc pci
	dev/ocs_fc/ocs_cam.c optional ocs_fc pci
	dev/ofw/ofw_bus_if.m optional fdt
	dev/ofw/ofw_bus_subr.c optional fdt
	dev/ofw/ofw_cpu.c optional fdt
	dev/ofw/ofw_fdt.c optional fdt
	dev/ofw/ofw_if.m optional fdt
	dev/ofw/ofw_graph.c optional fdt
	dev/ofw/ofw_subr.c optional fdt
	dev/ofw/ofwbus.c optional fdt
	dev/ofw/openfirm.c optional fdt
	dev/ofw/openfirmio.c optional fdt
	dev/ow/ow.c optional ow \
	dependency "owll_if.h" \
	dependency "own_if.h"
	dev/ow/owll_if.m optional ow
	dev/ow/own_if.m optional ow
	dev/ow/ow_temp.c optional ow_temp
	dev/ow/owc_gpiobus.c optional owc gpio
	dev/pbio/pbio.c optional pbio isa
	dev/pccard/card_if.m standard
	dev/pccard/pccard.c optional pccard
	dev/pccard/pccard_cis.c optional pccard
	dev/pccard/pccard_cis_quirks.c optional pccard
	dev/pccard/pccard_device.c optional pccard
	dev/pccard/power_if.m standard
	dev/pccbb/pccbb.c optional cbb
	dev/pccbb/pccbb_pci.c optional cbb pci
	dev/pcf/pcf.c optional pcf
	dev/pci/fixup_pci.c optional pci
	dev/pci/hostb_pci.c optional pci
	dev/pci/ignore_pci.c optional pci
	dev/pci/isa_pci.c optional pci isa
	dev/pci/pci.c optional pci
	dev/pci/pci_if.m standard
	dev/pci/pci_iov.c optional pci pci_iov
	dev/pci/pci_iov_if.m standard
	dev/pci/pci_iov_schema.c optional pci pci_iov
	dev/pci/pci_pci.c optional pci
	dev/pci/pci_subr.c optional pci
	dev/pci/pci_user.c optional pci
	dev/pci/pcib_if.m standard
	dev/pci/pcib_support.c standard
	dev/pci/vga_pci.c optional pci
	dev/pms/freebsd/driver/ini/src/agtiapi.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sadisc.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/mpi.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/saframe.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sahw.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sainit.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/saint.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sampicmd.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sampirsp.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/saphy.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/saport.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sasata.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sasmp.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sassp.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/satimer.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/sautil.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/saioctlcmd.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sallsdk/spc/mpidebug.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/discovery/dm/dminit.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/discovery/dm/dmsmp.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/discovery/dm/dmdisc.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/discovery/dm/dmport.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/discovery/dm/dmtimer.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/discovery/dm/dmmisc.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sat/src/sminit.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sat/src/smmisc.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sat/src/smsat.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sat/src/smsatcb.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sat/src/smsathw.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/sat/src/smtimer.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdinit.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdmisc.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdesgl.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdport.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdint.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdioctl.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdhw.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/ossacmnapi.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tddmcmnapi.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdsmcmnapi.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/common/tdtimers.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sas/ini/itdio.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sas/ini/itdcb.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sas/ini/itdinit.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sas/ini/itddisc.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sata/host/sat.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sata/host/ossasat.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/pms/RefTisa/tisa/sassata/sata/host/sathw.c optional pmspcv \
	compile-with "${NORMAL_C} -Wunused-variable -Woverflow -Wparentheses -w"
	dev/ppbus/if_plip.c optional plip
	dev/ppbus/lpbb.c optional lpbb
	dev/ppbus/lpt.c optional lpt
	dev/ppbus/pcfclock.c optional pcfclock
	dev/ppbus/ppb_1284.c optional ppbus
	dev/ppbus/ppb_base.c optional ppbus
	dev/ppbus/ppb_msq.c optional ppbus
	dev/ppbus/ppbconf.c optional ppbus
	dev/ppbus/ppbus_if.m optional ppbus
	dev/ppbus/ppi.c optional ppi
	dev/ppbus/pps.c optional pps
	dev/ppc/ppc.c optional ppc
	dev/ppc/ppc_acpi.c optional ppc acpi
	dev/ppc/ppc_isa.c optional ppc isa
	dev/ppc/ppc_pci.c optional ppc pci
	dev/ppc/ppc_puc.c optional ppc puc
	dev/proto/proto_bus_isa.c optional proto acpi \| proto isa
	dev/proto/proto_bus_pci.c optional proto pci
	dev/proto/proto_busdma.c optional proto
	dev/proto/proto_core.c optional proto
	dev/pst/pst-iop.c optional pst
	dev/pst/pst-pci.c optional pst pci
	dev/pst/pst-raid.c optional pst
	dev/pty/pty.c optional pty
	dev/puc/puc.c optional puc
	dev/puc/puc_cfg.c optional puc
	dev/puc/puc_pccard.c optional puc pccard
	dev/puc/puc_pci.c optional puc pci
	dev/pwm/pwmc.c optional pwm \| pwmc
	dev/pwm/pwmbus.c optional pwm \| pwmbus
	dev/pwm/pwmbus_if.m optional pwm \| pwmbus
	dev/pwm/ofw_pwm.c optional pwm fdt \| pwmbus fdt
	dev/pwm/ofw_pwmbus.c optional pwm fdt \| pwmbus fdt
	dev/quicc/quicc_core.c optional quicc
	dev/ral/rt2560.c optional ral
	dev/ral/rt2661.c optional ral
	dev/ral/rt2860.c optional ral
	dev/ral/if_ral_pci.c optional ral pci
	rt2561fw.c optional rt2561fw \| ralfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561.fw:rt2561fw -mrt2561 -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rt2561fw.c"
	rt2561fw.fwo optional rt2561fw \| ralfw \
	dependency "rt2561.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rt2561fw.fwo"
	rt2561.fw optional rt2561fw \| ralfw \
	dependency "$S/contrib/dev/ral/rt2561.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rt2561.fw"
	rt2561sfw.c optional rt2561sfw \| ralfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rt2561s.fw:rt2561sfw -mrt2561s -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rt2561sfw.c"
	rt2561sfw.fwo optional rt2561sfw \| ralfw \
	dependency "rt2561s.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rt2561sfw.fwo"
	rt2561s.fw optional rt2561sfw \| ralfw \
	dependency "$S/contrib/dev/ral/rt2561s.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rt2561s.fw"
	rt2661fw.c optional rt2661fw \| ralfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rt2661.fw:rt2661fw -mrt2661 -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rt2661fw.c"
	rt2661fw.fwo optional rt2661fw \| ralfw \
	dependency "rt2661.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rt2661fw.fwo"
	rt2661.fw optional rt2661fw \| ralfw \
	dependency "$S/contrib/dev/ral/rt2661.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rt2661.fw"
	rt2860fw.c optional rt2860fw \| ralfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rt2860.fw:rt2860fw -mrt2860 -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rt2860fw.c"
	rt2860fw.fwo optional rt2860fw \| ralfw \
	dependency "rt2860.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rt2860fw.fwo"
	rt2860.fw optional rt2860fw \| ralfw \
	dependency "$S/contrib/dev/ral/rt2860.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rt2860.fw"
	dev/random/random_infra.c standard
	dev/random/random_harvestq.c standard
	dev/random/randomdev.c optional !random_loadable
	dev/random/fortuna.c optional !random_loadable
	dev/random/hash.c optional !random_loadable
	-dev/rc/rc.c optional rc
	dev/rccgpio/rccgpio.c optional rccgpio gpio
	dev/re/if_re.c optional re
	dev/rl/if_rl.c optional rl pci
	dev/rndtest/rndtest.c optional rndtest
	-dev/rp/rp.c optional rp
	-dev/rp/rp_isa.c optional rp isa
	-dev/rp/rp_pci.c optional rp pci
	#
	dev/rtwn/if_rtwn.c optional rtwn
	dev/rtwn/if_rtwn_beacon.c optional rtwn
	dev/rtwn/if_rtwn_calib.c optional rtwn
	dev/rtwn/if_rtwn_cam.c optional rtwn
	dev/rtwn/if_rtwn_efuse.c optional rtwn
	dev/rtwn/if_rtwn_fw.c optional rtwn
	dev/rtwn/if_rtwn_rx.c optional rtwn
	dev/rtwn/if_rtwn_task.c optional rtwn
	dev/rtwn/if_rtwn_tx.c optional rtwn
	#
	dev/rtwn/pci/rtwn_pci_attach.c optional rtwn_pci pci
	dev/rtwn/pci/rtwn_pci_reg.c optional rtwn_pci pci
	dev/rtwn/pci/rtwn_pci_rx.c optional rtwn_pci pci
	dev/rtwn/pci/rtwn_pci_tx.c optional rtwn_pci pci
	#
	dev/rtwn/usb/rtwn_usb_attach.c optional rtwn_usb
	dev/rtwn/usb/rtwn_usb_ep.c optional rtwn_usb
	dev/rtwn/usb/rtwn_usb_reg.c optional rtwn_usb
	dev/rtwn/usb/rtwn_usb_rx.c optional rtwn_usb
	dev/rtwn/usb/rtwn_usb_tx.c optional rtwn_usb
	# RTL8188E
	dev/rtwn/rtl8188e/r88e_beacon.c optional rtwn
	dev/rtwn/rtl8188e/r88e_calib.c optional rtwn
	dev/rtwn/rtl8188e/r88e_chan.c optional rtwn
	dev/rtwn/rtl8188e/r88e_fw.c optional rtwn
	dev/rtwn/rtl8188e/r88e_init.c optional rtwn
	dev/rtwn/rtl8188e/r88e_led.c optional rtwn
	dev/rtwn/rtl8188e/r88e_tx.c optional rtwn
	dev/rtwn/rtl8188e/r88e_rf.c optional rtwn
	dev/rtwn/rtl8188e/r88e_rom.c optional rtwn
	dev/rtwn/rtl8188e/r88e_rx.c optional rtwn
	dev/rtwn/rtl8188e/pci/r88ee_attach.c optional rtwn_pci pci
	dev/rtwn/rtl8188e/pci/r88ee_init.c optional rtwn_pci pci
	dev/rtwn/rtl8188e/pci/r88ee_rx.c optional rtwn_pci pci
	dev/rtwn/rtl8188e/usb/r88eu_attach.c optional rtwn_usb
	dev/rtwn/rtl8188e/usb/r88eu_init.c optional rtwn_usb
	# RTL8192C
	dev/rtwn/rtl8192c/r92c_attach.c optional rtwn
	dev/rtwn/rtl8192c/r92c_beacon.c optional rtwn
	dev/rtwn/rtl8192c/r92c_calib.c optional rtwn
	dev/rtwn/rtl8192c/r92c_chan.c optional rtwn
	dev/rtwn/rtl8192c/r92c_fw.c optional rtwn
	dev/rtwn/rtl8192c/r92c_init.c optional rtwn
	dev/rtwn/rtl8192c/r92c_llt.c optional rtwn
	dev/rtwn/rtl8192c/r92c_rf.c optional rtwn
	dev/rtwn/rtl8192c/r92c_rom.c optional rtwn
	dev/rtwn/rtl8192c/r92c_rx.c optional rtwn
	dev/rtwn/rtl8192c/r92c_tx.c optional rtwn
	dev/rtwn/rtl8192c/pci/r92ce_attach.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/pci/r92ce_calib.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/pci/r92ce_fw.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/pci/r92ce_init.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/pci/r92ce_led.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/pci/r92ce_rx.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/pci/r92ce_tx.c optional rtwn_pci pci
	dev/rtwn/rtl8192c/usb/r92cu_attach.c optional rtwn_usb
	dev/rtwn/rtl8192c/usb/r92cu_init.c optional rtwn_usb
	dev/rtwn/rtl8192c/usb/r92cu_led.c optional rtwn_usb
	dev/rtwn/rtl8192c/usb/r92cu_rx.c optional rtwn_usb
	dev/rtwn/rtl8192c/usb/r92cu_tx.c optional rtwn_usb
	# RTL8192E
	dev/rtwn/rtl8192e/r92e_chan.c optional rtwn
	dev/rtwn/rtl8192e/r92e_fw.c optional rtwn
	dev/rtwn/rtl8192e/r92e_init.c optional rtwn
	dev/rtwn/rtl8192e/r92e_led.c optional rtwn
	dev/rtwn/rtl8192e/r92e_rf.c optional rtwn
	dev/rtwn/rtl8192e/r92e_rom.c optional rtwn
	dev/rtwn/rtl8192e/r92e_rx.c optional rtwn
	dev/rtwn/rtl8192e/usb/r92eu_attach.c optional rtwn_usb
	dev/rtwn/rtl8192e/usb/r92eu_init.c optional rtwn_usb
	# RTL8812A
	dev/rtwn/rtl8812a/r12a_beacon.c optional rtwn
	dev/rtwn/rtl8812a/r12a_calib.c optional rtwn
	dev/rtwn/rtl8812a/r12a_caps.c optional rtwn
	dev/rtwn/rtl8812a/r12a_chan.c optional rtwn
	dev/rtwn/rtl8812a/r12a_fw.c optional rtwn
	dev/rtwn/rtl8812a/r12a_init.c optional rtwn
	dev/rtwn/rtl8812a/r12a_led.c optional rtwn
	dev/rtwn/rtl8812a/r12a_rf.c optional rtwn
	dev/rtwn/rtl8812a/r12a_rom.c optional rtwn
	dev/rtwn/rtl8812a/r12a_rx.c optional rtwn
	dev/rtwn/rtl8812a/r12a_tx.c optional rtwn
	dev/rtwn/rtl8812a/usb/r12au_attach.c optional rtwn_usb
	dev/rtwn/rtl8812a/usb/r12au_init.c optional rtwn_usb
	dev/rtwn/rtl8812a/usb/r12au_rx.c optional rtwn_usb
	dev/rtwn/rtl8812a/usb/r12au_tx.c optional rtwn_usb
	# RTL8821A
	dev/rtwn/rtl8821a/r21a_beacon.c optional rtwn
	dev/rtwn/rtl8821a/r21a_calib.c optional rtwn
	dev/rtwn/rtl8821a/r21a_chan.c optional rtwn
	dev/rtwn/rtl8821a/r21a_fw.c optional rtwn
	dev/rtwn/rtl8821a/r21a_init.c optional rtwn
	dev/rtwn/rtl8821a/r21a_led.c optional rtwn
	dev/rtwn/rtl8821a/r21a_rom.c optional rtwn
	dev/rtwn/rtl8821a/r21a_rx.c optional rtwn
	dev/rtwn/rtl8821a/usb/r21au_attach.c optional rtwn_usb
	dev/rtwn/rtl8821a/usb/r21au_dfs.c optional rtwn_usb
	dev/rtwn/rtl8821a/usb/r21au_init.c optional rtwn_usb
	rtwn-rtl8188eefw.c optional rtwn-rtl8188eefw \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8188eefw.fw:rtwn-rtl8188eefw:111 -mrtwn-rtl8188eefw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8188eefw.c"
	rtwn-rtl8188eefw.fwo optional rtwn-rtl8188eefw \| rtwnfw \
	dependency "rtwn-rtl8188eefw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8188eefw.fwo"
	rtwn-rtl8188eefw.fw optional rtwn-rtl8188eefw \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8188eefw.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8188eefw.fw"
	rtwn-rtl8188eufw.c optional rtwn-rtl8188eufw \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8188eufw.fw:rtwn-rtl8188eufw:111 -mrtwn-rtl8188eufw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8188eufw.c"
	rtwn-rtl8188eufw.fwo optional rtwn-rtl8188eufw \| rtwnfw \
	dependency "rtwn-rtl8188eufw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8188eufw.fwo"
	rtwn-rtl8188eufw.fw optional rtwn-rtl8188eufw \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8188eufw.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8188eufw.fw"
	rtwn-rtl8192cfwE.c optional rtwn-rtl8192cfwE \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192cfwE.fw:rtwn-rtl8192cfwE:111 -mrtwn-rtl8192cfwE -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8192cfwE.c"
	rtwn-rtl8192cfwE.fwo optional rtwn-rtl8192cfwE \| rtwnfw \
	dependency "rtwn-rtl8192cfwE.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8192cfwE.fwo"
	rtwn-rtl8192cfwE.fw optional rtwn-rtl8192cfwE \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8192cfwE.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8192cfwE.fw"
	rtwn-rtl8192cfwE_B.c optional rtwn-rtl8192cfwE_B \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192cfwE_B.fw:rtwn-rtl8192cfwE_B:111 -mrtwn-rtl8192cfwE_B -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8192cfwE_B.c"
	rtwn-rtl8192cfwE_B.fwo optional rtwn-rtl8192cfwE_B \| rtwnfw \
	dependency "rtwn-rtl8192cfwE_B.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8192cfwE_B.fwo"
	rtwn-rtl8192cfwE_B.fw optional rtwn-rtl8192cfwE_B \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8192cfwE_B.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8192cfwE_B.fw"
	rtwn-rtl8192cfwT.c optional rtwn-rtl8192cfwT \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192cfwT.fw:rtwn-rtl8192cfwT:111 -mrtwn-rtl8192cfwT -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8192cfwT.c"
	rtwn-rtl8192cfwT.fwo optional rtwn-rtl8192cfwT \| rtwnfw \
	dependency "rtwn-rtl8192cfwT.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8192cfwT.fwo"
	rtwn-rtl8192cfwT.fw optional rtwn-rtl8192cfwT \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8192cfwT.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8192cfwT.fw"
	rtwn-rtl8192cfwU.c optional rtwn-rtl8192cfwU \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192cfwU.fw:rtwn-rtl8192cfwU:111 -mrtwn-rtl8192cfwU -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8192cfwU.c"
	rtwn-rtl8192cfwU.fwo optional rtwn-rtl8192cfwU \| rtwnfw \
	dependency "rtwn-rtl8192cfwU.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8192cfwU.fwo"
	rtwn-rtl8192cfwU.fw optional rtwn-rtl8192cfwU \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8192cfwU.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8192cfwU.fw"
	rtwn-rtl8192eufw.c optional rtwn-rtl8192eufw \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8192eufw.fw:rtwn-rtl8192eufw:111 -mrtwn-rtl8192eufw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8192eufw.c"
	rtwn-rtl8192eufw.fwo optional rtwn-rtl8192eufw \| rtwnfw \
	dependency "rtwn-rtl8192eufw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8192eufw.fwo"
	rtwn-rtl8192eufw.fw optional rtwn-rtl8192eufw \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8192eufw.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8192eufw.fw"
	rtwn-rtl8812aufw.c optional rtwn-rtl8812aufw \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8812aufw.fw:rtwn-rtl8812aufw:111 -mrtwn-rtl8812aufw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8812aufw.c"
	rtwn-rtl8812aufw.fwo optional rtwn-rtl8812aufw \| rtwnfw \
	dependency "rtwn-rtl8812aufw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8812aufw.fwo"
	rtwn-rtl8812aufw.fw optional rtwn-rtl8812aufw \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8812aufw.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8812aufw.fw"
	rtwn-rtl8821aufw.c optional rtwn-rtl8821aufw \| rtwnfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rtwn-rtl8821aufw.fw:rtwn-rtl8821aufw:111 -mrtwn-rtl8821aufw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rtwn-rtl8821aufw.c"
	rtwn-rtl8821aufw.fwo optional rtwn-rtl8821aufw \| rtwnfw \
	dependency "rtwn-rtl8821aufw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rtwn-rtl8821aufw.fwo"
	rtwn-rtl8821aufw.fw optional rtwn-rtl8821aufw \| rtwnfw \
	dependency "$S/contrib/dev/rtwn/rtwn-rtl8821aufw.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rtwn-rtl8821aufw.fw"
	dev/safe/safe.c optional safe
	dev/scc/scc_if.m optional scc
	dev/scc/scc_bfe_quicc.c optional scc quicc
	dev/scc/scc_core.c optional scc
	dev/scc/scc_dev_quicc.c optional scc quicc
	dev/scc/scc_dev_sab82532.c optional scc
	dev/scc/scc_dev_z8530.c optional scc
	dev/sdhci/sdhci.c optional sdhci
	dev/sdhci/sdhci_fdt.c optional sdhci fdt
	dev/sdhci/sdhci_fdt_gpio.c optional sdhci fdt gpio
	dev/sdhci/sdhci_if.m optional sdhci
	dev/sdhci/sdhci_acpi.c optional sdhci acpi
	dev/sdhci/sdhci_pci.c optional sdhci pci
	dev/sdio/sdio_if.m optional mmccam
	dev/sdio/sdio_subr.c optional mmccam
	dev/sdio/sdiob.c optional mmccam
	dev/sge/if_sge.c optional sge pci
	dev/siis/siis.c optional siis pci
	dev/sis/if_sis.c optional sis pci
	dev/sk/if_sk.c optional sk pci
	dev/smbus/smb.c optional smb
	dev/smbus/smbconf.c optional smbus
	dev/smbus/smbus.c optional smbus
	dev/smbus/smbus_if.m optional smbus
	dev/smc/if_smc.c optional smc
	+dev/smc/if_smc_acpi.c optional smc acpi
	dev/smc/if_smc_fdt.c optional smc fdt
	dev/snp/snp.c optional snp
	dev/sound/clone.c optional sound
	dev/sound/unit.c optional sound
	dev/sound/isa/ad1816.c optional snd_ad1816 isa
	dev/sound/isa/ess.c optional snd_ess isa
	dev/sound/isa/gusc.c optional snd_gusc isa
	dev/sound/isa/mss.c optional snd_mss isa
	dev/sound/isa/sb16.c optional snd_sb16 isa
	dev/sound/isa/sb8.c optional snd_sb8 isa
	dev/sound/isa/sbc.c optional snd_sbc isa
	dev/sound/isa/sndbuf_dma.c optional sound isa
	dev/sound/pci/als4000.c optional snd_als4000 pci
	dev/sound/pci/atiixp.c optional snd_atiixp pci
	dev/sound/pci/cmi.c optional snd_cmi pci
	dev/sound/pci/cs4281.c optional snd_cs4281 pci
	dev/sound/pci/csa.c optional snd_csa pci
	dev/sound/pci/csapcm.c optional snd_csa pci
	dev/sound/pci/ds1.c optional snd_ds1 pci
	dev/sound/pci/emu10k1.c optional snd_emu10k1 pci
	dev/sound/pci/emu10kx.c optional snd_emu10kx pci
	dev/sound/pci/emu10kx-pcm.c optional snd_emu10kx pci
	dev/sound/pci/emu10kx-midi.c optional snd_emu10kx pci
	dev/sound/pci/envy24.c optional snd_envy24 pci
	dev/sound/pci/envy24ht.c optional snd_envy24ht pci
	dev/sound/pci/es137x.c optional snd_es137x pci
	dev/sound/pci/fm801.c optional snd_fm801 pci
	dev/sound/pci/ich.c optional snd_ich pci
	dev/sound/pci/maestro.c optional snd_maestro pci
	dev/sound/pci/maestro3.c optional snd_maestro3 pci
	dev/sound/pci/neomagic.c optional snd_neomagic pci
	dev/sound/pci/solo.c optional snd_solo pci
	dev/sound/pci/spicds.c optional snd_spicds pci
	dev/sound/pci/t4dwave.c optional snd_t4dwave pci
	dev/sound/pci/via8233.c optional snd_via8233 pci
	dev/sound/pci/via82c686.c optional snd_via82c686 pci
	dev/sound/pci/vibes.c optional snd_vibes pci
	dev/sound/pci/hda/hdaa.c optional snd_hda pci
	dev/sound/pci/hda/hdaa_patches.c optional snd_hda pci
	dev/sound/pci/hda/hdac.c optional snd_hda pci
	dev/sound/pci/hda/hdac_if.m optional snd_hda pci
	dev/sound/pci/hda/hdacc.c optional snd_hda pci
	dev/sound/pci/hdspe.c optional snd_hdspe pci
	dev/sound/pci/hdspe-pcm.c optional snd_hdspe pci
	dev/sound/pcm/ac97.c optional sound
	dev/sound/pcm/ac97_if.m optional sound
	dev/sound/pcm/ac97_patch.c optional sound
	dev/sound/pcm/buffer.c optional sound \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/channel.c optional sound
	dev/sound/pcm/channel_if.m optional sound
	dev/sound/pcm/dsp.c optional sound
	dev/sound/pcm/feeder.c optional sound
	dev/sound/pcm/feeder_chain.c optional sound
	dev/sound/pcm/feeder_eq.c optional sound \
	dependency "feeder_eq_gen.h" \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/feeder_if.m optional sound
	dev/sound/pcm/feeder_format.c optional sound \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/feeder_matrix.c optional sound \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/feeder_mixer.c optional sound \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/feeder_rate.c optional sound \
	dependency "feeder_rate_gen.h" \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/feeder_volume.c optional sound \
	dependency "snd_fxdiv_gen.h"
	dev/sound/pcm/mixer.c optional sound
	dev/sound/pcm/mixer_if.m optional sound
	dev/sound/pcm/sndstat.c optional sound
	dev/sound/pcm/sound.c optional sound
	dev/sound/pcm/vchan.c optional sound
	dev/sound/usb/uaudio.c optional snd_uaudio usb
	dev/sound/usb/uaudio_pcm.c optional snd_uaudio usb
	dev/sound/midi/midi.c optional sound
	dev/sound/midi/mpu401.c optional sound
	dev/sound/midi/mpu_if.m optional sound
	dev/sound/midi/mpufoi_if.m optional sound
	dev/sound/midi/sequencer.c optional sound
	dev/sound/midi/synth_if.m optional sound
	dev/spibus/ofw_spibus.c optional fdt spibus
	dev/spibus/spibus.c optional spibus \
	dependency "spibus_if.h"
	dev/spibus/spigen.c optional spigen
	dev/spibus/spibus_if.m optional spibus
	dev/ste/if_ste.c optional ste pci
	dev/stge/if_stge.c optional stge
	dev/sym/sym_hipd.c optional sym \
	dependency "$S/dev/sym/sym_{conf,defs}.h"
	dev/syscons/blank/blank_saver.c optional blank_saver
	dev/syscons/daemon/daemon_saver.c optional daemon_saver
	dev/syscons/dragon/dragon_saver.c optional dragon_saver
	dev/syscons/fade/fade_saver.c optional fade_saver
	dev/syscons/fire/fire_saver.c optional fire_saver
	dev/syscons/green/green_saver.c optional green_saver
	dev/syscons/logo/logo.c optional logo_saver
	dev/syscons/logo/logo_saver.c optional logo_saver
	dev/syscons/rain/rain_saver.c optional rain_saver
	dev/syscons/schistory.c optional sc
	dev/syscons/scmouse.c optional sc
	dev/syscons/scterm.c optional sc
	dev/syscons/scterm-dumb.c optional sc !SC_NO_TERM_DUMB
	dev/syscons/scterm-sc.c optional sc !SC_NO_TERM_SC
	dev/syscons/scterm-teken.c optional sc !SC_NO_TERM_TEKEN
	dev/syscons/scvidctl.c optional sc
	dev/syscons/scvtb.c optional sc
	dev/syscons/snake/snake_saver.c optional snake_saver
	dev/syscons/star/star_saver.c optional star_saver
	dev/syscons/syscons.c optional sc
	dev/syscons/sysmouse.c optional sc
	dev/syscons/warp/warp_saver.c optional warp_saver
	dev/tcp_log/tcp_log_dev.c optional tcp_blackbox inet \| tcp_blackbox inet6
	dev/tdfx/tdfx_linux.c optional tdfx_linux tdfx compat_linux
	dev/tdfx/tdfx_pci.c optional tdfx pci
	dev/ti/if_ti.c optional ti pci
	dev/twa/tw_cl_init.c optional twa \
	compile-with "${NORMAL_C} -I$S/dev/twa"
	dev/twa/tw_cl_intr.c optional twa \
	compile-with "${NORMAL_C} -I$S/dev/twa"
	dev/twa/tw_cl_io.c optional twa \
	compile-with "${NORMAL_C} -I$S/dev/twa"
	dev/twa/tw_cl_misc.c optional twa \
	compile-with "${NORMAL_C} -I$S/dev/twa"
	dev/twa/tw_osl_cam.c optional twa \
	compile-with "${NORMAL_C} -I$S/dev/twa"
	dev/twa/tw_osl_freebsd.c optional twa \
	compile-with "${NORMAL_C} -I$S/dev/twa"
	dev/twe/twe.c optional twe
	dev/twe/twe_freebsd.c optional twe
	dev/tws/tws.c optional tws
	dev/tws/tws_cam.c optional tws
	dev/tws/tws_hdm.c optional tws
	dev/tws/tws_services.c optional tws
	dev/tws/tws_user.c optional tws
	dev/uart/uart_bus_acpi.c optional uart acpi
	dev/uart/uart_bus_fdt.c optional uart fdt
	dev/uart/uart_bus_isa.c optional uart isa
	dev/uart/uart_bus_pccard.c optional uart pccard
	dev/uart/uart_bus_pci.c optional uart pci
	dev/uart/uart_bus_puc.c optional uart puc
	dev/uart/uart_bus_scc.c optional uart scc
	dev/uart/uart_core.c optional uart
	dev/uart/uart_cpu_acpi.c optional uart acpi
	dev/uart/uart_dbg.c optional uart gdb
	dev/uart/uart_dev_imx.c optional uart uart_imx fdt
	dev/uart/uart_dev_msm.c optional uart uart_msm fdt
	dev/uart/uart_dev_mvebu.c optional uart uart_mvebu
	dev/uart/uart_dev_ns8250.c optional uart uart_ns8250 \| uart uart_snps
	dev/uart/uart_dev_pl011.c optional uart pl011
	dev/uart/uart_dev_quicc.c optional uart quicc
	dev/uart/uart_dev_sab82532.c optional uart uart_sab82532 \| uart scc
	dev/uart/uart_dev_snps.c optional uart uart_snps fdt
	dev/uart/uart_dev_z8530.c optional uart uart_z8530 \| uart scc
	dev/uart/uart_if.m optional uart
	dev/uart/uart_subr.c optional uart
	dev/uart/uart_tty.c optional uart
	#
	# USB controller drivers
	#
	dev/usb/controller/musb_otg.c optional musb
	dev/usb/controller/dwc_otg.c optional dwcotg
	dev/usb/controller/dwc_otg_fdt.c optional dwcotg fdt
	dev/usb/controller/dwc_otg_acpi.c optional dwcotg acpi
	dev/usb/controller/ehci.c optional ehci
	dev/usb/controller/ehci_msm.c optional ehci_msm fdt
	dev/usb/controller/ehci_pci.c optional ehci pci
	dev/usb/controller/ohci.c optional ohci
	dev/usb/controller/ohci_pci.c optional ohci pci
	dev/usb/controller/uhci.c optional uhci
	dev/usb/controller/uhci_pci.c optional uhci pci
	dev/usb/controller/xhci.c optional xhci
	dev/usb/controller/xhci_pci.c optional xhci pci
	dev/usb/controller/saf1761_otg.c optional saf1761otg
	dev/usb/controller/saf1761_otg_fdt.c optional saf1761otg fdt
	dev/usb/controller/uss820dci.c optional uss820dci
	dev/usb/controller/usb_controller.c optional usb
	#
	# USB storage drivers
	#
	dev/usb/storage/cfumass.c optional cfumass ctl
	dev/usb/storage/umass.c optional umass
	dev/usb/storage/urio.c optional urio
	dev/usb/storage/ustorage_fs.c optional usfs
	#
	# USB core
	#
	dev/usb/usb_busdma.c optional usb
	dev/usb/usb_core.c optional usb
	dev/usb/usb_debug.c optional usb
	dev/usb/usb_dev.c optional usb
	dev/usb/usb_device.c optional usb
	dev/usb/usb_dynamic.c optional usb
	dev/usb/usb_error.c optional usb
	dev/usb/usb_fdt_support.c optional usb fdt
	dev/usb/usb_generic.c optional usb
	dev/usb/usb_handle_request.c optional usb
	dev/usb/usb_hid.c optional usb
	dev/usb/usb_hub.c optional usb
	dev/usb/usb_hub_acpi.c optional uacpi acpi
	dev/usb/usb_if.m optional usb
	dev/usb/usb_lookup.c optional usb
	dev/usb/usb_mbuf.c optional usb
	dev/usb/usb_msctest.c optional usb
	dev/usb/usb_parse.c optional usb
	dev/usb/usb_pf.c optional usb
	dev/usb/usb_process.c optional usb
	dev/usb/usb_request.c optional usb
	dev/usb/usb_transfer.c optional usb
	dev/usb/usb_util.c optional usb
	#
	# USB network drivers
	#
	dev/usb/net/if_aue.c optional aue
	dev/usb/net/if_axe.c optional axe
	dev/usb/net/if_axge.c optional axge
	dev/usb/net/if_cdce.c optional cdce
	dev/usb/net/if_cdceem.c optional cdceem
	dev/usb/net/if_cue.c optional cue
	dev/usb/net/if_ipheth.c optional ipheth
	dev/usb/net/if_kue.c optional kue
	dev/usb/net/if_mos.c optional mos
	dev/usb/net/if_muge.c optional muge
	dev/usb/net/if_rue.c optional rue
	dev/usb/net/if_smsc.c optional smsc
	dev/usb/net/if_udav.c optional udav
	dev/usb/net/if_ure.c optional ure
	dev/usb/net/if_usie.c optional usie
	dev/usb/net/if_urndis.c optional urndis
	dev/usb/net/ruephy.c optional rue
	dev/usb/net/usb_ethernet.c optional uether \| aue \| axe \| axge \| cdce \| \
	cdceem \| cue \| ipheth \| kue \| mos \| \
	rue \| smsc \| udav \| ure \| urndis \| muge
	dev/usb/net/uhso.c optional uhso
	#
	# USB WLAN drivers
	#
	dev/usb/wlan/if_rsu.c optional rsu
	rsu-rtl8712fw.c optional rsu-rtl8712fw \| rsufw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk rsu-rtl8712fw.fw:rsu-rtl8712fw:120 -mrsu-rtl8712fw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "rsu-rtl8712fw.c"
	rsu-rtl8712fw.fwo optional rsu-rtl8712fw \| rsufw \
	dependency "rsu-rtl8712fw.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "rsu-rtl8712fw.fwo"
	rsu-rtl8712fw.fw optional rsu-rtl8712.fw \| rsufw \
	dependency "$S/contrib/dev/rsu/rsu-rtl8712fw.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "rsu-rtl8712fw.fw"
	dev/usb/wlan/if_rum.c optional rum
	dev/usb/wlan/if_run.c optional run
	runfw.c optional runfw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk run.fw:runfw -mrunfw -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "runfw.c"
	runfw.fwo optional runfw \
	dependency "run.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "runfw.fwo"
	run.fw optional runfw \
	dependency "$S/contrib/dev/run/rt2870.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "run.fw"
	dev/usb/wlan/if_uath.c optional uath
	dev/usb/wlan/if_upgt.c optional upgt
	dev/usb/wlan/if_ural.c optional ural
	dev/usb/wlan/if_urtw.c optional urtw
	dev/usb/wlan/if_zyd.c optional zyd
	#
	# USB serial and parallel port drivers
	#
	dev/usb/serial/u3g.c optional u3g
	dev/usb/serial/uark.c optional uark
	dev/usb/serial/ubsa.c optional ubsa
	dev/usb/serial/ubser.c optional ubser
	dev/usb/serial/uchcom.c optional uchcom
	dev/usb/serial/ucycom.c optional ucycom
	dev/usb/serial/ufoma.c optional ufoma
	dev/usb/serial/uftdi.c optional uftdi
	dev/usb/serial/ugensa.c optional ugensa
	dev/usb/serial/uipaq.c optional uipaq
	dev/usb/serial/ulpt.c optional ulpt
	dev/usb/serial/umcs.c optional umcs
	dev/usb/serial/umct.c optional umct
	dev/usb/serial/umodem.c optional umodem
	dev/usb/serial/umoscom.c optional umoscom
	dev/usb/serial/uplcom.c optional uplcom
	dev/usb/serial/uslcom.c optional uslcom
	dev/usb/serial/uvisor.c optional uvisor
	dev/usb/serial/uvscom.c optional uvscom
	dev/usb/serial/usb_serial.c optional ucom \| u3g \| uark \| ubsa \| ubser \| \
	uchcom \| ucycom \| ufoma \| uftdi \| \
	ugensa \| uipaq \| umcs \| umct \| \
	umodem \| umoscom \| uplcom \| usie \| \
	uslcom \| uvisor \| uvscom
	#
	# USB misc drivers
	#
	dev/usb/misc/ufm.c optional ufm
	dev/usb/misc/udbp.c optional udbp
	dev/usb/misc/ugold.c optional ugold
	dev/usb/misc/uled.c optional uled
	#
	# USB input drivers
	#
	dev/usb/input/atp.c optional atp
	dev/usb/input/uep.c optional uep
	dev/usb/input/uhid.c optional uhid
	dev/usb/input/uhid_snes.c optional uhid_snes
	dev/usb/input/ukbd.c optional ukbd
	dev/usb/input/ums.c optional ums
	dev/usb/input/wmt.c optional wmt
	dev/usb/input/wsp.c optional wsp
	#
	# USB quirks
	#
	dev/usb/quirk/usb_quirk.c optional usb
	#
	# USB templates
	#
	dev/usb/template/usb_template.c optional usb_template
	dev/usb/template/usb_template_audio.c optional usb_template
	dev/usb/template/usb_template_cdce.c optional usb_template
	dev/usb/template/usb_template_kbd.c optional usb_template
	dev/usb/template/usb_template_modem.c optional usb_template
	dev/usb/template/usb_template_mouse.c optional usb_template
	dev/usb/template/usb_template_msc.c optional usb_template
	dev/usb/template/usb_template_mtp.c optional usb_template
	dev/usb/template/usb_template_phone.c optional usb_template
	dev/usb/template/usb_template_serialnet.c optional usb_template
	dev/usb/template/usb_template_midi.c optional usb_template
	dev/usb/template/usb_template_multi.c optional usb_template
	dev/usb/template/usb_template_cdceem.c optional usb_template
	#
	# USB video drivers
	#
	dev/usb/video/udl.c optional udl
	#
	# USB END
	#
	dev/videomode/videomode.c optional videomode
	dev/videomode/edid.c optional videomode
	dev/videomode/pickmode.c optional videomode
	dev/videomode/vesagtf.c optional videomode
	dev/veriexec/verified_exec.c optional veriexec mac_veriexec
	dev/vge/if_vge.c optional vge
	dev/viapm/viapm.c optional viapm pci
	dev/virtio/virtio.c optional virtio
	dev/virtio/virtqueue.c optional virtio
	dev/virtio/virtio_bus_if.m optional virtio
	dev/virtio/virtio_if.m optional virtio
	dev/virtio/pci/virtio_pci.c optional virtio_pci
	dev/virtio/mmio/virtio_mmio.c optional virtio_mmio
	dev/virtio/mmio/virtio_mmio_acpi.c optional virtio_mmio acpi
	dev/virtio/mmio/virtio_mmio_fdt.c optional virtio_mmio fdt
	dev/virtio/mmio/virtio_mmio_if.m optional virtio_mmio
	dev/virtio/network/if_vtnet.c optional vtnet
	dev/virtio/block/virtio_blk.c optional virtio_blk
	dev/virtio/balloon/virtio_balloon.c optional virtio_balloon
	dev/virtio/scsi/virtio_scsi.c optional virtio_scsi
	dev/virtio/random/virtio_random.c optional virtio_random
	dev/virtio/console/virtio_console.c optional virtio_console
	dev/vkbd/vkbd.c optional vkbd
	dev/vmgenc/vmgenc_acpi.c optional acpi
	dev/vr/if_vr.c optional vr pci
	dev/vt/colors/vt_termcolors.c optional vt
	dev/vt/font/vt_font_default.c optional vt
	dev/vt/font/vt_mouse_cursor.c optional vt
	dev/vt/hw/efifb/efifb.c optional vt_efifb
	dev/vt/hw/fb/vt_fb.c optional vt
	dev/vt/hw/vga/vt_vga.c optional vt vt_vga
	dev/vt/logo/logo_freebsd.c optional vt splash
	dev/vt/logo/logo_beastie.c optional vt splash
	dev/vt/vt_buf.c optional vt
	dev/vt/vt_consolectl.c optional vt
	dev/vt/vt_core.c optional vt
	dev/vt/vt_cpulogos.c optional vt splash
	dev/vt/vt_font.c optional vt
	dev/vt/vt_sysmouse.c optional vt
	dev/vte/if_vte.c optional vte pci
	dev/watchdog/watchdog.c standard
	dev/wi/if_wi.c optional wi
	dev/wi/if_wi_pccard.c optional wi pccard
	dev/wi/if_wi_pci.c optional wi pci
	dev/wpi/if_wpi.c optional wpi pci
	wpifw.c optional wpifw \
	compile-with "${AWK} -f $S/tools/fw_stub.awk wpi.fw:wpifw:153229 -mwpi -c${.TARGET}" \
	no-implicit-rule before-depend local \
	clean "wpifw.c"
	wpifw.fwo optional wpifw \
	dependency "wpi.fw" \
	compile-with "${NORMAL_FWO}" \
	no-implicit-rule \
	clean "wpifw.fwo"
	wpi.fw optional wpifw \
	dependency "$S/contrib/dev/wpi/iwlwifi-3945-15.32.2.9.fw.uu" \
	compile-with "${NORMAL_FW}" \
	no-obj no-implicit-rule \
	clean "wpi.fw"
	dev/xdma/controller/pl330.c optional xdma pl330
	dev/xdma/xdma.c optional xdma
	dev/xdma/xdma_bank.c optional xdma
	dev/xdma/xdma_bio.c optional xdma
	dev/xdma/xdma_fdt_test.c optional xdma xdma_test fdt
	dev/xdma/xdma_if.m optional xdma
	dev/xdma/xdma_iommu.c optional xdma
	dev/xdma/xdma_mbuf.c optional xdma
	dev/xdma/xdma_queue.c optional xdma
	dev/xdma/xdma_sg.c optional xdma
	dev/xdma/xdma_sglist.c optional xdma
	dev/xen/balloon/balloon.c optional xenhvm
	dev/xen/blkfront/blkfront.c optional xenhvm
	dev/xen/blkback/blkback.c optional xenhvm
	dev/xen/console/xen_console.c optional xenhvm
	dev/xen/control/control.c optional xenhvm
	dev/xen/grant_table/grant_table.c optional xenhvm
	dev/xen/netback/netback.c optional xenhvm
	dev/xen/netfront/netfront.c optional xenhvm
	dev/xen/xenpci/xenpci.c optional xenpci
	dev/xen/timer/timer.c optional xenhvm
	dev/xen/pvcpu/pvcpu.c optional xenhvm
	dev/xen/xenstore/xenstore.c optional xenhvm
	dev/xen/xenstore/xenstore_dev.c optional xenhvm
	dev/xen/xenstore/xenstored_dev.c optional xenhvm
	dev/xen/evtchn/evtchn_dev.c optional xenhvm
	dev/xen/privcmd/privcmd.c optional xenhvm
	dev/xen/gntdev/gntdev.c optional xenhvm
	dev/xen/debug/debug.c optional xenhvm
	dev/xl/if_xl.c optional xl pci
	dev/xl/xlphy.c optional xl pci
	fs/autofs/autofs.c optional autofs
	fs/autofs/autofs_vfsops.c optional autofs
	fs/autofs/autofs_vnops.c optional autofs
	fs/deadfs/dead_vnops.c standard
	fs/devfs/devfs_devs.c standard
	fs/devfs/devfs_dir.c standard
	fs/devfs/devfs_rule.c standard
	fs/devfs/devfs_vfsops.c standard
	fs/devfs/devfs_vnops.c standard
	fs/fdescfs/fdesc_vfsops.c optional fdescfs
	fs/fdescfs/fdesc_vnops.c optional fdescfs
	fs/fifofs/fifo_vnops.c standard
	fs/cuse/cuse.c optional cuse
	fs/fuse/fuse_device.c optional fusefs
	fs/fuse/fuse_file.c optional fusefs
	fs/fuse/fuse_internal.c optional fusefs
	fs/fuse/fuse_io.c optional fusefs
	fs/fuse/fuse_ipc.c optional fusefs
	fs/fuse/fuse_main.c optional fusefs
	fs/fuse/fuse_node.c optional fusefs
	fs/fuse/fuse_vfsops.c optional fusefs
	fs/fuse/fuse_vnops.c optional fusefs
	fs/mntfs/mntfs_vnops.c standard
	fs/msdosfs/msdosfs_conv.c optional msdosfs
	fs/msdosfs/msdosfs_denode.c optional msdosfs
	fs/msdosfs/msdosfs_fat.c optional msdosfs
	fs/msdosfs/msdosfs_iconv.c optional msdosfs_iconv
	fs/msdosfs/msdosfs_lookup.c optional msdosfs
	fs/msdosfs/msdosfs_vfsops.c optional msdosfs
	fs/msdosfs/msdosfs_vnops.c optional msdosfs
	fs/nfs/nfs_commonkrpc.c optional nfscl \| nfslockd \| nfsd
	fs/nfs/nfs_commonsubs.c optional nfscl \| nfslockd \| nfsd
	fs/nfs/nfs_commonport.c optional nfscl \| nfslockd \| nfsd
	fs/nfs/nfs_commonacl.c optional nfscl \| nfslockd \| nfsd
	fs/nfsclient/nfs_clcomsubs.c optional nfscl
	fs/nfsclient/nfs_clsubs.c optional nfscl
	fs/nfsclient/nfs_clstate.c optional nfscl
	fs/nfsclient/nfs_clkrpc.c optional nfscl
	fs/nfsclient/nfs_clrpcops.c optional nfscl
	fs/nfsclient/nfs_clvnops.c optional nfscl
	fs/nfsclient/nfs_clnode.c optional nfscl
	fs/nfsclient/nfs_clvfsops.c optional nfscl
	fs/nfsclient/nfs_clport.c optional nfscl
	fs/nfsclient/nfs_clbio.c optional nfscl
	fs/nfsclient/nfs_clnfsiod.c optional nfscl
	fs/nfsserver/nfs_fha_new.c optional nfsd inet
	fs/nfsserver/nfs_nfsdsocket.c optional nfsd inet
	fs/nfsserver/nfs_nfsdsubs.c optional nfsd inet
	fs/nfsserver/nfs_nfsdstate.c optional nfsd inet
	fs/nfsserver/nfs_nfsdkrpc.c optional nfsd inet
	fs/nfsserver/nfs_nfsdserv.c optional nfsd inet
	fs/nfsserver/nfs_nfsdport.c optional nfsd inet
	fs/nfsserver/nfs_nfsdcache.c optional nfsd inet
	fs/nullfs/null_subr.c optional nullfs
	fs/nullfs/null_vfsops.c optional nullfs
	fs/nullfs/null_vnops.c optional nullfs
	fs/procfs/procfs.c optional procfs
	fs/procfs/procfs_dbregs.c optional procfs
	fs/procfs/procfs_fpregs.c optional procfs
	fs/procfs/procfs_map.c optional procfs
	fs/procfs/procfs_mem.c optional procfs
	fs/procfs/procfs_note.c optional procfs
	fs/procfs/procfs_osrel.c optional procfs
	fs/procfs/procfs_regs.c optional procfs
	fs/procfs/procfs_rlimit.c optional procfs
	fs/procfs/procfs_status.c optional procfs
	fs/procfs/procfs_type.c optional procfs
	fs/pseudofs/pseudofs.c optional pseudofs
	fs/pseudofs/pseudofs_fileno.c optional pseudofs
	fs/pseudofs/pseudofs_vncache.c optional pseudofs
	fs/pseudofs/pseudofs_vnops.c optional pseudofs
	fs/smbfs/smbfs_io.c optional smbfs
	fs/smbfs/smbfs_node.c optional smbfs
	fs/smbfs/smbfs_smb.c optional smbfs
	fs/smbfs/smbfs_subr.c optional smbfs
	fs/smbfs/smbfs_vfsops.c optional smbfs
	fs/smbfs/smbfs_vnops.c optional smbfs
	fs/udf/osta.c optional udf
	fs/udf/udf_iconv.c optional udf_iconv
	fs/udf/udf_vfsops.c optional udf
	fs/udf/udf_vnops.c optional udf
	fs/unionfs/union_subr.c optional unionfs
	fs/unionfs/union_vfsops.c optional unionfs
	fs/unionfs/union_vnops.c optional unionfs
	fs/tmpfs/tmpfs_vnops.c optional tmpfs
	fs/tmpfs/tmpfs_fifoops.c optional tmpfs
	fs/tmpfs/tmpfs_vfsops.c optional tmpfs
	fs/tmpfs/tmpfs_subr.c optional tmpfs
	gdb/gdb_cons.c optional gdb
	gdb/gdb_main.c optional gdb
	gdb/gdb_packet.c optional gdb
	gdb/netgdb.c optional ddb debugnet gdb netgdb inet
	geom/bde/g_bde.c optional geom_bde
	geom/bde/g_bde_crypt.c optional geom_bde
	geom/bde/g_bde_lock.c optional geom_bde
	geom/bde/g_bde_work.c optional geom_bde
	geom/cache/g_cache.c optional geom_cache
	geom/concat/g_concat.c optional geom_concat
	geom/eli/g_eli.c optional geom_eli
	geom/eli/g_eli_crypto.c optional geom_eli
	geom/eli/g_eli_ctl.c optional geom_eli
	geom/eli/g_eli_hmac.c optional geom_eli
	geom/eli/g_eli_integrity.c optional geom_eli
	geom/eli/g_eli_key.c optional geom_eli
	geom/eli/g_eli_key_cache.c optional geom_eli
	geom/eli/g_eli_privacy.c optional geom_eli
	geom/eli/pkcs5v2.c optional geom_eli
	geom/gate/g_gate.c optional geom_gate
	geom/geom_bsd_enc.c optional geom_part_bsd
	geom/geom_ccd.c optional ccd \| geom_ccd
	geom/geom_ctl.c standard
	geom/geom_dev.c standard
	geom/geom_disk.c standard
	geom/geom_dump.c standard
	geom/geom_event.c standard
	geom/geom_flashmap.c optional fdt cfi \| fdt mx25l \| mmcsd \| fdt n25q \| fdt at45d
	geom/geom_io.c standard
	geom/geom_kern.c standard
	geom/geom_map.c optional geom_map
	geom/geom_redboot.c optional geom_redboot
	geom/geom_slice.c standard
	geom/geom_subr.c standard
	geom/geom_vfs.c standard
	geom/journal/g_journal.c optional geom_journal
	geom/journal/g_journal_ufs.c optional geom_journal
	geom/label/g_label.c optional geom_label \| geom_label_gpt
	geom/label/g_label_ext2fs.c optional geom_label
	geom/label/g_label_flashmap.c optional geom_label
	geom/label/g_label_iso9660.c optional geom_label
	geom/label/g_label_msdosfs.c optional geom_label
	geom/label/g_label_ntfs.c optional geom_label
	geom/label/g_label_reiserfs.c optional geom_label
	geom/label/g_label_ufs.c optional geom_label
	geom/label/g_label_gpt.c optional geom_label \| geom_label_gpt
	geom/label/g_label_disk_ident.c optional geom_label
	geom/linux_lvm/g_linux_lvm.c optional geom_linux_lvm
	geom/mirror/g_mirror.c optional geom_mirror
	geom/mirror/g_mirror_ctl.c optional geom_mirror
	geom/mountver/g_mountver.c optional geom_mountver
	geom/multipath/g_multipath.c optional geom_multipath
	geom/nop/g_nop.c optional geom_nop
	geom/part/g_part.c standard
	geom/part/g_part_if.m standard
	geom/part/g_part_apm.c optional geom_part_apm
	geom/part/g_part_bsd.c optional geom_part_bsd
	geom/part/g_part_bsd64.c optional geom_part_bsd64
	geom/part/g_part_ebr.c optional geom_part_ebr
	geom/part/g_part_gpt.c optional geom_part_gpt
	geom/part/g_part_ldm.c optional geom_part_ldm
	geom/part/g_part_mbr.c optional geom_part_mbr
	geom/part/g_part_vtoc8.c optional geom_part_vtoc8
	geom/raid/g_raid.c optional geom_raid
	geom/raid/g_raid_ctl.c optional geom_raid
	geom/raid/g_raid_md_if.m optional geom_raid
	geom/raid/g_raid_tr_if.m optional geom_raid
	geom/raid/md_ddf.c optional geom_raid
	geom/raid/md_intel.c optional geom_raid
	geom/raid/md_jmicron.c optional geom_raid
	geom/raid/md_nvidia.c optional geom_raid
	geom/raid/md_promise.c optional geom_raid
	geom/raid/md_sii.c optional geom_raid
	geom/raid/tr_concat.c optional geom_raid
	geom/raid/tr_raid0.c optional geom_raid
	geom/raid/tr_raid1.c optional geom_raid
	geom/raid/tr_raid1e.c optional geom_raid
	geom/raid/tr_raid5.c optional geom_raid
	geom/raid3/g_raid3.c optional geom_raid3
	geom/raid3/g_raid3_ctl.c optional geom_raid3
	geom/shsec/g_shsec.c optional geom_shsec
	geom/stripe/g_stripe.c optional geom_stripe
	geom/uzip/g_uzip.c optional geom_uzip
	geom/uzip/g_uzip_lzma.c optional geom_uzip
	geom/uzip/g_uzip_wrkthr.c optional geom_uzip
	geom/uzip/g_uzip_zlib.c optional geom_uzip
	geom/uzip/g_uzip_zstd.c optional geom_uzip zstdio \
	compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd"
	geom/vinum/geom_vinum.c optional geom_vinum
	geom/vinum/geom_vinum_create.c optional geom_vinum
	geom/vinum/geom_vinum_drive.c optional geom_vinum
	geom/vinum/geom_vinum_plex.c optional geom_vinum
	geom/vinum/geom_vinum_volume.c optional geom_vinum
	geom/vinum/geom_vinum_subr.c optional geom_vinum
	geom/vinum/geom_vinum_raid5.c optional geom_vinum
	geom/vinum/geom_vinum_share.c optional geom_vinum
	geom/vinum/geom_vinum_list.c optional geom_vinum
	geom/vinum/geom_vinum_rm.c optional geom_vinum
	geom/vinum/geom_vinum_init.c optional geom_vinum
	geom/vinum/geom_vinum_state.c optional geom_vinum
	geom/vinum/geom_vinum_rename.c optional geom_vinum
	geom/vinum/geom_vinum_move.c optional geom_vinum
	geom/vinum/geom_vinum_events.c optional geom_vinum
	geom/virstor/binstream.c optional geom_virstor
	geom/virstor/g_virstor.c optional geom_virstor
	geom/virstor/g_virstor_md.c optional geom_virstor
	geom/zero/g_zero.c optional geom_zero
	fs/ext2fs/ext2_acl.c optional ext2fs
	fs/ext2fs/ext2_alloc.c optional ext2fs
	fs/ext2fs/ext2_balloc.c optional ext2fs
	fs/ext2fs/ext2_bmap.c optional ext2fs
	fs/ext2fs/ext2_csum.c optional ext2fs
	fs/ext2fs/ext2_extattr.c optional ext2fs
	fs/ext2fs/ext2_extents.c optional ext2fs
	fs/ext2fs/ext2_inode.c optional ext2fs
	fs/ext2fs/ext2_inode_cnv.c optional ext2fs
	fs/ext2fs/ext2_hash.c optional ext2fs
	fs/ext2fs/ext2_htree.c optional ext2fs
	fs/ext2fs/ext2_lookup.c optional ext2fs
	fs/ext2fs/ext2_subr.c optional ext2fs
	fs/ext2fs/ext2_vfsops.c optional ext2fs
	fs/ext2fs/ext2_vnops.c optional ext2fs
	#
	isa/isa_if.m standard
	isa/isa_common.c optional isa
	isa/isahint.c optional isa
	isa/pnp.c optional isa isapnp
	isa/pnpparse.c optional isa isapnp
	fs/cd9660/cd9660_bmap.c optional cd9660
	fs/cd9660/cd9660_lookup.c optional cd9660
	fs/cd9660/cd9660_node.c optional cd9660
	fs/cd9660/cd9660_rrip.c optional cd9660
	fs/cd9660/cd9660_util.c optional cd9660
	fs/cd9660/cd9660_vfsops.c optional cd9660
	fs/cd9660/cd9660_vnops.c optional cd9660
	fs/cd9660/cd9660_iconv.c optional cd9660_iconv
	gnu/gcov/gcc_4_7.c optional gcov \
	warning "kernel contains GPL licensed gcov support"
	gnu/gcov/gcov_fs.c optional gcov lindebugfs \
	compile-with "${LINUXKPI_C}"
	gnu/gcov/gcov_subr.c optional gcov

	kern/bus_if.m standard
	kern/clock_if.m standard
	kern/cpufreq_if.m standard
	kern/device_if.m standard
	kern/imgact_binmisc.c optional imagact_binmisc
	kern/imgact_elf.c standard
	kern/imgact_elf32.c optional compat_freebsd32
	kern/imgact_shell.c standard
	kern/init_main.c standard
	kern/init_sysent.c standard
	kern/ksched.c optional _kposix_priority_scheduling
	kern/kern_acct.c standard
	kern/kern_alq.c optional alq
	kern/kern_clock.c standard
	kern/kern_condvar.c standard
	kern/kern_conf.c standard
	kern/kern_cons.c standard
	kern/kern_cpu.c standard
	kern/kern_cpuset.c standard
	kern/kern_context.c standard
	kern/kern_descrip.c standard
	kern/kern_dtrace.c optional kdtrace_hooks
	kern/kern_dump.c standard
	kern/kern_environment.c standard
	kern/kern_et.c standard
	kern/kern_event.c standard
	kern/kern_exec.c standard
	kern/kern_exit.c standard
	kern/kern_fail.c standard
	kern/kern_ffclock.c standard
	kern/kern_fork.c standard
	kern/kern_hhook.c standard
	kern/kern_idle.c standard
	kern/kern_intr.c standard
	kern/kern_jail.c standard
	kern/kern_kcov.c optional kcov \
	compile-with "${NORMAL_C:N-fsanitize*}"
	kern/kern_khelp.c standard
	kern/kern_kthread.c standard
	kern/kern_ktr.c optional ktr
	kern/kern_ktrace.c standard
	kern/kern_linker.c standard
	kern/kern_lock.c standard
	kern/kern_lockf.c standard
	kern/kern_lockstat.c optional kdtrace_hooks
	kern/kern_loginclass.c standard
	kern/kern_malloc.c standard
	kern/kern_mbuf.c standard
	kern/kern_mib.c standard
	kern/kern_module.c standard
	kern/kern_mtxpool.c standard
	kern/kern_mutex.c standard
	kern/kern_ntptime.c standard
	kern/kern_osd.c standard
	kern/kern_physio.c standard
	kern/kern_pmc.c standard
	kern/kern_poll.c optional device_polling
	kern/kern_priv.c standard
	kern/kern_proc.c standard
	kern/kern_procctl.c standard
	kern/kern_prot.c standard
	kern/kern_racct.c standard
	kern/kern_rangelock.c standard
	kern/kern_rctl.c standard
	kern/kern_resource.c standard
	kern/kern_rmlock.c standard
	kern/kern_rwlock.c standard
	kern/kern_sdt.c optional kdtrace_hooks
	kern/kern_sema.c standard
	kern/kern_sendfile.c standard
	kern/kern_sharedpage.c standard
	kern/kern_shutdown.c standard
	kern/kern_sig.c standard
	kern/kern_switch.c standard
	kern/kern_sx.c standard
	kern/kern_synch.c standard
	kern/kern_syscalls.c standard
	kern/kern_sysctl.c standard
	kern/kern_tc.c standard
	kern/kern_thr.c standard
	kern/kern_thread.c standard
	kern/kern_time.c standard
	kern/kern_timeout.c standard
	kern/kern_tslog.c optional tslog
	kern/kern_ubsan.c optional kubsan
	kern/kern_umtx.c standard
	kern/kern_uuid.c standard
	kern/kern_xxx.c standard
	kern/link_elf.c standard
	kern/linker_if.m standard
	kern/md4c.c optional netsmb
	kern/md5c.c standard
	kern/p1003_1b.c standard
	kern/posix4_mib.c standard
	kern/sched_4bsd.c optional sched_4bsd
	kern/sched_ule.c optional sched_ule
	kern/serdev_if.m standard
	kern/stack_protector.c standard \
	compile-with "${NORMAL_C:N-fstack-protector*}"
	kern/subr_acl_nfs4.c optional ufs_acl \| zfs
	kern/subr_acl_posix1e.c optional ufs_acl
	kern/subr_autoconf.c standard
	kern/subr_blist.c standard
	kern/subr_boot.c standard
	kern/subr_bus.c standard
	kern/subr_bus_dma.c standard
	kern/subr_bufring.c standard
	kern/subr_capability.c standard
	kern/subr_clock.c standard
	kern/subr_compressor.c standard \
	compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd"
	kern/subr_coverage.c optional coverage \
	compile-with "${NORMAL_C:N-fsanitize*}"
	kern/subr_counter.c standard
	kern/subr_csan.c optional kcsan \
	compile-with "${NORMAL_C:N-fsanitize*}"
	kern/subr_devstat.c standard
	kern/subr_disk.c standard
	kern/subr_early.c standard
	kern/subr_epoch.c standard
	kern/subr_eventhandler.c standard
	kern/subr_fattime.c standard
	kern/subr_firmware.c optional firmware
	kern/subr_filter.c standard
	kern/subr_gtaskqueue.c standard
	kern/subr_hash.c standard
	kern/subr_hints.c standard
	kern/subr_kdb.c standard
	kern/subr_kobj.c standard
	kern/subr_lock.c standard
	kern/subr_log.c standard
	kern/subr_mchain.c optional libmchain
	kern/subr_module.c standard
	kern/subr_msgbuf.c standard
	kern/subr_param.c standard
	kern/subr_pcpu.c standard
	kern/subr_pctrie.c standard
	kern/subr_pidctrl.c standard
	kern/subr_power.c standard
	kern/subr_prf.c standard
	kern/subr_prof.c standard
	kern/subr_rangeset.c standard
	kern/subr_rman.c standard
	kern/subr_rtc.c standard
	kern/subr_sbuf.c standard
	kern/subr_scanf.c standard
	kern/subr_sglist.c standard
	kern/subr_sleepqueue.c standard
	kern/subr_smp.c standard
	kern/subr_smr.c standard
	kern/subr_stack.c optional ddb \| stack \| ktr
	kern/subr_stats.c optional stats
	kern/subr_taskqueue.c standard
	kern/subr_terminal.c optional vt
	kern/subr_trap.c standard
	kern/subr_turnstile.c standard
	kern/subr_uio.c standard
	kern/subr_unit.c standard
	kern/subr_vmem.c standard
	kern/subr_witness.c optional witness
	kern/sys_capability.c standard
	kern/sys_generic.c standard
	kern/sys_getrandom.c standard
	kern/sys_pipe.c standard
	kern/sys_procdesc.c standard
	kern/sys_process.c standard
	kern/sys_socket.c standard
	kern/syscalls.c standard
	kern/sysv_ipc.c standard
	kern/sysv_msg.c optional sysvmsg
	kern/sysv_sem.c optional sysvsem
	kern/sysv_shm.c optional sysvshm
	kern/tty.c standard
	kern/tty_compat.c optional compat_43tty
	kern/tty_info.c standard
	kern/tty_inq.c standard
	kern/tty_outq.c standard
	kern/tty_pts.c standard
	kern/tty_tty.c standard
	kern/tty_ttydisc.c standard
	kern/uipc_accf.c standard
	kern/uipc_debug.c optional ddb
	kern/uipc_domain.c standard
	kern/uipc_ktls.c optional kern_tls
	kern/uipc_mbuf.c standard
	kern/uipc_mbuf2.c standard
	kern/uipc_mbufhash.c standard
	kern/uipc_mqueue.c optional p1003_1b_mqueue
	kern/uipc_sem.c optional p1003_1b_semaphores
	kern/uipc_shm.c standard
	kern/uipc_sockbuf.c standard
	kern/uipc_socket.c standard
	kern/uipc_syscalls.c standard
	kern/uipc_usrreq.c standard
	kern/vfs_acl.c standard
	kern/vfs_aio.c standard
	kern/vfs_bio.c standard
	kern/vfs_cache.c standard
	kern/vfs_cluster.c standard
	kern/vfs_default.c standard
	kern/vfs_export.c standard
	kern/vfs_extattr.c standard
	kern/vfs_hash.c standard
	kern/vfs_init.c standard
	kern/vfs_lookup.c standard
	kern/vfs_mount.c standard
	kern/vfs_mountroot.c standard
	kern/vfs_subr.c standard
	kern/vfs_syscalls.c standard
	kern/vfs_vnops.c standard
	#
	# Kernel GSS-API
	#
	gssd.h optional kgssapi \
	dependency "$S/kgssapi/gssd.x" \
	compile-with "RPCGEN_CPP='${CPP}' rpcgen -hM $S/kgssapi/gssd.x \| grep -v pthread.h > gssd.h" \
	no-obj no-implicit-rule before-depend local \
	clean "gssd.h"
	gssd_xdr.c optional kgssapi \
	dependency "$S/kgssapi/gssd.x gssd.h" \
	compile-with "RPCGEN_CPP='${CPP}' rpcgen -c $S/kgssapi/gssd.x -o gssd_xdr.c" \
	no-implicit-rule before-depend local \
	clean "gssd_xdr.c"
	gssd_clnt.c optional kgssapi \
	dependency "$S/kgssapi/gssd.x gssd.h" \
	compile-with "RPCGEN_CPP='${CPP}' rpcgen -lM $S/kgssapi/gssd.x \| grep -v string.h > gssd_clnt.c" \
	no-implicit-rule before-depend local \
	clean "gssd_clnt.c"
	kgssapi/gss_accept_sec_context.c optional kgssapi
	kgssapi/gss_add_oid_set_member.c optional kgssapi
	kgssapi/gss_acquire_cred.c optional kgssapi
	kgssapi/gss_canonicalize_name.c optional kgssapi
	kgssapi/gss_create_empty_oid_set.c optional kgssapi
	kgssapi/gss_delete_sec_context.c optional kgssapi
	kgssapi/gss_display_status.c optional kgssapi
	kgssapi/gss_export_name.c optional kgssapi
	kgssapi/gss_get_mic.c optional kgssapi
	kgssapi/gss_init_sec_context.c optional kgssapi
	kgssapi/gss_impl.c optional kgssapi
	kgssapi/gss_import_name.c optional kgssapi
	kgssapi/gss_names.c optional kgssapi
	kgssapi/gss_pname_to_uid.c optional kgssapi
	kgssapi/gss_release_buffer.c optional kgssapi
	kgssapi/gss_release_cred.c optional kgssapi
	kgssapi/gss_release_name.c optional kgssapi
	kgssapi/gss_release_oid_set.c optional kgssapi
	kgssapi/gss_set_cred_option.c optional kgssapi
	kgssapi/gss_test_oid_set_member.c optional kgssapi
	kgssapi/gss_unwrap.c optional kgssapi
	kgssapi/gss_verify_mic.c optional kgssapi
	kgssapi/gss_wrap.c optional kgssapi
	kgssapi/gss_wrap_size_limit.c optional kgssapi
	kgssapi/gssd_prot.c optional kgssapi
	kgssapi/krb5/krb5_mech.c optional kgssapi
	kgssapi/krb5/kcrypto.c optional kgssapi
	kgssapi/krb5/kcrypto_aes.c optional kgssapi
	kgssapi/kgss_if.m optional kgssapi
	kgssapi/gsstest.c optional kgssapi_debug
	# These files in libkern/ are those needed by all architectures. Some
	# of the files in libkern/ are only needed on some architectures, e.g.,
	# libkern/divdi3.c is needed by i386 but not alpha. Also, some of these
	# routines may be optimized for a particular platform. In either case,
	# the file should be moved to conf/files.<arch> from here.
	#
	libkern/arc4random.c standard
	libkern/arc4random_uniform.c standard
	libkern/asprintf.c standard
	libkern/bcd.c standard
	libkern/bsearch.c standard
	libkern/explicit_bzero.c standard
	libkern/fnmatch.c standard
	libkern/gsb_crc32.c standard
	libkern/iconv.c optional libiconv
	libkern/iconv_converter_if.m optional libiconv
	libkern/iconv_ucs.c optional libiconv
	libkern/iconv_xlat.c optional libiconv
	libkern/iconv_xlat16.c optional libiconv
	libkern/inet_aton.c standard
	libkern/inet_ntoa.c standard
	libkern/inet_ntop.c standard
	libkern/inet_pton.c standard
	libkern/jenkins_hash.c standard
	libkern/murmur3_32.c standard
	libkern/mcount.c optional profiling-routine
	libkern/memcchr.c standard
	libkern/memchr.c standard
	libkern/memmem.c optional gdb
	libkern/qsort.c standard
	libkern/qsort_r.c standard
	libkern/random.c standard
	libkern/scanc.c standard
	libkern/strcasecmp.c standard
	libkern/strcat.c standard
	libkern/strchr.c standard
	libkern/strchrnul.c optional gdb
	libkern/strcmp.c standard
	libkern/strcpy.c standard
	libkern/strcspn.c standard
	libkern/strdup.c standard
	libkern/strndup.c standard
	libkern/strlcat.c standard
	libkern/strlcpy.c standard
	libkern/strlen.c standard
	libkern/strncat.c standard
	libkern/strncmp.c standard
	libkern/strncpy.c standard
	libkern/strnlen.c standard
	libkern/strrchr.c standard
	libkern/strsep.c standard
	libkern/strspn.c standard
	libkern/strstr.c standard
	libkern/strtol.c standard
	libkern/strtoq.c standard
	libkern/strtoul.c standard
	libkern/strtouq.c standard
	libkern/strvalid.c standard
	libkern/timingsafe_bcmp.c standard
	contrib/zlib/adler32.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	contrib/zlib/compress.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib \
	compile-with "${NORMAL_C} -Wno-cast-qual"
	contrib/zlib/crc32.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	contrib/zlib/deflate.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib \
	compile-with "${NORMAL_C} -Wno-cast-qual"
	contrib/zlib/inffast.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	contrib/zlib/inflate.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	contrib/zlib/inftrees.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	contrib/zlib/trees.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	contrib/zlib/uncompr.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib \
	compile-with "${NORMAL_C} -Wno-cast-qual"
	contrib/zlib/zutil.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	dev/zlib/zlib_mod.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	dev/zlib/zcalloc.c optional crypto \| geom_uzip \| ipsec \| \
	ipsec_support \| mxge \| ddb_ctf \| gzio \| zfs \| zlib
	net/altq/altq_cbq.c optional altq
	net/altq/altq_codel.c optional altq
	net/altq/altq_hfsc.c optional altq
	net/altq/altq_fairq.c optional altq
	net/altq/altq_priq.c optional altq
	net/altq/altq_red.c optional altq
	net/altq/altq_rio.c optional altq
	net/altq/altq_rmclass.c optional altq
	net/altq/altq_subr.c optional altq
	net/bpf.c standard
	net/bpf_buffer.c optional bpf
	net/bpf_jitter.c optional bpf_jitter
	net/bpf_filter.c optional bpf \| netgraph_bpf
	net/bpf_zerocopy.c optional bpf
	net/bridgestp.c optional bridge \| if_bridge
	net/ieee8023ad_lacp.c optional lagg
	net/if.c standard
	net/if_bridge.c optional bridge inet \| if_bridge inet
	net/if_clone.c standard
	net/if_dead.c standard
	net/if_debug.c optional ddb
	net/if_disc.c optional disc
	net/if_edsc.c optional edsc
	net/if_enc.c optional enc inet \| enc inet6
	net/if_epair.c optional epair
	net/if_ethersubr.c optional ether
	net/if_fwsubr.c optional fwip
	net/if_gif.c optional gif inet \| gif inet6 \| \
	netgraph_gif inet \| netgraph_gif inet6
	net/if_gre.c optional gre inet \| gre inet6
	net/if_ipsec.c optional inet ipsec \| inet6 ipsec
	net/if_lagg.c optional lagg
	net/if_loop.c optional loop
	net/if_llatbl.c standard
	net/if_me.c optional me inet
	net/if_media.c standard
	net/if_mib.c standard
	net/if_spppfr.c optional sppp \| netgraph_sppp
	net/if_spppsubr.c optional sppp \| netgraph_sppp
	net/if_stf.c optional stf inet inet6
	net/if_tuntap.c optional tuntap
	net/if_vlan.c optional vlan
	net/if_vxlan.c optional vxlan inet \| vxlan inet6
	net/ifdi_if.m optional ether pci iflib
	net/iflib.c optional ether pci iflib
	net/iflib_clone.c optional ether pci iflib
	net/mp_ring.c optional ether iflib
	net/mppcc.c optional netgraph_mppc_compression
	net/mppcd.c optional netgraph_mppc_compression
	net/netisr.c standard
	net/debugnet.c optional inet debugnet
	net/debugnet_inet.c optional inet debugnet
	net/pfil.c optional ether \| inet
	net/radix.c standard
	net/radix_mpath.c standard
	net/raw_cb.c standard
	net/raw_usrreq.c standard
	net/route.c standard
	net/route/nhop.c standard
	net/route/nhop_ctl.c standard
	net/route/nhop_utils.c standard
	net/route/route_ctl.c standard
	net/route/route_ddb.c optional ddb
	net/route/route_helpers.c standard
	net/route/route_temporal.c standard
	net/rss_config.c optional inet rss \| inet6 rss
	net/rtsock.c standard
	net/slcompress.c optional netgraph_vjc \| sppp \| \
	netgraph_sppp
	net/toeplitz.c optional inet rss \| inet6 rss
	net/vnet.c optional vimage
	net80211/ieee80211.c optional wlan
	net80211/ieee80211_acl.c optional wlan wlan_acl
	net80211/ieee80211_action.c optional wlan
	net80211/ieee80211_adhoc.c optional wlan \
	compile-with "${NORMAL_C} -Wno-unused-function"
	net80211/ieee80211_ageq.c optional wlan
	net80211/ieee80211_amrr.c optional wlan \| wlan_amrr
	net80211/ieee80211_crypto.c optional wlan \
	compile-with "${NORMAL_C} -Wno-unused-function"
	net80211/ieee80211_crypto_ccmp.c optional wlan wlan_ccmp
	net80211/ieee80211_crypto_none.c optional wlan
	net80211/ieee80211_crypto_tkip.c optional wlan wlan_tkip
	net80211/ieee80211_crypto_wep.c optional wlan wlan_wep
	net80211/ieee80211_ddb.c optional wlan ddb
	net80211/ieee80211_dfs.c optional wlan
	net80211/ieee80211_freebsd.c optional wlan
	net80211/ieee80211_hostap.c optional wlan \
	compile-with "${NORMAL_C} -Wno-unused-function"
	net80211/ieee80211_ht.c optional wlan
	net80211/ieee80211_hwmp.c optional wlan ieee80211_support_mesh
	net80211/ieee80211_input.c optional wlan
	net80211/ieee80211_ioctl.c optional wlan
	net80211/ieee80211_mesh.c optional wlan ieee80211_support_mesh \
	compile-with "${NORMAL_C} -Wno-unused-function"
	net80211/ieee80211_monitor.c optional wlan
	net80211/ieee80211_node.c optional wlan
	net80211/ieee80211_output.c optional wlan
	net80211/ieee80211_phy.c optional wlan
	net80211/ieee80211_power.c optional wlan
	net80211/ieee80211_proto.c optional wlan
	net80211/ieee80211_radiotap.c optional wlan
	net80211/ieee80211_ratectl.c optional wlan
	net80211/ieee80211_ratectl_none.c optional wlan
	net80211/ieee80211_regdomain.c optional wlan
	net80211/ieee80211_rssadapt.c optional wlan wlan_rssadapt
	net80211/ieee80211_scan.c optional wlan
	net80211/ieee80211_scan_sta.c optional wlan
	net80211/ieee80211_sta.c optional wlan \
	compile-with "${NORMAL_C} -Wno-unused-function"
	net80211/ieee80211_superg.c optional wlan ieee80211_support_superg
	net80211/ieee80211_scan_sw.c optional wlan
	net80211/ieee80211_tdma.c optional wlan ieee80211_support_tdma
	net80211/ieee80211_vht.c optional wlan
	net80211/ieee80211_wds.c optional wlan
	net80211/ieee80211_xauth.c optional wlan wlan_xauth
	net80211/ieee80211_alq.c optional wlan ieee80211_alq
	netgraph/atm/ccatm/ng_ccatm.c optional ngatm_ccatm \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	netgraph/atm/ngatmbase.c optional ngatm_atmbase \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	netgraph/atm/sscfu/ng_sscfu.c optional ngatm_sscfu \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	netgraph/atm/sscop/ng_sscop.c optional ngatm_sscop \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	netgraph/atm/uni/ng_uni.c optional ngatm_uni \
	compile-with "${NORMAL_C} -I$S/contrib/ngatm"
	netgraph/bluetooth/common/ng_bluetooth.c optional netgraph_bluetooth
	netgraph/bluetooth/drivers/bt3c/ng_bt3c_pccard.c optional netgraph_bluetooth_bt3c
	netgraph/bluetooth/drivers/h4/ng_h4.c optional netgraph_bluetooth_h4
	netgraph/bluetooth/drivers/ubt/ng_ubt.c optional netgraph_bluetooth_ubt usb
	netgraph/bluetooth/drivers/ubt/ng_ubt_intel.c optional netgraph_bluetooth_ubt usb
	netgraph/bluetooth/drivers/ubtbcmfw/ubtbcmfw.c optional netgraph_bluetooth_ubtbcmfw usb
	netgraph/bluetooth/hci/ng_hci_cmds.c optional netgraph_bluetooth_hci
	netgraph/bluetooth/hci/ng_hci_evnt.c optional netgraph_bluetooth_hci
	netgraph/bluetooth/hci/ng_hci_main.c optional netgraph_bluetooth_hci
	netgraph/bluetooth/hci/ng_hci_misc.c optional netgraph_bluetooth_hci
	netgraph/bluetooth/hci/ng_hci_ulpi.c optional netgraph_bluetooth_hci
	netgraph/bluetooth/l2cap/ng_l2cap_cmds.c optional netgraph_bluetooth_l2cap
	netgraph/bluetooth/l2cap/ng_l2cap_evnt.c optional netgraph_bluetooth_l2cap
	netgraph/bluetooth/l2cap/ng_l2cap_llpi.c optional netgraph_bluetooth_l2cap
	netgraph/bluetooth/l2cap/ng_l2cap_main.c optional netgraph_bluetooth_l2cap
	netgraph/bluetooth/l2cap/ng_l2cap_misc.c optional netgraph_bluetooth_l2cap
	netgraph/bluetooth/l2cap/ng_l2cap_ulpi.c optional netgraph_bluetooth_l2cap
	netgraph/bluetooth/socket/ng_btsocket.c optional netgraph_bluetooth_socket
	netgraph/bluetooth/socket/ng_btsocket_hci_raw.c optional netgraph_bluetooth_socket
	netgraph/bluetooth/socket/ng_btsocket_l2cap.c optional netgraph_bluetooth_socket
	netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c optional netgraph_bluetooth_socket
	netgraph/bluetooth/socket/ng_btsocket_rfcomm.c optional netgraph_bluetooth_socket
	netgraph/bluetooth/socket/ng_btsocket_sco.c optional netgraph_bluetooth_socket
	netgraph/netflow/netflow.c optional netgraph_netflow
	netgraph/netflow/netflow_v9.c optional netgraph_netflow
	netgraph/netflow/ng_netflow.c optional netgraph_netflow
	netgraph/ng_UI.c optional netgraph_UI
	netgraph/ng_async.c optional netgraph_async
	netgraph/ng_atmllc.c optional netgraph_atmllc
	netgraph/ng_base.c optional netgraph
	netgraph/ng_bpf.c optional netgraph_bpf
	netgraph/ng_bridge.c optional netgraph_bridge
	netgraph/ng_car.c optional netgraph_car
	netgraph/ng_checksum.c optional netgraph_checksum
	netgraph/ng_cisco.c optional netgraph_cisco
	netgraph/ng_deflate.c optional netgraph_deflate
	netgraph/ng_device.c optional netgraph_device
	netgraph/ng_echo.c optional netgraph_echo
	netgraph/ng_eiface.c optional netgraph_eiface
	netgraph/ng_ether.c optional netgraph_ether
	netgraph/ng_ether_echo.c optional netgraph_ether_echo
	netgraph/ng_frame_relay.c optional netgraph_frame_relay
	netgraph/ng_gif.c optional netgraph_gif inet6 \| netgraph_gif inet
	netgraph/ng_gif_demux.c optional netgraph_gif_demux
	netgraph/ng_hole.c optional netgraph_hole
	netgraph/ng_iface.c optional netgraph_iface
	netgraph/ng_ip_input.c optional netgraph_ip_input
	netgraph/ng_ipfw.c optional netgraph_ipfw inet ipfirewall
	netgraph/ng_ksocket.c optional netgraph_ksocket
	netgraph/ng_l2tp.c optional netgraph_l2tp
	netgraph/ng_lmi.c optional netgraph_lmi
	netgraph/ng_mppc.c optional netgraph_mppc_compression \| \
	netgraph_mppc_encryption
	netgraph/ng_nat.c optional netgraph_nat inet libalias
	netgraph/ng_one2many.c optional netgraph_one2many
	netgraph/ng_parse.c optional netgraph
	netgraph/ng_patch.c optional netgraph_patch
	netgraph/ng_pipe.c optional netgraph_pipe
	netgraph/ng_ppp.c optional netgraph_ppp
	netgraph/ng_pppoe.c optional netgraph_pppoe
	netgraph/ng_pptpgre.c optional netgraph_pptpgre
	netgraph/ng_pred1.c optional netgraph_pred1
	netgraph/ng_rfc1490.c optional netgraph_rfc1490
	netgraph/ng_socket.c optional netgraph_socket
	netgraph/ng_split.c optional netgraph_split
	netgraph/ng_sppp.c optional netgraph_sppp
	netgraph/ng_tag.c optional netgraph_tag
	netgraph/ng_tcpmss.c optional netgraph_tcpmss
	netgraph/ng_tee.c optional netgraph_tee
	netgraph/ng_tty.c optional netgraph_tty
	netgraph/ng_vjc.c optional netgraph_vjc
	netgraph/ng_vlan.c optional netgraph_vlan
	netinet/accf_data.c optional accept_filter_data inet
	netinet/accf_dns.c optional accept_filter_dns inet
	netinet/accf_http.c optional accept_filter_http inet
	netinet/if_ether.c optional inet ether
	netinet/igmp.c optional inet
	netinet/in.c optional inet
	netinet/in_debug.c optional inet ddb
	netinet/in_kdtrace.c optional inet \| inet6
	netinet/ip_carp.c optional inet carp \| inet6 carp
	netinet/in_fib.c optional inet
	netinet/in_gif.c optional gif inet \| netgraph_gif inet
	netinet/ip_gre.c optional gre inet
	netinet/ip_id.c optional inet
	netinet/in_jail.c optional inet
	netinet/in_mcast.c optional inet
	netinet/in_pcb.c optional inet \| inet6
	netinet/in_pcbgroup.c optional inet pcbgroup \| inet6 pcbgroup
	netinet/in_prot.c optional inet \| inet6
	netinet/in_proto.c optional inet \| inet6
	netinet/in_rmx.c optional inet
	netinet/in_rss.c optional inet rss
	netinet/ip_divert.c optional inet ipdivert ipfirewall
	netinet/ip_ecn.c optional inet \| inet6
	netinet/ip_encap.c optional inet \| inet6
	netinet/ip_fastfwd.c optional inet
	netinet/ip_icmp.c optional inet \| inet6
	netinet/ip_input.c optional inet
	netinet/ip_mroute.c optional mrouting inet
	netinet/ip_options.c optional inet
	netinet/ip_output.c optional inet
	netinet/ip_reass.c optional inet
	netinet/raw_ip.c optional inet \| inet6
	netinet/cc/cc.c optional inet \| inet6
	netinet/cc/cc_newreno.c optional inet \| inet6
	netinet/sctp_asconf.c optional inet sctp \| inet6 sctp
	netinet/sctp_auth.c optional inet sctp \| inet6 sctp
	netinet/sctp_bsd_addr.c optional inet sctp \| inet6 sctp
	netinet/sctp_cc_functions.c optional inet sctp \| inet6 sctp
	netinet/sctp_crc32.c optional inet \| inet6
	netinet/sctp_indata.c optional inet sctp \| inet6 sctp
	netinet/sctp_input.c optional inet sctp \| inet6 sctp
	netinet/sctp_kdtrace.c optional inet sctp \| inet6 sctp
	netinet/sctp_output.c optional inet sctp \| inet6 sctp
	netinet/sctp_pcb.c optional inet sctp \| inet6 sctp
	netinet/sctp_peeloff.c optional inet sctp \| inet6 sctp
	netinet/sctp_ss_functions.c optional inet sctp \| inet6 sctp
	netinet/sctp_syscalls.c optional inet sctp \| inet6 sctp
	netinet/sctp_sysctl.c optional inet sctp \| inet6 sctp
	netinet/sctp_timer.c optional inet sctp \| inet6 sctp
	netinet/sctp_usrreq.c optional inet sctp \| inet6 sctp
	netinet/sctputil.c optional inet sctp \| inet6 sctp
	netinet/siftr.c optional inet siftr alq \| inet6 siftr alq
	netinet/tcp_debug.c optional tcpdebug
	netinet/tcp_fastopen.c optional inet tcp_rfc7413 \| inet6 tcp_rfc7413
	netinet/tcp_hostcache.c optional inet \| inet6
	netinet/tcp_input.c optional inet \| inet6
	netinet/tcp_log_buf.c optional tcp_blackbox inet \| tcp_blackbox inet6
	netinet/tcp_lro.c optional inet \| inet6
	netinet/tcp_output.c optional inet \| inet6
	netinet/tcp_offload.c optional tcp_offload inet \| tcp_offload inet6
	netinet/tcp_hpts.c optional tcphpts inet \| tcphpts inet6
	netinet/tcp_ratelimit.c optional ratelimit inet \| ratelimit inet6
	netinet/tcp_pcap.c optional inet tcppcap \| inet6 tcppcap \
	compile-with "${NORMAL_C} ${NO_WNONNULL}"
	netinet/tcp_reass.c optional inet \| inet6
	netinet/tcp_sack.c optional inet \| inet6
	netinet/tcp_stats.c optional stats inet \| stats inet6
	netinet/tcp_subr.c optional inet \| inet6
	netinet/tcp_syncache.c optional inet \| inet6
	netinet/tcp_timer.c optional inet \| inet6
	netinet/tcp_timewait.c optional inet \| inet6
	netinet/tcp_usrreq.c optional inet \| inet6
	netinet/udp_usrreq.c optional inet \| inet6
	netinet/libalias/alias.c optional libalias inet \| netgraph_nat inet
	netinet/libalias/alias_db.c optional libalias inet \| netgraph_nat inet
	netinet/libalias/alias_mod.c optional libalias \| netgraph_nat
	netinet/libalias/alias_proxy.c optional libalias inet \| netgraph_nat inet
	netinet/libalias/alias_util.c optional libalias inet \| netgraph_nat inet
	netinet/libalias/alias_sctp.c optional libalias inet \| netgraph_nat inet
	netinet/netdump/netdump_client.c optional inet debugnet netdump
	netinet6/dest6.c optional inet6
	netinet6/frag6.c optional inet6
	netinet6/icmp6.c optional inet6
	netinet6/in6.c optional inet6
	netinet6/in6_cksum.c optional inet6
	netinet6/in6_fib.c optional inet6
	netinet6/in6_gif.c optional gif inet6 \| netgraph_gif inet6
	netinet6/in6_ifattach.c optional inet6
	netinet6/in6_jail.c optional inet6
	netinet6/in6_mcast.c optional inet6
	netinet6/in6_pcb.c optional inet6
	netinet6/in6_pcbgroup.c optional inet6 pcbgroup
	netinet6/in6_proto.c optional inet6
	netinet6/in6_rmx.c optional inet6
	netinet6/in6_rss.c optional inet6 rss
	netinet6/in6_src.c optional inet6
	netinet6/ip6_fastfwd.c optional inet6
	netinet6/ip6_forward.c optional inet6
	netinet6/ip6_gre.c optional gre inet6
	netinet6/ip6_id.c optional inet6
	netinet6/ip6_input.c optional inet6
	netinet6/ip6_mroute.c optional mrouting inet6
	netinet6/ip6_output.c optional inet6
	netinet6/mld6.c optional inet6
	netinet6/nd6.c optional inet6
	netinet6/nd6_nbr.c optional inet6
	netinet6/nd6_rtr.c optional inet6
	netinet6/raw_ip6.c optional inet6
	netinet6/route6.c optional inet6
	netinet6/scope6.c optional inet6
	netinet6/sctp6_usrreq.c optional inet6 sctp
	netinet6/udp6_usrreq.c optional inet6
	netipsec/ipsec.c optional ipsec inet \| ipsec inet6
	netipsec/ipsec_input.c optional ipsec inet \| ipsec inet6
	netipsec/ipsec_mbuf.c optional ipsec inet \| ipsec inet6
	netipsec/ipsec_mod.c optional ipsec inet \| ipsec inet6
	netipsec/ipsec_output.c optional ipsec inet \| ipsec inet6
	netipsec/ipsec_pcb.c optional ipsec inet \| ipsec inet6 \| \
	ipsec_support inet \| ipsec_support inet6
	netipsec/key.c optional ipsec inet \| ipsec inet6 \| \
	ipsec_support inet \| ipsec_support inet6
	netipsec/key_debug.c optional ipsec inet \| ipsec inet6 \| \
	ipsec_support inet \| ipsec_support inet6
	netipsec/keysock.c optional ipsec inet \| ipsec inet6 \| \
	ipsec_support inet \| ipsec_support inet6
	netipsec/subr_ipsec.c optional ipsec inet \| ipsec inet6 \| \
	ipsec_support inet \| ipsec_support inet6
	netipsec/udpencap.c optional ipsec inet
	netipsec/xform_ah.c optional ipsec inet \| ipsec inet6
	netipsec/xform_esp.c optional ipsec inet \| ipsec inet6
	netipsec/xform_ipcomp.c optional ipsec inet \| ipsec inet6
	netipsec/xform_tcp.c optional ipsec inet tcp_signature \| \
	ipsec inet6 tcp_signature \| ipsec_support inet tcp_signature \| \
	ipsec_support inet6 tcp_signature
	netpfil/ipfw/dn_aqm_codel.c optional inet dummynet
	netpfil/ipfw/dn_aqm_pie.c optional inet dummynet
	netpfil/ipfw/dn_heap.c optional inet dummynet
	netpfil/ipfw/dn_sched_fifo.c optional inet dummynet
	netpfil/ipfw/dn_sched_fq_codel.c optional inet dummynet
	netpfil/ipfw/dn_sched_fq_pie.c optional inet dummynet
	netpfil/ipfw/dn_sched_prio.c optional inet dummynet
	netpfil/ipfw/dn_sched_qfq.c optional inet dummynet
	netpfil/ipfw/dn_sched_rr.c optional inet dummynet
	netpfil/ipfw/dn_sched_wf2q.c optional inet dummynet
	netpfil/ipfw/ip_dummynet.c optional inet dummynet
	netpfil/ipfw/ip_dn_io.c optional inet dummynet
	netpfil/ipfw/ip_dn_glue.c optional inet dummynet
	netpfil/ipfw/ip_fw2.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_bpf.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_dynamic.c optional inet ipfirewall \
	compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	netpfil/ipfw/ip_fw_eaction.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_log.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_pfil.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_sockopt.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_table.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_table_algo.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_table_value.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_iface.c optional inet ipfirewall
	netpfil/ipfw/ip_fw_nat.c optional inet ipfirewall_nat
	netpfil/ipfw/nat64/ip_fw_nat64.c optional inet inet6 ipfirewall \
	ipfirewall_nat64
	netpfil/ipfw/nat64/nat64clat.c optional inet inet6 ipfirewall \
	ipfirewall_nat64
	netpfil/ipfw/nat64/nat64clat_control.c optional inet inet6 ipfirewall \
	ipfirewall_nat64
	netpfil/ipfw/nat64/nat64lsn.c optional inet inet6 ipfirewall \
	ipfirewall_nat64 compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	netpfil/ipfw/nat64/nat64lsn_control.c optional inet inet6 ipfirewall \
	ipfirewall_nat64 compile-with "${NORMAL_C} -I$S/contrib/ck/include"
	netpfil/ipfw/nat64/nat64stl.c optional inet inet6 ipfirewall \
	ipfirewall_nat64
	netpfil/ipfw/nat64/nat64stl_control.c optional inet inet6 ipfirewall \
	ipfirewall_nat64
	netpfil/ipfw/nat64/nat64_translate.c optional inet inet6 ipfirewall \
	ipfirewall_nat64
	netpfil/ipfw/nptv6/ip_fw_nptv6.c optional inet inet6 ipfirewall \
	ipfirewall_nptv6
	netpfil/ipfw/nptv6/nptv6.c optional inet inet6 ipfirewall \
	ipfirewall_nptv6
	netpfil/ipfw/pmod/ip_fw_pmod.c optional inet ipfirewall_pmod
	netpfil/ipfw/pmod/tcpmod.c optional inet ipfirewall_pmod
	netpfil/pf/if_pflog.c optional pflog pf inet
	netpfil/pf/if_pfsync.c optional pfsync pf inet
	netpfil/pf/pf.c optional pf inet
	netpfil/pf/pf_if.c optional pf inet
	netpfil/pf/pf_ioctl.c optional pf inet
	netpfil/pf/pf_lb.c optional pf inet
	netpfil/pf/pf_norm.c optional pf inet
	netpfil/pf/pf_osfp.c optional pf inet
	netpfil/pf/pf_ruleset.c optional pf inet
	netpfil/pf/pf_table.c optional pf inet
	netpfil/pf/in4_cksum.c optional pf inet
	netsmb/smb_conn.c optional netsmb
	netsmb/smb_crypt.c optional netsmb
	netsmb/smb_dev.c optional netsmb
	netsmb/smb_iod.c optional netsmb
	netsmb/smb_rq.c optional netsmb
	netsmb/smb_smb.c optional netsmb
	netsmb/smb_subr.c optional netsmb
	netsmb/smb_trantcp.c optional netsmb
	netsmb/smb_usr.c optional netsmb
	nfs/bootp_subr.c optional bootp nfscl
	nfs/krpc_subr.c optional bootp nfscl
	nfs/nfs_diskless.c optional nfscl nfs_root
	nfs/nfs_nfssvc.c optional nfscl \| nfslockd \| nfsd
	nlm/nlm_advlock.c optional nfslockd \| nfsd
	nlm/nlm_prot_clnt.c optional nfslockd \| nfsd
	nlm/nlm_prot_impl.c optional nfslockd \| nfsd
	nlm/nlm_prot_server.c optional nfslockd \| nfsd
	nlm/nlm_prot_svc.c optional nfslockd \| nfsd
	nlm/nlm_prot_xdr.c optional nfslockd \| nfsd
	nlm/sm_inter_xdr.c optional nfslockd \| nfsd

	# Linux Kernel Programming Interface
	compat/linuxkpi/common/src/linux_kmod.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_compat.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_current.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_hrtimer.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_kthread.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_lock.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_page.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_pci.c optional compat_linuxkpi pci \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_tasklet.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_idr.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_radix.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_rcu.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C} -I$S/contrib/ck/include"
	compat/linuxkpi/common/src/linux_schedule.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_shmemfs.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_slab.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_usb.c optional compat_linuxkpi usb \
	compile-with "${LINUXKPI_C}"
	compat/linuxkpi/common/src/linux_work.c optional compat_linuxkpi \
	compile-with "${LINUXKPI_C}"

	compat/linuxkpi/common/src/linux_seq_file.c optional compat_linuxkpi \| lindebugfs \
	compile-with "${LINUXKPI_C}"

	compat/lindebugfs/lindebugfs.c optional lindebugfs \
	compile-with "${LINUXKPI_C}"

	# OpenFabrics Enterprise Distribution (Infiniband)
	ofed/drivers/infiniband/core/ib_addr.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_agent.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_cache.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_cm.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_cma.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_cq.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_device.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_fmr_pool.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_iwcm.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_iwpm_msg.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_iwpm_util.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_mad.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_mad_rmpp.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_multicast.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_packer.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_sa_query.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_smi.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_sysfs.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_ucm.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_ucma.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_ud_header.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_umem.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_user_mad.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_uverbs_cmd.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_uverbs_main.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_uverbs_marshall.c optional ofed \
	compile-with "${OFED_C}"
	ofed/drivers/infiniband/core/ib_verbs.c optional ofed \
	compile-with "${OFED_C}"

	ofed/drivers/infiniband/ulp/ipoib/ipoib_cm.c optional ipoib \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
	#ofed/drivers/infiniband/ulp/ipoib/ipoib_fs.c optional ipoib \
	# compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
	ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c optional ipoib \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
	ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c optional ipoib \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
	ofed/drivers/infiniband/ulp/ipoib/ipoib_multicast.c optional ipoib \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
	ofed/drivers/infiniband/ulp/ipoib/ipoib_verbs.c optional ipoib \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"
	#ofed/drivers/infiniband/ulp/ipoib/ipoib_vlan.c optional ipoib \
	# compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/ipoib/"

	ofed/drivers/infiniband/ulp/sdp/sdp_bcopy.c optional sdp inet \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
	ofed/drivers/infiniband/ulp/sdp/sdp_main.c optional sdp inet \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
	ofed/drivers/infiniband/ulp/sdp/sdp_rx.c optional sdp inet \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
	ofed/drivers/infiniband/ulp/sdp/sdp_cma.c optional sdp inet \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"
	ofed/drivers/infiniband/ulp/sdp/sdp_tx.c optional sdp inet \
	compile-with "${OFED_C} -I$S/ofed/drivers/infiniband/ulp/sdp/"

	dev/mthca/mthca_allocator.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_av.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_catas.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_cmd.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_cq.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_eq.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_mad.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_main.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_mcg.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_memfree.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_mr.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_pd.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_profile.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_provider.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_qp.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_reset.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_srq.c optional mthca pci ofed \
	compile-with "${OFED_C}"
	dev/mthca/mthca_uar.c optional mthca pci ofed \
	compile-with "${OFED_C}"

	dev/mlx4/mlx4_ib/mlx4_ib_alias_GUID.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_mcg.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_sysfs.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_cm.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_ah.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_cq.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_doorbell.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_mad.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_main.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_mr.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_qp.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_srq.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_ib/mlx4_ib_wc.c optional mlx4ib pci ofed \
	compile-with "${OFED_C}"

	dev/mlx4/mlx4_core/mlx4_alloc.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_catas.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_cmd.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_cq.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_eq.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_fw.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_fw_qos.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_icm.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_intf.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_main.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_mcg.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_mr.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_pd.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_port.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_profile.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_qp.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_reset.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_sense.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_srq.c optional mlx4 pci \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_core/mlx4_resource_tracker.c optional mlx4 pci \
	compile-with "${OFED_C}"

	dev/mlx4/mlx4_en/mlx4_en_cq.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_en/mlx4_en_main.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_en/mlx4_en_netdev.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_en/mlx4_en_port.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_en/mlx4_en_resources.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_en/mlx4_en_rx.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx4/mlx4_en/mlx4_en_tx.c optional mlx4en pci inet inet6 \
	compile-with "${OFED_C}"

	dev/mlx5/mlx5_ib/mlx5_ib_ah.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_cong.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_cq.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_doorbell.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_gsi.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_mad.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_main.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_mem.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_mr.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_qp.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_srq.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_ib/mlx5_ib_virt.c optional mlx5ib pci ofed \
	compile-with "${OFED_C}"

	dev/mlx5/mlx5_core/mlx5_alloc.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_cmd.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_cq.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_diagnostics.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_eq.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_eswitch.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_fs_cmd.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_fs_tree.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_fw.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_fwdump.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_health.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_mad.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_main.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_mcg.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_mpfs.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_mr.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_pagealloc.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_pd.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_port.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_qp.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_rl.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_srq.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_tls.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_transobj.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_uar.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_vport.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_vsc.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_core/mlx5_wq.c optional mlx5 pci \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_lib/mlx5_gid.c optional mlx5 pci \
	compile-with "${OFED_C}"

	dev/mlx5/mlx5_en/mlx5_en_dim.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_ethtool.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_main.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_tx.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_flow_table.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_hw_tls.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_rx.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_rl.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_txrx.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"
	dev/mlx5/mlx5_en/mlx5_en_port_buffer.c optional mlx5en pci inet inet6 \
	compile-with "${OFED_C}"

	# crypto support
	opencrypto/criov.c optional crypto \| ipsec \| ipsec_support
	opencrypto/crypto.c optional crypto \| ipsec \| ipsec_support
	opencrypto/cryptodev.c optional cryptodev
	opencrypto/cryptodev_if.m optional crypto \| ipsec \| ipsec_support
	opencrypto/cryptosoft.c optional crypto \| ipsec \| ipsec_support
	opencrypto/cryptodeflate.c optional crypto \| ipsec \| ipsec_support
	opencrypto/gmac.c optional crypto \| ipsec \| ipsec_support
	opencrypto/gfmult.c optional crypto \| ipsec \| ipsec_support
	opencrypto/rmd160.c optional crypto \| ipsec \| ipsec_support
	opencrypto/xform.c optional crypto \| ipsec \| ipsec_support
	opencrypto/xform_poly1305.c optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_onetimeauth/poly1305/onetimeauth_poly1305.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_onetimeauth/poly1305/donna/poly1305_donna.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_verify/sodium/verify.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	crypto/libsodium/randombytes.c optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include -I$S/crypto/libsodium"
	crypto/libsodium/utils.c optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_core/hchacha20/core_hchacha20.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_stream/chacha20/stream_chacha20.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_stream/chacha20/ref/chacha20_ref.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_aead/chacha20poly1305/sodium/aead_chacha20poly1305.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"
	contrib/libsodium/src/libsodium/crypto_aead/xchacha20poly1305/sodium/aead_xchacha20poly1305.c \
	optional crypto \
	compile-with "${NORMAL_C} -I$S/contrib/libsodium/src/libsodium/include/sodium -I$S/crypto/libsodium"


	opencrypto/cbc_mac.c optional crypto
	opencrypto/xform_cbc_mac.c optional crypto
	rpc/auth_none.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/auth_unix.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/authunix_prot.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/clnt_bck.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/clnt_dg.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/clnt_rc.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/clnt_vc.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/getnetconfig.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/replay.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/rpc_callmsg.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/rpc_generic.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/rpc_prot.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/rpcb_clnt.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/rpcb_prot.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/svc.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/svc_auth.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/svc_auth_unix.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/svc_dg.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/svc_generic.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/svc_vc.c optional krpc \| nfslockd \| nfscl \| nfsd
	rpc/rpcsec_gss/rpcsec_gss.c optional krpc kgssapi \| nfslockd kgssapi \| nfscl kgssapi \| nfsd kgssapi
	rpc/rpcsec_gss/rpcsec_gss_conf.c optional krpc kgssapi \| nfslockd kgssapi \| nfscl kgssapi \| nfsd kgssapi
	rpc/rpcsec_gss/rpcsec_gss_misc.c optional krpc kgssapi \| nfslockd kgssapi \| nfscl kgssapi \| nfsd kgssapi
	rpc/rpcsec_gss/rpcsec_gss_prot.c optional krpc kgssapi \| nfslockd kgssapi \| nfscl kgssapi \| nfsd kgssapi
	rpc/rpcsec_gss/svc_rpcsec_gss.c optional krpc kgssapi \| nfslockd kgssapi \| nfscl kgssapi \| nfsd kgssapi
	security/audit/audit.c optional audit
	security/audit/audit_arg.c optional audit
	security/audit/audit_bsm.c optional audit
	security/audit/audit_bsm_db.c optional audit
	security/audit/audit_bsm_klib.c optional audit
	security/audit/audit_dtrace.c optional dtaudit audit \| dtraceall audit compile-with "${CDDL_C}"
	security/audit/audit_pipe.c optional audit
	security/audit/audit_syscalls.c standard
	security/audit/audit_trigger.c optional audit
	security/audit/audit_worker.c optional audit
	security/audit/bsm_domain.c optional audit
	security/audit/bsm_errno.c optional audit
	security/audit/bsm_fcntl.c optional audit
	security/audit/bsm_socket_type.c optional audit
	security/audit/bsm_token.c optional audit
	security/mac/mac_audit.c optional mac audit
	security/mac/mac_cred.c optional mac
	security/mac/mac_framework.c optional mac
	security/mac/mac_inet.c optional mac inet \| mac inet6
	security/mac/mac_inet6.c optional mac inet6
	security/mac/mac_label.c optional mac
	security/mac/mac_net.c optional mac
	security/mac/mac_pipe.c optional mac
	security/mac/mac_posix_sem.c optional mac
	security/mac/mac_posix_shm.c optional mac
	security/mac/mac_priv.c optional mac
	security/mac/mac_process.c optional mac
	security/mac/mac_socket.c optional mac
	security/mac/mac_syscalls.c standard
	security/mac/mac_system.c optional mac
	security/mac/mac_sysv_msg.c optional mac
	security/mac/mac_sysv_sem.c optional mac
	security/mac/mac_sysv_shm.c optional mac
	security/mac/mac_vfs.c optional mac
	security/mac_biba/mac_biba.c optional mac_biba
	security/mac_bsdextended/mac_bsdextended.c optional mac_bsdextended
	security/mac_bsdextended/ugidfw_system.c optional mac_bsdextended
	security/mac_bsdextended/ugidfw_vnode.c optional mac_bsdextended
	security/mac_ifoff/mac_ifoff.c optional mac_ifoff
	security/mac_lomac/mac_lomac.c optional mac_lomac
	security/mac_mls/mac_mls.c optional mac_mls
	security/mac_none/mac_none.c optional mac_none
	security/mac_ntpd/mac_ntpd.c optional mac_ntpd
	security/mac_partition/mac_partition.c optional mac_partition
	security/mac_portacl/mac_portacl.c optional mac_portacl
	security/mac_seeotheruids/mac_seeotheruids.c optional mac_seeotheruids
	security/mac_stub/mac_stub.c optional mac_stub
	security/mac_test/mac_test.c optional mac_test
	security/mac_veriexec/mac_veriexec.c optional mac_veriexec
	security/mac_veriexec/veriexec_fingerprint.c optional mac_veriexec
	security/mac_veriexec/veriexec_metadata.c optional mac_veriexec
	security/mac_veriexec_parser/mac_veriexec_parser.c optional mac_veriexec mac_veriexec_parser
	security/mac_veriexec/mac_veriexec_rmd160.c optional mac_veriexec_rmd160
	security/mac_veriexec/mac_veriexec_sha1.c optional mac_veriexec_sha1
	security/mac_veriexec/mac_veriexec_sha256.c optional mac_veriexec_sha256
	security/mac_veriexec/mac_veriexec_sha384.c optional mac_veriexec_sha384
	security/mac_veriexec/mac_veriexec_sha512.c optional mac_veriexec_sha512
	teken/teken.c optional sc !SC_NO_TERM_TEKEN \| vt
	ufs/ffs/ffs_alloc.c optional ffs
	ufs/ffs/ffs_balloc.c optional ffs
	ufs/ffs/ffs_inode.c optional ffs
	ufs/ffs/ffs_snapshot.c optional ffs
	ufs/ffs/ffs_softdep.c optional ffs
	ufs/ffs/ffs_subr.c optional ffs \| geom_label
	ufs/ffs/ffs_tables.c optional ffs \| geom_label
	ufs/ffs/ffs_vfsops.c optional ffs
	ufs/ffs/ffs_vnops.c optional ffs
	ufs/ffs/ffs_rawread.c optional ffs directio
	ufs/ffs/ffs_suspend.c optional ffs
	ufs/ufs/ufs_acl.c optional ffs
	ufs/ufs/ufs_bmap.c optional ffs
	ufs/ufs/ufs_dirhash.c optional ffs
	ufs/ufs/ufs_extattr.c optional ffs
	ufs/ufs/ufs_gjournal.c optional ffs UFS_GJOURNAL
	ufs/ufs/ufs_inode.c optional ffs
	ufs/ufs/ufs_lookup.c optional ffs
	ufs/ufs/ufs_quota.c optional ffs
	ufs/ufs/ufs_vfsops.c optional ffs
	ufs/ufs/ufs_vnops.c optional ffs
	vm/default_pager.c standard
	vm/device_pager.c standard
	vm/phys_pager.c standard
	vm/redzone.c optional DEBUG_REDZONE
	vm/sg_pager.c standard
	vm/swap_pager.c standard
	vm/uma_core.c standard
	vm/uma_dbg.c standard
	vm/memguard.c optional DEBUG_MEMGUARD
	vm/vm_domainset.c standard
	vm/vm_fault.c standard
	vm/vm_glue.c standard
	vm/vm_init.c standard
	vm/vm_kern.c standard
	vm/vm_map.c standard
	vm/vm_meter.c standard
	vm/vm_mmap.c standard
	vm/vm_object.c standard
	vm/vm_page.c standard
	vm/vm_pageout.c standard
	vm/vm_pager.c standard
	vm/vm_phys.c standard
	vm/vm_radix.c standard
	vm/vm_reserv.c standard
	vm/vm_swapout.c optional !NO_SWAPPING
	vm/vm_swapout_dummy.c optional NO_SWAPPING
	vm/vm_unix.c standard
	vm/vnode_pager.c standard
	xen/features.c optional xenhvm
	xen/xenbus/xenbus_if.m optional xenhvm
	xen/xenbus/xenbus.c optional xenhvm
	xen/xenbus/xenbusb_if.m optional xenhvm
	xen/xenbus/xenbusb.c optional xenhvm
	xen/xenbus/xenbusb_front.c optional xenhvm
	xen/xenbus/xenbusb_back.c optional xenhvm
	xen/xenmem/xenmem_if.m optional xenhvm
	xdr/xdr.c optional xdr \| krpc \| nfslockd \| nfscl \| nfsd
	xdr/xdr_array.c optional xdr \| krpc \| nfslockd \| nfscl \| nfsd
	xdr/xdr_mbuf.c optional xdr \| krpc \| nfslockd \| nfscl \| nfsd
	xdr/xdr_mem.c optional xdr \| krpc \| nfslockd \| nfscl \| nfsd
	xdr/xdr_reference.c optional xdr \| krpc \| nfslockd \| nfscl \| nfsd
	xdr/xdr_sizeof.c optional xdr \| krpc \| nfslockd \| nfscl \| nfsd
	diff --git a/sys/contrib/pcg-c/include/pcg_variants.h b/sys/contrib/pcg-c/include/pcg_variants.h
	new file mode 100644
	index 000000000000..768fb75ae93b
	--- /dev/null
	+++ b/sys/contrib/pcg-c/include/pcg_variants.h
	@@ -0,0 +1,2544 @@
	+/*
	+ * PCG Random Number Generation for C.
	+ *
	+ * Copyright 2014-2019 Melissa O'Neill <oneill@pcg-random.org>,
	+ * and the PCG Project contributors.
	+ *
	+ * SPDX-License-Identifier: (Apache-2.0 OR MIT)
	+ *
	+ * Licensed under the Apache License, Version 2.0 (provided in
	+ * LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0)
	+ * or under the MIT license (provided in LICENSE-MIT.txt and at
	+ * http://opensource.org/licenses/MIT), at your option. This file may not
	+ * be copied, modified, or distributed except according to those terms.
	+ *
	+ * Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either
	+ * express or implied. See your chosen license for details.
	+ *
	+ * For additional information about the PCG random number generation scheme,
	+ * visit http://www.pcg-random.org/.
	+ */
	+
	+/*
	+ * This code is derived from the canonical C++ PCG implementation, which
	+ * has many additional features and is preferable if you can use C++ in
	+ * your project.
	+ *
	+ * Much of the derivation was performed mechanically. In particular, the
	+ * output functions were generated by compiling the C++ output functions
	+ * into LLVM bitcode and then transforming that using the LLVM C backend
	+ * (from https://github.com/draperlaboratory/llvm-cbe), and then
	+ * postprocessing and hand editing the output.
	+ *
	+ * Much of the remaining code was generated by C-preprocessor metaprogramming.
	+ */
	+
	+#ifndef PCG_VARIANTS_H_INCLUDED
	+#define PCG_VARIANTS_H_INCLUDED 1
	+
	+#include <inttypes.h>
	+
	+#if __SIZEOF_INT128__
	+ typedef __uint128_t pcg128_t;
	+ #define PCG_128BIT_CONSTANT(high,low) \
	+ ((((pcg128_t)high) << 64) + low)
	+ #define PCG_HAS_128BIT_OPS 1
	+#endif
	+
	+#if __GNUC_GNU_INLINE__ && !defined(__cplusplus)
	+ #error Nonstandard GNU inlining semantics. Compile with -std=c99 or better.
	+ /* We could instead use macros PCG_INLINE and PCG_EXTERN_INLINE
	+ but better to just reject ancient C code. */
	+#endif
	+
	+#if __cplusplus
	+extern "C" {
	+#endif
	+
	+/*
	+ * Rotate helper functions.
	+ */
	+
	+inline uint8_t pcg_rotr_8(uint8_t value, unsigned int rot)
	+{
	+/* Unfortunately, clang is kinda pathetic when it comes to properly
	+ * recognizing idiomatic rotate code, so for clang we actually provide
	+ * assembler directives (enabled with PCG_USE_INLINE_ASM). Boo, hiss.
	+ */
	+#if PCG_USE_INLINE_ASM && __clang__ && (__x86_64__ \|\| __i386__)
	+ asm ("rorb %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
	+ return value;
	+#else
	+ return (value >> rot) \| (value << ((- rot) & 7));
	+#endif
	+}
	+
	+inline uint16_t pcg_rotr_16(uint16_t value, unsigned int rot)
	+{
	+#if PCG_USE_INLINE_ASM && __clang__ && (__x86_64__ \|\| __i386__)
	+ asm ("rorw %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
	+ return value;
	+#else
	+ return (value >> rot) \| (value << ((- rot) & 15));
	+#endif
	+}
	+
	+inline uint32_t pcg_rotr_32(uint32_t value, unsigned int rot)
	+{
	+#if PCG_USE_INLINE_ASM && __clang__ && (__x86_64__ \|\| __i386__)
	+ asm ("rorl %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
	+ return value;
	+#else
	+ return (value >> rot) \| (value << ((- rot) & 31));
	+#endif
	+}
	+
	+inline uint64_t pcg_rotr_64(uint64_t value, unsigned int rot)
	+{
	+#if 0 && PCG_USE_INLINE_ASM && __clang__ && __x86_64__
	+ /* For whatever reason, clang actually does generate rotq by
	+ itself, so we don't need this code. */
	+ asm ("rorq %%cl, %0" : "=r" (value) : "0" (value), "c" (rot));
	+ return value;
	+#else
	+ return (value >> rot) \| (value << ((- rot) & 63));
	+#endif
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_rotr_128(pcg128_t value, unsigned int rot)
	+{
	+ return (value >> rot) \| (value << ((- rot) & 127));
	+}
	+#endif
	+
	+/*
	+ * Output functions. These are the core of the PCG generation scheme.
	+ */
	+
	+/* XSH RS */
	+
	+inline uint8_t pcg_output_xsh_rs_16_8(uint16_t state)
	+{
	+ return (uint8_t)(((state >> 7u) ^ state) >> ((state >> 14u) + 3u));
	+}
	+
	+inline uint16_t pcg_output_xsh_rs_32_16(uint32_t state)
	+{
	+ return (uint16_t)(((state >> 11u) ^ state) >> ((state >> 30u) + 11u));
	+}
	+
	+inline uint32_t pcg_output_xsh_rs_64_32(uint64_t state)
	+{
	+
	+ return (uint32_t)(((state >> 22u) ^ state) >> ((state >> 61u) + 22u));
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_output_xsh_rs_128_64(pcg128_t state)
	+{
	+ return (uint64_t)(((state >> 43u) ^ state) >> ((state >> 124u) + 45u));
	+}
	+#endif
	+
	+/* XSH RR */
	+
	+inline uint8_t pcg_output_xsh_rr_16_8(uint16_t state)
	+{
	+ return pcg_rotr_8(((state >> 5u) ^ state) >> 5u, state >> 13u);
	+}
	+
	+inline uint16_t pcg_output_xsh_rr_32_16(uint32_t state)
	+{
	+ return pcg_rotr_16(((state >> 10u) ^ state) >> 12u, state >> 28u);
	+}
	+
	+inline uint32_t pcg_output_xsh_rr_64_32(uint64_t state)
	+{
	+ return pcg_rotr_32(((state >> 18u) ^ state) >> 27u, state >> 59u);
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_output_xsh_rr_128_64(pcg128_t state)
	+{
	+ return pcg_rotr_64(((state >> 35u) ^ state) >> 58u, state >> 122u);
	+}
	+#endif
	+
	+/* RXS M XS */
	+
	+inline uint8_t pcg_output_rxs_m_xs_8_8(uint8_t state)
	+{
	+ uint8_t word = ((state >> ((state >> 6u) + 2u)) ^ state) * 217u;
	+ return (word >> 6u) ^ word;
	+}
	+
	+inline uint16_t pcg_output_rxs_m_xs_16_16(uint16_t state)
	+{
	+ uint16_t word = ((state >> ((state >> 13u) + 3u)) ^ state) * 62169u;
	+ return (word >> 11u) ^ word;
	+}
	+
	+inline uint32_t pcg_output_rxs_m_xs_32_32(uint32_t state)
	+{
	+ uint32_t word = ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u;
	+ return (word >> 22u) ^ word;
	+}
	+
	+inline uint64_t pcg_output_rxs_m_xs_64_64(uint64_t state)
	+{
	+ uint64_t word = ((state >> ((state >> 59u) + 5u)) ^ state)
	+ * 12605985483714917081ull;
	+ return (word >> 43u) ^ word;
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_output_rxs_m_xs_128_128(pcg128_t state)
	+{
	+ pcg128_t word = ((state >> ((state >> 122u) + 6u)) ^ state)
	+ * (PCG_128BIT_CONSTANT(17766728186571221404ULL,
	+ 12605985483714917081ULL));
	+ /* 327738287884841127335028083622016905945 */
	+ return (word >> 86u) ^ word;
	+}
	+#endif
	+
	+/* RXS M */
	+
	+inline uint8_t pcg_output_rxs_m_16_8(uint16_t state)
	+{
	+ return (((state >> ((state >> 13u) + 3u)) ^ state) * 62169u) >> 8u;
	+}
	+
	+inline uint16_t pcg_output_rxs_m_32_16(uint32_t state)
	+{
	+ return (((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u) >> 16u;
	+}
	+
	+inline uint32_t pcg_output_rxs_m_64_32(uint64_t state)
	+{
	+ return (((state >> ((state >> 59u) + 5u)) ^ state)
	+ * 12605985483714917081ull) >> 32u;
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_output_rxs_m_128_64(pcg128_t state)
	+{
	+ return (((state >> ((state >> 122u) + 6u)) ^ state)
	+ * (PCG_128BIT_CONSTANT(17766728186571221404ULL,
	+ 12605985483714917081ULL))) >> 64u;
	+ /* 327738287884841127335028083622016905945 */
	+}
	+#endif
	+
	+/* XSL RR (only defined for >= 64 bits) */
	+
	+inline uint32_t pcg_output_xsl_rr_64_32(uint64_t state)
	+{
	+ return pcg_rotr_32(((uint32_t)(state >> 32u)) ^ (uint32_t)state,
	+ state >> 59u);
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_output_xsl_rr_128_64(pcg128_t state)
	+{
	+ return pcg_rotr_64(((uint64_t)(state >> 64u)) ^ (uint64_t)state,
	+ state >> 122u);
	+}
	+#endif
	+
	+/* XSL RR RR (only defined for >= 64 bits) */
	+
	+inline uint64_t pcg_output_xsl_rr_rr_64_64(uint64_t state)
	+{
	+ uint32_t rot1 = (uint32_t)(state >> 59u);
	+ uint32_t high = (uint32_t)(state >> 32u);
	+ uint32_t low = (uint32_t)state;
	+ uint32_t xored = high ^ low;
	+ uint32_t newlow = pcg_rotr_32(xored, rot1);
	+ uint32_t newhigh = pcg_rotr_32(high, newlow & 31u);
	+ return (((uint64_t)newhigh) << 32u) \| newlow;
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_output_xsl_rr_rr_128_128(pcg128_t state)
	+{
	+ uint32_t rot1 = (uint32_t)(state >> 122u);
	+ uint64_t high = (uint64_t)(state >> 64u);
	+ uint64_t low = (uint64_t)state;
	+ uint64_t xored = high ^ low;
	+ uint64_t newlow = pcg_rotr_64(xored, rot1);
	+ uint64_t newhigh = pcg_rotr_64(high, newlow & 63u);
	+ return (((pcg128_t)newhigh) << 64u) \| newlow;
	+}
	+#endif
	+
	+#define PCG_DEFAULT_MULTIPLIER_8 141U
	+#define PCG_DEFAULT_MULTIPLIER_16 12829U
	+#define PCG_DEFAULT_MULTIPLIER_32 747796405U
	+#define PCG_DEFAULT_MULTIPLIER_64 6364136223846793005ULL
	+
	+#define PCG_DEFAULT_INCREMENT_8 77U
	+#define PCG_DEFAULT_INCREMENT_16 47989U
	+#define PCG_DEFAULT_INCREMENT_32 2891336453U
	+#define PCG_DEFAULT_INCREMENT_64 1442695040888963407ULL
	+
	+#if PCG_HAS_128BIT_OPS
	+#define PCG_DEFAULT_MULTIPLIER_128 \
	+ PCG_128BIT_CONSTANT(2549297995355413924ULL,4865540595714422341ULL)
	+#define PCG_DEFAULT_INCREMENT_128 \
	+ PCG_128BIT_CONSTANT(6364136223846793005ULL,1442695040888963407ULL)
	+#endif
	+
	+/*
	+ * Static initialization constants (if you can't call srandom for some
	+ * bizarre reason).
	+ */
	+
	+#define PCG_STATE_ONESEQ_8_INITIALIZER { 0xd7U }
	+#define PCG_STATE_ONESEQ_16_INITIALIZER { 0x20dfU }
	+#define PCG_STATE_ONESEQ_32_INITIALIZER { 0x46b56677U }
	+#define PCG_STATE_ONESEQ_64_INITIALIZER { 0x4d595df4d0f33173ULL }
	+#if PCG_HAS_128BIT_OPS
	+#define PCG_STATE_ONESEQ_128_INITIALIZER \
	+ { PCG_128BIT_CONSTANT(0xb8dc10e158a92392ULL, 0x98046df007ec0a53ULL) }
	+#endif
	+
	+#define PCG_STATE_UNIQUE_8_INITIALIZER PCG_STATE_ONESEQ_8_INITIALIZER
	+#define PCG_STATE_UNIQUE_16_INITIALIZER PCG_STATE_ONESEQ_16_INITIALIZER
	+#define PCG_STATE_UNIQUE_32_INITIALIZER PCG_STATE_ONESEQ_32_INITIALIZER
	+#define PCG_STATE_UNIQUE_64_INITIALIZER PCG_STATE_ONESEQ_64_INITIALIZER
	+#if PCG_HAS_128BIT_OPS
	+#define PCG_STATE_UNIQUE_128_INITIALIZER PCG_STATE_ONESEQ_128_INITIALIZER
	+#endif
	+
	+#define PCG_STATE_MCG_8_INITIALIZER { 0xe5U }
	+#define PCG_STATE_MCG_16_INITIALIZER { 0xa5e5U }
	+#define PCG_STATE_MCG_32_INITIALIZER { 0xd15ea5e5U }
	+#define PCG_STATE_MCG_64_INITIALIZER { 0xcafef00dd15ea5e5ULL }
	+#if PCG_HAS_128BIT_OPS
	+#define PCG_STATE_MCG_128_INITIALIZER \
	+ { PCG_128BIT_CONSTANT(0x0000000000000000ULL, 0xcafef00dd15ea5e5ULL) }
	+#endif
	+
	+#define PCG_STATE_SETSEQ_8_INITIALIZER { 0x9bU, 0xdbU }
	+#define PCG_STATE_SETSEQ_16_INITIALIZER { 0xe39bU, 0x5bdbU }
	+#define PCG_STATE_SETSEQ_32_INITIALIZER { 0xec02d89bU, 0x94b95bdbU }
	+#define PCG_STATE_SETSEQ_64_INITIALIZER \
	+ { 0x853c49e6748fea9bULL, 0xda3e39cb94b95bdbULL }
	+#if PCG_HAS_128BIT_OPS
	+#define PCG_STATE_SETSEQ_128_INITIALIZER \
	+ { PCG_128BIT_CONSTANT(0x979c9a98d8462005ULL, 0x7d3e9cb6cfe0549bULL), \
	+ PCG_128BIT_CONSTANT(0x0000000000000001ULL, 0xda3e39cb94b95bdbULL) }
	+#endif
	+
	+/* Representations for the oneseq, mcg, and unique variants */
	+
	+struct pcg_state_8 {
	+ uint8_t state;
	+};
	+
	+struct pcg_state_16 {
	+ uint16_t state;
	+};
	+
	+struct pcg_state_32 {
	+ uint32_t state;
	+};
	+
	+struct pcg_state_64 {
	+ uint64_t state;
	+};
	+
	+#if PCG_HAS_128BIT_OPS
	+struct pcg_state_128 {
	+ pcg128_t state;
	+};
	+#endif
	+
	+/* Representations setseq variants */
	+
	+struct pcg_state_setseq_8 {
	+ uint8_t state;
	+ uint8_t inc;
	+};
	+
	+struct pcg_state_setseq_16 {
	+ uint16_t state;
	+ uint16_t inc;
	+};
	+
	+struct pcg_state_setseq_32 {
	+ uint32_t state;
	+ uint32_t inc;
	+};
	+
	+struct pcg_state_setseq_64 {
	+ uint64_t state;
	+ uint64_t inc;
	+};
	+
	+#if PCG_HAS_128BIT_OPS
	+struct pcg_state_setseq_128 {
	+ pcg128_t state;
	+ pcg128_t inc;
	+};
	+#endif
	+
	+/* Multi-step advance functions (jump-ahead, jump-back) */
	+
	+extern uint8_t pcg_advance_lcg_8(uint8_t state, uint8_t delta, uint8_t cur_mult,
	+ uint8_t cur_plus);
	+extern uint16_t pcg_advance_lcg_16(uint16_t state, uint16_t delta,
	+ uint16_t cur_mult, uint16_t cur_plus);
	+extern uint32_t pcg_advance_lcg_32(uint32_t state, uint32_t delta,
	+ uint32_t cur_mult, uint32_t cur_plus);
	+extern uint64_t pcg_advance_lcg_64(uint64_t state, uint64_t delta,
	+ uint64_t cur_mult, uint64_t cur_plus);
	+
	+#if PCG_HAS_128BIT_OPS
	+extern pcg128_t pcg_advance_lcg_128(pcg128_t state, pcg128_t delta,
	+ pcg128_t cur_mult, pcg128_t cur_plus);
	+#endif
	+
	+/* Functions to advance the underlying LCG, one version for each size and
	+ * each style. These functions are considered semi-private. There is rarely
	+ * a good reason to call them directly.
	+ */
	+
	+inline void pcg_oneseq_8_step_r(struct pcg_state_8* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_8
	+ + PCG_DEFAULT_INCREMENT_8;
	+}
	+
	+inline void pcg_oneseq_8_advance_r(struct pcg_state_8* rng, uint8_t delta)
	+{
	+ rng->state = pcg_advance_lcg_8(rng->state, delta, PCG_DEFAULT_MULTIPLIER_8,
	+ PCG_DEFAULT_INCREMENT_8);
	+}
	+
	+inline void pcg_mcg_8_step_r(struct pcg_state_8* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_8;
	+}
	+
	+inline void pcg_mcg_8_advance_r(struct pcg_state_8* rng, uint8_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_8(rng->state, delta, PCG_DEFAULT_MULTIPLIER_8, 0u);
	+}
	+
	+inline void pcg_unique_8_step_r(struct pcg_state_8* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_8
	+ + (uint8_t)(((intptr_t)rng) \| 1u);
	+}
	+
	+inline void pcg_unique_8_advance_r(struct pcg_state_8* rng, uint8_t delta)
	+{
	+ rng->state = pcg_advance_lcg_8(rng->state, delta, PCG_DEFAULT_MULTIPLIER_8,
	+ (uint8_t)(((intptr_t)rng) \| 1u));
	+}
	+
	+inline void pcg_setseq_8_step_r(struct pcg_state_setseq_8* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_8 + rng->inc;
	+}
	+
	+inline void pcg_setseq_8_advance_r(struct pcg_state_setseq_8* rng,
	+ uint8_t delta)
	+{
	+ rng->state = pcg_advance_lcg_8(rng->state, delta, PCG_DEFAULT_MULTIPLIER_8,
	+ rng->inc);
	+}
	+
	+inline void pcg_oneseq_16_step_r(struct pcg_state_16* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_16
	+ + PCG_DEFAULT_INCREMENT_16;
	+}
	+
	+inline void pcg_oneseq_16_advance_r(struct pcg_state_16* rng, uint16_t delta)
	+{
	+ rng->state = pcg_advance_lcg_16(
	+ rng->state, delta, PCG_DEFAULT_MULTIPLIER_16, PCG_DEFAULT_INCREMENT_16);
	+}
	+
	+inline void pcg_mcg_16_step_r(struct pcg_state_16* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_16;
	+}
	+
	+inline void pcg_mcg_16_advance_r(struct pcg_state_16* rng, uint16_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_16(rng->state, delta, PCG_DEFAULT_MULTIPLIER_16, 0u);
	+}
	+
	+inline void pcg_unique_16_step_r(struct pcg_state_16* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_16
	+ + (uint16_t)(((intptr_t)rng) \| 1u);
	+}
	+
	+inline void pcg_unique_16_advance_r(struct pcg_state_16* rng, uint16_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_16(rng->state, delta, PCG_DEFAULT_MULTIPLIER_16,
	+ (uint16_t)(((intptr_t)rng) \| 1u));
	+}
	+
	+inline void pcg_setseq_16_step_r(struct pcg_state_setseq_16* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_16 + rng->inc;
	+}
	+
	+inline void pcg_setseq_16_advance_r(struct pcg_state_setseq_16* rng,
	+ uint16_t delta)
	+{
	+ rng->state = pcg_advance_lcg_16(rng->state, delta,
	+ PCG_DEFAULT_MULTIPLIER_16, rng->inc);
	+}
	+
	+inline void pcg_oneseq_32_step_r(struct pcg_state_32* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_32
	+ + PCG_DEFAULT_INCREMENT_32;
	+}
	+
	+inline void pcg_oneseq_32_advance_r(struct pcg_state_32* rng, uint32_t delta)
	+{
	+ rng->state = pcg_advance_lcg_32(
	+ rng->state, delta, PCG_DEFAULT_MULTIPLIER_32, PCG_DEFAULT_INCREMENT_32);
	+}
	+
	+inline void pcg_mcg_32_step_r(struct pcg_state_32* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_32;
	+}
	+
	+inline void pcg_mcg_32_advance_r(struct pcg_state_32* rng, uint32_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_32(rng->state, delta, PCG_DEFAULT_MULTIPLIER_32, 0u);
	+}
	+
	+inline void pcg_unique_32_step_r(struct pcg_state_32* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_32
	+ + (uint32_t)(((intptr_t)rng) \| 1u);
	+}
	+
	+inline void pcg_unique_32_advance_r(struct pcg_state_32* rng, uint32_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_32(rng->state, delta, PCG_DEFAULT_MULTIPLIER_32,
	+ (uint32_t)(((intptr_t)rng) \| 1u));
	+}
	+
	+inline void pcg_setseq_32_step_r(struct pcg_state_setseq_32* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_32 + rng->inc;
	+}
	+
	+inline void pcg_setseq_32_advance_r(struct pcg_state_setseq_32* rng,
	+ uint32_t delta)
	+{
	+ rng->state = pcg_advance_lcg_32(rng->state, delta,
	+ PCG_DEFAULT_MULTIPLIER_32, rng->inc);
	+}
	+
	+inline void pcg_oneseq_64_step_r(struct pcg_state_64* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_64
	+ + PCG_DEFAULT_INCREMENT_64;
	+}
	+
	+inline void pcg_oneseq_64_advance_r(struct pcg_state_64* rng, uint64_t delta)
	+{
	+ rng->state = pcg_advance_lcg_64(
	+ rng->state, delta, PCG_DEFAULT_MULTIPLIER_64, PCG_DEFAULT_INCREMENT_64);
	+}
	+
	+inline void pcg_mcg_64_step_r(struct pcg_state_64* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_64;
	+}
	+
	+inline void pcg_mcg_64_advance_r(struct pcg_state_64* rng, uint64_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_64(rng->state, delta, PCG_DEFAULT_MULTIPLIER_64, 0u);
	+}
	+
	+inline void pcg_unique_64_step_r(struct pcg_state_64* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_64
	+ + (uint64_t)(((intptr_t)rng) \| 1u);
	+}
	+
	+inline void pcg_unique_64_advance_r(struct pcg_state_64* rng, uint64_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_64(rng->state, delta, PCG_DEFAULT_MULTIPLIER_64,
	+ (uint64_t)(((intptr_t)rng) \| 1u));
	+}
	+
	+inline void pcg_setseq_64_step_r(struct pcg_state_setseq_64* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_64 + rng->inc;
	+}
	+
	+inline void pcg_setseq_64_advance_r(struct pcg_state_setseq_64* rng,
	+ uint64_t delta)
	+{
	+ rng->state = pcg_advance_lcg_64(rng->state, delta,
	+ PCG_DEFAULT_MULTIPLIER_64, rng->inc);
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_oneseq_128_step_r(struct pcg_state_128* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_128
	+ + PCG_DEFAULT_INCREMENT_128;
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_oneseq_128_advance_r(struct pcg_state_128* rng, pcg128_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_128(rng->state, delta, PCG_DEFAULT_MULTIPLIER_128,
	+ PCG_DEFAULT_INCREMENT_128);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_mcg_128_step_r(struct pcg_state_128* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_128;
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_mcg_128_advance_r(struct pcg_state_128* rng, pcg128_t delta)
	+{
	+ rng->state = pcg_advance_lcg_128(rng->state, delta,
	+ PCG_DEFAULT_MULTIPLIER_128, 0u);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_unique_128_step_r(struct pcg_state_128* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_128
	+ + (pcg128_t)(((intptr_t)rng) \| 1u);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_unique_128_advance_r(struct pcg_state_128* rng, pcg128_t delta)
	+{
	+ rng->state
	+ = pcg_advance_lcg_128(rng->state, delta, PCG_DEFAULT_MULTIPLIER_128,
	+ (pcg128_t)(((intptr_t)rng) \| 1u));
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_setseq_128_step_r(struct pcg_state_setseq_128* rng)
	+{
	+ rng->state = rng->state * PCG_DEFAULT_MULTIPLIER_128 + rng->inc;
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_setseq_128_advance_r(struct pcg_state_setseq_128* rng,
	+ pcg128_t delta)
	+{
	+ rng->state = pcg_advance_lcg_128(rng->state, delta,
	+ PCG_DEFAULT_MULTIPLIER_128, rng->inc);
	+}
	+#endif
	+
	+/* Functions to seed the RNG state, one version for each size and each
	+ * style. Unlike the step functions, regular users can and should call
	+ * these functions.
	+ */
	+
	+inline void pcg_oneseq_8_srandom_r(struct pcg_state_8* rng, uint8_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_oneseq_8_step_r(rng);
	+ rng->state += initstate;
	+ pcg_oneseq_8_step_r(rng);
	+}
	+
	+inline void pcg_mcg_8_srandom_r(struct pcg_state_8* rng, uint8_t initstate)
	+{
	+ rng->state = initstate \| 1u;
	+}
	+
	+inline void pcg_unique_8_srandom_r(struct pcg_state_8* rng, uint8_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_unique_8_step_r(rng);
	+ rng->state += initstate;
	+ pcg_unique_8_step_r(rng);
	+}
	+
	+inline void pcg_setseq_8_srandom_r(struct pcg_state_setseq_8* rng,
	+ uint8_t initstate, uint8_t initseq)
	+{
	+ rng->state = 0U;
	+ rng->inc = (initseq << 1u) \| 1u;
	+ pcg_setseq_8_step_r(rng);
	+ rng->state += initstate;
	+ pcg_setseq_8_step_r(rng);
	+}
	+
	+inline void pcg_oneseq_16_srandom_r(struct pcg_state_16* rng,
	+ uint16_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_oneseq_16_step_r(rng);
	+ rng->state += initstate;
	+ pcg_oneseq_16_step_r(rng);
	+}
	+
	+inline void pcg_mcg_16_srandom_r(struct pcg_state_16* rng, uint16_t initstate)
	+{
	+ rng->state = initstate \| 1u;
	+}
	+
	+inline void pcg_unique_16_srandom_r(struct pcg_state_16* rng,
	+ uint16_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_unique_16_step_r(rng);
	+ rng->state += initstate;
	+ pcg_unique_16_step_r(rng);
	+}
	+
	+inline void pcg_setseq_16_srandom_r(struct pcg_state_setseq_16* rng,
	+ uint16_t initstate, uint16_t initseq)
	+{
	+ rng->state = 0U;
	+ rng->inc = (initseq << 1u) \| 1u;
	+ pcg_setseq_16_step_r(rng);
	+ rng->state += initstate;
	+ pcg_setseq_16_step_r(rng);
	+}
	+
	+inline void pcg_oneseq_32_srandom_r(struct pcg_state_32* rng,
	+ uint32_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_oneseq_32_step_r(rng);
	+ rng->state += initstate;
	+ pcg_oneseq_32_step_r(rng);
	+}
	+
	+inline void pcg_mcg_32_srandom_r(struct pcg_state_32* rng, uint32_t initstate)
	+{
	+ rng->state = initstate \| 1u;
	+}
	+
	+inline void pcg_unique_32_srandom_r(struct pcg_state_32* rng,
	+ uint32_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_unique_32_step_r(rng);
	+ rng->state += initstate;
	+ pcg_unique_32_step_r(rng);
	+}
	+
	+inline void pcg_setseq_32_srandom_r(struct pcg_state_setseq_32* rng,
	+ uint32_t initstate, uint32_t initseq)
	+{
	+ rng->state = 0U;
	+ rng->inc = (initseq << 1u) \| 1u;
	+ pcg_setseq_32_step_r(rng);
	+ rng->state += initstate;
	+ pcg_setseq_32_step_r(rng);
	+}
	+
	+inline void pcg_oneseq_64_srandom_r(struct pcg_state_64* rng,
	+ uint64_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_oneseq_64_step_r(rng);
	+ rng->state += initstate;
	+ pcg_oneseq_64_step_r(rng);
	+}
	+
	+inline void pcg_mcg_64_srandom_r(struct pcg_state_64* rng, uint64_t initstate)
	+{
	+ rng->state = initstate \| 1u;
	+}
	+
	+inline void pcg_unique_64_srandom_r(struct pcg_state_64* rng,
	+ uint64_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_unique_64_step_r(rng);
	+ rng->state += initstate;
	+ pcg_unique_64_step_r(rng);
	+}
	+
	+inline void pcg_setseq_64_srandom_r(struct pcg_state_setseq_64* rng,
	+ uint64_t initstate, uint64_t initseq)
	+{
	+ rng->state = 0U;
	+ rng->inc = (initseq << 1u) \| 1u;
	+ pcg_setseq_64_step_r(rng);
	+ rng->state += initstate;
	+ pcg_setseq_64_step_r(rng);
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_oneseq_128_srandom_r(struct pcg_state_128* rng,
	+ pcg128_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_oneseq_128_step_r(rng);
	+ rng->state += initstate;
	+ pcg_oneseq_128_step_r(rng);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_mcg_128_srandom_r(struct pcg_state_128* rng, pcg128_t initstate)
	+{
	+ rng->state = initstate \| 1u;
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_unique_128_srandom_r(struct pcg_state_128* rng,
	+ pcg128_t initstate)
	+{
	+ rng->state = 0U;
	+ pcg_unique_128_step_r(rng);
	+ rng->state += initstate;
	+ pcg_unique_128_step_r(rng);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline void pcg_setseq_128_srandom_r(struct pcg_state_setseq_128* rng,
	+ pcg128_t initstate, pcg128_t initseq)
	+{
	+ rng->state = 0U;
	+ rng->inc = (initseq << 1u) \| 1u;
	+ pcg_setseq_128_step_r(rng);
	+ rng->state += initstate;
	+ pcg_setseq_128_step_r(rng);
	+}
	+#endif
	+
	+/* Now, finally we create each of the individual generators. We provide
	+ * a random_r function that provides a random number of the appropriate
	+ * type (using the full range of the type) and a boundedrand_r version
	+ * that provides
	+ *
	+ * Implementation notes for boundedrand_r:
	+ *
	+ * To avoid bias, we need to make the range of the RNG a multiple of
	+ * bound, which we do by dropping output less than a threshold.
	+ * Let's consider a 32-bit case... A naive scheme to calculate the
	+ * threshold would be to do
	+ *
	+ * uint32_t threshold = 0x100000000ull % bound;
	+ *
	+ * but 64-bit div/mod is slower than 32-bit div/mod (especially on
	+ * 32-bit platforms). In essence, we do
	+ *
	+ * uint32_t threshold = (0x100000000ull-bound) % bound;
	+ *
	+ * because this version will calculate the same modulus, but the LHS
	+ * value is less than 2^32.
	+ *
	+ * (Note that using modulo is only wise for good RNGs, poorer RNGs
	+ * such as raw LCGs do better using a technique based on division.)
	+ * Empricical tests show that division is preferable to modulus for
	+ * reducting the range of an RNG. It's faster, and sometimes it can
	+ * even be statistically prefereable.
	+ */
	+
	+/* Generation functions for XSH RS */
	+
	+inline uint8_t pcg_oneseq_16_xsh_rs_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_oneseq_16_step_r(rng);
	+ return pcg_output_xsh_rs_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_oneseq_16_xsh_rs_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_oneseq_16_xsh_rs_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_oneseq_32_xsh_rs_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_oneseq_32_step_r(rng);
	+ return pcg_output_xsh_rs_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_oneseq_32_xsh_rs_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_oneseq_32_xsh_rs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_oneseq_64_xsh_rs_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_oneseq_64_step_r(rng);
	+ return pcg_output_xsh_rs_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_oneseq_64_xsh_rs_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_oneseq_64_xsh_rs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_oneseq_128_xsh_rs_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_oneseq_128_step_r(rng);
	+ return pcg_output_xsh_rs_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_oneseq_128_xsh_rs_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_oneseq_128_xsh_rs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_unique_16_xsh_rs_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_unique_16_step_r(rng);
	+ return pcg_output_xsh_rs_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_unique_16_xsh_rs_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_unique_16_xsh_rs_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_unique_32_xsh_rs_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_unique_32_step_r(rng);
	+ return pcg_output_xsh_rs_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_unique_32_xsh_rs_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_unique_32_xsh_rs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_unique_64_xsh_rs_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_unique_64_step_r(rng);
	+ return pcg_output_xsh_rs_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_unique_64_xsh_rs_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_unique_64_xsh_rs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_unique_128_xsh_rs_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_unique_128_step_r(rng);
	+ return pcg_output_xsh_rs_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_unique_128_xsh_rs_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_unique_128_xsh_rs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_setseq_16_xsh_rs_8_random_r(struct pcg_state_setseq_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_setseq_16_step_r(rng);
	+ return pcg_output_xsh_rs_16_8(oldstate);
	+}
	+
	+inline uint8_t
	+pcg_setseq_16_xsh_rs_8_boundedrand_r(struct pcg_state_setseq_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_setseq_16_xsh_rs_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t
	+pcg_setseq_32_xsh_rs_16_random_r(struct pcg_state_setseq_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_setseq_32_step_r(rng);
	+ return pcg_output_xsh_rs_32_16(oldstate);
	+}
	+
	+inline uint16_t
	+pcg_setseq_32_xsh_rs_16_boundedrand_r(struct pcg_state_setseq_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_setseq_32_xsh_rs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t
	+pcg_setseq_64_xsh_rs_32_random_r(struct pcg_state_setseq_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_setseq_64_step_r(rng);
	+ return pcg_output_xsh_rs_64_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_setseq_64_xsh_rs_32_boundedrand_r(struct pcg_state_setseq_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_setseq_64_xsh_rs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_xsh_rs_64_random_r(struct pcg_state_setseq_128* rng)
	+{
	+ pcg_setseq_128_step_r(rng);
	+ return pcg_output_xsh_rs_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_xsh_rs_64_boundedrand_r(struct pcg_state_setseq_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_setseq_128_xsh_rs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_mcg_16_xsh_rs_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_mcg_16_step_r(rng);
	+ return pcg_output_xsh_rs_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_mcg_16_xsh_rs_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_mcg_16_xsh_rs_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_mcg_32_xsh_rs_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_mcg_32_step_r(rng);
	+ return pcg_output_xsh_rs_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_mcg_32_xsh_rs_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_mcg_32_xsh_rs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_mcg_64_xsh_rs_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_mcg_64_step_r(rng);
	+ return pcg_output_xsh_rs_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_mcg_64_xsh_rs_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_mcg_64_xsh_rs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_xsh_rs_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_mcg_128_step_r(rng);
	+ return pcg_output_xsh_rs_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_xsh_rs_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_mcg_128_xsh_rs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+/* Generation functions for XSH RR */
	+
	+inline uint8_t pcg_oneseq_16_xsh_rr_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_oneseq_16_step_r(rng);
	+ return pcg_output_xsh_rr_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_oneseq_16_xsh_rr_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_oneseq_16_xsh_rr_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_oneseq_32_xsh_rr_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_oneseq_32_step_r(rng);
	+ return pcg_output_xsh_rr_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_oneseq_32_xsh_rr_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_oneseq_32_xsh_rr_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_oneseq_64_xsh_rr_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_oneseq_64_step_r(rng);
	+ return pcg_output_xsh_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_oneseq_64_xsh_rr_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_oneseq_64_xsh_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_oneseq_128_xsh_rr_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_oneseq_128_step_r(rng);
	+ return pcg_output_xsh_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_oneseq_128_xsh_rr_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_oneseq_128_xsh_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_unique_16_xsh_rr_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_unique_16_step_r(rng);
	+ return pcg_output_xsh_rr_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_unique_16_xsh_rr_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_unique_16_xsh_rr_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_unique_32_xsh_rr_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_unique_32_step_r(rng);
	+ return pcg_output_xsh_rr_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_unique_32_xsh_rr_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_unique_32_xsh_rr_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_unique_64_xsh_rr_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_unique_64_step_r(rng);
	+ return pcg_output_xsh_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_unique_64_xsh_rr_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_unique_64_xsh_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_unique_128_xsh_rr_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_unique_128_step_r(rng);
	+ return pcg_output_xsh_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_unique_128_xsh_rr_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_unique_128_xsh_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_setseq_16_xsh_rr_8_random_r(struct pcg_state_setseq_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_setseq_16_step_r(rng);
	+ return pcg_output_xsh_rr_16_8(oldstate);
	+}
	+
	+inline uint8_t
	+pcg_setseq_16_xsh_rr_8_boundedrand_r(struct pcg_state_setseq_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_setseq_16_xsh_rr_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t
	+pcg_setseq_32_xsh_rr_16_random_r(struct pcg_state_setseq_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_setseq_32_step_r(rng);
	+ return pcg_output_xsh_rr_32_16(oldstate);
	+}
	+
	+inline uint16_t
	+pcg_setseq_32_xsh_rr_16_boundedrand_r(struct pcg_state_setseq_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_setseq_32_xsh_rr_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t
	+pcg_setseq_64_xsh_rr_32_random_r(struct pcg_state_setseq_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_setseq_64_step_r(rng);
	+ return pcg_output_xsh_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_setseq_64_xsh_rr_32_boundedrand_r(struct pcg_state_setseq_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_setseq_64_xsh_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_xsh_rr_64_random_r(struct pcg_state_setseq_128* rng)
	+{
	+ pcg_setseq_128_step_r(rng);
	+ return pcg_output_xsh_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_xsh_rr_64_boundedrand_r(struct pcg_state_setseq_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_setseq_128_xsh_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_mcg_16_xsh_rr_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_mcg_16_step_r(rng);
	+ return pcg_output_xsh_rr_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_mcg_16_xsh_rr_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_mcg_16_xsh_rr_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_mcg_32_xsh_rr_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_mcg_32_step_r(rng);
	+ return pcg_output_xsh_rr_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_mcg_32_xsh_rr_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_mcg_32_xsh_rr_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_mcg_64_xsh_rr_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_mcg_64_step_r(rng);
	+ return pcg_output_xsh_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_mcg_64_xsh_rr_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_mcg_64_xsh_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_xsh_rr_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_mcg_128_step_r(rng);
	+ return pcg_output_xsh_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_xsh_rr_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_mcg_128_xsh_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+/* Generation functions for RXS M XS (no MCG versions because they
	+ * don't make sense when you want to use the entire state)
	+ */
	+
	+inline uint8_t pcg_oneseq_8_rxs_m_xs_8_random_r(struct pcg_state_8* rng)
	+{
	+ uint8_t oldstate = rng->state;
	+ pcg_oneseq_8_step_r(rng);
	+ return pcg_output_rxs_m_xs_8_8(oldstate);
	+}
	+
	+inline uint8_t pcg_oneseq_8_rxs_m_xs_8_boundedrand_r(struct pcg_state_8* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_oneseq_8_rxs_m_xs_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_oneseq_16_rxs_m_xs_16_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_oneseq_16_step_r(rng);
	+ return pcg_output_rxs_m_xs_16_16(oldstate);
	+}
	+
	+inline uint16_t
	+pcg_oneseq_16_rxs_m_xs_16_boundedrand_r(struct pcg_state_16* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_oneseq_16_rxs_m_xs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_oneseq_32_rxs_m_xs_32_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_oneseq_32_step_r(rng);
	+ return pcg_output_rxs_m_xs_32_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_oneseq_32_rxs_m_xs_32_boundedrand_r(struct pcg_state_32* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_oneseq_32_rxs_m_xs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint64_t pcg_oneseq_64_rxs_m_xs_64_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_oneseq_64_step_r(rng);
	+ return pcg_output_rxs_m_xs_64_64(oldstate);
	+}
	+
	+inline uint64_t
	+pcg_oneseq_64_rxs_m_xs_64_boundedrand_r(struct pcg_state_64* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_oneseq_64_rxs_m_xs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_oneseq_128_rxs_m_xs_128_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_oneseq_128_step_r(rng);
	+ return pcg_output_rxs_m_xs_128_128(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_oneseq_128_rxs_m_xs_128_boundedrand_r(struct pcg_state_128* rng,
	+ pcg128_t bound)
	+{
	+ pcg128_t threshold = -bound % bound;
	+ for (;;) {
	+ pcg128_t r = pcg_oneseq_128_rxs_m_xs_128_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint16_t pcg_unique_16_rxs_m_xs_16_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_unique_16_step_r(rng);
	+ return pcg_output_rxs_m_xs_16_16(oldstate);
	+}
	+
	+inline uint16_t
	+pcg_unique_16_rxs_m_xs_16_boundedrand_r(struct pcg_state_16* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_unique_16_rxs_m_xs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_unique_32_rxs_m_xs_32_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_unique_32_step_r(rng);
	+ return pcg_output_rxs_m_xs_32_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_unique_32_rxs_m_xs_32_boundedrand_r(struct pcg_state_32* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_unique_32_rxs_m_xs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint64_t pcg_unique_64_rxs_m_xs_64_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_unique_64_step_r(rng);
	+ return pcg_output_rxs_m_xs_64_64(oldstate);
	+}
	+
	+inline uint64_t
	+pcg_unique_64_rxs_m_xs_64_boundedrand_r(struct pcg_state_64* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_unique_64_rxs_m_xs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_unique_128_rxs_m_xs_128_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_unique_128_step_r(rng);
	+ return pcg_output_rxs_m_xs_128_128(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_unique_128_rxs_m_xs_128_boundedrand_r(struct pcg_state_128* rng,
	+ pcg128_t bound)
	+{
	+ pcg128_t threshold = -bound % bound;
	+ for (;;) {
	+ pcg128_t r = pcg_unique_128_rxs_m_xs_128_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_setseq_8_rxs_m_xs_8_random_r(struct pcg_state_setseq_8* rng)
	+{
	+ uint8_t oldstate = rng->state;
	+ pcg_setseq_8_step_r(rng);
	+ return pcg_output_rxs_m_xs_8_8(oldstate);
	+}
	+
	+inline uint8_t
	+pcg_setseq_8_rxs_m_xs_8_boundedrand_r(struct pcg_state_setseq_8* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_setseq_8_rxs_m_xs_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t
	+pcg_setseq_16_rxs_m_xs_16_random_r(struct pcg_state_setseq_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_setseq_16_step_r(rng);
	+ return pcg_output_rxs_m_xs_16_16(oldstate);
	+}
	+
	+inline uint16_t
	+pcg_setseq_16_rxs_m_xs_16_boundedrand_r(struct pcg_state_setseq_16* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_setseq_16_rxs_m_xs_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t
	+pcg_setseq_32_rxs_m_xs_32_random_r(struct pcg_state_setseq_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_setseq_32_step_r(rng);
	+ return pcg_output_rxs_m_xs_32_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_setseq_32_rxs_m_xs_32_boundedrand_r(struct pcg_state_setseq_32* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_setseq_32_rxs_m_xs_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint64_t
	+pcg_setseq_64_rxs_m_xs_64_random_r(struct pcg_state_setseq_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_setseq_64_step_r(rng);
	+ return pcg_output_rxs_m_xs_64_64(oldstate);
	+}
	+
	+inline uint64_t
	+pcg_setseq_64_rxs_m_xs_64_boundedrand_r(struct pcg_state_setseq_64* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_setseq_64_rxs_m_xs_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_setseq_128_rxs_m_xs_128_random_r(struct pcg_state_setseq_128* rng)
	+{
	+ pcg_setseq_128_step_r(rng);
	+ return pcg_output_rxs_m_xs_128_128(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_setseq_128_rxs_m_xs_128_boundedrand_r(struct pcg_state_setseq_128* rng,
	+ pcg128_t bound)
	+{
	+ pcg128_t threshold = -bound % bound;
	+ for (;;) {
	+ pcg128_t r = pcg_setseq_128_rxs_m_xs_128_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+/* Generation functions for RXS M */
	+
	+inline uint8_t pcg_oneseq_16_rxs_m_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_oneseq_16_step_r(rng);
	+ return pcg_output_rxs_m_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_oneseq_16_rxs_m_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_oneseq_16_rxs_m_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_oneseq_32_rxs_m_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_oneseq_32_step_r(rng);
	+ return pcg_output_rxs_m_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_oneseq_32_rxs_m_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_oneseq_32_rxs_m_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_oneseq_64_rxs_m_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_oneseq_64_step_r(rng);
	+ return pcg_output_rxs_m_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_oneseq_64_rxs_m_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_oneseq_64_rxs_m_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_oneseq_128_rxs_m_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_oneseq_128_step_r(rng);
	+ return pcg_output_rxs_m_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_oneseq_128_rxs_m_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_oneseq_128_rxs_m_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_unique_16_rxs_m_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_unique_16_step_r(rng);
	+ return pcg_output_rxs_m_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_unique_16_rxs_m_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_unique_16_rxs_m_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_unique_32_rxs_m_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_unique_32_step_r(rng);
	+ return pcg_output_rxs_m_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_unique_32_rxs_m_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_unique_32_rxs_m_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_unique_64_rxs_m_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_unique_64_step_r(rng);
	+ return pcg_output_rxs_m_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_unique_64_rxs_m_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_unique_64_rxs_m_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_unique_128_rxs_m_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_unique_128_step_r(rng);
	+ return pcg_output_rxs_m_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_unique_128_rxs_m_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_unique_128_rxs_m_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_setseq_16_rxs_m_8_random_r(struct pcg_state_setseq_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_setseq_16_step_r(rng);
	+ return pcg_output_rxs_m_16_8(oldstate);
	+}
	+
	+inline uint8_t
	+pcg_setseq_16_rxs_m_8_boundedrand_r(struct pcg_state_setseq_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_setseq_16_rxs_m_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_setseq_32_rxs_m_16_random_r(struct pcg_state_setseq_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_setseq_32_step_r(rng);
	+ return pcg_output_rxs_m_32_16(oldstate);
	+}
	+
	+inline uint16_t
	+pcg_setseq_32_rxs_m_16_boundedrand_r(struct pcg_state_setseq_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_setseq_32_rxs_m_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_setseq_64_rxs_m_32_random_r(struct pcg_state_setseq_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_setseq_64_step_r(rng);
	+ return pcg_output_rxs_m_64_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_setseq_64_rxs_m_32_boundedrand_r(struct pcg_state_setseq_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_setseq_64_rxs_m_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_rxs_m_64_random_r(struct pcg_state_setseq_128* rng)
	+{
	+ pcg_setseq_128_step_r(rng);
	+ return pcg_output_rxs_m_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_rxs_m_64_boundedrand_r(struct pcg_state_setseq_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_setseq_128_rxs_m_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint8_t pcg_mcg_16_rxs_m_8_random_r(struct pcg_state_16* rng)
	+{
	+ uint16_t oldstate = rng->state;
	+ pcg_mcg_16_step_r(rng);
	+ return pcg_output_rxs_m_16_8(oldstate);
	+}
	+
	+inline uint8_t pcg_mcg_16_rxs_m_8_boundedrand_r(struct pcg_state_16* rng,
	+ uint8_t bound)
	+{
	+ uint8_t threshold = ((uint8_t)(-bound)) % bound;
	+ for (;;) {
	+ uint8_t r = pcg_mcg_16_rxs_m_8_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint16_t pcg_mcg_32_rxs_m_16_random_r(struct pcg_state_32* rng)
	+{
	+ uint32_t oldstate = rng->state;
	+ pcg_mcg_32_step_r(rng);
	+ return pcg_output_rxs_m_32_16(oldstate);
	+}
	+
	+inline uint16_t pcg_mcg_32_rxs_m_16_boundedrand_r(struct pcg_state_32* rng,
	+ uint16_t bound)
	+{
	+ uint16_t threshold = ((uint16_t)(-bound)) % bound;
	+ for (;;) {
	+ uint16_t r = pcg_mcg_32_rxs_m_16_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+inline uint32_t pcg_mcg_64_rxs_m_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_mcg_64_step_r(rng);
	+ return pcg_output_rxs_m_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_mcg_64_rxs_m_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_mcg_64_rxs_m_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_rxs_m_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_mcg_128_step_r(rng);
	+ return pcg_output_rxs_m_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_rxs_m_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_mcg_128_rxs_m_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+/* Generation functions for XSL RR (only defined for "large" types) */
	+
	+inline uint32_t pcg_oneseq_64_xsl_rr_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_oneseq_64_step_r(rng);
	+ return pcg_output_xsl_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_oneseq_64_xsl_rr_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_oneseq_64_xsl_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_oneseq_128_xsl_rr_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_oneseq_128_step_r(rng);
	+ return pcg_output_xsl_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_oneseq_128_xsl_rr_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_oneseq_128_xsl_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint32_t pcg_unique_64_xsl_rr_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_unique_64_step_r(rng);
	+ return pcg_output_xsl_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_unique_64_xsl_rr_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_unique_64_xsl_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_unique_128_xsl_rr_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_unique_128_step_r(rng);
	+ return pcg_output_xsl_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_unique_128_xsl_rr_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_unique_128_xsl_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint32_t
	+pcg_setseq_64_xsl_rr_32_random_r(struct pcg_state_setseq_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_setseq_64_step_r(rng);
	+ return pcg_output_xsl_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t
	+pcg_setseq_64_xsl_rr_32_boundedrand_r(struct pcg_state_setseq_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_setseq_64_xsl_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_xsl_rr_64_random_r(struct pcg_state_setseq_128* rng)
	+{
	+ pcg_setseq_128_step_r(rng);
	+ return pcg_output_xsl_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t
	+pcg_setseq_128_xsl_rr_64_boundedrand_r(struct pcg_state_setseq_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_setseq_128_xsl_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint32_t pcg_mcg_64_xsl_rr_32_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_mcg_64_step_r(rng);
	+ return pcg_output_xsl_rr_64_32(oldstate);
	+}
	+
	+inline uint32_t pcg_mcg_64_xsl_rr_32_boundedrand_r(struct pcg_state_64* rng,
	+ uint32_t bound)
	+{
	+ uint32_t threshold = -bound % bound;
	+ for (;;) {
	+ uint32_t r = pcg_mcg_64_xsl_rr_32_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_xsl_rr_64_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_mcg_128_step_r(rng);
	+ return pcg_output_xsl_rr_128_64(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline uint64_t pcg_mcg_128_xsl_rr_64_boundedrand_r(struct pcg_state_128* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_mcg_128_xsl_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+/* Generation functions for XSL RR RR (only defined for "large" types) */
	+
	+inline uint64_t pcg_oneseq_64_xsl_rr_rr_64_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_oneseq_64_step_r(rng);
	+ return pcg_output_xsl_rr_rr_64_64(oldstate);
	+}
	+
	+inline uint64_t
	+pcg_oneseq_64_xsl_rr_rr_64_boundedrand_r(struct pcg_state_64* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_oneseq_64_xsl_rr_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_oneseq_128_xsl_rr_rr_128_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_oneseq_128_step_r(rng);
	+ return pcg_output_xsl_rr_rr_128_128(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_oneseq_128_xsl_rr_rr_128_boundedrand_r(struct pcg_state_128* rng,
	+ pcg128_t bound)
	+{
	+ pcg128_t threshold = -bound % bound;
	+ for (;;) {
	+ pcg128_t r = pcg_oneseq_128_xsl_rr_rr_128_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint64_t pcg_unique_64_xsl_rr_rr_64_random_r(struct pcg_state_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_unique_64_step_r(rng);
	+ return pcg_output_xsl_rr_rr_64_64(oldstate);
	+}
	+
	+inline uint64_t
	+pcg_unique_64_xsl_rr_rr_64_boundedrand_r(struct pcg_state_64* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_unique_64_xsl_rr_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t pcg_unique_128_xsl_rr_rr_128_random_r(struct pcg_state_128* rng)
	+{
	+ pcg_unique_128_step_r(rng);
	+ return pcg_output_xsl_rr_rr_128_128(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_unique_128_xsl_rr_rr_128_boundedrand_r(struct pcg_state_128* rng,
	+ pcg128_t bound)
	+{
	+ pcg128_t threshold = -bound % bound;
	+ for (;;) {
	+ pcg128_t r = pcg_unique_128_xsl_rr_rr_128_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+inline uint64_t
	+pcg_setseq_64_xsl_rr_rr_64_random_r(struct pcg_state_setseq_64* rng)
	+{
	+ uint64_t oldstate = rng->state;
	+ pcg_setseq_64_step_r(rng);
	+ return pcg_output_xsl_rr_rr_64_64(oldstate);
	+}
	+
	+inline uint64_t
	+pcg_setseq_64_xsl_rr_rr_64_boundedrand_r(struct pcg_state_setseq_64* rng,
	+ uint64_t bound)
	+{
	+ uint64_t threshold = -bound % bound;
	+ for (;;) {
	+ uint64_t r = pcg_setseq_64_xsl_rr_rr_64_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_setseq_128_xsl_rr_rr_128_random_r(struct pcg_state_setseq_128* rng)
	+{
	+ pcg_setseq_128_step_r(rng);
	+ return pcg_output_xsl_rr_rr_128_128(rng->state);
	+}
	+#endif
	+
	+#if PCG_HAS_128BIT_OPS
	+inline pcg128_t
	+pcg_setseq_128_xsl_rr_rr_128_boundedrand_r(struct pcg_state_setseq_128* rng,
	+ pcg128_t bound)
	+{
	+ pcg128_t threshold = -bound % bound;
	+ for (;;) {
	+ pcg128_t r = pcg_setseq_128_xsl_rr_rr_128_random_r(rng);
	+ if (r >= threshold)
	+ return r % bound;
	+ }
	+}
	+#endif
	+
	+/*** Typedefs */
	+typedef struct pcg_state_setseq_64 pcg32_random_t;
	+typedef struct pcg_state_64 pcg32s_random_t;
	+typedef struct pcg_state_64 pcg32u_random_t;
	+typedef struct pcg_state_64 pcg32f_random_t;
	+/*** random_r */
	+#define pcg32_random_r pcg_setseq_64_xsh_rr_32_random_r
	+#define pcg32s_random_r pcg_oneseq_64_xsh_rr_32_random_r
	+#define pcg32u_random_r pcg_unique_64_xsh_rr_32_random_r
	+#define pcg32f_random_r pcg_mcg_64_xsh_rs_32_random_r
	+/*** boundedrand_r */
	+#define pcg32_boundedrand_r pcg_setseq_64_xsh_rr_32_boundedrand_r
	+#define pcg32s_boundedrand_r pcg_oneseq_64_xsh_rr_32_boundedrand_r
	+#define pcg32u_boundedrand_r pcg_unique_64_xsh_rr_32_boundedrand_r
	+#define pcg32f_boundedrand_r pcg_mcg_64_xsh_rs_32_boundedrand_r
	+/*** srandom_r */
	+#define pcg32_srandom_r pcg_setseq_64_srandom_r
	+#define pcg32s_srandom_r pcg_oneseq_64_srandom_r
	+#define pcg32u_srandom_r pcg_unique_64_srandom_r
	+#define pcg32f_srandom_r pcg_mcg_64_srandom_r
	+/*** advance_r */
	+#define pcg32_advance_r pcg_setseq_64_advance_r
	+#define pcg32s_advance_r pcg_oneseq_64_advance_r
	+#define pcg32u_advance_r pcg_unique_64_advance_r
	+#define pcg32f_advance_r pcg_mcg_64_advance_r
	+
	+#if PCG_HAS_128BIT_OPS
	+/*** Typedefs */
	+typedef struct pcg_state_setseq_128 pcg64_random_t;
	+typedef struct pcg_state_128 pcg64s_random_t;
	+typedef struct pcg_state_128 pcg64u_random_t;
	+typedef struct pcg_state_128 pcg64f_random_t;
	+/*** random_r */
	+#define pcg64_random_r pcg_setseq_128_xsl_rr_64_random_r
	+#define pcg64s_random_r pcg_oneseq_128_xsl_rr_64_random_r
	+#define pcg64u_random_r pcg_unique_128_xsl_rr_64_random_r
	+#define pcg64f_random_r pcg_mcg_128_xsl_rr_64_random_r
	+/*** boundedrand_r */
	+#define pcg64_boundedrand_r pcg_setseq_128_xsl_rr_64_boundedrand_r
	+#define pcg64s_boundedrand_r pcg_oneseq_128_xsl_rr_64_boundedrand_r
	+#define pcg64u_boundedrand_r pcg_unique_128_xsl_rr_64_boundedrand_r
	+#define pcg64f_boundedrand_r pcg_mcg_128_xsl_rr_64_boundedrand_r
	+/*** srandom_r */
	+#define pcg64_srandom_r pcg_setseq_128_srandom_r
	+#define pcg64s_srandom_r pcg_oneseq_128_srandom_r
	+#define pcg64u_srandom_r pcg_unique_128_srandom_r
	+#define pcg64f_srandom_r pcg_mcg_128_srandom_r
	+/*** advance_r */
	+#define pcg64_advance_r pcg_setseq_128_advance_r
	+#define pcg64s_advance_r pcg_oneseq_128_advance_r
	+#define pcg64u_advance_r pcg_unique_128_advance_r
	+#define pcg64f_advance_r pcg_mcg_128_advance_r
	+#endif
	+
	+/*** Typedefs */
	+typedef struct pcg_state_8 pcg8si_random_t;
	+typedef struct pcg_state_16 pcg16si_random_t;
	+typedef struct pcg_state_32 pcg32si_random_t;
	+typedef struct pcg_state_64 pcg64si_random_t;
	+/*** random_r */
	+#define pcg8si_random_r pcg_oneseq_8_rxs_m_xs_8_random_r
	+#define pcg16si_random_r pcg_oneseq_16_rxs_m_xs_16_random_r
	+#define pcg32si_random_r pcg_oneseq_32_rxs_m_xs_32_random_r
	+#define pcg64si_random_r pcg_oneseq_64_rxs_m_xs_64_random_r
	+/*** boundedrand_r */
	+#define pcg8si_boundedrand_r pcg_oneseq_8_rxs_m_xs_8_boundedrand_r
	+#define pcg16si_boundedrand_r pcg_oneseq_16_rxs_m_xs_16_boundedrand_r
	+#define pcg32si_boundedrand_r pcg_oneseq_32_rxs_m_xs_32_boundedrand_r
	+#define pcg64si_boundedrand_r pcg_oneseq_64_rxs_m_xs_64_boundedrand_r
	+/*** srandom_r */
	+#define pcg8si_srandom_r pcg_oneseq_8_srandom_r
	+#define pcg16si_srandom_r pcg_oneseq_16_srandom_r
	+#define pcg32si_srandom_r pcg_oneseq_32_srandom_r
	+#define pcg64si_srandom_r pcg_oneseq_64_srandom_r
	+/*** advance_r */
	+#define pcg8si_advance_r pcg_oneseq_8_advance_r
	+#define pcg16si_advance_r pcg_oneseq_16_advance_r
	+#define pcg32si_advance_r pcg_oneseq_32_advance_r
	+#define pcg64si_advance_r pcg_oneseq_64_advance_r
	+
	+#if PCG_HAS_128BIT_OPS
	+typedef struct pcg_state_128 pcg128si_random_t;
	+#define pcg128si_random_r pcg_oneseq_128_rxs_m_xs_128_random_r
	+#define pcg128si_boundedrand_r pcg_oneseq_128_rxs_m_xs_128_boundedrand_r
	+#define pcg128si_srandom_r pcg_oneseq_128_srandom_r
	+#define pcg128si_advance_r pcg_oneseq_128_advance_r
	+#endif
	+
	+/*** Typedefs */
	+typedef struct pcg_state_setseq_8 pcg8i_random_t;
	+typedef struct pcg_state_setseq_16 pcg16i_random_t;
	+typedef struct pcg_state_setseq_32 pcg32i_random_t;
	+typedef struct pcg_state_setseq_64 pcg64i_random_t;
	+/*** random_r */
	+#define pcg8i_random_r pcg_setseq_8_rxs_m_xs_8_random_r
	+#define pcg16i_random_r pcg_setseq_16_rxs_m_xs_16_random_r
	+#define pcg32i_random_r pcg_setseq_32_rxs_m_xs_32_random_r
	+#define pcg64i_random_r pcg_setseq_64_rxs_m_xs_64_random_r
	+/*** boundedrand_r */
	+#define pcg8i_boundedrand_r pcg_setseq_8_rxs_m_xs_8_boundedrand_r
	+#define pcg16i_boundedrand_r pcg_setseq_16_rxs_m_xs_16_boundedrand_r
	+#define pcg32i_boundedrand_r pcg_setseq_32_rxs_m_xs_32_boundedrand_r
	+#define pcg64i_boundedrand_r pcg_setseq_64_rxs_m_xs_64_boundedrand_r
	+/*** srandom_r */
	+#define pcg8i_srandom_r pcg_setseq_8_srandom_r
	+#define pcg16i_srandom_r pcg_setseq_16_srandom_r
	+#define pcg32i_srandom_r pcg_setseq_32_srandom_r
	+#define pcg64i_srandom_r pcg_setseq_64_srandom_r
	+/*** advance_r */
	+#define pcg8i_advance_r pcg_setseq_8_advance_r
	+#define pcg16i_advance_r pcg_setseq_16_advance_r
	+#define pcg32i_advance_r pcg_setseq_32_advance_r
	+#define pcg64i_advance_r pcg_setseq_64_advance_r
	+
	+#if PCG_HAS_128BIT_OPS
	+typedef struct pcg_state_setseq_128 pcg128i_random_t;
	+#define pcg128i_random_r pcg_setseq_128_rxs_m_xs_128_random_r
	+#define pcg128i_boundedrand_r pcg_setseq_128_rxs_m_xs_128_boundedrand_r
	+#define pcg128i_srandom_r pcg_setseq_128_srandom_r
	+#define pcg128i_advance_r pcg_setseq_128_advance_r
	+#endif
	+
	+extern uint32_t pcg32_random(void);
	+extern uint32_t pcg32_boundedrand(uint32_t bound);
	+extern void pcg32_srandom(uint64_t seed, uint64_t seq);
	+extern void pcg32_advance(uint64_t delta);
	+
	+#if PCG_HAS_128BIT_OPS
	+extern uint64_t pcg64_random(void);
	+extern uint64_t pcg64_boundedrand(uint64_t bound);
	+extern void pcg64_srandom(pcg128_t seed, pcg128_t seq);
	+extern void pcg64_advance(pcg128_t delta);
	+#endif
	+
	+/*
	+ * Static initialization constants (if you can't call srandom for some
	+ * bizarre reason).
	+ */
	+
	+#define PCG32_INITIALIZER PCG_STATE_SETSEQ_64_INITIALIZER
	+#define PCG32U_INITIALIZER PCG_STATE_UNIQUE_64_INITIALIZER
	+#define PCG32S_INITIALIZER PCG_STATE_ONESEQ_64_INITIALIZER
	+#define PCG32F_INITIALIZER PCG_STATE_MCG_64_INITIALIZER
	+
	+#if PCG_HAS_128BIT_OPS
	+#define PCG64_INITIALIZER PCG_STATE_SETSEQ_128_INITIALIZER
	+#define PCG64U_INITIALIZER PCG_STATE_UNIQUE_128_INITIALIZER
	+#define PCG64S_INITIALIZER PCG_STATE_ONESEQ_128_INITIALIZER
	+#define PCG64F_INITIALIZER PCG_STATE_MCG_128_INITIALIZER
	+#endif
	+
	+#define PCG8SI_INITIALIZER PCG_STATE_ONESEQ_8_INITIALIZER
	+#define PCG16SI_INITIALIZER PCG_STATE_ONESEQ_16_INITIALIZER
	+#define PCG32SI_INITIALIZER PCG_STATE_ONESEQ_32_INITIALIZER
	+#define PCG64SI_INITIALIZER PCG_STATE_ONESEQ_64_INITIALIZER
	+#if PCG_HAS_128BIT_OPS
	+#define PCG128SI_INITIALIZER PCG_STATE_ONESEQ_128_INITIALIZER
	+#endif
	+
	+#define PCG8I_INITIALIZER PCG_STATE_SETSEQ_8_INITIALIZER
	+#define PCG16I_INITIALIZER PCG_STATE_SETSEQ_16_INITIALIZER
	+#define PCG32I_INITIALIZER PCG_STATE_SETSEQ_32_INITIALIZER
	+#define PCG64I_INITIALIZER PCG_STATE_SETSEQ_64_INITIALIZER
	+#if PCG_HAS_128BIT_OPS
	+#define PCG128I_INITIALIZER PCG_STATE_SETSEQ_128_INITIALIZER
	+#endif
	+
	+#if __cplusplus
	+}
	+#endif
	+
	+#endif /* PCG_VARIANTS_H_INCLUDED */
	+
	diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
	index c3da5eb4cbbe..abe517263706 100644
	--- a/sys/dev/acpica/acpi.c
	+++ b/sys/dev/acpica/acpi.c
	@@ -1,4327 +1,4358 @@
	/*-
	* Copyright (c) 2000 Takanori Watanabe <takawata@jp.freebsd.org>
	* Copyright (c) 2000 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
	* Copyright (c) 2000, 2001 Michael Smith
	* Copyright (c) 2000 BSDi
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_acpi.h"

	#include <sys/param.h>
	#include <sys/eventhandler.h>
	#include <sys/kernel.h>
	#include <sys/proc.h>
	#include <sys/fcntl.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/ioccom.h>
	#include <sys/reboot.h>
	#include <sys/sysctl.h>
	#include <sys/ctype.h>
	#include <sys/linker.h>
	#include <sys/power.h>
	#include <sys/sbuf.h>
	#include <sys/sched.h>
	#include <sys/smp.h>
	#include <sys/timetc.h>

	#if defined(__i386__) \|\| defined(__amd64__)
	#include <machine/clock.h>
	#include <machine/pci_cfgreg.h>
	#endif
	#include <machine/resource.h>
	#include <machine/bus.h>
	#include <sys/rman.h>
	#include <isa/isavar.h>
	#include <isa/pnpvar.h>

	#include <contrib/dev/acpica/include/acpi.h>
	#include <contrib/dev/acpica/include/accommon.h>
	#include <contrib/dev/acpica/include/acnamesp.h>

	#include <dev/acpica/acpivar.h>
	#include <dev/acpica/acpiio.h>

	#include <dev/pci/pcivar.h>

	#include <vm/vm_param.h>

	static MALLOC_DEFINE(M_ACPIDEV, "acpidev", "ACPI devices");

	/* Hooks for the ACPI CA debugging infrastructure */
	#define _COMPONENT ACPI_BUS
	ACPI_MODULE_NAME("ACPI")

	static d_open_t acpiopen;
	static d_close_t acpiclose;
	static d_ioctl_t acpiioctl;

	static struct cdevsw acpi_cdevsw = {
	.d_version = D_VERSION,
	.d_open = acpiopen,
	.d_close = acpiclose,
	.d_ioctl = acpiioctl,
	.d_name = "acpi",
	};

	struct acpi_interface {
	ACPI_STRING *data;
	int num;
	};

	static char *sysres_ids[] = { "PNP0C01", "PNP0C02", NULL };
	static char *pcilink_ids[] = { "PNP0C0F", NULL };

	/* Global mutex for locking access to the ACPI subsystem. */
	struct mtx acpi_mutex;
	struct callout acpi_sleep_timer;

	/* Bitmap of device quirks. */
	int acpi_quirks;

	/* Supported sleep states. */
	static BOOLEAN acpi_sleep_states[ACPI_S_STATE_COUNT];

	static void acpi_lookup(void arg, const char name, device_t *dev);
	static int acpi_modevent(struct module mod, int event, void junk);
	static int acpi_probe(device_t dev);
	static int acpi_attach(device_t dev);
	static int acpi_suspend(device_t dev);
	static int acpi_resume(device_t dev);
	static int acpi_shutdown(device_t dev);
	static device_t acpi_add_child(device_t bus, u_int order, const char *name,
	int unit);
	static int acpi_print_child(device_t bus, device_t child);
	static void acpi_probe_nomatch(device_t bus, device_t child);
	static void acpi_driver_added(device_t dev, driver_t *driver);
	static void acpi_child_deleted(device_t dev, device_t child);
	static int acpi_read_ivar(device_t dev, device_t child, int index,
	uintptr_t *result);
	static int acpi_write_ivar(device_t dev, device_t child, int index,
	uintptr_t value);
	static struct resource_list *acpi_get_rlist(device_t dev, device_t child);
	static void acpi_reserve_resources(device_t dev);
	static int acpi_sysres_alloc(device_t dev);
	static int acpi_set_resource(device_t dev, device_t child, int type,
	int rid, rman_res_t start, rman_res_t count);
	static struct resource *acpi_alloc_resource(device_t bus, device_t child,
	int type, int *rid, rman_res_t start, rman_res_t end,
	rman_res_t count, u_int flags);
	static int acpi_adjust_resource(device_t bus, device_t child, int type,
	struct resource *r, rman_res_t start, rman_res_t end);
	static int acpi_release_resource(device_t bus, device_t child, int type,
	int rid, struct resource *r);
	static void acpi_delete_resource(device_t bus, device_t child, int type,
	int rid);
	static uint32_t acpi_isa_get_logicalid(device_t dev);
	static int acpi_isa_get_compatid(device_t dev, uint32_t *cids, int count);
	static int acpi_device_id_probe(device_t bus, device_t dev, char ids, char match);
	static ACPI_STATUS acpi_device_eval_obj(device_t bus, device_t dev,
	ACPI_STRING pathname, ACPI_OBJECT_LIST *parameters,
	ACPI_BUFFER *ret);
	static ACPI_STATUS acpi_device_scan_cb(ACPI_HANDLE h, UINT32 level,
	void context, void *retval);
	static ACPI_STATUS acpi_device_scan_children(device_t bus, device_t dev,
	int max_depth, acpi_scan_cb_t user_fn, void *arg);
	static int acpi_isa_pnp_probe(device_t bus, device_t child,
	struct isa_pnp_id *ids);
	+static void acpi_platform_osc(device_t dev);
	static void acpi_probe_children(device_t bus);
	static void acpi_probe_order(ACPI_HANDLE handle, int *order);
	static ACPI_STATUS acpi_probe_child(ACPI_HANDLE handle, UINT32 level,
	void context, void *status);
	static void acpi_sleep_enable(void *arg);
	static ACPI_STATUS acpi_sleep_disable(struct acpi_softc *sc);
	static ACPI_STATUS acpi_EnterSleepState(struct acpi_softc *sc, int state);
	static void acpi_shutdown_final(void *arg, int howto);
	static void acpi_enable_fixed_events(struct acpi_softc *sc);
	static void acpi_resync_clock(struct acpi_softc *sc);
	static int acpi_wake_sleep_prep(ACPI_HANDLE handle, int sstate);
	static int acpi_wake_run_prep(ACPI_HANDLE handle, int sstate);
	static int acpi_wake_prep_walk(int sstate);
	static int acpi_wake_sysctl_walk(device_t dev);
	static int acpi_wake_set_sysctl(SYSCTL_HANDLER_ARGS);
	static void acpi_system_eventhandler_sleep(void *arg, int state);
	static void acpi_system_eventhandler_wakeup(void *arg, int state);
	static int acpi_sname2sstate(const char *sname);
	static const char *acpi_sstate2sname(int sstate);
	static int acpi_supported_sleep_state_sysctl(SYSCTL_HANDLER_ARGS);
	static int acpi_sleep_state_sysctl(SYSCTL_HANDLER_ARGS);
	static int acpi_debug_objects_sysctl(SYSCTL_HANDLER_ARGS);
	static int acpi_pm_func(u_long cmd, void *arg, ...);
	static int acpi_child_location_str_method(device_t acdev, device_t child,
	char *buf, size_t buflen);
	static int acpi_child_pnpinfo_str_method(device_t acdev, device_t child,
	char *buf, size_t buflen);
	static void acpi_enable_pcie(void);
	static void acpi_hint_device_unit(device_t acdev, device_t child,
	const char name, int unitp);
	static void acpi_reset_interfaces(device_t dev);

	static device_method_t acpi_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, acpi_probe),
	DEVMETHOD(device_attach, acpi_attach),
	DEVMETHOD(device_shutdown, acpi_shutdown),
	DEVMETHOD(device_detach, bus_generic_detach),
	DEVMETHOD(device_suspend, acpi_suspend),
	DEVMETHOD(device_resume, acpi_resume),

	/* Bus interface */
	DEVMETHOD(bus_add_child, acpi_add_child),
	DEVMETHOD(bus_print_child, acpi_print_child),
	DEVMETHOD(bus_probe_nomatch, acpi_probe_nomatch),
	DEVMETHOD(bus_driver_added, acpi_driver_added),
	DEVMETHOD(bus_child_deleted, acpi_child_deleted),
	DEVMETHOD(bus_read_ivar, acpi_read_ivar),
	DEVMETHOD(bus_write_ivar, acpi_write_ivar),
	DEVMETHOD(bus_get_resource_list, acpi_get_rlist),
	DEVMETHOD(bus_set_resource, acpi_set_resource),
	DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
	DEVMETHOD(bus_alloc_resource, acpi_alloc_resource),
	DEVMETHOD(bus_adjust_resource, acpi_adjust_resource),
	DEVMETHOD(bus_release_resource, acpi_release_resource),
	DEVMETHOD(bus_delete_resource, acpi_delete_resource),
	DEVMETHOD(bus_child_pnpinfo_str, acpi_child_pnpinfo_str_method),
	DEVMETHOD(bus_child_location_str, acpi_child_location_str_method),
	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
	DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
	DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
	DEVMETHOD(bus_hint_device_unit, acpi_hint_device_unit),
	DEVMETHOD(bus_get_cpus, acpi_get_cpus),
	DEVMETHOD(bus_get_domain, acpi_get_domain),

	/* ACPI bus */
	DEVMETHOD(acpi_id_probe, acpi_device_id_probe),
	DEVMETHOD(acpi_evaluate_object, acpi_device_eval_obj),
	DEVMETHOD(acpi_pwr_for_sleep, acpi_device_pwr_for_sleep),
	DEVMETHOD(acpi_scan_children, acpi_device_scan_children),

	/* ISA emulation */
	DEVMETHOD(isa_pnp_probe, acpi_isa_pnp_probe),

	DEVMETHOD_END
	};

	static driver_t acpi_driver = {
	"acpi",
	acpi_methods,
	sizeof(struct acpi_softc),
	};

	static devclass_t acpi_devclass;
	EARLY_DRIVER_MODULE(acpi, nexus, acpi_driver, acpi_devclass, acpi_modevent, 0,
	BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
	MODULE_VERSION(acpi, 1);

	ACPI_SERIAL_DECL(acpi, "ACPI root bus");

	/* Local pools for managing system resources for ACPI child devices. */
	static struct rman acpi_rman_io, acpi_rman_mem;

	#define ACPI_MINIMUM_AWAKETIME 5

	/* Holds the description of the acpi0 device. */
	static char acpi_desc[ACPI_OEM_ID_SIZE + ACPI_OEM_TABLE_ID_SIZE + 2];

	SYSCTL_NODE(_debug, OID_AUTO, acpi, CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"ACPI debugging");
	static char acpi_ca_version[12];
	SYSCTL_STRING(_debug_acpi, OID_AUTO, acpi_ca_version, CTLFLAG_RD,
	acpi_ca_version, 0, "Version of Intel ACPI-CA");

	/*
	* Allow overriding _OSI methods.
	*/
	static char acpi_install_interface[256];
	TUNABLE_STR("hw.acpi.install_interface", acpi_install_interface,
	sizeof(acpi_install_interface));
	static char acpi_remove_interface[256];
	TUNABLE_STR("hw.acpi.remove_interface", acpi_remove_interface,
	sizeof(acpi_remove_interface));

	/* Allow users to dump Debug objects without ACPI debugger. */
	static int acpi_debug_objects;
	TUNABLE_INT("debug.acpi.enable_debug_objects", &acpi_debug_objects);
	SYSCTL_PROC(_debug_acpi, OID_AUTO, enable_debug_objects,
	CTLFLAG_RW \| CTLTYPE_INT \| CTLFLAG_NEEDGIANT, NULL, 0,
	acpi_debug_objects_sysctl, "I",
	"Enable Debug objects");

	/* Allow the interpreter to ignore common mistakes in BIOS. */
	static int acpi_interpreter_slack = 1;
	TUNABLE_INT("debug.acpi.interpreter_slack", &acpi_interpreter_slack);
	SYSCTL_INT(_debug_acpi, OID_AUTO, interpreter_slack, CTLFLAG_RDTUN,
	&acpi_interpreter_slack, 1, "Turn on interpreter slack mode.");

	/* Ignore register widths set by FADT and use default widths instead. */
	static int acpi_ignore_reg_width = 1;
	TUNABLE_INT("debug.acpi.default_register_width", &acpi_ignore_reg_width);
	SYSCTL_INT(_debug_acpi, OID_AUTO, default_register_width, CTLFLAG_RDTUN,
	&acpi_ignore_reg_width, 1, "Ignore register widths set by FADT");

	/* Allow users to override quirks. */
	TUNABLE_INT("debug.acpi.quirks", &acpi_quirks);

	int acpi_susp_bounce;
	SYSCTL_INT(_debug_acpi, OID_AUTO, suspend_bounce, CTLFLAG_RW,
	&acpi_susp_bounce, 0, "Don't actually suspend, just test devices.");

	/*
	* ACPI can only be loaded as a module by the loader; activating it after
	* system bootstrap time is not useful, and can be fatal to the system.
	* It also cannot be unloaded, since the entire system bus hierarchy hangs
	* off it.
	*/
	static int
	acpi_modevent(struct module mod, int event, void junk)
	{
	switch (event) {
	case MOD_LOAD:
	if (!cold) {
	printf("The ACPI driver cannot be loaded after boot.\n");
	return (EPERM);
	}
	break;
	case MOD_UNLOAD:
	if (!cold && power_pm_get_type() == POWER_PM_TYPE_ACPI)
	return (EBUSY);
	break;
	default:
	break;
	}
	return (0);
	}

	/*
	* Perform early initialization.
	*/
	ACPI_STATUS
	acpi_Startup(void)
	{
	static int started = 0;
	ACPI_STATUS status;
	int val;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	/* Only run the startup code once. The MADT driver also calls this. */
	if (started)
	return_VALUE (AE_OK);
	started = 1;

	/*
	* Initialize the ACPICA subsystem.
	*/
	if (ACPI_FAILURE(status = AcpiInitializeSubsystem())) {
	printf("ACPI: Could not initialize Subsystem: %s\n",
	AcpiFormatException(status));
	return_VALUE (status);
	}

	/*
	* Pre-allocate space for RSDT/XSDT and DSDT tables and allow resizing
	* if more tables exist.
	*/
	if (ACPI_FAILURE(status = AcpiInitializeTables(NULL, 2, TRUE))) {
	printf("ACPI: Table initialisation failed: %s\n",
	AcpiFormatException(status));
	return_VALUE (status);
	}

	/* Set up any quirks we have for this system. */
	if (acpi_quirks == ACPI_Q_OK)
	acpi_table_quirks(&acpi_quirks);

	/* If the user manually set the disabled hint to 0, force-enable ACPI. */
	if (resource_int_value("acpi", 0, "disabled", &val) == 0 && val == 0)
	acpi_quirks &= ~ACPI_Q_BROKEN;
	if (acpi_quirks & ACPI_Q_BROKEN) {
	printf("ACPI disabled by blacklist. Contact your BIOS vendor.\n");
	status = AE_SUPPORT;
	}

	return_VALUE (status);
	}

	/*
	* Detect ACPI and perform early initialisation.
	*/
	int
	acpi_identify(void)
	{
	ACPI_TABLE_RSDP *rsdp;
	ACPI_TABLE_HEADER *rsdt;
	ACPI_PHYSICAL_ADDRESS paddr;
	struct sbuf sb;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	if (!cold)
	return (ENXIO);

	/* Check that we haven't been disabled with a hint. */
	if (resource_disabled("acpi", 0))
	return (ENXIO);

	/* Check for other PM systems. */
	if (power_pm_get_type() != POWER_PM_TYPE_NONE &&
	power_pm_get_type() != POWER_PM_TYPE_ACPI) {
	printf("ACPI identify failed, other PM system enabled.\n");
	return (ENXIO);
	}

	/* Initialize root tables. */
	if (ACPI_FAILURE(acpi_Startup())) {
	printf("ACPI: Try disabling either ACPI or apic support.\n");
	return (ENXIO);
	}

	if ((paddr = AcpiOsGetRootPointer()) == 0 \|\|
	(rsdp = AcpiOsMapMemory(paddr, sizeof(ACPI_TABLE_RSDP))) == NULL)
	return (ENXIO);
	if (rsdp->Revision > 1 && rsdp->XsdtPhysicalAddress != 0)
	paddr = (ACPI_PHYSICAL_ADDRESS)rsdp->XsdtPhysicalAddress;
	else
	paddr = (ACPI_PHYSICAL_ADDRESS)rsdp->RsdtPhysicalAddress;
	AcpiOsUnmapMemory(rsdp, sizeof(ACPI_TABLE_RSDP));

	if ((rsdt = AcpiOsMapMemory(paddr, sizeof(ACPI_TABLE_HEADER))) == NULL)
	return (ENXIO);
	sbuf_new(&sb, acpi_desc, sizeof(acpi_desc), SBUF_FIXEDLEN);
	sbuf_bcat(&sb, rsdt->OemId, ACPI_OEM_ID_SIZE);
	sbuf_trim(&sb);
	sbuf_putc(&sb, ' ');
	sbuf_bcat(&sb, rsdt->OemTableId, ACPI_OEM_TABLE_ID_SIZE);
	sbuf_trim(&sb);
	sbuf_finish(&sb);
	sbuf_delete(&sb);
	AcpiOsUnmapMemory(rsdt, sizeof(ACPI_TABLE_HEADER));

	snprintf(acpi_ca_version, sizeof(acpi_ca_version), "%x", ACPI_CA_VERSION);

	return (0);
	}

	/*
	* Fetch some descriptive data from ACPI to put in our attach message.
	*/
	static int
	acpi_probe(device_t dev)
	{

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	device_set_desc(dev, acpi_desc);

	return_VALUE (BUS_PROBE_NOWILDCARD);
	}

	static int
	acpi_attach(device_t dev)
	{
	struct acpi_softc *sc;
	ACPI_STATUS status;
	int error, state;
	UINT32 flags;
	UINT8 TypeA, TypeB;
	char *env;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	sc = device_get_softc(dev);
	sc->acpi_dev = dev;
	callout_init(&sc->susp_force_to, 1);

	error = ENXIO;

	/* Initialize resource manager. */
	acpi_rman_io.rm_type = RMAN_ARRAY;
	acpi_rman_io.rm_start = 0;
	acpi_rman_io.rm_end = 0xffff;
	acpi_rman_io.rm_descr = "ACPI I/O ports";
	if (rman_init(&acpi_rman_io) != 0)
	panic("acpi rman_init IO ports failed");
	acpi_rman_mem.rm_type = RMAN_ARRAY;
	acpi_rman_mem.rm_descr = "ACPI I/O memory addresses";
	if (rman_init(&acpi_rman_mem) != 0)
	panic("acpi rman_init memory failed");

	/* Initialise the ACPI mutex */
	mtx_init(&acpi_mutex, "ACPI global lock", NULL, MTX_DEF);

	/*
	* Set the globals from our tunables. This is needed because ACPI-CA
	* uses UINT8 for some values and we have no tunable_byte.
	*/
	AcpiGbl_EnableInterpreterSlack = acpi_interpreter_slack ? TRUE : FALSE;
	AcpiGbl_EnableAmlDebugObject = acpi_debug_objects ? TRUE : FALSE;
	AcpiGbl_UseDefaultRegisterWidths = acpi_ignore_reg_width ? TRUE : FALSE;

	#ifndef ACPI_DEBUG
	/*
	* Disable all debugging layers and levels.
	*/
	AcpiDbgLayer = 0;
	AcpiDbgLevel = 0;
	#endif

	/* Override OS interfaces if the user requested. */
	acpi_reset_interfaces(dev);

	/* Load ACPI name space. */
	status = AcpiLoadTables();
	if (ACPI_FAILURE(status)) {
	device_printf(dev, "Could not load Namespace: %s\n",
	AcpiFormatException(status));
	goto out;
	}

	/* Handle MCFG table if present. */
	acpi_enable_pcie();

	/*
	* Note that some systems (specifically, those with namespace evaluation
	* issues that require the avoidance of parts of the namespace) must
	* avoid running _INI and _STA on everything, as well as dodging the final
	* object init pass.
	*
	* For these devices, we set ACPI_NO_DEVICE_INIT and ACPI_NO_OBJECT_INIT).
	*
	* XXX We should arrange for the object init pass after we have attached
	* all our child devices, but on many systems it works here.
	*/
	flags = 0;
	if (testenv("debug.acpi.avoid"))
	flags = ACPI_NO_DEVICE_INIT \| ACPI_NO_OBJECT_INIT;

	/* Bring the hardware and basic handlers online. */
	if (ACPI_FAILURE(status = AcpiEnableSubsystem(flags))) {
	device_printf(dev, "Could not enable ACPI: %s\n",
	AcpiFormatException(status));
	goto out;
	}

	/*
	* Call the ECDT probe function to provide EC functionality before
	* the namespace has been evaluated.
	*
	* XXX This happens before the sysresource devices have been probed and
	* attached so its resources come from nexus0. In practice, this isn't
	* a problem but should be addressed eventually.
	*/
	acpi_ec_ecdt_probe(dev);

	/* Bring device objects and regions online. */
	if (ACPI_FAILURE(status = AcpiInitializeObjects(flags))) {
	device_printf(dev, "Could not initialize ACPI objects: %s\n",
	AcpiFormatException(status));
	goto out;
	}

	/*
	* Setup our sysctl tree.
	*
	* XXX: This doesn't check to make sure that none of these fail.
	*/
	sysctl_ctx_init(&sc->acpi_sysctl_ctx);
	sc->acpi_sysctl_tree = SYSCTL_ADD_NODE(&sc->acpi_sysctl_ctx,
	SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO, device_get_name(dev),
	CTLFLAG_RD \| CTLFLAG_MPSAFE, 0, "");
	SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "supported_sleep_state",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	0, 0, acpi_supported_sleep_state_sysctl, "A",
	"List supported ACPI sleep states.");
	SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "power_button_state",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&sc->acpi_power_button_sx, 0, acpi_sleep_state_sysctl, "A",
	"Power button ACPI sleep state.");
	SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "sleep_button_state",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&sc->acpi_sleep_button_sx, 0, acpi_sleep_state_sysctl, "A",
	"Sleep button ACPI sleep state.");
	SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "lid_switch_state",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&sc->acpi_lid_switch_sx, 0, acpi_sleep_state_sysctl, "A",
	"Lid ACPI sleep state. Set to S3 if you want to suspend your laptop when close the Lid.");
	SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "standby_state",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&sc->acpi_standby_sx, 0, acpi_sleep_state_sysctl, "A", "");
	SYSCTL_ADD_PROC(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "suspend_state",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&sc->acpi_suspend_sx, 0, acpi_sleep_state_sysctl, "A", "");
	SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "sleep_delay", CTLFLAG_RW, &sc->acpi_sleep_delay, 0,
	"sleep delay in seconds");
	SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "s4bios", CTLFLAG_RW, &sc->acpi_s4bios, 0, "S4BIOS mode");
	SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "verbose", CTLFLAG_RW, &sc->acpi_verbose, 0, "verbose mode");
	SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "disable_on_reboot", CTLFLAG_RW,
	&sc->acpi_do_disable, 0, "Disable ACPI when rebooting/halting system");
	SYSCTL_ADD_INT(&sc->acpi_sysctl_ctx, SYSCTL_CHILDREN(sc->acpi_sysctl_tree),
	OID_AUTO, "handle_reboot", CTLFLAG_RW,
	&sc->acpi_handle_reboot, 0, "Use ACPI Reset Register to reboot");

	/*
	* Default to 1 second before sleeping to give some machines time to
	* stabilize.
	*/
	sc->acpi_sleep_delay = 1;
	if (bootverbose)
	sc->acpi_verbose = 1;
	if ((env = kern_getenv("hw.acpi.verbose")) != NULL) {
	if (strcmp(env, "0") != 0)
	sc->acpi_verbose = 1;
	freeenv(env);
	}

	/* Only enable reboot by default if the FADT says it is available. */
	if (AcpiGbl_FADT.Flags & ACPI_FADT_RESET_REGISTER)
	sc->acpi_handle_reboot = 1;

	#if !ACPI_REDUCED_HARDWARE
	/* Only enable S4BIOS by default if the FACS says it is available. */
	if (AcpiGbl_FACS != NULL && AcpiGbl_FACS->Flags & ACPI_FACS_S4_BIOS_PRESENT)
	sc->acpi_s4bios = 1;
	#endif

	/* Probe all supported sleep states. */
	acpi_sleep_states[ACPI_STATE_S0] = TRUE;
	for (state = ACPI_STATE_S1; state < ACPI_S_STATE_COUNT; state++)
	if (ACPI_SUCCESS(AcpiEvaluateObject(ACPI_ROOT_OBJECT,
	__DECONST(char *, AcpiGbl_SleepStateNames[state]), NULL, NULL)) &&
	ACPI_SUCCESS(AcpiGetSleepTypeData(state, &TypeA, &TypeB)))
	acpi_sleep_states[state] = TRUE;

	/*
	* Dispatch the default sleep state to devices. The lid switch is set
	* to UNKNOWN by default to avoid surprising users.
	*/
	sc->acpi_power_button_sx = acpi_sleep_states[ACPI_STATE_S5] ?
	ACPI_STATE_S5 : ACPI_STATE_UNKNOWN;
	sc->acpi_lid_switch_sx = ACPI_STATE_UNKNOWN;
	sc->acpi_standby_sx = acpi_sleep_states[ACPI_STATE_S1] ?
	ACPI_STATE_S1 : ACPI_STATE_UNKNOWN;
	sc->acpi_suspend_sx = acpi_sleep_states[ACPI_STATE_S3] ?
	ACPI_STATE_S3 : ACPI_STATE_UNKNOWN;

	/* Pick the first valid sleep state for the sleep button default. */
	sc->acpi_sleep_button_sx = ACPI_STATE_UNKNOWN;
	for (state = ACPI_STATE_S1; state <= ACPI_STATE_S4; state++)
	if (acpi_sleep_states[state]) {
	sc->acpi_sleep_button_sx = state;
	break;
	}

	acpi_enable_fixed_events(sc);

	/*
	* Scan the namespace and attach/initialise children.
	*/

	/* Register our shutdown handler. */
	EVENTHANDLER_REGISTER(shutdown_final, acpi_shutdown_final, sc,
	SHUTDOWN_PRI_LAST);

	/*
	* Register our acpi event handlers.
	* XXX should be configurable eg. via userland policy manager.
	*/
	EVENTHANDLER_REGISTER(acpi_sleep_event, acpi_system_eventhandler_sleep,
	sc, ACPI_EVENT_PRI_LAST);
	EVENTHANDLER_REGISTER(acpi_wakeup_event, acpi_system_eventhandler_wakeup,
	sc, ACPI_EVENT_PRI_LAST);

	/* Flag our initial states. */
	sc->acpi_enabled = TRUE;
	sc->acpi_sstate = ACPI_STATE_S0;
	sc->acpi_sleep_disabled = TRUE;

	/* Create the control device */
	sc->acpi_dev_t = make_dev(&acpi_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0664,
	"acpi");
	sc->acpi_dev_t->si_drv1 = sc;

	if ((error = acpi_machdep_init(dev)))
	goto out;

	/* Register ACPI again to pass the correct argument of pm_func. */
	power_pm_register(POWER_PM_TYPE_ACPI, acpi_pm_func, sc);

	+ acpi_platform_osc(dev);
	+
	if (!acpi_disabled("bus")) {
	EVENTHANDLER_REGISTER(dev_lookup, acpi_lookup, NULL, 1000);
	acpi_probe_children(dev);
	}

	/* Update all GPEs and enable runtime GPEs. */
	status = AcpiUpdateAllGpes();
	if (ACPI_FAILURE(status))
	device_printf(dev, "Could not update all GPEs: %s\n",
	AcpiFormatException(status));

	/* Allow sleep request after a while. */
	callout_init_mtx(&acpi_sleep_timer, &acpi_mutex, 0);
	callout_reset(&acpi_sleep_timer, hz * ACPI_MINIMUM_AWAKETIME,
	acpi_sleep_enable, sc);

	error = 0;

	out:
	return_VALUE (error);
	}

	static void
	acpi_set_power_children(device_t dev, int state)
	{
	device_t child;
	device_t *devlist;
	int dstate, i, numdevs;

	if (device_get_children(dev, &devlist, &numdevs) != 0)
	return;

	/*
	* Retrieve and set D-state for the sleep state if _SxD is present.
	* Skip children who aren't attached since they are handled separately.
	*/
	for (i = 0; i < numdevs; i++) {
	child = devlist[i];
	dstate = state;
	if (device_is_attached(child) &&
	acpi_device_pwr_for_sleep(dev, child, &dstate) == 0)
	acpi_set_powerstate(child, dstate);
	}
	free(devlist, M_TEMP);
	}

	static int
	acpi_suspend(device_t dev)
	{
	int error;

	GIANT_REQUIRED;

	error = bus_generic_suspend(dev);
	if (error == 0)
	acpi_set_power_children(dev, ACPI_STATE_D3);

	return (error);
	}

	static int
	acpi_resume(device_t dev)
	{

	GIANT_REQUIRED;

	acpi_set_power_children(dev, ACPI_STATE_D0);

	return (bus_generic_resume(dev));
	}

	static int
	acpi_shutdown(device_t dev)
	{

	GIANT_REQUIRED;

	/* Allow children to shutdown first. */
	bus_generic_shutdown(dev);

	/*
	* Enable any GPEs that are able to power-on the system (i.e., RTC).
	* Also, disable any that are not valid for this state (most).
	*/
	acpi_wake_prep_walk(ACPI_STATE_S5);

	return (0);
	}

	/*
	* Handle a new device being added
	*/
	static device_t
	acpi_add_child(device_t bus, u_int order, const char *name, int unit)
	{
	struct acpi_device *ad;
	device_t child;

	if ((ad = malloc(sizeof(*ad), M_ACPIDEV, M_NOWAIT \| M_ZERO)) == NULL)
	return (NULL);

	resource_list_init(&ad->ad_rl);

	child = device_add_child_ordered(bus, order, name, unit);
	if (child != NULL)
	device_set_ivars(child, ad);
	else
	free(ad, M_ACPIDEV);
	return (child);
	}

	static int
	acpi_print_child(device_t bus, device_t child)
	{
	struct acpi_device *adev = device_get_ivars(child);
	struct resource_list *rl = &adev->ad_rl;
	int retval = 0;

	retval += bus_print_child_header(bus, child);
	retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
	retval += resource_list_print_type(rl, "iomem", SYS_RES_MEMORY, "%#jx");
	retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
	retval += resource_list_print_type(rl, "drq", SYS_RES_DRQ, "%jd");
	if (device_get_flags(child))
	retval += printf(" flags %#x", device_get_flags(child));
	retval += bus_print_child_domain(bus, child);
	retval += bus_print_child_footer(bus, child);

	return (retval);
	}

	/*
	* If this device is an ACPI child but no one claimed it, attempt
	* to power it off. We'll power it back up when a driver is added.
	*
	* XXX Disabled for now since many necessary devices (like fdc and
	* ATA) don't claim the devices we created for them but still expect
	* them to be powered up.
	*/
	static void
	acpi_probe_nomatch(device_t bus, device_t child)
	{
	#ifdef ACPI_ENABLE_POWERDOWN_NODRIVER
	acpi_set_powerstate(child, ACPI_STATE_D3);
	#endif
	}

	/*
	* If a new driver has a chance to probe a child, first power it up.
	*
	* XXX Disabled for now (see acpi_probe_nomatch for details).
	*/
	static void
	acpi_driver_added(device_t dev, driver_t *driver)
	{
	device_t child, *devlist;
	int i, numdevs;

	DEVICE_IDENTIFY(driver, dev);
	if (device_get_children(dev, &devlist, &numdevs))
	return;
	for (i = 0; i < numdevs; i++) {
	child = devlist[i];
	if (device_get_state(child) == DS_NOTPRESENT) {
	#ifdef ACPI_ENABLE_POWERDOWN_NODRIVER
	acpi_set_powerstate(child, ACPI_STATE_D0);
	if (device_probe_and_attach(child) != 0)
	acpi_set_powerstate(child, ACPI_STATE_D3);
	#else
	device_probe_and_attach(child);
	#endif
	}
	}
	free(devlist, M_TEMP);
	}

	/* Location hint for devctl(8) */
	static int
	acpi_child_location_str_method(device_t cbdev, device_t child, char *buf,
	size_t buflen)
	{
	struct acpi_device *dinfo = device_get_ivars(child);
	char buf2[32];
	int pxm;

	if (dinfo->ad_handle) {
	snprintf(buf, buflen, "handle=%s", acpi_name(dinfo->ad_handle));
	if (ACPI_SUCCESS(acpi_GetInteger(dinfo->ad_handle, "_PXM", &pxm))) {
	snprintf(buf2, 32, " _PXM=%d", pxm);
	strlcat(buf, buf2, buflen);
	}
	} else {
	snprintf(buf, buflen, "");
	}
	return (0);
	}

	/* PnP information for devctl(8) */
	int
	acpi_pnpinfo_str(ACPI_HANDLE handle, char *buf, size_t buflen)
	{
	ACPI_DEVICE_INFO *adinfo;

	if (ACPI_FAILURE(AcpiGetObjectInfo(handle, &adinfo))) {
	snprintf(buf, buflen, "unknown");
	return (0);
	}

	snprintf(buf, buflen, "_HID=%s _UID=%lu _CID=%s",
	(adinfo->Valid & ACPI_VALID_HID) ?
	adinfo->HardwareId.String : "none",
	(adinfo->Valid & ACPI_VALID_UID) ?
	strtoul(adinfo->UniqueId.String, NULL, 10) : 0UL,
	((adinfo->Valid & ACPI_VALID_CID) &&
	adinfo->CompatibleIdList.Count > 0) ?
	adinfo->CompatibleIdList.Ids[0].String : "none");
	AcpiOsFree(adinfo);

	return (0);
	}

	static int
	acpi_child_pnpinfo_str_method(device_t cbdev, device_t child, char *buf,
	size_t buflen)
	{
	struct acpi_device *dinfo = device_get_ivars(child);

	return (acpi_pnpinfo_str(dinfo->ad_handle, buf, buflen));
	}

	/*
	* Handle device deletion.
	*/
	static void
	acpi_child_deleted(device_t dev, device_t child)
	{
	struct acpi_device *dinfo = device_get_ivars(child);

	if (acpi_get_device(dinfo->ad_handle) == child)
	AcpiDetachData(dinfo->ad_handle, acpi_fake_objhandler);
	}

	/*
	* Handle per-device ivars
	*/
	static int
	acpi_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
	{
	struct acpi_device *ad;

	if ((ad = device_get_ivars(child)) == NULL) {
	device_printf(child, "device has no ivars\n");
	return (ENOENT);
	}

	/* ACPI and ISA compatibility ivars */
	switch(index) {
	case ACPI_IVAR_HANDLE:
	(ACPI_HANDLE )result = ad->ad_handle;
	break;
	case ACPI_IVAR_PRIVATE:
	(void *)result = ad->ad_private;
	break;
	case ACPI_IVAR_FLAGS:
	(int )result = ad->ad_flags;
	break;
	case ISA_IVAR_VENDORID:
	case ISA_IVAR_SERIAL:
	case ISA_IVAR_COMPATID:
	(int )result = -1;
	break;
	case ISA_IVAR_LOGICALID:
	(int )result = acpi_isa_get_logicalid(child);
	break;
	case PCI_IVAR_CLASS:
	(uint8_t)result = (ad->ad_cls_class >> 16) & 0xff;
	break;
	case PCI_IVAR_SUBCLASS:
	(uint8_t)result = (ad->ad_cls_class >> 8) & 0xff;
	break;
	case PCI_IVAR_PROGIF:
	(uint8_t)result = (ad->ad_cls_class >> 0) & 0xff;
	break;
	default:
	return (ENOENT);
	}

	return (0);
	}

	static int
	acpi_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
	{
	struct acpi_device *ad;

	if ((ad = device_get_ivars(child)) == NULL) {
	device_printf(child, "device has no ivars\n");
	return (ENOENT);
	}

	switch(index) {
	case ACPI_IVAR_HANDLE:
	ad->ad_handle = (ACPI_HANDLE)value;
	break;
	case ACPI_IVAR_PRIVATE:
	ad->ad_private = (void *)value;
	break;
	case ACPI_IVAR_FLAGS:
	ad->ad_flags = (int)value;
	break;
	default:
	panic("bad ivar write request (%d)", index);
	return (ENOENT);
	}

	return (0);
	}

	/*
	* Handle child resource allocation/removal
	*/
	static struct resource_list *
	acpi_get_rlist(device_t dev, device_t child)
	{
	struct acpi_device *ad;

	ad = device_get_ivars(child);
	return (&ad->ad_rl);
	}

	static int
	acpi_match_resource_hint(device_t dev, int type, long value)
	{
	struct acpi_device *ad = device_get_ivars(dev);
	struct resource_list *rl = &ad->ad_rl;
	struct resource_list_entry *rle;

	STAILQ_FOREACH(rle, rl, link) {
	if (rle->type != type)
	continue;
	if (rle->start <= value && rle->end >= value)
	return (1);
	}
	return (0);
	}

	/*
	* Wire device unit numbers based on resource matches in hints.
	*/
	static void
	acpi_hint_device_unit(device_t acdev, device_t child, const char *name,
	int *unitp)
	{
	const char *s;
	long value;
	int line, matches, unit;

	/*
	* Iterate over all the hints for the devices with the specified
	* name to see if one's resources are a subset of this device.
	*/
	line = 0;
	while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
	/* Must have an "at" for acpi or isa. */
	resource_string_value(name, unit, "at", &s);
	if (!(strcmp(s, "acpi0") == 0 \|\| strcmp(s, "acpi") == 0 \|\|
	strcmp(s, "isa0") == 0 \|\| strcmp(s, "isa") == 0))
	continue;

	/*
	* Check for matching resources. We must have at least one match.
	* Since I/O and memory resources cannot be shared, if we get a
	* match on either of those, ignore any mismatches in IRQs or DRQs.
	*
	* XXX: We may want to revisit this to be more lenient and wire
	* as long as it gets one match.
	*/
	matches = 0;
	if (resource_long_value(name, unit, "port", &value) == 0) {
	/*
	* Floppy drive controllers are notorious for having a
	* wide variety of resources not all of which include the
	* first port that is specified by the hint (typically
	* 0x3f0) (see the comment above fdc_isa_alloc_resources()
	* in fdc_isa.c). However, they do all seem to include
	* port + 2 (e.g. 0x3f2) so for a floppy device, look for
	* 'value + 2' in the port resources instead of the hint
	* value.
	*/
	if (strcmp(name, "fdc") == 0)
	value += 2;
	if (acpi_match_resource_hint(child, SYS_RES_IOPORT, value))
	matches++;
	else
	continue;
	}
	if (resource_long_value(name, unit, "maddr", &value) == 0) {
	if (acpi_match_resource_hint(child, SYS_RES_MEMORY, value))
	matches++;
	else
	continue;
	}
	if (matches > 0)
	goto matched;
	if (resource_long_value(name, unit, "irq", &value) == 0) {
	if (acpi_match_resource_hint(child, SYS_RES_IRQ, value))
	matches++;
	else
	continue;
	}
	if (resource_long_value(name, unit, "drq", &value) == 0) {
	if (acpi_match_resource_hint(child, SYS_RES_DRQ, value))
	matches++;
	else
	continue;
	}

	matched:
	if (matches > 0) {
	/* We have a winner! */
	*unitp = unit;
	break;
	}
	}
	}

	/*
	* Fetch the NUMA domain for a device by mapping the value returned by
	* _PXM to a NUMA domain. If the device does not have a _PXM method,
	* -2 is returned. If any other error occurs, -1 is returned.
	*/
	static int
	acpi_parse_pxm(device_t dev)
	{
	#ifdef NUMA
	#if defined(__i386__) \|\| defined(__amd64__)
	ACPI_HANDLE handle;
	ACPI_STATUS status;
	int pxm;

	handle = acpi_get_handle(dev);
	if (handle == NULL)
	return (-2);
	status = acpi_GetInteger(handle, "_PXM", &pxm);
	if (ACPI_SUCCESS(status))
	return (acpi_map_pxm_to_vm_domainid(pxm));
	if (status == AE_NOT_FOUND)
	return (-2);
	#endif
	#endif
	return (-1);
	}

	int
	acpi_get_cpus(device_t dev, device_t child, enum cpu_sets op, size_t setsize,
	cpuset_t *cpuset)
	{
	int d, error;

	d = acpi_parse_pxm(child);
	if (d < 0)
	return (bus_generic_get_cpus(dev, child, op, setsize, cpuset));

	switch (op) {
	case LOCAL_CPUS:
	if (setsize != sizeof(cpuset_t))
	return (EINVAL);
	*cpuset = cpuset_domain[d];
	return (0);
	case INTR_CPUS:
	error = bus_generic_get_cpus(dev, child, op, setsize, cpuset);
	if (error != 0)
	return (error);
	if (setsize != sizeof(cpuset_t))
	return (EINVAL);
	CPU_AND(cpuset, &cpuset_domain[d]);
	return (0);
	default:
	return (bus_generic_get_cpus(dev, child, op, setsize, cpuset));
	}
	}

	/*
	* Fetch the NUMA domain for the given device 'dev'.
	*
	* If a device has a _PXM method, map that to a NUMA domain.
	* Otherwise, pass the request up to the parent.
	* If there's no matching domain or the domain cannot be
	* determined, return ENOENT.
	*/
	int
	acpi_get_domain(device_t dev, device_t child, int *domain)
	{
	int d;

	d = acpi_parse_pxm(child);
	if (d >= 0) {
	*domain = d;
	return (0);
	}
	if (d == -1)
	return (ENOENT);

	/* No _PXM node; go up a level */
	return (bus_generic_get_domain(dev, child, domain));
	}

	/*
	* Pre-allocate/manage all memory and IO resources. Since rman can't handle
	* duplicates, we merge any in the sysresource attach routine.
	*/
	static int
	acpi_sysres_alloc(device_t dev)
	{
	struct resource *res;
	struct resource_list *rl;
	struct resource_list_entry *rle;
	struct rman *rm;
	device_t *children;
	int child_count, i;

	/*
	* Probe/attach any sysresource devices. This would be unnecessary if we
	* had multi-pass probe/attach.
	*/
	if (device_get_children(dev, &children, &child_count) != 0)
	return (ENXIO);
	for (i = 0; i < child_count; i++) {
	if (ACPI_ID_PROBE(dev, children[i], sysres_ids, NULL) <= 0)
	device_probe_and_attach(children[i]);
	}
	free(children, M_TEMP);

	rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
	STAILQ_FOREACH(rle, rl, link) {
	if (rle->res != NULL) {
	device_printf(dev, "duplicate resource for %jx\n", rle->start);
	continue;
	}

	/* Only memory and IO resources are valid here. */
	switch (rle->type) {
	case SYS_RES_IOPORT:
	rm = &acpi_rman_io;
	break;
	case SYS_RES_MEMORY:
	rm = &acpi_rman_mem;
	break;
	default:
	continue;
	}

	/* Pre-allocate resource and add to our rman pool. */
	res = BUS_ALLOC_RESOURCE(device_get_parent(dev), dev, rle->type,
	&rle->rid, rle->start, rle->start + rle->count - 1, rle->count, 0);
	if (res != NULL) {
	rman_manage_region(rm, rman_get_start(res), rman_get_end(res));
	rle->res = res;
	} else if (bootverbose)
	device_printf(dev, "reservation of %jx, %jx (%d) failed\n",
	rle->start, rle->count, rle->type);
	}
	return (0);
	}

	/*
	* Reserve declared resources for devices found during attach once system
	* resources have been allocated.
	*/
	static void
	acpi_reserve_resources(device_t dev)
	{
	struct resource_list_entry *rle;
	struct resource_list *rl;
	struct acpi_device *ad;
	struct acpi_softc *sc;
	device_t *children;
	int child_count, i;

	sc = device_get_softc(dev);
	if (device_get_children(dev, &children, &child_count) != 0)
	return;
	for (i = 0; i < child_count; i++) {
	ad = device_get_ivars(children[i]);
	rl = &ad->ad_rl;

	/* Don't reserve system resources. */
	if (ACPI_ID_PROBE(dev, children[i], sysres_ids, NULL) <= 0)
	continue;

	STAILQ_FOREACH(rle, rl, link) {
	/*
	* Don't reserve IRQ resources. There are many sticky things
	* to get right otherwise (e.g. IRQs for psm, atkbd, and HPET
	* when using legacy routing).
	*/
	if (rle->type == SYS_RES_IRQ)
	continue;

	/*
	* Don't reserve the resource if it is already allocated.
	* The acpi_ec(4) driver can allocate its resources early
	* if ECDT is present.
	*/
	if (rle->res != NULL)
	continue;

	/*
	* Try to reserve the resource from our parent. If this
	* fails because the resource is a system resource, just
	* let it be. The resource range is already reserved so
	* that other devices will not use it. If the driver
	* needs to allocate the resource, then
	* acpi_alloc_resource() will sub-alloc from the system
	* resource.
	*/
	resource_list_reserve(rl, dev, children[i], rle->type, &rle->rid,
	rle->start, rle->end, rle->count, 0);
	}
	}
	free(children, M_TEMP);
	sc->acpi_resources_reserved = 1;
	}

	static int
	acpi_set_resource(device_t dev, device_t child, int type, int rid,
	rman_res_t start, rman_res_t count)
	{
	struct acpi_softc *sc = device_get_softc(dev);
	struct acpi_device *ad = device_get_ivars(child);
	struct resource_list *rl = &ad->ad_rl;
	ACPI_DEVICE_INFO *devinfo;
	rman_res_t end;
	int allow;

	/* Ignore IRQ resources for PCI link devices. */
	if (type == SYS_RES_IRQ &&
	ACPI_ID_PROBE(dev, child, pcilink_ids, NULL) <= 0)
	return (0);

	/*
	* Ignore most resources for PCI root bridges. Some BIOSes
	* incorrectly enumerate the memory ranges they decode as plain
	* memory resources instead of as ResourceProducer ranges. Other
	* BIOSes incorrectly list system resource entries for I/O ranges
	* under the PCI bridge. Do allow the one known-correct case on
	* x86 of a PCI bridge claiming the I/O ports used for PCI config
	* access.
	*/
	if (type == SYS_RES_MEMORY \|\| type == SYS_RES_IOPORT) {
	if (ACPI_SUCCESS(AcpiGetObjectInfo(ad->ad_handle, &devinfo))) {
	if ((devinfo->Flags & ACPI_PCI_ROOT_BRIDGE) != 0) {
	#if defined(__i386__) \|\| defined(__amd64__)
	allow = (type == SYS_RES_IOPORT && start == CONF1_ADDR_PORT);
	#else
	allow = 0;
	#endif
	if (!allow) {
	AcpiOsFree(devinfo);
	return (0);
	}
	}
	AcpiOsFree(devinfo);
	}
	}

	#ifdef INTRNG
	/* map with default for now */
	if (type == SYS_RES_IRQ)
	start = (rman_res_t)acpi_map_intr(child, (u_int)start,
	acpi_get_handle(child));
	#endif

	/* If the resource is already allocated, fail. */
	if (resource_list_busy(rl, type, rid))
	return (EBUSY);

	/* If the resource is already reserved, release it. */
	if (resource_list_reserved(rl, type, rid))
	resource_list_unreserve(rl, dev, child, type, rid);

	/* Add the resource. */
	end = (start + count - 1);
	resource_list_add(rl, type, rid, start, end, count);

	/* Don't reserve resources until the system resources are allocated. */
	if (!sc->acpi_resources_reserved)
	return (0);

	/* Don't reserve system resources. */
	if (ACPI_ID_PROBE(dev, child, sysres_ids, NULL) <= 0)
	return (0);

	/*
	* Don't reserve IRQ resources. There are many sticky things to
	* get right otherwise (e.g. IRQs for psm, atkbd, and HPET when
	* using legacy routing).
	*/
	if (type == SYS_RES_IRQ)
	return (0);

	/*
	* Don't reserve resources for CPU devices. Some of these
	* resources need to be allocated as shareable, but reservations
	* are always non-shareable.
	*/
	if (device_get_devclass(child) == devclass_find("cpu"))
	return (0);

	/*
	* Reserve the resource.
	*
	* XXX: Ignores failure for now. Failure here is probably a
	* BIOS/firmware bug?
	*/
	resource_list_reserve(rl, dev, child, type, &rid, start, end, count, 0);
	return (0);
	}

	static struct resource *
	acpi_alloc_resource(device_t bus, device_t child, int type, int *rid,
	rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
	{
	#ifndef INTRNG
	ACPI_RESOURCE ares;
	#endif
	struct acpi_device *ad;
	struct resource_list_entry *rle;
	struct resource_list *rl;
	struct resource *res;
	int isdefault = RMAN_IS_DEFAULT_RANGE(start, end);

	/*
	* First attempt at allocating the resource. For direct children,
	* use resource_list_alloc() to handle reserved resources. For
	* other devices, pass the request up to our parent.
	*/
	if (bus == device_get_parent(child)) {
	ad = device_get_ivars(child);
	rl = &ad->ad_rl;

	/*
	* Simulate the behavior of the ISA bus for direct children
	* devices. That is, if a non-default range is specified for
	* a resource that doesn't exist, use bus_set_resource() to
	* add the resource before allocating it. Note that these
	* resources will not be reserved.
	*/
	if (!isdefault && resource_list_find(rl, type, *rid) == NULL)
	resource_list_add(rl, type, *rid, start, end, count);
	res = resource_list_alloc(rl, bus, child, type, rid, start, end, count,
	flags);
	#ifndef INTRNG
	if (res != NULL && type == SYS_RES_IRQ) {
	/*
	* Since bus_config_intr() takes immediate effect, we cannot
	* configure the interrupt associated with a device when we
	* parse the resources but have to defer it until a driver
	* actually allocates the interrupt via bus_alloc_resource().
	*
	* XXX: Should we handle the lookup failing?
	*/
	if (ACPI_SUCCESS(acpi_lookup_irq_resource(child, *rid, res, &ares)))
	acpi_config_intr(child, &ares);
	}
	#endif

	/*
	* If this is an allocation of the "default" range for a given
	* RID, fetch the exact bounds for this resource from the
	* resource list entry to try to allocate the range from the
	* system resource regions.
	*/
	if (res == NULL && isdefault) {
	rle = resource_list_find(rl, type, *rid);
	if (rle != NULL) {
	start = rle->start;
	end = rle->end;
	count = rle->count;
	}
	}
	} else
	res = BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type, rid,
	start, end, count, flags);

	/*
	* If the first attempt failed and this is an allocation of a
	* specific range, try to satisfy the request via a suballocation
	* from our system resource regions.
	*/
	if (res == NULL && start + count - 1 == end)
	res = acpi_alloc_sysres(child, type, rid, start, end, count, flags);
	return (res);
	}

	/*
	* Attempt to allocate a specific resource range from the system
	* resource ranges. Note that we only handle memory and I/O port
	* system resources.
	*/
	struct resource *
	acpi_alloc_sysres(device_t child, int type, int *rid, rman_res_t start,
	rman_res_t end, rman_res_t count, u_int flags)
	{
	struct rman *rm;
	struct resource *res;

	switch (type) {
	case SYS_RES_IOPORT:
	rm = &acpi_rman_io;
	break;
	case SYS_RES_MEMORY:
	rm = &acpi_rman_mem;
	break;
	default:
	return (NULL);
	}

	KASSERT(start + count - 1 == end, ("wildcard resource range"));
	res = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
	child);
	if (res == NULL)
	return (NULL);

	rman_set_rid(res, *rid);

	/* If requested, activate the resource using the parent's method. */
	if (flags & RF_ACTIVE)
	if (bus_activate_resource(child, type, *rid, res) != 0) {
	rman_release_resource(res);
	return (NULL);
	}

	return (res);
	}

	static int
	acpi_is_resource_managed(int type, struct resource *r)
	{

	/* We only handle memory and IO resources through rman. */
	switch (type) {
	case SYS_RES_IOPORT:
	return (rman_is_region_manager(r, &acpi_rman_io));
	case SYS_RES_MEMORY:
	return (rman_is_region_manager(r, &acpi_rman_mem));
	}
	return (0);
	}

	static int
	acpi_adjust_resource(device_t bus, device_t child, int type, struct resource *r,
	rman_res_t start, rman_res_t end)
	{

	if (acpi_is_resource_managed(type, r))
	return (rman_adjust_resource(r, start, end));
	return (bus_generic_adjust_resource(bus, child, type, r, start, end));
	}

	static int
	acpi_release_resource(device_t bus, device_t child, int type, int rid,
	struct resource *r)
	{
	int ret;

	/*
	* If this resource belongs to one of our internal managers,
	* deactivate it and release it to the local pool.
	*/
	if (acpi_is_resource_managed(type, r)) {
	if (rman_get_flags(r) & RF_ACTIVE) {
	ret = bus_deactivate_resource(child, type, rid, r);
	if (ret != 0)
	return (ret);
	}
	return (rman_release_resource(r));
	}

	return (bus_generic_rl_release_resource(bus, child, type, rid, r));
	}

	static void
	acpi_delete_resource(device_t bus, device_t child, int type, int rid)
	{
	struct resource_list *rl;

	rl = acpi_get_rlist(bus, child);
	if (resource_list_busy(rl, type, rid)) {
	device_printf(bus, "delete_resource: Resource still owned by child"
	" (type=%d, rid=%d)\n", type, rid);
	return;
	}
	resource_list_unreserve(rl, bus, child, type, rid);
	resource_list_delete(rl, type, rid);
	}

	/* Allocate an IO port or memory resource, given its GAS. */
	int
	acpi_bus_alloc_gas(device_t dev, int type, int rid, ACPI_GENERIC_ADDRESS *gas,
	struct resource **res, u_int flags)
	{
	int error, res_type;

	error = ENOMEM;
	if (type == NULL \|\| rid == NULL \|\| gas == NULL \|\| res == NULL)
	return (EINVAL);

	/* We only support memory and IO spaces. */
	switch (gas->SpaceId) {
	case ACPI_ADR_SPACE_SYSTEM_MEMORY:
	res_type = SYS_RES_MEMORY;
	break;
	case ACPI_ADR_SPACE_SYSTEM_IO:
	res_type = SYS_RES_IOPORT;
	break;
	default:
	return (EOPNOTSUPP);
	}

	/*
	* If the register width is less than 8, assume the BIOS author means
	* it is a bit field and just allocate a byte.
	*/
	if (gas->BitWidth && gas->BitWidth < 8)
	gas->BitWidth = 8;

	/* Validate the address after we're sure we support the space. */
	if (gas->Address == 0 \|\| gas->BitWidth == 0)
	return (EINVAL);

	bus_set_resource(dev, res_type, *rid, gas->Address,
	gas->BitWidth / 8);
	*res = bus_alloc_resource_any(dev, res_type, rid, RF_ACTIVE \| flags);
	if (*res != NULL) {
	*type = res_type;
	error = 0;
	} else
	bus_delete_resource(dev, res_type, *rid);

	return (error);
	}

	/* Probe _HID and _CID for compatible ISA PNP ids. */
	static uint32_t
	acpi_isa_get_logicalid(device_t dev)
	{
	ACPI_DEVICE_INFO *devinfo;
	ACPI_HANDLE h;
	uint32_t pnpid;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	/* Fetch and validate the HID. */
	if ((h = acpi_get_handle(dev)) == NULL \|\|
	ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
	return_VALUE (0);

	pnpid = (devinfo->Valid & ACPI_VALID_HID) != 0 &&
	devinfo->HardwareId.Length >= ACPI_EISAID_STRING_SIZE ?
	PNP_EISAID(devinfo->HardwareId.String) : 0;
	AcpiOsFree(devinfo);

	return_VALUE (pnpid);
	}

	static int
	acpi_isa_get_compatid(device_t dev, uint32_t *cids, int count)
	{
	ACPI_DEVICE_INFO *devinfo;
	ACPI_PNP_DEVICE_ID *ids;
	ACPI_HANDLE h;
	uint32_t *pnpid;
	int i, valid;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	pnpid = cids;

	/* Fetch and validate the CID */
	if ((h = acpi_get_handle(dev)) == NULL \|\|
	ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
	return_VALUE (0);

	if ((devinfo->Valid & ACPI_VALID_CID) == 0) {
	AcpiOsFree(devinfo);
	return_VALUE (0);
	}

	if (devinfo->CompatibleIdList.Count < count)
	count = devinfo->CompatibleIdList.Count;
	ids = devinfo->CompatibleIdList.Ids;
	for (i = 0, valid = 0; i < count; i++)
	if (ids[i].Length >= ACPI_EISAID_STRING_SIZE &&
	strncmp(ids[i].String, "PNP", 3) == 0) {
	*pnpid++ = PNP_EISAID(ids[i].String);
	valid++;
	}
	AcpiOsFree(devinfo);

	return_VALUE (valid);
	}

	static int
	acpi_device_id_probe(device_t bus, device_t dev, char ids, char match)
	{
	ACPI_HANDLE h;
	ACPI_OBJECT_TYPE t;
	int rv;
	int i;

	h = acpi_get_handle(dev);
	if (ids == NULL \|\| h == NULL)
	return (ENXIO);
	t = acpi_get_type(dev);
	if (t != ACPI_TYPE_DEVICE && t != ACPI_TYPE_PROCESSOR)
	return (ENXIO);

	/* Try to match one of the array of IDs with a HID or CID. */
	for (i = 0; ids[i] != NULL; i++) {
	rv = acpi_MatchHid(h, ids[i]);
	if (rv == ACPI_MATCHHID_NOMATCH)
	continue;

	if (match != NULL) {
	*match = ids[i];
	}
	return ((rv == ACPI_MATCHHID_HID)?
	BUS_PROBE_DEFAULT : BUS_PROBE_LOW_PRIORITY);
	}
	return (ENXIO);
	}

	static ACPI_STATUS
	acpi_device_eval_obj(device_t bus, device_t dev, ACPI_STRING pathname,
	ACPI_OBJECT_LIST parameters, ACPI_BUFFER ret)
	{
	ACPI_HANDLE h;

	if (dev == NULL)
	h = ACPI_ROOT_OBJECT;
	else if ((h = acpi_get_handle(dev)) == NULL)
	return (AE_BAD_PARAMETER);
	return (AcpiEvaluateObject(h, pathname, parameters, ret));
	}

	int
	acpi_device_pwr_for_sleep(device_t bus, device_t dev, int *dstate)
	{
	struct acpi_softc *sc;
	ACPI_HANDLE handle;
	ACPI_STATUS status;
	char sxd[8];

	handle = acpi_get_handle(dev);

	/*
	* XXX If we find these devices, don't try to power them down.
	* The serial and IRDA ports on my T23 hang the system when
	* set to D3 and it appears that such legacy devices may
	* need special handling in their drivers.
	*/
	if (dstate == NULL \|\| handle == NULL \|\|
	acpi_MatchHid(handle, "PNP0500") \|\|
	acpi_MatchHid(handle, "PNP0501") \|\|
	acpi_MatchHid(handle, "PNP0502") \|\|
	acpi_MatchHid(handle, "PNP0510") \|\|
	acpi_MatchHid(handle, "PNP0511"))
	return (ENXIO);

	/*
	* Override next state with the value from _SxD, if present.
	* Note illegal _S0D is evaluated because some systems expect this.
	*/
	sc = device_get_softc(bus);
	snprintf(sxd, sizeof(sxd), "_S%dD", sc->acpi_sstate);
	status = acpi_GetInteger(handle, sxd, dstate);
	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
	device_printf(dev, "failed to get %s on %s: %s\n", sxd,
	acpi_name(handle), AcpiFormatException(status));
	return (ENXIO);
	}

	return (0);
	}

	/* Callback arg for our implementation of walking the namespace. */
	struct acpi_device_scan_ctx {
	acpi_scan_cb_t user_fn;
	void *arg;
	ACPI_HANDLE parent;
	};

	static ACPI_STATUS
	acpi_device_scan_cb(ACPI_HANDLE h, UINT32 level, void arg, void *retval)
	{
	struct acpi_device_scan_ctx *ctx;
	device_t dev, old_dev;
	ACPI_STATUS status;
	ACPI_OBJECT_TYPE type;

	/*
	* Skip this device if we think we'll have trouble with it or it is
	* the parent where the scan began.
	*/
	ctx = (struct acpi_device_scan_ctx *)arg;
	if (acpi_avoid(h) \|\| h == ctx->parent)
	return (AE_OK);

	/* If this is not a valid device type (e.g., a method), skip it. */
	if (ACPI_FAILURE(AcpiGetType(h, &type)))
	return (AE_OK);
	if (type != ACPI_TYPE_DEVICE && type != ACPI_TYPE_PROCESSOR &&
	type != ACPI_TYPE_THERMAL && type != ACPI_TYPE_POWER)
	return (AE_OK);

	/*
	* Call the user function with the current device. If it is unchanged
	* afterwards, return. Otherwise, we update the handle to the new dev.
	*/
	old_dev = acpi_get_device(h);
	dev = old_dev;
	status = ctx->user_fn(h, &dev, level, ctx->arg);
	if (ACPI_FAILURE(status) \|\| old_dev == dev)
	return (status);

	/* Remove the old child and its connection to the handle. */
	if (old_dev != NULL)
	device_delete_child(device_get_parent(old_dev), old_dev);

	/* Recreate the handle association if the user created a device. */
	if (dev != NULL)
	AcpiAttachData(h, acpi_fake_objhandler, dev);

	return (AE_OK);
	}

	static ACPI_STATUS
	acpi_device_scan_children(device_t bus, device_t dev, int max_depth,
	acpi_scan_cb_t user_fn, void *arg)
	{
	ACPI_HANDLE h;
	struct acpi_device_scan_ctx ctx;

	if (acpi_disabled("children"))
	return (AE_OK);

	if (dev == NULL)
	h = ACPI_ROOT_OBJECT;
	else if ((h = acpi_get_handle(dev)) == NULL)
	return (AE_BAD_PARAMETER);
	ctx.user_fn = user_fn;
	ctx.arg = arg;
	ctx.parent = h;
	return (AcpiWalkNamespace(ACPI_TYPE_ANY, h, max_depth,
	acpi_device_scan_cb, NULL, &ctx, NULL));
	}

	/*
	* Even though ACPI devices are not PCI, we use the PCI approach for setting
	* device power states since it's close enough to ACPI.
	*/
	int
	acpi_set_powerstate(device_t child, int state)
	{
	ACPI_HANDLE h;
	ACPI_STATUS status;

	h = acpi_get_handle(child);
	if (state < ACPI_STATE_D0 \|\| state > ACPI_D_STATES_MAX)
	return (EINVAL);
	if (h == NULL)
	return (0);

	/* Ignore errors if the power methods aren't present. */
	status = acpi_pwr_switch_consumer(h, state);
	if (ACPI_SUCCESS(status)) {
	if (bootverbose)
	device_printf(child, "set ACPI power state D%d on %s\n",
	state, acpi_name(h));
	} else if (status != AE_NOT_FOUND)
	device_printf(child,
	"failed to set ACPI power state D%d on %s: %s\n", state,
	acpi_name(h), AcpiFormatException(status));

	return (0);
	}

	static int
	acpi_isa_pnp_probe(device_t bus, device_t child, struct isa_pnp_id *ids)
	{
	int result, cid_count, i;
	uint32_t lid, cids[8];

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	/*
	* ISA-style drivers attached to ACPI may persist and
	* probe manually if we return ENOENT. We never want
	* that to happen, so don't ever return it.
	*/
	result = ENXIO;

	/* Scan the supplied IDs for a match */
	lid = acpi_isa_get_logicalid(child);
	cid_count = acpi_isa_get_compatid(child, cids, 8);
	while (ids && ids->ip_id) {
	if (lid == ids->ip_id) {
	result = 0;
	goto out;
	}
	for (i = 0; i < cid_count; i++) {
	if (cids[i] == ids->ip_id) {
	result = 0;
	goto out;
	}
	}
	ids++;
	}

	out:
	if (result == 0 && ids->ip_desc)
	device_set_desc(child, ids->ip_desc);

	return_VALUE (result);
	}

	/*
	* Look for a MCFG table. If it is present, use the settings for
	* domain (segment) 0 to setup PCI config space access via the memory
	* map.
	*
	* On non-x86 architectures (arm64 for now), this will be done from the
	* PCI host bridge driver.
	*/
	static void
	acpi_enable_pcie(void)
	{
	#if defined(__i386__) \|\| defined(__amd64__)
	ACPI_TABLE_HEADER *hdr;
	ACPI_MCFG_ALLOCATION alloc, end;
	ACPI_STATUS status;

	status = AcpiGetTable(ACPI_SIG_MCFG, 1, &hdr);
	if (ACPI_FAILURE(status))
	return;

	end = (ACPI_MCFG_ALLOCATION )((char )hdr + hdr->Length);
	alloc = (ACPI_MCFG_ALLOCATION )((ACPI_TABLE_MCFG )hdr + 1);
	while (alloc < end) {
	if (alloc->PciSegment == 0) {
	pcie_cfgregopen(alloc->Address, alloc->StartBusNumber,
	alloc->EndBusNumber);
	return;
	}
	alloc++;
	}
	#endif
	}

	+static void
	+acpi_platform_osc(device_t dev)
	+{
	+ ACPI_HANDLE sb_handle;
	+ ACPI_STATUS status;
	+ uint32_t cap_set[2];
	+
	+ /* 0811B06E-4A27-44F9-8D60-3CBBC22E7B48 */
	+ static uint8_t acpi_platform_uuid[ACPI_UUID_LENGTH] = {
	+ 0x6e, 0xb0, 0x11, 0x08, 0x27, 0x4a, 0xf9, 0x44,
	+ 0x8d, 0x60, 0x3c, 0xbb, 0xc2, 0x2e, 0x7b, 0x48
	+ };
	+
	+ if (ACPI_FAILURE(AcpiGetHandle(ACPI_ROOT_OBJECT, "\\_SB_", &sb_handle)))
	+ return;
	+
	+ cap_set[1] = 0x10; /* APEI Support */
	+ status = acpi_EvaluateOSC(sb_handle, acpi_platform_uuid, 1,
	+ nitems(cap_set), cap_set, cap_set, false);
	+ if (ACPI_FAILURE(status)) {
	+ if (status == AE_NOT_FOUND)
	+ return;
	+ device_printf(dev, "_OSC failed: %s\n",
	+ AcpiFormatException(status));
	+ return;
	+ }
	+}
	+
	/*
	* Scan all of the ACPI namespace and attach child devices.
	*
	* We should only expect to find devices in the \_PR, \_TZ, \_SI, and
	* \_SB scopes, and \_PR and \_TZ became obsolete in the ACPI 2.0 spec.
	* However, in violation of the spec, some systems place their PCI link
	* devices in \, so we have to walk the whole namespace. We check the
	* type of namespace nodes, so this should be ok.
	*/
	static void
	acpi_probe_children(device_t bus)
	{

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	/*
	* Scan the namespace and insert placeholders for all the devices that
	* we find. We also probe/attach any early devices.
	*
	* Note that we use AcpiWalkNamespace rather than AcpiGetDevices because
	* we want to create nodes for all devices, not just those that are
	* currently present. (This assumes that we don't want to create/remove
	* devices as they appear, which might be smarter.)
	*/
	ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "namespace scan\n"));
	AcpiWalkNamespace(ACPI_TYPE_ANY, ACPI_ROOT_OBJECT, 100, acpi_probe_child,
	NULL, bus, NULL);

	/* Pre-allocate resources for our rman from any sysresource devices. */
	acpi_sysres_alloc(bus);

	/* Reserve resources already allocated to children. */
	acpi_reserve_resources(bus);

	/* Create any static children by calling device identify methods. */
	ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "device identify routines\n"));
	bus_generic_probe(bus);

	/* Probe/attach all children, created statically and from the namespace. */
	ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "acpi bus_generic_attach\n"));
	bus_generic_attach(bus);

	/* Attach wake sysctls. */
	acpi_wake_sysctl_walk(bus);

	ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "done attaching children\n"));
	return_VOID;
	}

	/*
	* Determine the probe order for a given device.
	*/
	static void
	acpi_probe_order(ACPI_HANDLE handle, int *order)
	{
	ACPI_OBJECT_TYPE type;

	/*
	* 0. CPUs
	* 1. I/O port and memory system resource holders
	* 2. Clocks and timers (to handle early accesses)
	* 3. Embedded controllers (to handle early accesses)
	* 4. PCI Link Devices
	*/
	AcpiGetType(handle, &type);
	if (type == ACPI_TYPE_PROCESSOR)
	*order = 0;
	else if (acpi_MatchHid(handle, "PNP0C01") \|\|
	acpi_MatchHid(handle, "PNP0C02"))
	*order = 1;
	else if (acpi_MatchHid(handle, "PNP0100") \|\|
	acpi_MatchHid(handle, "PNP0103") \|\|
	acpi_MatchHid(handle, "PNP0B00"))
	*order = 2;
	else if (acpi_MatchHid(handle, "PNP0C09"))
	*order = 3;
	else if (acpi_MatchHid(handle, "PNP0C0F"))
	*order = 4;
	}

	/*
	* Evaluate a child device and determine whether we might attach a device to
	* it.
	*/
	static ACPI_STATUS
	acpi_probe_child(ACPI_HANDLE handle, UINT32 level, void context, void *status)
	{
	ACPI_DEVICE_INFO *devinfo;
	struct acpi_device *ad;
	struct acpi_prw_data prw;
	ACPI_OBJECT_TYPE type;
	ACPI_HANDLE h;
	device_t bus, child;
	char *handle_str;
	int order;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	if (acpi_disabled("children"))
	return_ACPI_STATUS (AE_OK);

	/* Skip this device if we think we'll have trouble with it. */
	if (acpi_avoid(handle))
	return_ACPI_STATUS (AE_OK);

	bus = (device_t)context;
	if (ACPI_SUCCESS(AcpiGetType(handle, &type))) {
	handle_str = acpi_name(handle);
	switch (type) {
	case ACPI_TYPE_DEVICE:
	/*
	* Since we scan from \, be sure to skip system scope objects.
	* \_SB_ and \_TZ_ are defined in ACPICA as devices to work around
	* BIOS bugs. For example, \_SB_ is to allow \_SB_._INI to be run
	* during the initialization and \_TZ_ is to support Notify() on it.
	*/
	if (strcmp(handle_str, "\\_SB_") == 0 \|\|
	strcmp(handle_str, "\\_TZ_") == 0)
	break;
	if (acpi_parse_prw(handle, &prw) == 0)
	AcpiSetupGpeForWake(handle, prw.gpe_handle, prw.gpe_bit);

	/*
	* Ignore devices that do not have a _HID or _CID. They should
	* be discovered by other buses (e.g. the PCI bus driver).
	*/
	if (!acpi_has_hid(handle))
	break;
	/* FALLTHROUGH */
	case ACPI_TYPE_PROCESSOR:
	case ACPI_TYPE_THERMAL:
	case ACPI_TYPE_POWER:
	/*
	* Create a placeholder device for this node. Sort the
	* placeholder so that the probe/attach passes will run
	* breadth-first. Orders less than ACPI_DEV_BASE_ORDER
	* are reserved for special objects (i.e., system
	* resources).
	*/
	ACPI_DEBUG_PRINT((ACPI_DB_OBJECTS, "scanning '%s'\n", handle_str));
	order = level * 10 + ACPI_DEV_BASE_ORDER;
	acpi_probe_order(handle, &order);
	child = BUS_ADD_CHILD(bus, order, NULL, -1);
	if (child == NULL)
	break;

	/* Associate the handle with the device_t and vice versa. */
	acpi_set_handle(child, handle);
	AcpiAttachData(handle, acpi_fake_objhandler, child);

	/*
	* Check that the device is present. If it's not present,
	* leave it disabled (so that we have a device_t attached to
	* the handle, but we don't probe it).
	*
	* XXX PCI link devices sometimes report "present" but not
	* "functional" (i.e. if disabled). Go ahead and probe them
	* anyway since we may enable them later.
	*/
	if (type == ACPI_TYPE_DEVICE && !acpi_DeviceIsPresent(child)) {
	/* Never disable PCI link devices. */
	if (acpi_MatchHid(handle, "PNP0C0F"))
	break;
	/*
	* Docking stations should remain enabled since the system
	* may be undocked at boot.
	*/
	if (ACPI_SUCCESS(AcpiGetHandle(handle, "_DCK", &h)))
	break;

	device_disable(child);
	break;
	}

	/*
	* Get the device's resource settings and attach them.
	* Note that if the device has _PRS but no _CRS, we need
	* to decide when it's appropriate to try to configure the
	* device. Ignore the return value here; it's OK for the
	* device not to have any resources.
	*/
	acpi_parse_resources(child, handle, &acpi_res_parse_set, NULL);

	ad = device_get_ivars(child);
	ad->ad_cls_class = 0xffffff;
	if (ACPI_SUCCESS(AcpiGetObjectInfo(handle, &devinfo))) {
	if ((devinfo->Valid & ACPI_VALID_CLS) != 0 &&
	devinfo->ClassCode.Length >= ACPI_PCICLS_STRING_SIZE) {
	ad->ad_cls_class = strtoul(devinfo->ClassCode.String,
	NULL, 16);
	}
	AcpiOsFree(devinfo);
	}
	break;
	}
	}

	return_ACPI_STATUS (AE_OK);
	}

	/*
	* AcpiAttachData() requires an object handler but never uses it. This is a
	* placeholder object handler so we can store a device_t in an ACPI_HANDLE.
	*/
	void
	acpi_fake_objhandler(ACPI_HANDLE h, void *data)
	{
	}

	static void
	acpi_shutdown_final(void *arg, int howto)
	{
	struct acpi_softc sc = (struct acpi_softc )arg;
	register_t intr;
	ACPI_STATUS status;

	/*
	* XXX Shutdown code should only run on the BSP (cpuid 0).
	* Some chipsets do not power off the system correctly if called from
	* an AP.
	*/
	if ((howto & RB_POWEROFF) != 0) {
	status = AcpiEnterSleepStatePrep(ACPI_STATE_S5);
	if (ACPI_FAILURE(status)) {
	device_printf(sc->acpi_dev, "AcpiEnterSleepStatePrep failed - %s\n",
	AcpiFormatException(status));
	return;
	}
	device_printf(sc->acpi_dev, "Powering system off\n");
	intr = intr_disable();
	status = AcpiEnterSleepState(ACPI_STATE_S5);
	if (ACPI_FAILURE(status)) {
	intr_restore(intr);
	device_printf(sc->acpi_dev, "power-off failed - %s\n",
	AcpiFormatException(status));
	} else {
	DELAY(1000000);
	intr_restore(intr);
	device_printf(sc->acpi_dev, "power-off failed - timeout\n");
	}
	} else if ((howto & RB_HALT) == 0 && sc->acpi_handle_reboot) {
	/* Reboot using the reset register. */
	status = AcpiReset();
	if (ACPI_SUCCESS(status)) {
	DELAY(1000000);
	device_printf(sc->acpi_dev, "reset failed - timeout\n");
	} else if (status != AE_NOT_EXIST)
	device_printf(sc->acpi_dev, "reset failed - %s\n",
	AcpiFormatException(status));
	} else if (sc->acpi_do_disable && !KERNEL_PANICKED()) {
	/*
	* Only disable ACPI if the user requested. On some systems, writing
	* the disable value to SMI_CMD hangs the system.
	*/
	device_printf(sc->acpi_dev, "Shutting down\n");
	AcpiTerminate();
	}
	}

	static void
	acpi_enable_fixed_events(struct acpi_softc *sc)
	{
	static int first_time = 1;

	/* Enable and clear fixed events and install handlers. */
	if ((AcpiGbl_FADT.Flags & ACPI_FADT_POWER_BUTTON) == 0) {
	AcpiClearEvent(ACPI_EVENT_POWER_BUTTON);
	AcpiInstallFixedEventHandler(ACPI_EVENT_POWER_BUTTON,
	acpi_event_power_button_sleep, sc);
	if (first_time)
	device_printf(sc->acpi_dev, "Power Button (fixed)\n");
	}
	if ((AcpiGbl_FADT.Flags & ACPI_FADT_SLEEP_BUTTON) == 0) {
	AcpiClearEvent(ACPI_EVENT_SLEEP_BUTTON);
	AcpiInstallFixedEventHandler(ACPI_EVENT_SLEEP_BUTTON,
	acpi_event_sleep_button_sleep, sc);
	if (first_time)
	device_printf(sc->acpi_dev, "Sleep Button (fixed)\n");
	}

	first_time = 0;
	}

	/*
	* Returns true if the device is actually present and should
	* be attached to. This requires the present, enabled, UI-visible
	* and diagnostics-passed bits to be set.
	*/
	BOOLEAN
	acpi_DeviceIsPresent(device_t dev)
	{
	ACPI_HANDLE h;
	UINT32 s;
	ACPI_STATUS status;

	h = acpi_get_handle(dev);
	if (h == NULL)
	return (FALSE);
	/*
	* Certain Treadripper boards always returns 0 for FreeBSD because it
	* only returns non-zero for the OS string "Windows 2015". Otherwise it
	* will return zero. Force them to always be treated as present.
	* Beata versions were worse: they always returned 0.
	*/
	if (acpi_MatchHid(h, "AMDI0020") \|\| acpi_MatchHid(h, "AMDI0010"))
	return (TRUE);

	status = acpi_GetInteger(h, "_STA", &s);

	/*
	* If no _STA method or if it failed, then assume that
	* the device is present.
	*/
	if (ACPI_FAILURE(status))
	return (TRUE);

	return (ACPI_DEVICE_PRESENT(s) ? TRUE : FALSE);
	}

	/*
	* Returns true if the battery is actually present and inserted.
	*/
	BOOLEAN
	acpi_BatteryIsPresent(device_t dev)
	{
	ACPI_HANDLE h;
	UINT32 s;
	ACPI_STATUS status;

	h = acpi_get_handle(dev);
	if (h == NULL)
	return (FALSE);
	status = acpi_GetInteger(h, "_STA", &s);

	/*
	* If no _STA method or if it failed, then assume that
	* the device is present.
	*/
	if (ACPI_FAILURE(status))
	return (TRUE);

	return (ACPI_BATTERY_PRESENT(s) ? TRUE : FALSE);
	}

	/*
	* Returns true if a device has at least one valid device ID.
	*/
	BOOLEAN
	acpi_has_hid(ACPI_HANDLE h)
	{
	ACPI_DEVICE_INFO *devinfo;
	BOOLEAN ret;

	if (h == NULL \|\|
	ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
	return (FALSE);

	ret = FALSE;
	if ((devinfo->Valid & ACPI_VALID_HID) != 0)
	ret = TRUE;
	else if ((devinfo->Valid & ACPI_VALID_CID) != 0)
	if (devinfo->CompatibleIdList.Count > 0)
	ret = TRUE;

	AcpiOsFree(devinfo);
	return (ret);
	}

	/*
	* Match a HID string against a handle
	* returns ACPI_MATCHHID_HID if _HID match
	* ACPI_MATCHHID_CID if _CID match and not _HID match.
	* ACPI_MATCHHID_NOMATCH=0 if no match.
	*/
	int
	acpi_MatchHid(ACPI_HANDLE h, const char *hid)
	{
	ACPI_DEVICE_INFO *devinfo;
	BOOLEAN ret;
	int i;

	if (hid == NULL \|\| h == NULL \|\|
	ACPI_FAILURE(AcpiGetObjectInfo(h, &devinfo)))
	return (ACPI_MATCHHID_NOMATCH);

	ret = ACPI_MATCHHID_NOMATCH;
	if ((devinfo->Valid & ACPI_VALID_HID) != 0 &&
	strcmp(hid, devinfo->HardwareId.String) == 0)
	ret = ACPI_MATCHHID_HID;
	else if ((devinfo->Valid & ACPI_VALID_CID) != 0)
	for (i = 0; i < devinfo->CompatibleIdList.Count; i++) {
	if (strcmp(hid, devinfo->CompatibleIdList.Ids[i].String) == 0) {
	ret = ACPI_MATCHHID_CID;
	break;
	}
	}

	AcpiOsFree(devinfo);
	return (ret);
	}

	/*
	* Return the handle of a named object within our scope, ie. that of (parent)
	* or one if its parents.
	*/
	ACPI_STATUS
	acpi_GetHandleInScope(ACPI_HANDLE parent, char path, ACPI_HANDLE result)
	{
	ACPI_HANDLE r;
	ACPI_STATUS status;

	/* Walk back up the tree to the root */
	for (;;) {
	status = AcpiGetHandle(parent, path, &r);
	if (ACPI_SUCCESS(status)) {
	*result = r;
	return (AE_OK);
	}
	/* XXX Return error here? */
	if (status != AE_NOT_FOUND)
	return (AE_OK);
	if (ACPI_FAILURE(AcpiGetParent(parent, &r)))
	return (AE_NOT_FOUND);
	parent = r;
	}
	}

	/*
	* Allocate a buffer with a preset data size.
	*/
	ACPI_BUFFER *
	acpi_AllocBuffer(int size)
	{
	ACPI_BUFFER *buf;

	if ((buf = malloc(size + sizeof(*buf), M_ACPIDEV, M_NOWAIT)) == NULL)
	return (NULL);
	buf->Length = size;
	buf->Pointer = (void *)(buf + 1);
	return (buf);
	}

	ACPI_STATUS
	acpi_SetInteger(ACPI_HANDLE handle, char *path, UINT32 number)
	{
	ACPI_OBJECT arg1;
	ACPI_OBJECT_LIST args;

	arg1.Type = ACPI_TYPE_INTEGER;
	arg1.Integer.Value = number;
	args.Count = 1;
	args.Pointer = &arg1;

	return (AcpiEvaluateObject(handle, path, &args, NULL));
	}

	/*
	* Evaluate a path that should return an integer.
	*/
	ACPI_STATUS
	acpi_GetInteger(ACPI_HANDLE handle, char path, UINT32 number)
	{
	ACPI_STATUS status;
	ACPI_BUFFER buf;
	ACPI_OBJECT param;

	if (handle == NULL)
	handle = ACPI_ROOT_OBJECT;

	/*
	* Assume that what we've been pointed at is an Integer object, or
	* a method that will return an Integer.
	*/
	buf.Pointer = &param;
	buf.Length = sizeof(param);
	status = AcpiEvaluateObject(handle, path, NULL, &buf);
	if (ACPI_SUCCESS(status)) {
	if (param.Type == ACPI_TYPE_INTEGER)
	*number = param.Integer.Value;
	else
	status = AE_TYPE;
	}

	/*
	* In some applications, a method that's expected to return an Integer
	* may instead return a Buffer (probably to simplify some internal
	* arithmetic). We'll try to fetch whatever it is, and if it's a Buffer,
	* convert it into an Integer as best we can.
	*
	* This is a hack.
	*/
	if (status == AE_BUFFER_OVERFLOW) {
	if ((buf.Pointer = AcpiOsAllocate(buf.Length)) == NULL) {
	status = AE_NO_MEMORY;
	} else {
	status = AcpiEvaluateObject(handle, path, NULL, &buf);
	if (ACPI_SUCCESS(status))
	status = acpi_ConvertBufferToInteger(&buf, number);
	AcpiOsFree(buf.Pointer);
	}
	}
	return (status);
	}

	ACPI_STATUS
	acpi_ConvertBufferToInteger(ACPI_BUFFER bufp, UINT32 number)
	{
	ACPI_OBJECT *p;
	UINT8 *val;
	int i;

	p = (ACPI_OBJECT *)bufp->Pointer;
	if (p->Type == ACPI_TYPE_INTEGER) {
	*number = p->Integer.Value;
	return (AE_OK);
	}
	if (p->Type != ACPI_TYPE_BUFFER)
	return (AE_TYPE);
	if (p->Buffer.Length > sizeof(int))
	return (AE_BAD_DATA);

	*number = 0;
	val = p->Buffer.Pointer;
	for (i = 0; i < p->Buffer.Length; i++)
	number += val[i] << (i 8);
	return (AE_OK);
	}

	/*
	* Iterate over the elements of an a package object, calling the supplied
	* function for each element.
	*
	* XXX possible enhancement might be to abort traversal on error.
	*/
	ACPI_STATUS
	acpi_ForeachPackageObject(ACPI_OBJECT *pkg,
	void (func)(ACPI_OBJECT comp, void arg), void arg)
	{
	ACPI_OBJECT *comp;
	int i;

	if (pkg == NULL \|\| pkg->Type != ACPI_TYPE_PACKAGE)
	return (AE_BAD_PARAMETER);

	/* Iterate over components */
	i = 0;
	comp = pkg->Package.Elements;
	for (; i < pkg->Package.Count; i++, comp++)
	func(comp, arg);

	return (AE_OK);
	}

	/*
	* Find the (index)th resource object in a set.
	*/
	ACPI_STATUS
	acpi_FindIndexedResource(ACPI_BUFFER buf, int index, ACPI_RESOURCE *resp)
	{
	ACPI_RESOURCE *rp;
	int i;

	rp = (ACPI_RESOURCE *)buf->Pointer;
	i = index;
	while (i-- > 0) {
	/* Range check */
	if (rp > (ACPI_RESOURCE )((u_int8_t )buf->Pointer + buf->Length))
	return (AE_BAD_PARAMETER);

	/* Check for terminator */
	if (rp->Type == ACPI_RESOURCE_TYPE_END_TAG \|\| rp->Length == 0)
	return (AE_NOT_FOUND);
	rp = ACPI_NEXT_RESOURCE(rp);
	}
	if (resp != NULL)
	*resp = rp;

	return (AE_OK);
	}

	/*
	* Append an ACPI_RESOURCE to an ACPI_BUFFER.
	*
	* Given a pointer to an ACPI_RESOURCE structure, expand the ACPI_BUFFER
	* provided to contain it. If the ACPI_BUFFER is empty, allocate a sensible
	* backing block. If the ACPI_RESOURCE is NULL, return an empty set of
	* resources.
	*/
	#define ACPI_INITIAL_RESOURCE_BUFFER_SIZE 512

	ACPI_STATUS
	acpi_AppendBufferResource(ACPI_BUFFER buf, ACPI_RESOURCE res)
	{
	ACPI_RESOURCE *rp;
	void *newp;

	/* Initialise the buffer if necessary. */
	if (buf->Pointer == NULL) {
	buf->Length = ACPI_INITIAL_RESOURCE_BUFFER_SIZE;
	if ((buf->Pointer = AcpiOsAllocate(buf->Length)) == NULL)
	return (AE_NO_MEMORY);
	rp = (ACPI_RESOURCE *)buf->Pointer;
	rp->Type = ACPI_RESOURCE_TYPE_END_TAG;
	rp->Length = ACPI_RS_SIZE_MIN;
	}
	if (res == NULL)
	return (AE_OK);

	/*
	* Scan the current buffer looking for the terminator.
	* This will either find the terminator or hit the end
	* of the buffer and return an error.
	*/
	rp = (ACPI_RESOURCE *)buf->Pointer;
	for (;;) {
	/* Range check, don't go outside the buffer */
	if (rp >= (ACPI_RESOURCE )((u_int8_t )buf->Pointer + buf->Length))
	return (AE_BAD_PARAMETER);
	if (rp->Type == ACPI_RESOURCE_TYPE_END_TAG \|\| rp->Length == 0)
	break;
	rp = ACPI_NEXT_RESOURCE(rp);
	}

	/*
	* Check the size of the buffer and expand if required.
	*
	* Required size is:
	* size of existing resources before terminator +
	* size of new resource and header +
	* size of terminator.
	*
	* Note that this loop should really only run once, unless
	* for some reason we are stuffing a really huge resource.
	*/
	while ((((u_int8_t )rp - (u_int8_t )buf->Pointer) +
	res->Length + ACPI_RS_SIZE_NO_DATA +
	ACPI_RS_SIZE_MIN) >= buf->Length) {
	if ((newp = AcpiOsAllocate(buf->Length * 2)) == NULL)
	return (AE_NO_MEMORY);
	bcopy(buf->Pointer, newp, buf->Length);
	rp = (ACPI_RESOURCE )((u_int8_t )newp +
	((u_int8_t )rp - (u_int8_t )buf->Pointer));
	AcpiOsFree(buf->Pointer);
	buf->Pointer = newp;
	buf->Length += buf->Length;
	}

	/* Insert the new resource. */
	bcopy(res, rp, res->Length + ACPI_RS_SIZE_NO_DATA);

	/* And add the terminator. */
	rp = ACPI_NEXT_RESOURCE(rp);
	rp->Type = ACPI_RESOURCE_TYPE_END_TAG;
	rp->Length = ACPI_RS_SIZE_MIN;

	return (AE_OK);
	}

	UINT8
	acpi_DSMQuery(ACPI_HANDLE h, uint8_t *uuid, int revision)
	{
	/*
	* ACPI spec 9.1.1 defines this.
	*
	* "Arg2: Function Index Represents a specific function whose meaning is
	* specific to the UUID and Revision ID. Function indices should start
	* with 1. Function number zero is a query function (see the special
	* return code defined below)."
	*/
	ACPI_BUFFER buf;
	ACPI_OBJECT *obj;
	UINT8 ret = 0;

	if (!ACPI_SUCCESS(acpi_EvaluateDSM(h, uuid, revision, 0, NULL, &buf))) {
	ACPI_INFO(("Failed to enumerate DSM functions\n"));
	return (0);
	}

	obj = (ACPI_OBJECT *)buf.Pointer;
	KASSERT(obj, ("Object not allowed to be NULL\n"));

	/*
	* From ACPI 6.2 spec 9.1.1:
	* If Function Index = 0, a Buffer containing a function index bitfield.
	* Otherwise, the return value and type depends on the UUID and revision
	* ID (see below).
	*/
	switch (obj->Type) {
	case ACPI_TYPE_BUFFER:
	ret = (uint8_t )obj->Buffer.Pointer;
	break;
	case ACPI_TYPE_INTEGER:
	ACPI_BIOS_WARNING((AE_INFO,
	"Possibly buggy BIOS with ACPI_TYPE_INTEGER for function enumeration\n"));
	ret = obj->Integer.Value & 0xFF;
	break;
	default:
	ACPI_WARNING((AE_INFO, "Unexpected return type %u\n", obj->Type));
	};

	AcpiOsFree(obj);
	return ret;
	}

	/*
	* DSM may return multiple types depending on the function. It is therefore
	* unsafe to use the typed evaluation. It is highly recommended that the caller
	* check the type of the returned object.
	*/
	ACPI_STATUS
	acpi_EvaluateDSM(ACPI_HANDLE handle, uint8_t *uuid, int revision,
	uint64_t function, union acpi_object package, ACPI_BUFFER out_buf)
	{
	ACPI_OBJECT arg[4];
	ACPI_OBJECT_LIST arglist;
	ACPI_BUFFER buf;
	ACPI_STATUS status;

	if (out_buf == NULL)
	return (AE_NO_MEMORY);

	arg[0].Type = ACPI_TYPE_BUFFER;
	arg[0].Buffer.Length = ACPI_UUID_LENGTH;
	arg[0].Buffer.Pointer = uuid;
	arg[1].Type = ACPI_TYPE_INTEGER;
	arg[1].Integer.Value = revision;
	arg[2].Type = ACPI_TYPE_INTEGER;
	arg[2].Integer.Value = function;
	if (package) {
	arg[3] = *package;
	} else {
	arg[3].Type = ACPI_TYPE_PACKAGE;
	arg[3].Package.Count = 0;
	arg[3].Package.Elements = NULL;
	}

	arglist.Pointer = arg;
	arglist.Count = 4;
	buf.Pointer = NULL;
	buf.Length = ACPI_ALLOCATE_BUFFER;
	status = AcpiEvaluateObject(handle, "_DSM", &arglist, &buf);
	if (ACPI_FAILURE(status))
	return (status);

	KASSERT(ACPI_SUCCESS(status), ("Unexpected status"));

	*out_buf = buf;
	return (status);
	}

	ACPI_STATUS
	acpi_EvaluateOSC(ACPI_HANDLE handle, uint8_t *uuid, int revision, int count,
	uint32_t caps_in, uint32_t caps_out, bool query)
	{
	ACPI_OBJECT arg[4], *ret;
	ACPI_OBJECT_LIST arglist;
	ACPI_BUFFER buf;
	ACPI_STATUS status;

	arglist.Pointer = arg;
	arglist.Count = 4;
	arg[0].Type = ACPI_TYPE_BUFFER;
	arg[0].Buffer.Length = ACPI_UUID_LENGTH;
	arg[0].Buffer.Pointer = uuid;
	arg[1].Type = ACPI_TYPE_INTEGER;
	arg[1].Integer.Value = revision;
	arg[2].Type = ACPI_TYPE_INTEGER;
	arg[2].Integer.Value = count;
	arg[3].Type = ACPI_TYPE_BUFFER;
	arg[3].Buffer.Length = count * sizeof(*caps_in);
	arg[3].Buffer.Pointer = (uint8_t *)caps_in;
	caps_in[0] = query ? 1 : 0;
	buf.Pointer = NULL;
	buf.Length = ACPI_ALLOCATE_BUFFER;
	status = AcpiEvaluateObjectTyped(handle, "_OSC", &arglist, &buf,
	ACPI_TYPE_BUFFER);
	if (ACPI_FAILURE(status))
	return (status);
	if (caps_out != NULL) {
	ret = buf.Pointer;
	if (ret->Buffer.Length != count * sizeof(*caps_out)) {
	AcpiOsFree(buf.Pointer);
	return (AE_BUFFER_OVERFLOW);
	}
	bcopy(ret->Buffer.Pointer, caps_out, ret->Buffer.Length);
	}
	AcpiOsFree(buf.Pointer);
	return (status);
	}

	/*
	* Set interrupt model.
	*/
	ACPI_STATUS
	acpi_SetIntrModel(int model)
	{

	return (acpi_SetInteger(ACPI_ROOT_OBJECT, "_PIC", model));
	}

	/*
	* Walk subtables of a table and call a callback routine for each
	* subtable. The caller should provide the first subtable and a
	* pointer to the end of the table. This can be used to walk tables
	* such as MADT and SRAT that use subtable entries.
	*/
	void
	acpi_walk_subtables(void first, void end, acpi_subtable_handler *handler,
	void *arg)
	{
	ACPI_SUBTABLE_HEADER *entry;

	for (entry = first; (void *)entry < end; ) {
	/* Avoid an infinite loop if we hit a bogus entry. */
	if (entry->Length < sizeof(ACPI_SUBTABLE_HEADER))
	return;

	handler(entry, arg);
	entry = ACPI_ADD_PTR(ACPI_SUBTABLE_HEADER, entry, entry->Length);
	}
	}

	/*
	* DEPRECATED. This interface has serious deficiencies and will be
	* removed.
	*
	* Immediately enter the sleep state. In the old model, acpiconf(8) ran
	* rc.suspend and rc.resume so we don't have to notify devd(8) to do this.
	*/
	ACPI_STATUS
	acpi_SetSleepState(struct acpi_softc *sc, int state)
	{
	static int once;

	if (!once) {
	device_printf(sc->acpi_dev,
	"warning: acpi_SetSleepState() deprecated, need to update your software\n");
	once = 1;
	}
	return (acpi_EnterSleepState(sc, state));
	}

	#if defined(__amd64__) \|\| defined(__i386__)
	static void
	acpi_sleep_force_task(void *context)
	{
	struct acpi_softc sc = (struct acpi_softc )context;

	if (ACPI_FAILURE(acpi_EnterSleepState(sc, sc->acpi_next_sstate)))
	device_printf(sc->acpi_dev, "force sleep state S%d failed\n",
	sc->acpi_next_sstate);
	}

	static void
	acpi_sleep_force(void *arg)
	{
	struct acpi_softc sc = (struct acpi_softc )arg;

	device_printf(sc->acpi_dev,
	"suspend request timed out, forcing sleep now\n");
	/*
	* XXX Suspending from callout causes freezes in DEVICE_SUSPEND().
	* Suspend from acpi_task thread instead.
	*/
	if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
	acpi_sleep_force_task, sc)))
	device_printf(sc->acpi_dev, "AcpiOsExecute() for sleeping failed\n");
	}
	#endif

	/*
	* Request that the system enter the given suspend state. All /dev/apm
	* devices and devd(8) will be notified. Userland then has a chance to
	* save state and acknowledge the request. The system sleeps once all
	* acks are in.
	*/
	int
	acpi_ReqSleepState(struct acpi_softc *sc, int state)
	{
	#if defined(__amd64__) \|\| defined(__i386__)
	struct apm_clone_data *clone;
	ACPI_STATUS status;

	if (state < ACPI_STATE_S1 \|\| state > ACPI_S_STATES_MAX)
	return (EINVAL);
	if (!acpi_sleep_states[state])
	return (EOPNOTSUPP);

	/*
	* If a reboot/shutdown/suspend request is already in progress or
	* suspend is blocked due to an upcoming shutdown, just return.
	*/
	if (rebooting \|\| sc->acpi_next_sstate != 0 \|\| suspend_blocked) {
	return (0);
	}

	/* Wait until sleep is enabled. */
	while (sc->acpi_sleep_disabled) {
	AcpiOsSleep(1000);
	}

	ACPI_LOCK(acpi);

	sc->acpi_next_sstate = state;

	/* S5 (soft-off) should be entered directly with no waiting. */
	if (state == ACPI_STATE_S5) {
	ACPI_UNLOCK(acpi);
	status = acpi_EnterSleepState(sc, state);
	return (ACPI_SUCCESS(status) ? 0 : ENXIO);
	}

	/* Record the pending state and notify all apm devices. */
	STAILQ_FOREACH(clone, &sc->apm_cdevs, entries) {
	clone->notify_status = APM_EV_NONE;
	if ((clone->flags & ACPI_EVF_DEVD) == 0) {
	selwakeuppri(&clone->sel_read, PZERO);
	KNOTE_LOCKED(&clone->sel_read.si_note, 0);
	}
	}

	/* If devd(8) is not running, immediately enter the sleep state. */
	if (!devctl_process_running()) {
	ACPI_UNLOCK(acpi);
	status = acpi_EnterSleepState(sc, state);
	return (ACPI_SUCCESS(status) ? 0 : ENXIO);
	}

	/*
	* Set a timeout to fire if userland doesn't ack the suspend request
	* in time. This way we still eventually go to sleep if we were
	* overheating or running low on battery, even if userland is hung.
	* We cancel this timeout once all userland acks are in or the
	* suspend request is aborted.
	*/
	callout_reset(&sc->susp_force_to, 10 * hz, acpi_sleep_force, sc);
	ACPI_UNLOCK(acpi);

	/* Now notify devd(8) also. */
	acpi_UserNotify("Suspend", ACPI_ROOT_OBJECT, state);

	return (0);
	#else
	/* This platform does not support acpi suspend/resume. */
	return (EOPNOTSUPP);
	#endif
	}

	/*
	* Acknowledge (or reject) a pending sleep state. The caller has
	* prepared for suspend and is now ready for it to proceed. If the
	* error argument is non-zero, it indicates suspend should be cancelled
	* and gives an errno value describing why. Once all votes are in,
	* we suspend the system.
	*/
	int
	acpi_AckSleepState(struct apm_clone_data *clone, int error)
	{
	#if defined(__amd64__) \|\| defined(__i386__)
	struct acpi_softc *sc;
	int ret, sleeping;

	/* If no pending sleep state, return an error. */
	ACPI_LOCK(acpi);
	sc = clone->acpi_sc;
	if (sc->acpi_next_sstate == 0) {
	ACPI_UNLOCK(acpi);
	return (ENXIO);
	}

	/* Caller wants to abort suspend process. */
	if (error) {
	sc->acpi_next_sstate = 0;
	callout_stop(&sc->susp_force_to);
	device_printf(sc->acpi_dev,
	"listener on %s cancelled the pending suspend\n",
	devtoname(clone->cdev));
	ACPI_UNLOCK(acpi);
	return (0);
	}

	/*
	* Mark this device as acking the suspend request. Then, walk through
	* all devices, seeing if they agree yet. We only count devices that
	* are writable since read-only devices couldn't ack the request.
	*/
	sleeping = TRUE;
	clone->notify_status = APM_EV_ACKED;
	STAILQ_FOREACH(clone, &sc->apm_cdevs, entries) {
	if ((clone->flags & ACPI_EVF_WRITE) != 0 &&
	clone->notify_status != APM_EV_ACKED) {
	sleeping = FALSE;
	break;
	}
	}

	/* If all devices have voted "yes", we will suspend now. */
	if (sleeping)
	callout_stop(&sc->susp_force_to);
	ACPI_UNLOCK(acpi);
	ret = 0;
	if (sleeping) {
	if (ACPI_FAILURE(acpi_EnterSleepState(sc, sc->acpi_next_sstate)))
	ret = ENODEV;
	}
	return (ret);
	#else
	/* This platform does not support acpi suspend/resume. */
	return (EOPNOTSUPP);
	#endif
	}

	static void
	acpi_sleep_enable(void *arg)
	{
	struct acpi_softc sc = (struct acpi_softc )arg;

	ACPI_LOCK_ASSERT(acpi);

	/* Reschedule if the system is not fully up and running. */
	if (!AcpiGbl_SystemAwakeAndRunning) {
	callout_schedule(&acpi_sleep_timer, hz * ACPI_MINIMUM_AWAKETIME);
	return;
	}

	sc->acpi_sleep_disabled = FALSE;
	}

	static ACPI_STATUS
	acpi_sleep_disable(struct acpi_softc *sc)
	{
	ACPI_STATUS status;

	/* Fail if the system is not fully up and running. */
	if (!AcpiGbl_SystemAwakeAndRunning)
	return (AE_ERROR);

	ACPI_LOCK(acpi);
	status = sc->acpi_sleep_disabled ? AE_ERROR : AE_OK;
	sc->acpi_sleep_disabled = TRUE;
	ACPI_UNLOCK(acpi);

	return (status);
	}

	enum acpi_sleep_state {
	ACPI_SS_NONE,
	ACPI_SS_GPE_SET,
	ACPI_SS_DEV_SUSPEND,
	ACPI_SS_SLP_PREP,
	ACPI_SS_SLEPT,
	};

	/*
	* Enter the desired system sleep state.
	*
	* Currently we support S1-S5 but S4 is only S4BIOS
	*/
	static ACPI_STATUS
	acpi_EnterSleepState(struct acpi_softc *sc, int state)
	{
	register_t intr;
	ACPI_STATUS status;
	ACPI_EVENT_STATUS power_button_status;
	enum acpi_sleep_state slp_state;
	int sleep_result;

	ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);

	if (state < ACPI_STATE_S1 \|\| state > ACPI_S_STATES_MAX)
	return_ACPI_STATUS (AE_BAD_PARAMETER);
	if (!acpi_sleep_states[state]) {
	device_printf(sc->acpi_dev, "Sleep state S%d not supported by BIOS\n",
	state);
	return (AE_SUPPORT);
	}

	/* Re-entry once we're suspending is not allowed. */
	status = acpi_sleep_disable(sc);
	if (ACPI_FAILURE(status)) {
	device_printf(sc->acpi_dev,
	"suspend request ignored (not ready yet)\n");
	return (status);
	}

	if (state == ACPI_STATE_S5) {
	/*
	* Shut down cleanly and power off. This will call us back through the
	* shutdown handlers.
	*/
	shutdown_nice(RB_POWEROFF);
	return_ACPI_STATUS (AE_OK);
	}

	EVENTHANDLER_INVOKE(power_suspend_early);
	stop_all_proc();
	EVENTHANDLER_INVOKE(power_suspend);

	#ifdef EARLY_AP_STARTUP
	MPASS(mp_ncpus == 1 \|\| smp_started);
	thread_lock(curthread);
	sched_bind(curthread, 0);
	thread_unlock(curthread);
	#else
	if (smp_started) {
	thread_lock(curthread);
	sched_bind(curthread, 0);
	thread_unlock(curthread);
	}
	#endif

	/*
	* Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
	* drivers need this.
	*/
	mtx_lock(&Giant);

	slp_state = ACPI_SS_NONE;

	sc->acpi_sstate = state;

	/* Enable any GPEs as appropriate and requested by the user. */
	acpi_wake_prep_walk(state);
	slp_state = ACPI_SS_GPE_SET;

	/*
	* Inform all devices that we are going to sleep. If at least one
	* device fails, DEVICE_SUSPEND() automatically resumes the tree.
	*
	* XXX Note that a better two-pass approach with a 'veto' pass
	* followed by a "real thing" pass would be better, but the current
	* bus interface does not provide for this.
	*/
	if (DEVICE_SUSPEND(root_bus) != 0) {
	device_printf(sc->acpi_dev, "device_suspend failed\n");
	goto backout;
	}
	slp_state = ACPI_SS_DEV_SUSPEND;

	status = AcpiEnterSleepStatePrep(state);
	if (ACPI_FAILURE(status)) {
	device_printf(sc->acpi_dev, "AcpiEnterSleepStatePrep failed - %s\n",
	AcpiFormatException(status));
	goto backout;
	}
	slp_state = ACPI_SS_SLP_PREP;

	if (sc->acpi_sleep_delay > 0)
	DELAY(sc->acpi_sleep_delay * 1000000);

	suspendclock();
	intr = intr_disable();
	if (state != ACPI_STATE_S1) {
	sleep_result = acpi_sleep_machdep(sc, state);
	acpi_wakeup_machdep(sc, state, sleep_result, 0);

	/*
	* XXX According to ACPI specification SCI_EN bit should be restored
	* by ACPI platform (BIOS, firmware) to its pre-sleep state.
	* Unfortunately some BIOSes fail to do that and that leads to
	* unexpected and serious consequences during wake up like a system
	* getting stuck in SMI handlers.
	* This hack is picked up from Linux, which claims that it follows
	* Windows behavior.
	*/
	if (sleep_result == 1 && state != ACPI_STATE_S4)
	AcpiWriteBitRegister(ACPI_BITREG_SCI_ENABLE, ACPI_ENABLE_EVENT);

	if (sleep_result == 1 && state == ACPI_STATE_S3) {
	/*
	* Prevent mis-interpretation of the wakeup by power button
	* as a request for power off.
	* Ideally we should post an appropriate wakeup event,
	* perhaps using acpi_event_power_button_wake or alike.
	*
	* Clearing of power button status after wakeup is mandated
	* by ACPI specification in section "Fixed Power Button".
	*
	* XXX As of ACPICA 20121114 AcpiGetEventStatus provides
	* status as 0/1 corressponding to inactive/active despite
	* its type being ACPI_EVENT_STATUS. In other words,
	* we should not test for ACPI_EVENT_FLAG_SET for time being.
	*/
	if (ACPI_SUCCESS(AcpiGetEventStatus(ACPI_EVENT_POWER_BUTTON,
	&power_button_status)) && power_button_status != 0) {
	AcpiClearEvent(ACPI_EVENT_POWER_BUTTON);
	device_printf(sc->acpi_dev,
	"cleared fixed power button status\n");
	}
	}

	intr_restore(intr);

	/* call acpi_wakeup_machdep() again with interrupt enabled */
	acpi_wakeup_machdep(sc, state, sleep_result, 1);

	AcpiLeaveSleepStatePrep(state);

	if (sleep_result == -1)
	goto backout;

	/* Re-enable ACPI hardware on wakeup from sleep state 4. */
	if (state == ACPI_STATE_S4)
	AcpiEnable();
	} else {
	status = AcpiEnterSleepState(state);
	intr_restore(intr);
	AcpiLeaveSleepStatePrep(state);
	if (ACPI_FAILURE(status)) {
	device_printf(sc->acpi_dev, "AcpiEnterSleepState failed - %s\n",
	AcpiFormatException(status));
	goto backout;
	}
	}
	slp_state = ACPI_SS_SLEPT;

	/*
	* Back out state according to how far along we got in the suspend
	* process. This handles both the error and success cases.
	*/
	backout:
	if (slp_state >= ACPI_SS_SLP_PREP)
	resumeclock();
	if (slp_state >= ACPI_SS_GPE_SET) {
	acpi_wake_prep_walk(state);
	sc->acpi_sstate = ACPI_STATE_S0;
	}
	if (slp_state >= ACPI_SS_DEV_SUSPEND)
	DEVICE_RESUME(root_bus);
	if (slp_state >= ACPI_SS_SLP_PREP)
	AcpiLeaveSleepState(state);
	if (slp_state >= ACPI_SS_SLEPT) {
	#if defined(__i386__) \|\| defined(__amd64__)
	/* NB: we are still using ACPI timecounter at this point. */
	resume_TSC();
	#endif
	acpi_resync_clock(sc);
	acpi_enable_fixed_events(sc);
	}
	sc->acpi_next_sstate = 0;

	mtx_unlock(&Giant);

	#ifdef EARLY_AP_STARTUP
	thread_lock(curthread);
	sched_unbind(curthread);
	thread_unlock(curthread);
	#else
	if (smp_started) {
	thread_lock(curthread);
	sched_unbind(curthread);
	thread_unlock(curthread);
	}
	#endif

	resume_all_proc();

	EVENTHANDLER_INVOKE(power_resume);

	/* Allow another sleep request after a while. */
	callout_schedule(&acpi_sleep_timer, hz * ACPI_MINIMUM_AWAKETIME);

	/* Run /etc/rc.resume after we are back. */
	if (devctl_process_running())
	acpi_UserNotify("Resume", ACPI_ROOT_OBJECT, state);

	return_ACPI_STATUS (status);
	}

	static void
	acpi_resync_clock(struct acpi_softc *sc)
	{

	/*
	* Warm up timecounter again and reset system clock.
	*/
	(void)timecounter->tc_get_timecount(timecounter);
	inittodr(time_second + sc->acpi_sleep_delay);
	}

	/* Enable or disable the device's wake GPE. */
	int
	acpi_wake_set_enable(device_t dev, int enable)
	{
	struct acpi_prw_data prw;
	ACPI_STATUS status;
	int flags;

	/* Make sure the device supports waking the system and get the GPE. */
	if (acpi_parse_prw(acpi_get_handle(dev), &prw) != 0)
	return (ENXIO);

	flags = acpi_get_flags(dev);
	if (enable) {
	status = AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit,
	ACPI_GPE_ENABLE);
	if (ACPI_FAILURE(status)) {
	device_printf(dev, "enable wake failed\n");
	return (ENXIO);
	}
	acpi_set_flags(dev, flags \| ACPI_FLAG_WAKE_ENABLED);
	} else {
	status = AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit,
	ACPI_GPE_DISABLE);
	if (ACPI_FAILURE(status)) {
	device_printf(dev, "disable wake failed\n");
	return (ENXIO);
	}
	acpi_set_flags(dev, flags & ~ACPI_FLAG_WAKE_ENABLED);
	}

	return (0);
	}

	static int
	acpi_wake_sleep_prep(ACPI_HANDLE handle, int sstate)
	{
	struct acpi_prw_data prw;
	device_t dev;

	/* Check that this is a wake-capable device and get its GPE. */
	if (acpi_parse_prw(handle, &prw) != 0)
	return (ENXIO);
	dev = acpi_get_device(handle);

	/*
	* The destination sleep state must be less than (i.e., higher power)
	* or equal to the value specified by _PRW. If this GPE cannot be
	* enabled for the next sleep state, then disable it. If it can and
	* the user requested it be enabled, turn on any required power resources
	* and set _PSW.
	*/
	if (sstate > prw.lowest_wake) {
	AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit, ACPI_GPE_DISABLE);
	if (bootverbose)
	device_printf(dev, "wake_prep disabled wake for %s (S%d)\n",
	acpi_name(handle), sstate);
	} else if (dev && (acpi_get_flags(dev) & ACPI_FLAG_WAKE_ENABLED) != 0) {
	acpi_pwr_wake_enable(handle, 1);
	acpi_SetInteger(handle, "_PSW", 1);
	if (bootverbose)
	device_printf(dev, "wake_prep enabled for %s (S%d)\n",
	acpi_name(handle), sstate);
	}

	return (0);
	}

	static int
	acpi_wake_run_prep(ACPI_HANDLE handle, int sstate)
	{
	struct acpi_prw_data prw;
	device_t dev;

	/*
	* Check that this is a wake-capable device and get its GPE. Return
	* now if the user didn't enable this device for wake.
	*/
	if (acpi_parse_prw(handle, &prw) != 0)
	return (ENXIO);
	dev = acpi_get_device(handle);
	if (dev == NULL \|\| (acpi_get_flags(dev) & ACPI_FLAG_WAKE_ENABLED) == 0)
	return (0);

	/*
	* If this GPE couldn't be enabled for the previous sleep state, it was
	* disabled before going to sleep so re-enable it. If it was enabled,
	* clear _PSW and turn off any power resources it used.
	*/
	if (sstate > prw.lowest_wake) {
	AcpiSetGpeWakeMask(prw.gpe_handle, prw.gpe_bit, ACPI_GPE_ENABLE);
	if (bootverbose)
	device_printf(dev, "run_prep re-enabled %s\n", acpi_name(handle));
	} else {
	acpi_SetInteger(handle, "_PSW", 0);
	acpi_pwr_wake_enable(handle, 0);
	if (bootverbose)
	device_printf(dev, "run_prep cleaned up for %s\n",
	acpi_name(handle));
	}

	return (0);
	}

	static ACPI_STATUS
	acpi_wake_prep(ACPI_HANDLE handle, UINT32 level, void context, void *status)
	{
	int sstate;

	/* If suspending, run the sleep prep function, otherwise wake. */
	sstate = (int )context;
	if (AcpiGbl_SystemAwakeAndRunning)
	acpi_wake_sleep_prep(handle, sstate);
	else
	acpi_wake_run_prep(handle, sstate);
	return (AE_OK);
	}

	/* Walk the tree rooted at acpi0 to prep devices for suspend/resume. */
	static int
	acpi_wake_prep_walk(int sstate)
	{
	ACPI_HANDLE sb_handle;

	if (ACPI_SUCCESS(AcpiGetHandle(ACPI_ROOT_OBJECT, "\\_SB_", &sb_handle)))
	AcpiWalkNamespace(ACPI_TYPE_DEVICE, sb_handle, 100,
	acpi_wake_prep, NULL, &sstate, NULL);
	return (0);
	}

	/* Walk the tree rooted at acpi0 to attach per-device wake sysctls. */
	static int
	acpi_wake_sysctl_walk(device_t dev)
	{
	int error, i, numdevs;
	device_t *devlist;
	device_t child;
	ACPI_STATUS status;

	error = device_get_children(dev, &devlist, &numdevs);
	if (error != 0 \|\| numdevs == 0) {
	if (numdevs == 0)
	free(devlist, M_TEMP);
	return (error);
	}
	for (i = 0; i < numdevs; i++) {
	child = devlist[i];
	acpi_wake_sysctl_walk(child);
	if (!device_is_attached(child))
	continue;
	status = AcpiEvaluateObject(acpi_get_handle(child), "_PRW", NULL, NULL);
	if (ACPI_SUCCESS(status)) {
	SYSCTL_ADD_PROC(device_get_sysctl_ctx(child),
	SYSCTL_CHILDREN(device_get_sysctl_tree(child)), OID_AUTO,
	"wake", CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, child, 0,
	acpi_wake_set_sysctl, "I", "Device set to wake the system");
	}
	}
	free(devlist, M_TEMP);

	return (0);
	}

	/* Enable or disable wake from userland. */
	static int
	acpi_wake_set_sysctl(SYSCTL_HANDLER_ARGS)
	{
	int enable, error;
	device_t dev;

	dev = (device_t)arg1;
	enable = (acpi_get_flags(dev) & ACPI_FLAG_WAKE_ENABLED) ? 1 : 0;

	error = sysctl_handle_int(oidp, &enable, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	if (enable != 0 && enable != 1)
	return (EINVAL);

	return (acpi_wake_set_enable(dev, enable));
	}

	/* Parse a device's _PRW into a structure. */
	int
	acpi_parse_prw(ACPI_HANDLE h, struct acpi_prw_data *prw)
	{
	ACPI_STATUS status;
	ACPI_BUFFER prw_buffer;
	ACPI_OBJECT res, res2;
	int error, i, power_count;

	if (h == NULL \|\| prw == NULL)
	return (EINVAL);

	/*
	* The _PRW object (7.2.9) is only required for devices that have the
	* ability to wake the system from a sleeping state.
	*/
	error = EINVAL;
	prw_buffer.Pointer = NULL;
	prw_buffer.Length = ACPI_ALLOCATE_BUFFER;
	status = AcpiEvaluateObject(h, "_PRW", NULL, &prw_buffer);
	if (ACPI_FAILURE(status))
	return (ENOENT);
	res = (ACPI_OBJECT *)prw_buffer.Pointer;
	if (res == NULL)
	return (ENOENT);
	if (!ACPI_PKG_VALID(res, 2))
	goto out;

	/*
	* Element 1 of the _PRW object:
	* The lowest power system sleeping state that can be entered while still
	* providing wake functionality. The sleeping state being entered must
	* be less than (i.e., higher power) or equal to this value.
	*/
	if (acpi_PkgInt32(res, 1, &prw->lowest_wake) != 0)
	goto out;

	/*
	* Element 0 of the _PRW object:
	*/
	switch (res->Package.Elements[0].Type) {
	case ACPI_TYPE_INTEGER:
	/*
	* If the data type of this package element is numeric, then this
	* _PRW package element is the bit index in the GPEx_EN, in the
	* GPE blocks described in the FADT, of the enable bit that is
	* enabled for the wake event.
	*/
	prw->gpe_handle = NULL;
	prw->gpe_bit = res->Package.Elements[0].Integer.Value;
	error = 0;
	break;
	case ACPI_TYPE_PACKAGE:
	/*
	* If the data type of this package element is a package, then this
	* _PRW package element is itself a package containing two
	* elements. The first is an object reference to the GPE Block
	* device that contains the GPE that will be triggered by the wake
	* event. The second element is numeric and it contains the bit
	* index in the GPEx_EN, in the GPE Block referenced by the
	* first element in the package, of the enable bit that is enabled for
	* the wake event.
	*
	* For example, if this field is a package then it is of the form:
	* Package() {\_SB.PCI0.ISA.GPE, 2}
	*/
	res2 = &res->Package.Elements[0];
	if (!ACPI_PKG_VALID(res2, 2))
	goto out;
	prw->gpe_handle = acpi_GetReference(NULL, &res2->Package.Elements[0]);
	if (prw->gpe_handle == NULL)
	goto out;
	if (acpi_PkgInt32(res2, 1, &prw->gpe_bit) != 0)
	goto out;
	error = 0;
	break;
	default:
	goto out;
	}

	/* Elements 2 to N of the _PRW object are power resources. */
	power_count = res->Package.Count - 2;
	if (power_count > ACPI_PRW_MAX_POWERRES) {
	printf("ACPI device %s has too many power resources\n", acpi_name(h));
	power_count = 0;
	}
	prw->power_res_count = power_count;
	for (i = 0; i < power_count; i++)
	prw->power_res[i] = res->Package.Elements[i];

	out:
	if (prw_buffer.Pointer != NULL)
	AcpiOsFree(prw_buffer.Pointer);
	return (error);
	}

	/*
	* ACPI Event Handlers
	*/

	/* System Event Handlers (registered by EVENTHANDLER_REGISTER) */

	static void
	acpi_system_eventhandler_sleep(void *arg, int state)
	{
	struct acpi_softc sc = (struct acpi_softc )arg;
	int ret;

	ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);

	/* Check if button action is disabled or unknown. */
	if (state == ACPI_STATE_UNKNOWN)
	return;

	/* Request that the system prepare to enter the given suspend state. */
	ret = acpi_ReqSleepState(sc, state);
	if (ret != 0)
	device_printf(sc->acpi_dev,
	"request to enter state S%d failed (err %d)\n", state, ret);

	return_VOID;
	}

	static void
	acpi_system_eventhandler_wakeup(void *arg, int state)
	{

	ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);

	/* Currently, nothing to do for wakeup. */

	return_VOID;
	}

	/*
	* ACPICA Event Handlers (FixedEvent, also called from button notify handler)
	*/
	static void
	acpi_invoke_sleep_eventhandler(void *context)
	{

	EVENTHANDLER_INVOKE(acpi_sleep_event, (int )context);
	}

	static void
	acpi_invoke_wake_eventhandler(void *context)
	{

	EVENTHANDLER_INVOKE(acpi_wakeup_event, (int )context);
	}

	UINT32
	acpi_event_power_button_sleep(void *context)
	{
	struct acpi_softc sc = (struct acpi_softc )context;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
	acpi_invoke_sleep_eventhandler, &sc->acpi_power_button_sx)))
	return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
	return_VALUE (ACPI_INTERRUPT_HANDLED);
	}

	UINT32
	acpi_event_power_button_wake(void *context)
	{
	struct acpi_softc sc = (struct acpi_softc )context;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
	acpi_invoke_wake_eventhandler, &sc->acpi_power_button_sx)))
	return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
	return_VALUE (ACPI_INTERRUPT_HANDLED);
	}

	UINT32
	acpi_event_sleep_button_sleep(void *context)
	{
	struct acpi_softc sc = (struct acpi_softc )context;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
	acpi_invoke_sleep_eventhandler, &sc->acpi_sleep_button_sx)))
	return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
	return_VALUE (ACPI_INTERRUPT_HANDLED);
	}

	UINT32
	acpi_event_sleep_button_wake(void *context)
	{
	struct acpi_softc sc = (struct acpi_softc )context;

	ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);

	if (ACPI_FAILURE(AcpiOsExecute(OSL_NOTIFY_HANDLER,
	acpi_invoke_wake_eventhandler, &sc->acpi_sleep_button_sx)))
	return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
	return_VALUE (ACPI_INTERRUPT_HANDLED);
	}

	/*
	* XXX This static buffer is suboptimal. There is no locking so only
	* use this for single-threaded callers.
	*/
	char *
	acpi_name(ACPI_HANDLE handle)
	{
	ACPI_BUFFER buf;
	static char data[256];

	buf.Length = sizeof(data);
	buf.Pointer = data;

	if (handle && ACPI_SUCCESS(AcpiGetName(handle, ACPI_FULL_PATHNAME, &buf)))
	return (data);
	return ("(unknown)");
	}

	/*
	* Debugging/bug-avoidance. Avoid trying to fetch info on various
	* parts of the namespace.
	*/
	int
	acpi_avoid(ACPI_HANDLE handle)
	{
	char cp, env, *np;
	int len;

	np = acpi_name(handle);
	if (*np == '\\')
	np++;
	if ((env = kern_getenv("debug.acpi.avoid")) == NULL)
	return (0);

	/* Scan the avoid list checking for a match */
	cp = env;
	for (;;) {
	while (cp != 0 && isspace(cp))
	cp++;
	if (*cp == 0)
	break;
	len = 0;
	while (cp[len] != 0 && !isspace(cp[len]))
	len++;
	if (!strncmp(cp, np, len)) {
	freeenv(env);
	return(1);
	}
	cp += len;
	}
	freeenv(env);

	return (0);
	}

	/*
	* Debugging/bug-avoidance. Disable ACPI subsystem components.
	*/
	int
	acpi_disabled(char *subsys)
	{
	char cp, env;
	int len;

	if ((env = kern_getenv("debug.acpi.disabled")) == NULL)
	return (0);
	if (strcmp(env, "all") == 0) {
	freeenv(env);
	return (1);
	}

	/* Scan the disable list, checking for a match. */
	cp = env;
	for (;;) {
	while (cp != '\0' && isspace(cp))
	cp++;
	if (*cp == '\0')
	break;
	len = 0;
	while (cp[len] != '\0' && !isspace(cp[len]))
	len++;
	if (strncmp(cp, subsys, len) == 0) {
	freeenv(env);
	return (1);
	}
	cp += len;
	}
	freeenv(env);

	return (0);
	}

	static void
	acpi_lookup(void arg, const char name, device_t *dev)
	{
	ACPI_HANDLE handle;

	if (*dev != NULL)
	return;

	/*
	* Allow any handle name that is specified as an absolute path and
	* starts with '\'. We could restrict this to \_SB and friends,
	* but see acpi_probe_children() for notes on why we scan the entire
	* namespace for devices.
	*
	* XXX: The pathname argument to AcpiGetHandle() should be fixed to
	* be const.
	*/
	if (name[0] != '\\')
	return;
	if (ACPI_FAILURE(AcpiGetHandle(ACPI_ROOT_OBJECT, __DECONST(char *, name),
	&handle)))
	return;
	*dev = acpi_get_device(handle);
	}

	/*
	* Control interface.
	*
	* We multiplex ioctls for all participating ACPI devices here. Individual
	* drivers wanting to be accessible via /dev/acpi should use the
	* register/deregister interface to make their handlers visible.
	*/
	struct acpi_ioctl_hook
	{
	TAILQ_ENTRY(acpi_ioctl_hook) link;
	u_long cmd;
	acpi_ioctl_fn fn;
	void *arg;
	};

	static TAILQ_HEAD(,acpi_ioctl_hook) acpi_ioctl_hooks;
	static int acpi_ioctl_hooks_initted;

	int
	acpi_register_ioctl(u_long cmd, acpi_ioctl_fn fn, void *arg)
	{
	struct acpi_ioctl_hook *hp;

	if ((hp = malloc(sizeof(*hp), M_ACPIDEV, M_NOWAIT)) == NULL)
	return (ENOMEM);
	hp->cmd = cmd;
	hp->fn = fn;
	hp->arg = arg;

	ACPI_LOCK(acpi);
	if (acpi_ioctl_hooks_initted == 0) {
	TAILQ_INIT(&acpi_ioctl_hooks);
	acpi_ioctl_hooks_initted = 1;
	}
	TAILQ_INSERT_TAIL(&acpi_ioctl_hooks, hp, link);
	ACPI_UNLOCK(acpi);

	return (0);
	}

	void
	acpi_deregister_ioctl(u_long cmd, acpi_ioctl_fn fn)
	{
	struct acpi_ioctl_hook *hp;

	ACPI_LOCK(acpi);
	TAILQ_FOREACH(hp, &acpi_ioctl_hooks, link)
	if (hp->cmd == cmd && hp->fn == fn)
	break;

	if (hp != NULL) {
	TAILQ_REMOVE(&acpi_ioctl_hooks, hp, link);
	free(hp, M_ACPIDEV);
	}
	ACPI_UNLOCK(acpi);
	}

	static int
	acpiopen(struct cdev dev, int flag, int fmt, struct thread td)
	{
	return (0);
	}

	static int
	acpiclose(struct cdev dev, int flag, int fmt, struct thread td)
	{
	return (0);
	}

	static int
	acpiioctl(struct cdev dev, u_long cmd, caddr_t addr, int flag, struct thread td)
	{
	struct acpi_softc *sc;
	struct acpi_ioctl_hook *hp;
	int error, state;

	error = 0;
	hp = NULL;
	sc = dev->si_drv1;

	/*
	* Scan the list of registered ioctls, looking for handlers.
	*/
	ACPI_LOCK(acpi);
	if (acpi_ioctl_hooks_initted)
	TAILQ_FOREACH(hp, &acpi_ioctl_hooks, link) {
	if (hp->cmd == cmd)
	break;
	}
	ACPI_UNLOCK(acpi);
	if (hp)
	return (hp->fn(cmd, addr, hp->arg));

	/*
	* Core ioctls are not permitted for non-writable user.
	* Currently, other ioctls just fetch information.
	* Not changing system behavior.
	*/
	if ((flag & FWRITE) == 0)
	return (EPERM);

	/* Core system ioctls. */
	switch (cmd) {
	case ACPIIO_REQSLPSTATE:
	state = (int )addr;
	if (state != ACPI_STATE_S5)
	return (acpi_ReqSleepState(sc, state));
	device_printf(sc->acpi_dev, "power off via acpi ioctl not supported\n");
	error = EOPNOTSUPP;
	break;
	case ACPIIO_ACKSLPSTATE:
	error = (int )addr;
	error = acpi_AckSleepState(sc->acpi_clone, error);
	break;
	case ACPIIO_SETSLPSTATE: /* DEPRECATED */
	state = (int )addr;
	if (state < ACPI_STATE_S0 \|\| state > ACPI_S_STATES_MAX)
	return (EINVAL);
	if (!acpi_sleep_states[state])
	return (EOPNOTSUPP);
	if (ACPI_FAILURE(acpi_SetSleepState(sc, state)))
	error = ENXIO;
	break;
	default:
	error = ENXIO;
	break;
	}

	return (error);
	}

	static int
	acpi_sname2sstate(const char *sname)
	{
	int sstate;

	if (toupper(sname[0]) == 'S') {
	sstate = sname[1] - '0';
	if (sstate >= ACPI_STATE_S0 && sstate <= ACPI_STATE_S5 &&
	sname[2] == '\0')
	return (sstate);
	} else if (strcasecmp(sname, "NONE") == 0)
	return (ACPI_STATE_UNKNOWN);
	return (-1);
	}

	static const char *
	acpi_sstate2sname(int sstate)
	{
	static const char *snames[] = { "S0", "S1", "S2", "S3", "S4", "S5" };

	if (sstate >= ACPI_STATE_S0 && sstate <= ACPI_STATE_S5)
	return (snames[sstate]);
	else if (sstate == ACPI_STATE_UNKNOWN)
	return ("NONE");
	return (NULL);
	}

	static int
	acpi_supported_sleep_state_sysctl(SYSCTL_HANDLER_ARGS)
	{
	int error;
	struct sbuf sb;
	UINT8 state;

	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
	for (state = ACPI_STATE_S1; state < ACPI_S_STATE_COUNT; state++)
	if (acpi_sleep_states[state])
	sbuf_printf(&sb, "%s ", acpi_sstate2sname(state));
	sbuf_trim(&sb);
	sbuf_finish(&sb);
	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
	sbuf_delete(&sb);
	return (error);
	}

	static int
	acpi_sleep_state_sysctl(SYSCTL_HANDLER_ARGS)
	{
	char sleep_state[10];
	int error, new_state, old_state;

	old_state = (int )oidp->oid_arg1;
	strlcpy(sleep_state, acpi_sstate2sname(old_state), sizeof(sleep_state));
	error = sysctl_handle_string(oidp, sleep_state, sizeof(sleep_state), req);
	if (error == 0 && req->newptr != NULL) {
	new_state = acpi_sname2sstate(sleep_state);
	if (new_state < ACPI_STATE_S1)
	return (EINVAL);
	if (new_state < ACPI_S_STATE_COUNT && !acpi_sleep_states[new_state])
	return (EOPNOTSUPP);
	if (new_state != old_state)
	(int )oidp->oid_arg1 = new_state;
	}
	return (error);
	}

	/* Inform devctl(4) when we receive a Notify. */
	void
	acpi_UserNotify(const char *subsystem, ACPI_HANDLE h, uint8_t notify)
	{
	char notify_buf[16];
	ACPI_BUFFER handle_buf;
	ACPI_STATUS status;

	if (subsystem == NULL)
	return;

	handle_buf.Pointer = NULL;
	handle_buf.Length = ACPI_ALLOCATE_BUFFER;
	status = AcpiNsHandleToPathname(h, &handle_buf, FALSE);
	if (ACPI_FAILURE(status))
	return;
	snprintf(notify_buf, sizeof(notify_buf), "notify=0x%02x", notify);
	devctl_notify("ACPI", subsystem, handle_buf.Pointer, notify_buf);
	AcpiOsFree(handle_buf.Pointer);
	}

	#ifdef ACPI_DEBUG
	/*
	* Support for parsing debug options from the kernel environment.
	*
	* Bits may be set in the AcpiDbgLayer and AcpiDbgLevel debug registers
	* by specifying the names of the bits in the debug.acpi.layer and
	* debug.acpi.level environment variables. Bits may be unset by
	* prefixing the bit name with !.
	*/
	struct debugtag
	{
	char *name;
	UINT32 value;
	};

	static struct debugtag dbg_layer[] = {
	{"ACPI_UTILITIES", ACPI_UTILITIES},
	{"ACPI_HARDWARE", ACPI_HARDWARE},
	{"ACPI_EVENTS", ACPI_EVENTS},
	{"ACPI_TABLES", ACPI_TABLES},
	{"ACPI_NAMESPACE", ACPI_NAMESPACE},
	{"ACPI_PARSER", ACPI_PARSER},
	{"ACPI_DISPATCHER", ACPI_DISPATCHER},
	{"ACPI_EXECUTER", ACPI_EXECUTER},
	{"ACPI_RESOURCES", ACPI_RESOURCES},
	{"ACPI_CA_DEBUGGER", ACPI_CA_DEBUGGER},
	{"ACPI_OS_SERVICES", ACPI_OS_SERVICES},
	{"ACPI_CA_DISASSEMBLER", ACPI_CA_DISASSEMBLER},
	{"ACPI_ALL_COMPONENTS", ACPI_ALL_COMPONENTS},

	{"ACPI_AC_ADAPTER", ACPI_AC_ADAPTER},
	{"ACPI_BATTERY", ACPI_BATTERY},
	{"ACPI_BUS", ACPI_BUS},
	{"ACPI_BUTTON", ACPI_BUTTON},
	{"ACPI_EC", ACPI_EC},
	{"ACPI_FAN", ACPI_FAN},
	{"ACPI_POWERRES", ACPI_POWERRES},
	{"ACPI_PROCESSOR", ACPI_PROCESSOR},
	{"ACPI_THERMAL", ACPI_THERMAL},
	{"ACPI_TIMER", ACPI_TIMER},
	{"ACPI_ALL_DRIVERS", ACPI_ALL_DRIVERS},
	{NULL, 0}
	};

	static struct debugtag dbg_level[] = {
	{"ACPI_LV_INIT", ACPI_LV_INIT},
	{"ACPI_LV_DEBUG_OBJECT", ACPI_LV_DEBUG_OBJECT},
	{"ACPI_LV_INFO", ACPI_LV_INFO},
	{"ACPI_LV_REPAIR", ACPI_LV_REPAIR},
	{"ACPI_LV_ALL_EXCEPTIONS", ACPI_LV_ALL_EXCEPTIONS},

	/* Trace verbosity level 1 [Standard Trace Level] */
	{"ACPI_LV_INIT_NAMES", ACPI_LV_INIT_NAMES},
	{"ACPI_LV_PARSE", ACPI_LV_PARSE},
	{"ACPI_LV_LOAD", ACPI_LV_LOAD},
	{"ACPI_LV_DISPATCH", ACPI_LV_DISPATCH},
	{"ACPI_LV_EXEC", ACPI_LV_EXEC},
	{"ACPI_LV_NAMES", ACPI_LV_NAMES},
	{"ACPI_LV_OPREGION", ACPI_LV_OPREGION},
	{"ACPI_LV_BFIELD", ACPI_LV_BFIELD},
	{"ACPI_LV_TABLES", ACPI_LV_TABLES},
	{"ACPI_LV_VALUES", ACPI_LV_VALUES},
	{"ACPI_LV_OBJECTS", ACPI_LV_OBJECTS},
	{"ACPI_LV_RESOURCES", ACPI_LV_RESOURCES},
	{"ACPI_LV_USER_REQUESTS", ACPI_LV_USER_REQUESTS},
	{"ACPI_LV_PACKAGE", ACPI_LV_PACKAGE},
	{"ACPI_LV_VERBOSITY1", ACPI_LV_VERBOSITY1},

	/* Trace verbosity level 2 [Function tracing and memory allocation] */
	{"ACPI_LV_ALLOCATIONS", ACPI_LV_ALLOCATIONS},
	{"ACPI_LV_FUNCTIONS", ACPI_LV_FUNCTIONS},
	{"ACPI_LV_OPTIMIZATIONS", ACPI_LV_OPTIMIZATIONS},
	{"ACPI_LV_VERBOSITY2", ACPI_LV_VERBOSITY2},
	{"ACPI_LV_ALL", ACPI_LV_ALL},

	/* Trace verbosity level 3 [Threading, I/O, and Interrupts] */
	{"ACPI_LV_MUTEX", ACPI_LV_MUTEX},
	{"ACPI_LV_THREADS", ACPI_LV_THREADS},
	{"ACPI_LV_IO", ACPI_LV_IO},
	{"ACPI_LV_INTERRUPTS", ACPI_LV_INTERRUPTS},
	{"ACPI_LV_VERBOSITY3", ACPI_LV_VERBOSITY3},

	/* Exceptionally verbose output -- also used in the global "DebugLevel" */
	{"ACPI_LV_AML_DISASSEMBLE", ACPI_LV_AML_DISASSEMBLE},
	{"ACPI_LV_VERBOSE_INFO", ACPI_LV_VERBOSE_INFO},
	{"ACPI_LV_FULL_TABLES", ACPI_LV_FULL_TABLES},
	{"ACPI_LV_EVENTS", ACPI_LV_EVENTS},
	{"ACPI_LV_VERBOSE", ACPI_LV_VERBOSE},
	{NULL, 0}
	};

	static void
	acpi_parse_debug(char cp, struct debugtag tag, UINT32 *flag)
	{
	char *ep;
	int i, l;
	int set;

	while (*cp) {
	if (isspace(*cp)) {
	cp++;
	continue;
	}
	ep = cp;
	while (ep && !isspace(ep))
	ep++;
	if (*cp == '!') {
	set = 0;
	cp++;
	if (cp == ep)
	continue;
	} else {
	set = 1;
	}
	l = ep - cp;
	for (i = 0; tag[i].name != NULL; i++) {
	if (!strncmp(cp, tag[i].name, l)) {
	if (set)
	*flag \|= tag[i].value;
	else
	*flag &= ~tag[i].value;
	}
	}
	cp = ep;
	}
	}

	static void
	acpi_set_debugging(void *junk)
	{
	char layer, level;

	if (cold) {
	AcpiDbgLayer = 0;
	AcpiDbgLevel = 0;
	}

	layer = kern_getenv("debug.acpi.layer");
	level = kern_getenv("debug.acpi.level");
	if (layer == NULL && level == NULL)
	return;

	printf("ACPI set debug");
	if (layer != NULL) {
	if (strcmp("NONE", layer) != 0)
	printf(" layer '%s'", layer);
	acpi_parse_debug(layer, &dbg_layer[0], &AcpiDbgLayer);
	freeenv(layer);
	}
	if (level != NULL) {
	if (strcmp("NONE", level) != 0)
	printf(" level '%s'", level);
	acpi_parse_debug(level, &dbg_level[0], &AcpiDbgLevel);
	freeenv(level);
	}
	printf("\n");
	}

	SYSINIT(acpi_debugging, SI_SUB_TUNABLES, SI_ORDER_ANY, acpi_set_debugging,
	NULL);

	static int
	acpi_debug_sysctl(SYSCTL_HANDLER_ARGS)
	{
	int error, *dbg;
	struct debugtag *tag;
	struct sbuf sb;
	char temp[128];

	if (sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND) == NULL)
	return (ENOMEM);
	if (strcmp(oidp->oid_arg1, "debug.acpi.layer") == 0) {
	tag = &dbg_layer[0];
	dbg = &AcpiDbgLayer;
	} else {
	tag = &dbg_level[0];
	dbg = &AcpiDbgLevel;
	}

	/* Get old values if this is a get request. */
	ACPI_SERIAL_BEGIN(acpi);
	if (*dbg == 0) {
	sbuf_cpy(&sb, "NONE");
	} else if (req->newptr == NULL) {
	for (; tag->name != NULL; tag++) {
	if ((*dbg & tag->value) == tag->value)
	sbuf_printf(&sb, "%s ", tag->name);
	}
	}
	sbuf_trim(&sb);
	sbuf_finish(&sb);
	strlcpy(temp, sbuf_data(&sb), sizeof(temp));
	sbuf_delete(&sb);

	error = sysctl_handle_string(oidp, temp, sizeof(temp), req);

	/* Check for error or no change */
	if (error == 0 && req->newptr != NULL) {
	*dbg = 0;
	kern_setenv((char *)oidp->oid_arg1, temp);
	acpi_set_debugging(NULL);
	}
	ACPI_SERIAL_END(acpi);

	return (error);
	}

	SYSCTL_PROC(_debug_acpi, OID_AUTO, layer,
	CTLFLAG_RW \| CTLTYPE_STRING \| CTLFLAG_NEEDGIANT, "debug.acpi.layer", 0,
	acpi_debug_sysctl, "A",
	"");
	SYSCTL_PROC(_debug_acpi, OID_AUTO, level,
	CTLFLAG_RW \| CTLTYPE_STRING \| CTLFLAG_NEEDGIANT, "debug.acpi.level", 0,
	acpi_debug_sysctl, "A",
	"");
	#endif /* ACPI_DEBUG */

	static int
	acpi_debug_objects_sysctl(SYSCTL_HANDLER_ARGS)
	{
	int error;
	int old;

	old = acpi_debug_objects;
	error = sysctl_handle_int(oidp, &acpi_debug_objects, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	if (old == acpi_debug_objects \|\| (old && acpi_debug_objects))
	return (0);

	ACPI_SERIAL_BEGIN(acpi);
	AcpiGbl_EnableAmlDebugObject = acpi_debug_objects ? TRUE : FALSE;
	ACPI_SERIAL_END(acpi);

	return (0);
	}

	static int
	acpi_parse_interfaces(char str, struct acpi_interface iface)
	{
	char *p;
	size_t len;
	int i, j;

	p = str;
	while (isspace(p) \|\| p == ',')
	p++;
	len = strlen(p);
	if (len == 0)
	return (0);
	p = strdup(p, M_TEMP);
	for (i = 0; i < len; i++)
	if (p[i] == ',')
	p[i] = '\0';
	i = j = 0;
	while (i < len)
	if (isspace(p[i]) \|\| p[i] == '\0')
	i++;
	else {
	i += strlen(p + i) + 1;
	j++;
	}
	if (j == 0) {
	free(p, M_TEMP);
	return (0);
	}
	iface->data = malloc(sizeof(iface->data) j, M_TEMP, M_WAITOK);
	iface->num = j;
	i = j = 0;
	while (i < len)
	if (isspace(p[i]) \|\| p[i] == '\0')
	i++;
	else {
	iface->data[j] = p + i;
	i += strlen(p + i) + 1;
	j++;
	}

	return (j);
	}

	static void
	acpi_free_interfaces(struct acpi_interface *iface)
	{

	free(iface->data[0], M_TEMP);
	free(iface->data, M_TEMP);
	}

	static void
	acpi_reset_interfaces(device_t dev)
	{
	struct acpi_interface list;
	ACPI_STATUS status;
	int i;

	if (acpi_parse_interfaces(acpi_install_interface, &list) > 0) {
	for (i = 0; i < list.num; i++) {
	status = AcpiInstallInterface(list.data[i]);
	if (ACPI_FAILURE(status))
	device_printf(dev,
	"failed to install _OSI(\"%s\"): %s\n",
	list.data[i], AcpiFormatException(status));
	else if (bootverbose)
	device_printf(dev, "installed _OSI(\"%s\")\n",
	list.data[i]);
	}
	acpi_free_interfaces(&list);
	}
	if (acpi_parse_interfaces(acpi_remove_interface, &list) > 0) {
	for (i = 0; i < list.num; i++) {
	status = AcpiRemoveInterface(list.data[i]);
	if (ACPI_FAILURE(status))
	device_printf(dev,
	"failed to remove _OSI(\"%s\"): %s\n",
	list.data[i], AcpiFormatException(status));
	else if (bootverbose)
	device_printf(dev, "removed _OSI(\"%s\")\n",
	list.data[i]);
	}
	acpi_free_interfaces(&list);
	}
	}

	static int
	acpi_pm_func(u_long cmd, void *arg, ...)
	{
	int state, acpi_state;
	int error;
	struct acpi_softc *sc;
	va_list ap;

	error = 0;
	switch (cmd) {
	case POWER_CMD_SUSPEND:
	sc = (struct acpi_softc *)arg;
	if (sc == NULL) {
	error = EINVAL;
	goto out;
	}

	va_start(ap, arg);
	state = va_arg(ap, int);
	va_end(ap);

	switch (state) {
	case POWER_SLEEP_STATE_STANDBY:
	acpi_state = sc->acpi_standby_sx;
	break;
	case POWER_SLEEP_STATE_SUSPEND:
	acpi_state = sc->acpi_suspend_sx;
	break;
	case POWER_SLEEP_STATE_HIBERNATE:
	acpi_state = ACPI_STATE_S4;
	break;
	default:
	error = EINVAL;
	goto out;
	}

	if (ACPI_FAILURE(acpi_EnterSleepState(sc, acpi_state)))
	error = ENXIO;
	break;
	default:
	error = EINVAL;
	goto out;
	}

	out:
	return (error);
	}

	static void
	acpi_pm_register(void *arg)
	{
	if (!cold \|\| resource_disabled("acpi", 0))
	return;

	power_pm_register(POWER_PM_TYPE_ACPI, acpi_pm_func, NULL);
	}

	SYSINIT(power, SI_SUB_KLD, SI_ORDER_ANY, acpi_pm_register, NULL);
	diff --git a/sys/dev/acpica/acpi_apei.c b/sys/dev/acpica/acpi_apei.c
	new file mode 100644
	index 000000000000..8cdf09ee150c
	--- /dev/null
	+++ b/sys/dev/acpica/acpi_apei.c
	@@ -0,0 +1,684 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2020 Alexander Motin <mav@FreeBSD.org>
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include "opt_acpi.h"
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/bus.h>
	+#include <sys/callout.h>
	+#include <sys/interrupt.h>
	+#include <sys/kernel.h>
	+#include <sys/malloc.h>
	+#include <sys/module.h>
	+#include <sys/queue.h>
	+#include <sys/rman.h>
	+#include <vm/vm.h>
	+#include <vm/pmap.h>
	+
	+#include <contrib/dev/acpica/include/acpi.h>
	+#include <contrib/dev/acpica/include/accommon.h>
	+#include <contrib/dev/acpica/include/aclocal.h>
	+#include <contrib/dev/acpica/include/actables.h>
	+
	+#include <dev/acpica/acpivar.h>
	+#include <dev/pci/pcireg.h>
	+#include <dev/pci/pcivar.h>
	+
	+struct apei_ge {
	+ union {
	+ ACPI_HEST_GENERIC v1;
	+ ACPI_HEST_GENERIC_V2 v2;
	+ };
	+ int res_type;
	+ int res_rid;
	+ struct resource *res;
	+ int res2_type;
	+ int res2_rid;
	+ struct resource *res2;
	+ uint8_t buf, copybuf;
	+ TAILQ_ENTRY(apei_ge) link;
	+ struct callout poll;
	+ void *swi_ih;
	+} *apei_nmi_ge;
	+
	+struct apei_softc {
	+ ACPI_TABLE_HEST *hest;
	+ TAILQ_HEAD(, apei_ge) ges;
	+};
	+
	+struct apei_mem_error {
	+ uint64_t ValidationBits;
	+ uint64_t ErrorStatus;
	+ uint64_t PhysicalAddress;
	+ uint64_t PhysicalAddressMask;
	+ uint16_t Node;
	+ uint16_t Card;
	+ uint16_t Module;
	+ uint16_t Bank;
	+ uint16_t Device;
	+ uint16_t Row;
	+ uint16_t Column;
	+ uint16_t BitPosition;
	+ uint64_t RequesterID;
	+ uint64_t ResponderID;
	+ uint64_t TargetID;
	+ uint8_t MemoryErrorType;
	+ uint8_t Extended;
	+ uint16_t RankNumber;
	+ uint16_t CardHandle;
	+ uint16_t ModuleHandle;
	+};
	+
	+struct apei_pcie_error {
	+ uint64_t ValidationBits;
	+ uint32_t PortType;
	+ uint32_t Version;
	+ uint32_t CommandStatus;
	+ uint32_t Reserved;
	+ uint8_t DeviceID[16];
	+ uint8_t DeviceSerialNumber[8];
	+ uint8_t BridgeControlStatus[4];
	+ uint8_t CapabilityStructure[60];
	+ uint8_t AERInfo[96];
	+};
	+
	+#ifdef __i386__
	+static __inline uint64_t
	+apei_bus_read_8(struct resource *res, bus_size_t offset)
	+{
	+ return (bus_read_4(res, offset) \|
	+ ((uint64_t)bus_read_4(res, offset + 4)) << 32);
	+}
	+static __inline void
	+apei_bus_write_8(struct resource *res, bus_size_t offset, uint64_t val)
	+{
	+ bus_write_4(res, offset, val);
	+ bus_write_4(res, offset + 4, val >> 32);
	+}
	+#define READ8(r, o) apei_bus_read_8((r), (o))
	+#define WRITE8(r, o, v) apei_bus_write_8((r), (o), (v))
	+#else
	+#define READ8(r, o) bus_read_8((r), (o))
	+#define WRITE8(r, o, v) bus_write_8((r), (o), (v))
	+#endif
	+
	+int apei_nmi_handler(void);
	+
	+static const char *
	+apei_severity(uint32_t s)
	+{
	+ switch (s) {
	+ case ACPI_HEST_GEN_ERROR_RECOVERABLE:
	+ return ("Recoverable");
	+ case ACPI_HEST_GEN_ERROR_FATAL:
	+ return ("Fatal");
	+ case ACPI_HEST_GEN_ERROR_CORRECTED:
	+ return ("Corrected");
	+ case ACPI_HEST_GEN_ERROR_NONE:
	+ return ("Informational");
	+ }
	+ return ("???");
	+}
	+
	+static int
	+apei_mem_handler(ACPI_HEST_GENERIC_DATA *ged)
	+{
	+ struct apei_mem_error p = (struct apei_mem_error )(ged + 1);
	+
	+ printf("APEI %s Memory Error:\n", apei_severity(ged->ErrorSeverity));
	+ if (p->ValidationBits & 0x01)
	+ printf(" Error Status: 0x%jx\n", p->ErrorStatus);
	+ if (p->ValidationBits & 0x02)
	+ printf(" Physical Address: 0x%jx\n", p->PhysicalAddress);
	+ if (p->ValidationBits & 0x04)
	+ printf(" Physical Address Mask: 0x%jx\n", p->PhysicalAddressMask);
	+ if (p->ValidationBits & 0x08)
	+ printf(" Node: %u\n", p->Node);
	+ if (p->ValidationBits & 0x10)
	+ printf(" Card: %u\n", p->Card);
	+ if (p->ValidationBits & 0x20)
	+ printf(" Module: %u\n", p->Module);
	+ if (p->ValidationBits & 0x40)
	+ printf(" Bank: %u\n", p->Bank);
	+ if (p->ValidationBits & 0x80)
	+ printf(" Device: %u\n", p->Device);
	+ if (p->ValidationBits & 0x100)
	+ printf(" Row: %u\n", p->Row);
	+ if (p->ValidationBits & 0x200)
	+ printf(" Column: %u\n", p->Column);
	+ if (p->ValidationBits & 0x400)
	+ printf(" Bit Position: %u\n", p->BitPosition);
	+ if (p->ValidationBits & 0x800)
	+ printf(" Requester ID: 0x%jx\n", p->RequesterID);
	+ if (p->ValidationBits & 0x1000)
	+ printf(" Responder ID: 0x%jx\n", p->ResponderID);
	+ if (p->ValidationBits & 0x2000)
	+ printf(" Target ID: 0x%jx\n", p->TargetID);
	+ if (p->ValidationBits & 0x4000)
	+ printf(" Memory Error Type: %u\n", p->MemoryErrorType);
	+ if (p->ValidationBits & 0x8000)
	+ printf(" Rank Number: %u\n", p->RankNumber);
	+ if (p->ValidationBits & 0x10000)
	+ printf(" Card Handle: 0x%x\n", p->CardHandle);
	+ if (p->ValidationBits & 0x20000)
	+ printf(" Module Handle: 0x%x\n", p->ModuleHandle);
	+ if (p->ValidationBits & 0x40000)
	+ printf(" Extended Row: %u\n",
	+ (uint32_t)(p->Extended & 0x3) << 16 \| p->Row);
	+ if (p->ValidationBits & 0x80000)
	+ printf(" Bank Group: %u\n", p->Bank >> 8);
	+ if (p->ValidationBits & 0x100000)
	+ printf(" Bank Address: %u\n", p->Bank & 0xff);
	+ if (p->ValidationBits & 0x200000)
	+ printf(" Chip Identification: %u\n", (p->Extended >> 5) & 0x7);
	+
	+ return (0);
	+}
	+
	+static int
	+apei_pcie_handler(ACPI_HEST_GENERIC_DATA *ged)
	+{
	+ struct apei_pcie_error p = (struct apei_pcie_error )(ged + 1);
	+ device_t dev;
	+ int h = 0, off, sev;
	+
	+ if ((p->ValidationBits & 0x8) == 0x8) {
	+ mtx_lock(&Giant);
	+ dev = pci_find_dbsf((uint32_t)p->DeviceID[10] << 8 \|
	+ p->DeviceID[9], p->DeviceID[11], p->DeviceID[8],
	+ p->DeviceID[7]);
	+ if (dev != NULL) {
	+ switch (ged->ErrorSeverity) {
	+ case ACPI_HEST_GEN_ERROR_FATAL:
	+ sev = PCIEM_STA_FATAL_ERROR;
	+ break;
	+ case ACPI_HEST_GEN_ERROR_RECOVERABLE:
	+ sev = PCIEM_STA_NON_FATAL_ERROR;
	+ break;
	+ default:
	+ sev = PCIEM_STA_CORRECTABLE_ERROR;
	+ break;
	+ }
	+ pcie_apei_error(dev, sev,
	+ (p->ValidationBits & 0x80) ? p->AERInfo : NULL);
	+ h = 1;
	+ }
	+ mtx_unlock(&Giant);
	+ }
	+ if (h)
	+ return (h);
	+
	+ printf("APEI %s PCIe Error:\n", apei_severity(ged->ErrorSeverity));
	+ if (p->ValidationBits & 0x01)
	+ printf(" Port Type: %u\n", p->PortType);
	+ if (p->ValidationBits & 0x02)
	+ printf(" Version: %x\n", p->Version);
	+ if (p->ValidationBits & 0x04)
	+ printf(" Command Status: 0x%08x\n", p->CommandStatus);
	+ if (p->ValidationBits & 0x08) {
	+ printf(" DeviceID:");
	+ for (off = 0; off < sizeof(p->DeviceID); off++)
	+ printf(" %02x", p->DeviceID[off]);
	+ printf("\n");
	+ }
	+ if (p->ValidationBits & 0x10) {
	+ printf(" Device Serial Number:");
	+ for (off = 0; off < sizeof(p->DeviceSerialNumber); off++)
	+ printf(" %02x", p->DeviceSerialNumber[off]);
	+ printf("\n");
	+ }
	+ if (p->ValidationBits & 0x20) {
	+ printf(" Bridge Control Status:");
	+ for (off = 0; off < sizeof(p->BridgeControlStatus); off++)
	+ printf(" %02x", p->BridgeControlStatus[off]);
	+ printf("\n");
	+ }
	+ if (p->ValidationBits & 0x40) {
	+ printf(" Capability Structure:\n");
	+ for (off = 0; off < sizeof(p->CapabilityStructure); off++) {
	+ printf(" %02x", p->CapabilityStructure[off]);
	+ if ((off % 16) == 15 \|\|
	+ off + 1 == sizeof(p->CapabilityStructure))
	+ printf("\n");
	+ }
	+ }
	+ if (p->ValidationBits & 0x80) {
	+ printf(" AER Info:\n");
	+ for (off = 0; off < sizeof(p->AERInfo); off++) {
	+ printf(" %02x", p->AERInfo[off]);
	+ if ((off % 16) == 15 \|\| off + 1 == sizeof(p->AERInfo))
	+ printf("\n");
	+ }
	+ }
	+ return (h);
	+}
	+
	+static void
	+apei_ged_handler(ACPI_HEST_GENERIC_DATA *ged)
	+{
	+ ACPI_HEST_GENERIC_DATA_V300 ged3 = (ACPI_HEST_GENERIC_DATA_V300 )ged;
	+ /* A5BC1114-6F64-4EDE-B863-3E83ED7C83B1 */
	+ static uint8_t mem_uuid[ACPI_UUID_LENGTH] = {
	+ 0x14, 0x11, 0xBC, 0xA5, 0x64, 0x6F, 0xDE, 0x4E,
	+ 0xB8, 0x63, 0x3E, 0x83, 0xED, 0x7C, 0x83, 0xB1
	+ };
	+ /* D995E954-BBC1-430F-AD91-B44DCB3C6F35 */
	+ static uint8_t pcie_uuid[ACPI_UUID_LENGTH] = {
	+ 0x54, 0xE9, 0x95, 0xD9, 0xC1, 0xBB, 0x0F, 0x43,
	+ 0xAD, 0x91, 0xB4, 0x4D, 0xCB, 0x3C, 0x6F, 0x35
	+ };
	+ uint8_t *t;
	+ int h = 0, off;
	+
	+ if (memcmp(mem_uuid, ged->SectionType, ACPI_UUID_LENGTH) == 0) {
	+ h = apei_mem_handler(ged);
	+ } else if (memcmp(pcie_uuid, ged->SectionType, ACPI_UUID_LENGTH) == 0) {
	+ h = apei_pcie_handler(ged);
	+ } else {
	+ t = ged->SectionType;
	+ printf("APEI %s Error %02x%02x%02x%02x-%02x%02x-"
	+ "%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x:\n",
	+ apei_severity(ged->ErrorSeverity),
	+ t[3], t[2], t[1], t[0], t[5], t[4], t[7], t[6],
	+ t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15]);
	+ printf(" Error Data:\n");
	+ t = (uint8_t *)(ged + 1);
	+ for (off = 0; off < ged->ErrorDataLength; off++) {
	+ printf(" %02x", t[off]);
	+ if ((off % 16) == 15 \|\| off + 1 == ged->ErrorDataLength)
	+ printf("\n");
	+ }
	+ }
	+ if (h)
	+ return;
	+
	+ printf(" Flags: 0x%x\n", ged->Flags);
	+ if (ged->ValidationBits & ACPI_HEST_GEN_VALID_FRU_ID) {
	+ t = ged->FruId;
	+ printf(" FRU Id: %02x%02x%02x%02x-%02x%02x-%02x%02x-"
	+ "%02x%02x-%02x%02x%02x%02x%02x%02x\n",
	+ t[3], t[2], t[1], t[0], t[5], t[4], t[7], t[6],
	+ t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15]);
	+ }
	+ if (ged->ValidationBits & ACPI_HEST_GEN_VALID_FRU_STRING)
	+ printf(" FRU Text: %.20s", ged->FruText);
	+ if (ged->Revision == 0x300 &&
	+ ged->ValidationBits & ACPI_HEST_GEN_VALID_TIMESTAMP)
	+ printf(" Timestamp: %016jx", ged3->TimeStamp);
	+}
	+
	+static int
	+apei_ge_handler(struct apei_ge *ge, bool copy)
	+{
	+ uint8_t *buf = copy ? ge->copybuf : ge->buf;
	+ ACPI_HEST_GENERIC_STATUS ges = (ACPI_HEST_GENERIC_STATUS )buf;
	+ ACPI_HEST_GENERIC_DATA *ged;
	+ uint32_t sev;
	+ int i, c, off;
	+
	+ if (ges->BlockStatus == 0)
	+ return (0);
	+
	+ c = (ges->BlockStatus >> 4) & 0x3ff;
	+ sev = ges->ErrorSeverity;
	+
	+ /* Process error entries. */
	+ for (off = i = 0; i < c && off + sizeof(*ged) <= ges->DataLength; i++) {
	+ ged = (ACPI_HEST_GENERIC_DATA )&buf[sizeof(ges) + off];
	+ apei_ged_handler(ged);
	+ off += sizeof(*ged) + ged->ErrorDataLength;
	+ }
	+
	+ /* Acknowledge the error has been processed. */
	+ ges->BlockStatus = 0;
	+ if (!copy && ge->v1.Header.Type == ACPI_HEST_TYPE_GENERIC_ERROR_V2) {
	+ uint64_t val = READ8(ge->res2, 0);
	+ val &= ge->v2.ReadAckPreserve;
	+ val \|= ge->v2.ReadAckWrite;
	+ WRITE8(ge->res2, 0, val);
	+ }
	+
	+ /* If ACPI told the error is fatal -- make it so. */
	+ if (sev == ACPI_HEST_GEN_ERROR_FATAL)
	+ panic("APEI Fatal Hardware Error!");
	+
	+ return (1);
	+}
	+
	+static void
	+apei_nmi_swi(void *arg)
	+{
	+ struct apei_ge *ge = arg;
	+
	+ apei_ge_handler(ge, true);
	+}
	+
	+int
	+apei_nmi_handler(void)
	+{
	+ struct apei_ge *ge = apei_nmi_ge;
	+ ACPI_HEST_GENERIC_STATUS ges, gesc;
	+
	+ if (ge == NULL)
	+ return (0);
	+
	+ ges = (ACPI_HEST_GENERIC_STATUS *)ge->buf;
	+ if (ges->BlockStatus == 0)
	+ return (0);
	+
	+ /* If ACPI told the error is fatal -- make it so. */
	+ if (ges->ErrorSeverity == ACPI_HEST_GEN_ERROR_FATAL)
	+ panic("APEI Fatal Hardware Error!");
	+
	+ /* Copy the buffer for later processing. */
	+ gesc = (ACPI_HEST_GENERIC_STATUS *)ge->copybuf;
	+ if (gesc->BlockStatus == 0)
	+ memcpy(ge->copybuf, ge->buf, ge->v1.ErrorBlockLength);
	+
	+ /* Acknowledge the error has been processed. */
	+ ges->BlockStatus = 0;
	+ if (ge->v1.Header.Type == ACPI_HEST_TYPE_GENERIC_ERROR_V2) {
	+ uint64_t val = READ8(ge->res2, 0);
	+ val &= ge->v2.ReadAckPreserve;
	+ val \|= ge->v2.ReadAckWrite;
	+ WRITE8(ge->res2, 0, val);
	+ }
	+
	+ /* Schedule SWI for real handling. */
	+ swi_sched(ge->swi_ih, SWI_FROMNMI);
	+
	+ return (1);
	+}
	+
	+static void
	+apei_callout_handler(void *context)
	+{
	+ struct apei_ge *ge = context;
	+
	+ apei_ge_handler(ge, false);
	+ callout_schedule(&ge->poll, ge->v1.Notify.PollInterval * hz / 1000);
	+}
	+
	+static void
	+apei_notify_handler(ACPI_HANDLE h, UINT32 notify, void *context)
	+{
	+ device_t dev = context;
	+ struct apei_softc *sc = device_get_softc(dev);
	+ struct apei_ge *ge;
	+
	+ TAILQ_FOREACH(ge, &sc->ges, link) {
	+ if (ge->v1.Notify.Type == ACPI_HEST_NOTIFY_SCI \|\|
	+ ge->v1.Notify.Type == ACPI_HEST_NOTIFY_GPIO \|\|
	+ ge->v1.Notify.Type == ACPI_HEST_NOTIFY_GSIV)
	+ apei_ge_handler(ge, false);
	+ }
	+}
	+
	+static int
	+hest_parse_structure(struct apei_softc sc, void addr, int remaining)
	+{
	+ ACPI_HEST_HEADER *hdr = addr;
	+ struct apei_ge *ge;
	+
	+ if (remaining < (int)sizeof(ACPI_HEST_HEADER))
	+ return (-1);
	+
	+ switch (hdr->Type) {
	+ case ACPI_HEST_TYPE_IA32_CHECK: {
	+ ACPI_HEST_IA_MACHINE_CHECK *s = addr;
	+ return (sizeof(s) + s->NumHardwareBanks
	+ sizeof(ACPI_HEST_IA_ERROR_BANK));
	+ }
	+ case ACPI_HEST_TYPE_IA32_CORRECTED_CHECK: {
	+ ACPI_HEST_IA_CORRECTED *s = addr;
	+ return (sizeof(s) + s->NumHardwareBanks
	+ sizeof(ACPI_HEST_IA_ERROR_BANK));
	+ }
	+ case ACPI_HEST_TYPE_IA32_NMI: {
	+ ACPI_HEST_IA_NMI *s = addr;
	+ return (sizeof(*s));
	+ }
	+ case ACPI_HEST_TYPE_AER_ROOT_PORT: {
	+ ACPI_HEST_AER_ROOT *s = addr;
	+ return (sizeof(*s));
	+ }
	+ case ACPI_HEST_TYPE_AER_ENDPOINT: {
	+ ACPI_HEST_AER *s = addr;
	+ return (sizeof(*s));
	+ }
	+ case ACPI_HEST_TYPE_AER_BRIDGE: {
	+ ACPI_HEST_AER_BRIDGE *s = addr;
	+ return (sizeof(*s));
	+ }
	+ case ACPI_HEST_TYPE_GENERIC_ERROR: {
	+ ACPI_HEST_GENERIC *s = addr;
	+ ge = malloc(sizeof(*ge), M_DEVBUF, M_WAITOK \| M_ZERO);
	+ ge->v1 = *s;
	+ TAILQ_INSERT_TAIL(&sc->ges, ge, link);
	+ return (sizeof(*s));
	+ }
	+ case ACPI_HEST_TYPE_GENERIC_ERROR_V2: {
	+ ACPI_HEST_GENERIC_V2 *s = addr;
	+ ge = malloc(sizeof(*ge), M_DEVBUF, M_WAITOK \| M_ZERO);
	+ ge->v2 = *s;
	+ TAILQ_INSERT_TAIL(&sc->ges, ge, link);
	+ return (sizeof(*s));
	+ }
	+ case ACPI_HEST_TYPE_IA32_DEFERRED_CHECK: {
	+ ACPI_HEST_IA_DEFERRED_CHECK *s = addr;
	+ return (sizeof(s) + s->NumHardwareBanks
	+ sizeof(ACPI_HEST_IA_ERROR_BANK));
	+ }
	+ default:
	+ return (-1);
	+ }
	+}
	+
	+static void
	+hest_parse_table(struct apei_softc *sc)
	+{
	+ ACPI_TABLE_HEST *hest = sc->hest;
	+ char *cp;
	+ int remaining, consumed;
	+
	+ remaining = hest->Header.Length - sizeof(ACPI_TABLE_HEST);
	+ while (remaining > 0) {
	+ cp = (char *)hest + hest->Header.Length - remaining;
	+ consumed = hest_parse_structure(sc, cp, remaining);
	+ if (consumed <= 0)
	+ break;
	+ else
	+ remaining -= consumed;
	+ }
	+}
	+
	+static char *apei_ids[] = { "PNP0C33", NULL };
	+static devclass_t apei_devclass;
	+
	+static ACPI_STATUS
	+apei_find(ACPI_HANDLE handle, UINT32 level, void *context,
	+ void **status)
	+{
	+ int found = (int )status;
	+ char **ids;
	+
	+ for (ids = apei_ids; *ids != NULL; ids++) {
	+ if (acpi_MatchHid(handle, *ids)) {
	+ *found = 1;
	+ break;
	+ }
	+ }
	+ return (AE_OK);
	+}
	+
	+static void
	+apei_identify(driver_t *driver, device_t parent)
	+{
	+ device_t child;
	+ int found;
	+
	+ if (acpi_disabled("apei"))
	+ return;
	+ if (acpi_find_table(ACPI_SIG_HEST) == 0)
	+ return;
	+ /* Only one APEI device can exist. */
	+ if (devclass_get_device(apei_devclass, 0))
	+ return;
	+ /* Search for ACPI error device to be used. */
	+ found = 0;
	+ AcpiWalkNamespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
	+ 100, apei_find, NULL, NULL, (void *)&found);
	+ if (found)
	+ return;
	+ /* If not found - create a fake one. */
	+ child = BUS_ADD_CHILD(parent, 2, "apei", 0);
	+ if (child == NULL)
	+ printf("%s: can't add child\n", __func__);
	+}
	+
	+static int
	+apei_probe(device_t dev)
	+{
	+ int rv;
	+
	+ if (acpi_disabled("apei"))
	+ return (ENXIO);
	+ if (acpi_find_table(ACPI_SIG_HEST) == 0)
	+ return (ENXIO);
	+ if (acpi_get_handle(dev) != NULL)
	+ rv = ACPI_ID_PROBE(device_get_parent(dev), dev, apei_ids, NULL);
	+ else
	+ rv = 0;
	+ if (rv <= 0)
	+ device_set_desc(dev, "Platform Error Interface");
	+ return (rv);
	+}
	+
	+static int
	+apei_attach(device_t dev)
	+{
	+ struct apei_softc *sc = device_get_softc(dev);
	+ struct apei_ge *ge;
	+ ACPI_STATUS status;
	+ int rid;
	+
	+ TAILQ_INIT(&sc->ges);
	+
	+ /* Search and parse HEST table. */
	+ status = AcpiGetTable(ACPI_SIG_HEST, 0, (ACPI_TABLE_HEADER **)&sc->hest);
	+ if (ACPI_FAILURE(status))
	+ return (ENXIO);
	+ hest_parse_table(sc);
	+ AcpiPutTable((ACPI_TABLE_HEADER *)sc->hest);
	+
	+ rid = 0;
	+ TAILQ_FOREACH(ge, &sc->ges, link) {
	+ ge->res_rid = rid++;
	+ acpi_bus_alloc_gas(dev, &ge->res_type, &ge->res_rid,
	+ &ge->v1.ErrorStatusAddress, &ge->res, 0);
	+ if (ge->v1.Header.Type == ACPI_HEST_TYPE_GENERIC_ERROR_V2) {
	+ ge->res2_rid = rid++;
	+ acpi_bus_alloc_gas(dev, &ge->res2_type, &ge->res2_rid,
	+ &ge->v2.ReadAckRegister, &ge->res2, 0);
	+ }
	+ ge->buf = pmap_mapdev_attr(READ8(ge->res, 0),
	+ ge->v1.ErrorBlockLength, VM_MEMATTR_WRITE_COMBINING);
	+ if (ge->v1.Notify.Type == ACPI_HEST_NOTIFY_POLLED) {
	+ callout_init(&ge->poll, 1);
	+ callout_reset(&ge->poll,
	+ ge->v1.Notify.PollInterval * hz / 1000,
	+ apei_callout_handler, ge);
	+ } else if (ge->v1.Notify.Type == ACPI_HEST_NOTIFY_NMI) {
	+ ge->copybuf = malloc(ge->v1.ErrorBlockLength,
	+ M_DEVBUF, M_WAITOK \| M_ZERO);
	+ swi_add(&clk_intr_event, "apei", apei_nmi_swi, ge,
	+ SWI_CLOCK, INTR_MPSAFE, &ge->swi_ih);
	+ apei_nmi_ge = ge;
	+ apei_nmi = apei_nmi_handler;
	+ }
	+ }
	+
	+ if (acpi_get_handle(dev) != NULL) {
	+ AcpiInstallNotifyHandler(acpi_get_handle(dev),
	+ ACPI_DEVICE_NOTIFY, apei_notify_handler, dev);
	+ }
	+ return (0);
	+}
	+
	+static int
	+apei_detach(device_t dev)
	+{
	+ struct apei_softc *sc = device_get_softc(dev);
	+ struct apei_ge *ge;
	+
	+ apei_nmi = NULL;
	+ apei_nmi_ge = NULL;
	+ if (acpi_get_handle(dev) != NULL) {
	+ AcpiRemoveNotifyHandler(acpi_get_handle(dev),
	+ ACPI_DEVICE_NOTIFY, apei_notify_handler);
	+ }
	+
	+ while ((ge = TAILQ_FIRST(&sc->ges)) != NULL) {
	+ TAILQ_REMOVE(&sc->ges, ge, link);
	+ bus_release_resource(dev, ge->res_type, ge->res_rid, ge->res);
	+ if (ge->res2) {
	+ bus_release_resource(dev, ge->res2_type,
	+ ge->res2_rid, ge->res2);
	+ }
	+ if (ge->v1.Notify.Type == ACPI_HEST_NOTIFY_POLLED) {
	+ callout_drain(&ge->poll);
	+ } else if (ge->v1.Notify.Type == ACPI_HEST_NOTIFY_NMI) {
	+ swi_remove(&ge->swi_ih);
	+ free(ge->copybuf, M_DEVBUF);
	+ }
	+ pmap_unmapdev((vm_offset_t)ge->buf, ge->v1.ErrorBlockLength);
	+ free(ge, M_DEVBUF);
	+ }
	+ return (0);
	+}
	+
	+static device_method_t apei_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_identify, apei_identify),
	+ DEVMETHOD(device_probe, apei_probe),
	+ DEVMETHOD(device_attach, apei_attach),
	+ DEVMETHOD(device_detach, apei_detach),
	+ DEVMETHOD_END
	+};
	+
	+static driver_t apei_driver = {
	+ "apei",
	+ apei_methods,
	+ sizeof(struct apei_softc),
	+};
	+
	+DRIVER_MODULE(apei, acpi, apei_driver, apei_devclass, 0, 0);
	+MODULE_DEPEND(apei, acpi, 1, 1, 1);
	diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
	index 938d042c66b4..b088b5899dc0 100644
	--- a/sys/dev/cxgbe/t4_main.c
	+++ b/sys/dev/cxgbe/t4_main.c
	@@ -1,11367 +1,11351 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2011 Chelsio Communications, Inc.
	* All rights reserved.
	* Written by: Navdeep Parhar <np@FreeBSD.org>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_ddb.h"
	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_kern_tls.h"
	#include "opt_ratelimit.h"
	#include "opt_rss.h"

	#include <sys/param.h>
	#include <sys/conf.h>
	#include <sys/priv.h>
	#include <sys/kernel.h>
	#include <sys/bus.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/queue.h>
	#include <sys/taskqueue.h>
	#include <sys/pciio.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	#include <dev/pci/pci_private.h>
	#include <sys/firmware.h>
	#include <sys/sbuf.h>
	#include <sys/smp.h>
	#include <sys/socket.h>
	#include <sys/sockio.h>
	#include <sys/sysctl.h>
	#include <net/ethernet.h>
	#include <net/if.h>
	#include <net/if_types.h>
	#include <net/if_dl.h>
	#include <net/if_vlan_var.h>
	#ifdef RSS
	#include <net/rss_config.h>
	#endif
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#ifdef KERN_TLS
	#include <netinet/tcp_seq.h>
	#endif
	#if defined(__i386__) \|\| defined(__amd64__)
	#include <machine/md_var.h>
	#include <machine/cputypes.h>
	#include <vm/vm.h>
	#include <vm/pmap.h>
	#endif
	#ifdef DDB
	#include <ddb/ddb.h>
	#include <ddb/db_lex.h>
	#endif

	#include "common/common.h"
	#include "common/t4_msg.h"
	#include "common/t4_regs.h"
	#include "common/t4_regs_values.h"
	#include "cudbg/cudbg.h"
	#include "t4_clip.h"
	#include "t4_ioctl.h"
	#include "t4_l2t.h"
	#include "t4_mp_ring.h"
	#include "t4_if.h"
	#include "t4_smt.h"

	/* T4 bus driver interface */
	static int t4_probe(device_t);
	static int t4_attach(device_t);
	static int t4_detach(device_t);
	static int t4_child_location_str(device_t, device_t, char *, size_t);
	static int t4_ready(device_t);
	static int t4_read_port_device(device_t, int, device_t *);
	static device_method_t t4_methods[] = {
	DEVMETHOD(device_probe, t4_probe),
	DEVMETHOD(device_attach, t4_attach),
	DEVMETHOD(device_detach, t4_detach),

	DEVMETHOD(bus_child_location_str, t4_child_location_str),

	DEVMETHOD(t4_is_main_ready, t4_ready),
	DEVMETHOD(t4_read_port_device, t4_read_port_device),

	DEVMETHOD_END
	};
	static driver_t t4_driver = {
	"t4nex",
	t4_methods,
	sizeof(struct adapter)
	};


	/* T4 port (cxgbe) interface */
	static int cxgbe_probe(device_t);
	static int cxgbe_attach(device_t);
	static int cxgbe_detach(device_t);
	device_method_t cxgbe_methods[] = {
	DEVMETHOD(device_probe, cxgbe_probe),
	DEVMETHOD(device_attach, cxgbe_attach),
	DEVMETHOD(device_detach, cxgbe_detach),
	{ 0, 0 }
	};
	static driver_t cxgbe_driver = {
	"cxgbe",
	cxgbe_methods,
	sizeof(struct port_info)
	};

	/* T4 VI (vcxgbe) interface */
	static int vcxgbe_probe(device_t);
	static int vcxgbe_attach(device_t);
	static int vcxgbe_detach(device_t);
	static device_method_t vcxgbe_methods[] = {
	DEVMETHOD(device_probe, vcxgbe_probe),
	DEVMETHOD(device_attach, vcxgbe_attach),
	DEVMETHOD(device_detach, vcxgbe_detach),
	{ 0, 0 }
	};
	static driver_t vcxgbe_driver = {
	"vcxgbe",
	vcxgbe_methods,
	sizeof(struct vi_info)
	};

	static d_ioctl_t t4_ioctl;

	static struct cdevsw t4_cdevsw = {
	.d_version = D_VERSION,
	.d_ioctl = t4_ioctl,
	.d_name = "t4nex",
	};

	/* T5 bus driver interface */
	static int t5_probe(device_t);
	static device_method_t t5_methods[] = {
	DEVMETHOD(device_probe, t5_probe),
	DEVMETHOD(device_attach, t4_attach),
	DEVMETHOD(device_detach, t4_detach),

	DEVMETHOD(bus_child_location_str, t4_child_location_str),

	DEVMETHOD(t4_is_main_ready, t4_ready),
	DEVMETHOD(t4_read_port_device, t4_read_port_device),

	DEVMETHOD_END
	};
	static driver_t t5_driver = {
	"t5nex",
	t5_methods,
	sizeof(struct adapter)
	};


	/* T5 port (cxl) interface */
	static driver_t cxl_driver = {
	"cxl",
	cxgbe_methods,
	sizeof(struct port_info)
	};

	/* T5 VI (vcxl) interface */
	static driver_t vcxl_driver = {
	"vcxl",
	vcxgbe_methods,
	sizeof(struct vi_info)
	};

	/* T6 bus driver interface */
	static int t6_probe(device_t);
	static device_method_t t6_methods[] = {
	DEVMETHOD(device_probe, t6_probe),
	DEVMETHOD(device_attach, t4_attach),
	DEVMETHOD(device_detach, t4_detach),

	DEVMETHOD(bus_child_location_str, t4_child_location_str),

	DEVMETHOD(t4_is_main_ready, t4_ready),
	DEVMETHOD(t4_read_port_device, t4_read_port_device),

	DEVMETHOD_END
	};
	static driver_t t6_driver = {
	"t6nex",
	t6_methods,
	sizeof(struct adapter)
	};


	/* T6 port (cc) interface */
	static driver_t cc_driver = {
	"cc",
	cxgbe_methods,
	sizeof(struct port_info)
	};

	/* T6 VI (vcc) interface */
	static driver_t vcc_driver = {
	"vcc",
	vcxgbe_methods,
	sizeof(struct vi_info)
	};

	/* ifnet interface */
	static void cxgbe_init(void *);
	static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
	static int cxgbe_transmit(struct ifnet , struct mbuf );
	static void cxgbe_qflush(struct ifnet *);
	#if defined(KERN_TLS) \|\| defined(RATELIMIT)
	static int cxgbe_snd_tag_alloc(struct ifnet , union if_snd_tag_alloc_params ,
	struct m_snd_tag **);
	static int cxgbe_snd_tag_modify(struct m_snd_tag *,
	union if_snd_tag_modify_params *);
	static int cxgbe_snd_tag_query(struct m_snd_tag *,
	union if_snd_tag_query_params *);
	static void cxgbe_snd_tag_free(struct m_snd_tag *);
	#endif

	MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");

	/*
	* Correct lock order when you need to acquire multiple locks is t4_list_lock,
	* then ADAPTER_LOCK, then t4_uld_list_lock.
	*/
	static struct sx t4_list_lock;
	SLIST_HEAD(, adapter) t4_list;
	#ifdef TCP_OFFLOAD
	static struct sx t4_uld_list_lock;
	SLIST_HEAD(, uld_info) t4_uld_list;
	#endif

	/*
	* Tunables. See tweak_tunables() too.
	*
	* Each tunable is set to a default value here if it's known at compile-time.
	* Otherwise it is set to -n as an indication to tweak_tunables() that it should
	* provide a reasonable default (upto n) when the driver is loaded.
	*
	* Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to
	* T5 are under hw.cxl.
	*/
	SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"cxgbe(4) parameters");
	SYSCTL_NODE(_hw, OID_AUTO, cxl, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"cxgbe(4) T5+ parameters");
	SYSCTL_NODE(_hw_cxgbe, OID_AUTO, toe, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"cxgbe(4) TOE parameters");

	/*
	* Number of queues for tx and rx, NIC and offload.
	*/
	#define NTXQ 16
	int t4_ntxq = -NTXQ;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq, CTLFLAG_RDTUN, &t4_ntxq, 0,
	"Number of TX queues per port");
	TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq); /* Old name, undocumented */

	#define NRXQ 8
	int t4_nrxq = -NRXQ;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq, CTLFLAG_RDTUN, &t4_nrxq, 0,
	"Number of RX queues per port");
	TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq); /* Old name, undocumented */

	#define NTXQ_VI 1
	static int t4_ntxq_vi = -NTXQ_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq_vi, CTLFLAG_RDTUN, &t4_ntxq_vi, 0,
	"Number of TX queues per VI");

	#define NRXQ_VI 1
	static int t4_nrxq_vi = -NRXQ_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq_vi, CTLFLAG_RDTUN, &t4_nrxq_vi, 0,
	"Number of RX queues per VI");

	static int t4_rsrv_noflowq = 0;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_noflowq, CTLFLAG_RDTUN, &t4_rsrv_noflowq,
	0, "Reserve TX queue 0 of each VI for non-flowid packets");

	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	#define NOFLDTXQ 8
	static int t4_nofldtxq = -NOFLDTXQ;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq, CTLFLAG_RDTUN, &t4_nofldtxq, 0,
	"Number of offload TX queues per port");

	#define NOFLDRXQ 2
	static int t4_nofldrxq = -NOFLDRXQ;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq, CTLFLAG_RDTUN, &t4_nofldrxq, 0,
	"Number of offload RX queues per port");

	#define NOFLDTXQ_VI 1
	static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq_vi, CTLFLAG_RDTUN, &t4_nofldtxq_vi, 0,
	"Number of offload TX queues per VI");

	#define NOFLDRXQ_VI 1
	static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq_vi, CTLFLAG_RDTUN, &t4_nofldrxq_vi, 0,
	"Number of offload RX queues per VI");

	#define TMR_IDX_OFLD 1
	int t4_tmr_idx_ofld = TMR_IDX_OFLD;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_ofld, CTLFLAG_RDTUN,
	&t4_tmr_idx_ofld, 0, "Holdoff timer index for offload queues");

	#define PKTC_IDX_OFLD (-1)
	int t4_pktc_idx_ofld = PKTC_IDX_OFLD;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_ofld, CTLFLAG_RDTUN,
	&t4_pktc_idx_ofld, 0, "holdoff packet counter index for offload queues");

	/* 0 means chip/fw default, non-zero number is value in microseconds */
	static u_long t4_toe_keepalive_idle = 0;
	SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_idle, CTLFLAG_RDTUN,
	&t4_toe_keepalive_idle, 0, "TOE keepalive idle timer (us)");

	/* 0 means chip/fw default, non-zero number is value in microseconds */
	static u_long t4_toe_keepalive_interval = 0;
	SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_interval, CTLFLAG_RDTUN,
	&t4_toe_keepalive_interval, 0, "TOE keepalive interval timer (us)");

	/* 0 means chip/fw default, non-zero number is # of keepalives before abort */
	static int t4_toe_keepalive_count = 0;
	SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, keepalive_count, CTLFLAG_RDTUN,
	&t4_toe_keepalive_count, 0, "Number of TOE keepalive probes before abort");

	/* 0 means chip/fw default, non-zero number is value in microseconds */
	static u_long t4_toe_rexmt_min = 0;
	SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_min, CTLFLAG_RDTUN,
	&t4_toe_rexmt_min, 0, "Minimum TOE retransmit interval (us)");

	/* 0 means chip/fw default, non-zero number is value in microseconds */
	static u_long t4_toe_rexmt_max = 0;
	SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_max, CTLFLAG_RDTUN,
	&t4_toe_rexmt_max, 0, "Maximum TOE retransmit interval (us)");

	/* 0 means chip/fw default, non-zero number is # of rexmt before abort */
	static int t4_toe_rexmt_count = 0;
	SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, rexmt_count, CTLFLAG_RDTUN,
	&t4_toe_rexmt_count, 0, "Number of TOE retransmissions before abort");

	/* -1 means chip/fw default, other values are raw backoff values to use */
	static int t4_toe_rexmt_backoff[16] = {
	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
	};
	SYSCTL_NODE(_hw_cxgbe_toe, OID_AUTO, rexmt_backoff,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"cxgbe(4) TOE retransmit backoff values");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 0, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[0], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 1, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[1], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 2, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[2], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 3, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[3], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 4, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[4], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 5, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[5], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 6, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[6], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 7, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[7], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 8, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[8], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 9, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[9], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 10, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[10], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 11, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[11], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 12, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[12], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 13, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[13], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 14, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[14], 0, "");
	SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 15, CTLFLAG_RDTUN,
	&t4_toe_rexmt_backoff[15], 0, "");
	#endif

	#ifdef DEV_NETMAP
	#define NN_MAIN_VI (1 << 0) /* Native netmap on the main VI */
	#define NN_EXTRA_VI (1 << 1) /* Native netmap on the extra VI(s) */
	static int t4_native_netmap = NN_EXTRA_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, native_netmap, CTLFLAG_RDTUN, &t4_native_netmap,
	0, "Native netmap support. bit 0 = main VI, bit 1 = extra VIs");

	#define NNMTXQ 8
	static int t4_nnmtxq = -NNMTXQ;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq, CTLFLAG_RDTUN, &t4_nnmtxq, 0,
	"Number of netmap TX queues");

	#define NNMRXQ 8
	static int t4_nnmrxq = -NNMRXQ;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq, CTLFLAG_RDTUN, &t4_nnmrxq, 0,
	"Number of netmap RX queues");

	#define NNMTXQ_VI 2
	static int t4_nnmtxq_vi = -NNMTXQ_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq_vi, CTLFLAG_RDTUN, &t4_nnmtxq_vi, 0,
	"Number of netmap TX queues per VI");

	#define NNMRXQ_VI 2
	static int t4_nnmrxq_vi = -NNMRXQ_VI;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq_vi, CTLFLAG_RDTUN, &t4_nnmrxq_vi, 0,
	"Number of netmap RX queues per VI");
	#endif

	/*
	* Holdoff parameters for ports.
	*/
	#define TMR_IDX 1
	int t4_tmr_idx = TMR_IDX;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx, CTLFLAG_RDTUN, &t4_tmr_idx,
	0, "Holdoff timer index");
	TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx); /* Old name */

	#define PKTC_IDX (-1)
	int t4_pktc_idx = PKTC_IDX;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx, CTLFLAG_RDTUN, &t4_pktc_idx,
	0, "Holdoff packet counter index");
	TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx); /* Old name */

	/*
	* Size (# of entries) of each tx and rx queue.
	*/
	unsigned int t4_qsize_txq = TX_EQ_QSIZE;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN, &t4_qsize_txq, 0,
	"Number of descriptors in each TX queue");

	unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN, &t4_qsize_rxq, 0,
	"Number of descriptors in each RX queue");

	/*
	* Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
	*/
	int t4_intr_types = INTR_MSIX \| INTR_MSI \| INTR_INTX;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &t4_intr_types,
	0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)");

	/*
	* Configuration file. All the _CF names here are special.
	*/
	#define DEFAULT_CF "default"
	#define BUILTIN_CF "built-in"
	#define FLASH_CF "flash"
	#define UWIRE_CF "uwire"
	#define FPGA_CF "fpga"
	static char t4_cfg_file[32] = DEFAULT_CF;
	SYSCTL_STRING(_hw_cxgbe, OID_AUTO, config_file, CTLFLAG_RDTUN, t4_cfg_file,
	sizeof(t4_cfg_file), "Firmware configuration file");

	/*
	* PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively).
	* rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
	* tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
	* mark or when signalled to do so, 0 to never emit PAUSE.
	* pause_autoneg = 1 means PAUSE will be negotiated if possible and the
	* negotiated settings will override rx_pause/tx_pause.
	* Otherwise rx_pause/tx_pause are applied forcibly.
	*/
	static int t4_pause_settings = PAUSE_RX \| PAUSE_TX \| PAUSE_AUTONEG;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, pause_settings, CTLFLAG_RDTUN,
	&t4_pause_settings, 0,
	"PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)");

	/*
	* Forward Error Correction settings (bit 0, 1 = RS, BASER respectively).
	* -1 to run with the firmware default. Same as FEC_AUTO (bit 5)
	* 0 to disable FEC.
	*/
	static int t4_fec = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, fec, CTLFLAG_RDTUN, &t4_fec, 0,
	"Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)");

	/*
	* Link autonegotiation.
	* -1 to run with the firmware default.
	* 0 to disable.
	* 1 to enable.
	*/
	static int t4_autoneg = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, autoneg, CTLFLAG_RDTUN, &t4_autoneg, 0,
	"Link autonegotiation");

	/*
	* Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
	* encouraged respectively). '-n' is the same as 'n' except the firmware
	* version used in the checks is read from the firmware bundled with the driver.
	*/
	static int t4_fw_install = 1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, fw_install, CTLFLAG_RDTUN, &t4_fw_install, 0,
	"Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)");

	/*
	* ASIC features that will be used. Disable the ones you don't want so that the
	* chip resources aren't wasted on features that will not be used.
	*/
	static int t4_nbmcaps_allowed = 0;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, nbmcaps_allowed, CTLFLAG_RDTUN,
	&t4_nbmcaps_allowed, 0, "Default NBM capabilities");

	static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, linkcaps_allowed, CTLFLAG_RDTUN,
	&t4_linkcaps_allowed, 0, "Default link capabilities");

	static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS \|
	FW_CAPS_CONFIG_SWITCH_EGRESS;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, switchcaps_allowed, CTLFLAG_RDTUN,
	&t4_switchcaps_allowed, 0, "Default switch capabilities");

	#ifdef RATELIMIT
	static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC \|
	FW_CAPS_CONFIG_NIC_HASHFILTER \| FW_CAPS_CONFIG_NIC_ETHOFLD;
	#else
	static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC \|
	FW_CAPS_CONFIG_NIC_HASHFILTER;
	#endif
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, niccaps_allowed, CTLFLAG_RDTUN,
	&t4_niccaps_allowed, 0, "Default NIC capabilities");

	static int t4_toecaps_allowed = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, toecaps_allowed, CTLFLAG_RDTUN,
	&t4_toecaps_allowed, 0, "Default TCP offload capabilities");

	static int t4_rdmacaps_allowed = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, rdmacaps_allowed, CTLFLAG_RDTUN,
	&t4_rdmacaps_allowed, 0, "Default RDMA capabilities");

	static int t4_cryptocaps_allowed = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, cryptocaps_allowed, CTLFLAG_RDTUN,
	&t4_cryptocaps_allowed, 0, "Default crypto capabilities");

	static int t4_iscsicaps_allowed = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, iscsicaps_allowed, CTLFLAG_RDTUN,
	&t4_iscsicaps_allowed, 0, "Default iSCSI capabilities");

	static int t4_fcoecaps_allowed = 0;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, fcoecaps_allowed, CTLFLAG_RDTUN,
	&t4_fcoecaps_allowed, 0, "Default FCoE capabilities");

	static int t5_write_combine = 0;
	SYSCTL_INT(_hw_cxl, OID_AUTO, write_combine, CTLFLAG_RDTUN, &t5_write_combine,
	0, "Use WC instead of UC for BAR2");

	static int t4_num_vis = 1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, num_vis, CTLFLAG_RDTUN, &t4_num_vis, 0,
	"Number of VIs per port");

	/*
	* PCIe Relaxed Ordering.
	* -1: driver should figure out a good value.
	* 0: disable RO.
	* 1: enable RO.
	* 2: leave RO alone.
	*/
	static int pcie_relaxed_ordering = -1;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, pcie_relaxed_ordering, CTLFLAG_RDTUN,
	&pcie_relaxed_ordering, 0,
	"PCIe Relaxed Ordering: 0 = disable, 1 = enable, 2 = leave alone");

	static int t4_panic_on_fatal_err = 0;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, panic_on_fatal_err, CTLFLAG_RDTUN,
	&t4_panic_on_fatal_err, 0, "panic on fatal errors");

	#ifdef TCP_OFFLOAD
	/*
	* TOE tunables.
	*/
	static int t4_cop_managed_offloading = 0;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN,
	&t4_cop_managed_offloading, 0,
	"COP (Connection Offload Policy) controls all TOE offload");
	#endif

	#ifdef KERN_TLS
	/*
	* This enables KERN_TLS for all adapters if set.
	*/
	static int t4_kern_tls = 0;
	SYSCTL_INT(_hw_cxgbe, OID_AUTO, kern_tls, CTLFLAG_RDTUN, &t4_kern_tls, 0,
	"Enable KERN_TLS mode for all supported adapters");

	SYSCTL_NODE(_hw_cxgbe, OID_AUTO, tls, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"cxgbe(4) KERN_TLS parameters");

	static int t4_tls_inline_keys = 0;
	SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, inline_keys, CTLFLAG_RDTUN,
	&t4_tls_inline_keys, 0,
	"Always pass TLS keys in work requests (1) or attempt to store TLS keys "
	"in card memory.");

	static int t4_tls_combo_wrs = 0;
	SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, combo_wrs, CTLFLAG_RDTUN, &t4_tls_combo_wrs,
	0, "Attempt to combine TCB field updates with TLS record work requests.");
	#endif

	/* Functions used by VIs to obtain unique MAC addresses for each VI. */
	static int vi_mac_funcs[] = {
	FW_VI_FUNC_ETH,
	FW_VI_FUNC_OFLD,
	FW_VI_FUNC_IWARP,
	FW_VI_FUNC_OPENISCSI,
	FW_VI_FUNC_OPENFCOE,
	FW_VI_FUNC_FOISCSI,
	FW_VI_FUNC_FOFCOE,
	};

	struct intrs_and_queues {
	uint16_t intr_type; /* INTx, MSI, or MSI-X */
	uint16_t num_vis; /* number of VIs for each port */
	uint16_t nirq; /* Total # of vectors */
	uint16_t ntxq; /* # of NIC txq's for each port */
	uint16_t nrxq; /* # of NIC rxq's for each port */
	uint16_t nofldtxq; /* # of TOE/ETHOFLD txq's for each port */
	uint16_t nofldrxq; /* # of TOE rxq's for each port */
	uint16_t nnmtxq; /* # of netmap txq's */
	uint16_t nnmrxq; /* # of netmap rxq's */

	/* The vcxgbe/vcxl interfaces use these and not the ones above. */
	uint16_t ntxq_vi; /* # of NIC txq's */
	uint16_t nrxq_vi; /* # of NIC rxq's */
	uint16_t nofldtxq_vi; /* # of TOE txq's */
	uint16_t nofldrxq_vi; /* # of TOE rxq's */
	uint16_t nnmtxq_vi; /* # of netmap txq's */
	uint16_t nnmrxq_vi; /* # of netmap rxq's */
	};

	static void setup_memwin(struct adapter *);
	static void position_memwin(struct adapter *, int, uint32_t);
	static int validate_mem_range(struct adapter *, uint32_t, uint32_t);
	static int fwmtype_to_hwmtype(int);
	static int validate_mt_off_len(struct adapter *, int, uint32_t, uint32_t,
	uint32_t *);
	static int fixup_devlog_params(struct adapter *);
	static int cfg_itype_and_nqueues(struct adapter , struct intrs_and_queues );
	static int contact_firmware(struct adapter *);
	static int partition_resources(struct adapter *);
	static int get_params__pre_init(struct adapter *);
	static int set_params__pre_init(struct adapter *);
	static int get_params__post_init(struct adapter *);
	static int set_params__post_init(struct adapter *);
	static void t4_set_desc(struct adapter *);
	static bool fixed_ifmedia(struct port_info *);
	static void build_medialist(struct port_info *);
	static void init_link_config(struct port_info *);
	static int fixup_link_config(struct port_info *);
	static int apply_link_config(struct port_info *);
	static int cxgbe_init_synchronized(struct vi_info *);
	static int cxgbe_uninit_synchronized(struct vi_info *);
	static void quiesce_txq(struct adapter , struct sge_txq );
	static void quiesce_wrq(struct adapter , struct sge_wrq );
	static void quiesce_iq(struct adapter , struct sge_iq );
	static void quiesce_fl(struct adapter , struct sge_fl );
	static int t4_alloc_irq(struct adapter , struct irq , int rid,
	driver_intr_t , void , char *);
	static int t4_free_irq(struct adapter , struct irq );
	static void t4_init_atid_table(struct adapter *);
	static void t4_free_atid_table(struct adapter *);
	static void get_regs(struct adapter , struct t4_regdump , uint8_t *);
	static void vi_refresh_stats(struct adapter , struct vi_info );
	static void cxgbe_refresh_stats(struct adapter , struct port_info );
	static void cxgbe_tick(void *);
	static void cxgbe_sysctls(struct port_info *);
	static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
	static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS);
	static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS);
	static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
	static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
	static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
	static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
	static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
	static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
	static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
	static int sysctl_fec(SYSCTL_HANDLER_ARGS);
	static int sysctl_module_fec(SYSCTL_HANDLER_ARGS);
	static int sysctl_autoneg(SYSCTL_HANDLER_ARGS);
	static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
	static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
	static int sysctl_vdd(SYSCTL_HANDLER_ARGS);
	static int sysctl_reset_sensor(SYSCTL_HANDLER_ARGS);
	static int sysctl_loadavg(SYSCTL_HANDLER_ARGS);
	static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
	static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
	static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
	static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
	static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
	static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
	static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
	static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
	static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
	static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
	static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
	static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS);
	static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
	static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_tids(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
	static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
	static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
	static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
	static int sysctl_cpus(SYSCTL_HANDLER_ARGS);
	#ifdef TCP_OFFLOAD
	static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS);
	static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS);
	static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS);
	static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS);
	#endif
	static int get_sge_context(struct adapter , struct t4_sge_context );
	static int load_fw(struct adapter , struct t4_data );
	static int load_cfg(struct adapter , struct t4_data );
	static int load_boot(struct adapter , struct t4_bootrom );
	static int load_bootcfg(struct adapter , struct t4_data );
	static int cudbg_dump(struct adapter , struct t4_cudbg_dump );
	static void free_offload_policy(struct t4_offload_policy *);
	static int set_offload_policy(struct adapter , struct t4_offload_policy );
	static int read_card_mem(struct adapter , int, struct t4_mem_range );
	static int read_i2c(struct adapter , struct t4_i2c_data );
	static int clear_stats(struct adapter *, u_int);
	#ifdef TCP_OFFLOAD
	static int toe_capability(struct vi_info *, int);
	static void t4_async_event(void *, int);
	#endif
	static int mod_event(module_t, int, void *);
	static int notify_siblings(device_t, int);

	struct {
	uint16_t device;
	char *desc;
	} t4_pciids[] = {
	{0xa000, "Chelsio Terminator 4 FPGA"},
	{0x4400, "Chelsio T440-dbg"},
	{0x4401, "Chelsio T420-CR"},
	{0x4402, "Chelsio T422-CR"},
	{0x4403, "Chelsio T440-CR"},
	{0x4404, "Chelsio T420-BCH"},
	{0x4405, "Chelsio T440-BCH"},
	{0x4406, "Chelsio T440-CH"},
	{0x4407, "Chelsio T420-SO"},
	{0x4408, "Chelsio T420-CX"},
	{0x4409, "Chelsio T420-BT"},
	{0x440a, "Chelsio T404-BT"},
	{0x440e, "Chelsio T440-LP-CR"},
	}, t5_pciids[] = {
	{0xb000, "Chelsio Terminator 5 FPGA"},
	{0x5400, "Chelsio T580-dbg"},
	{0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
	{0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
	{0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
	{0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
	{0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
	{0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
	{0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
	{0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
	{0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
	{0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
	{0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
	{0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
	{0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
	{0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */
	{0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */
	{0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */
	{0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */

	/* Custom */
	{0x5483, "Custom T540-CR"},
	{0x5484, "Custom T540-BT"},
	}, t6_pciids[] = {
	{0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */
	{0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */
	{0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */
	{0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */
	{0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */
	{0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */
	{0x6405, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */
	{0x6406, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */
	{0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */
	{0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */
	{0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */
	{0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */
	{0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */
	{0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */
	{0x6414, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */
	{0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */

	/* Custom */
	{0x6480, "Custom T6225-CR"},
	{0x6481, "Custom T62100-CR"},
	{0x6482, "Custom T6225-CR"},
	{0x6483, "Custom T62100-CR"},
	{0x6484, "Custom T64100-CR"},
	{0x6485, "Custom T6240-SO"},
	{0x6486, "Custom T6225-SO-CR"},
	{0x6487, "Custom T6225-CR"},
	};

	#ifdef TCP_OFFLOAD
	/*
	* service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should
	* be exactly the same for both rxq and ofld_rxq.
	*/
	CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
	CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
	#endif
	CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);

	static int
	t4_probe(device_t dev)
	{
	int i;
	uint16_t v = pci_get_vendor(dev);
	uint16_t d = pci_get_device(dev);
	uint8_t f = pci_get_function(dev);

	if (v != PCI_VENDOR_ID_CHELSIO)
	return (ENXIO);

	/* Attach only to PF0 of the FPGA */
	if (d == 0xa000 && f != 0)
	return (ENXIO);

	for (i = 0; i < nitems(t4_pciids); i++) {
	if (d == t4_pciids[i].device) {
	device_set_desc(dev, t4_pciids[i].desc);
	return (BUS_PROBE_DEFAULT);
	}
	}

	return (ENXIO);
	}

	static int
	t5_probe(device_t dev)
	{
	int i;
	uint16_t v = pci_get_vendor(dev);
	uint16_t d = pci_get_device(dev);
	uint8_t f = pci_get_function(dev);

	if (v != PCI_VENDOR_ID_CHELSIO)
	return (ENXIO);

	/* Attach only to PF0 of the FPGA */
	if (d == 0xb000 && f != 0)
	return (ENXIO);

	for (i = 0; i < nitems(t5_pciids); i++) {
	if (d == t5_pciids[i].device) {
	device_set_desc(dev, t5_pciids[i].desc);
	return (BUS_PROBE_DEFAULT);
	}
	}

	return (ENXIO);
	}

	static int
	t6_probe(device_t dev)
	{
	int i;
	uint16_t v = pci_get_vendor(dev);
	uint16_t d = pci_get_device(dev);

	if (v != PCI_VENDOR_ID_CHELSIO)
	return (ENXIO);

	for (i = 0; i < nitems(t6_pciids); i++) {
	if (d == t6_pciids[i].device) {
	device_set_desc(dev, t6_pciids[i].desc);
	return (BUS_PROBE_DEFAULT);
	}
	}

	return (ENXIO);
	}

	static void
	t5_attribute_workaround(device_t dev)
	{
	device_t root_port;
	uint32_t v;

	/*
	* The T5 chips do not properly echo the No Snoop and Relaxed
	* Ordering attributes when replying to a TLP from a Root
	* Port. As a workaround, find the parent Root Port and
	* disable No Snoop and Relaxed Ordering. Note that this
	* affects all devices under this root port.
	*/
	root_port = pci_find_pcie_root_port(dev);
	if (root_port == NULL) {
	device_printf(dev, "Unable to find parent root port\n");
	return;
	}

	v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
	PCIEM_CTL_RELAXED_ORD_ENABLE \| PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
	if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE \| PCIEM_CTL_NOSNOOP_ENABLE)) !=
	0)
	device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
	device_get_nameunit(root_port));
	}

	static const struct devnames devnames[] = {
	{
	.nexus_name = "t4nex",
	.ifnet_name = "cxgbe",
	.vi_ifnet_name = "vcxgbe",
	.pf03_drv_name = "t4iov",
	.vf_nexus_name = "t4vf",
	.vf_ifnet_name = "cxgbev"
	}, {
	.nexus_name = "t5nex",
	.ifnet_name = "cxl",
	.vi_ifnet_name = "vcxl",
	.pf03_drv_name = "t5iov",
	.vf_nexus_name = "t5vf",
	.vf_ifnet_name = "cxlv"
	}, {
	.nexus_name = "t6nex",
	.ifnet_name = "cc",
	.vi_ifnet_name = "vcc",
	.pf03_drv_name = "t6iov",
	.vf_nexus_name = "t6vf",
	.vf_ifnet_name = "ccv"
	}
	};

	void
	t4_init_devnames(struct adapter *sc)
	{
	int id;

	id = chip_id(sc);
	if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
	sc->names = &devnames[id - CHELSIO_T4];
	else {
	device_printf(sc->dev, "chip id %d is not supported.\n", id);
	sc->names = NULL;
	}
	}

	static int
	t4_ifnet_unit(struct adapter sc, struct port_info pi)
	{
	const char parent, name;
	long value;
	int line, unit;

	line = 0;
	parent = device_get_nameunit(sc->dev);
	name = sc->names->ifnet_name;
	while (resource_find_dev(&line, name, &unit, "at", parent) == 0) {
	if (resource_long_value(name, unit, "port", &value) == 0 &&
	value == pi->port_id)
	return (unit);
	}
	return (-1);
	}

	static int
	t4_attach(device_t dev)
	{
	struct adapter *sc;
	int rc = 0, i, j, rqidx, tqidx, nports;
	struct make_dev_args mda;
	struct intrs_and_queues iaq;
	struct sge *s;
	uint32_t *buf;
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	int ofld_tqidx;
	#endif
	#ifdef TCP_OFFLOAD
	int ofld_rqidx;
	#endif
	#ifdef DEV_NETMAP
	int nm_rqidx, nm_tqidx;
	#endif
	int num_vis;

	sc = device_get_softc(dev);
	sc->dev = dev;
	TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);

	if ((pci_get_device(dev) & 0xff00) == 0x5400)
	t5_attribute_workaround(dev);
	pci_enable_busmaster(dev);
	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
	uint32_t v;

	pci_set_max_read_req(dev, 4096);
	v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
	sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
	if (pcie_relaxed_ordering == 0 &&
	(v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) {
	v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE;
	pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
	} else if (pcie_relaxed_ordering == 1 &&
	(v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) {
	v \|= PCIEM_CTL_RELAXED_ORD_ENABLE;
	pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
	}
	}

	sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
	sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
	sc->traceq = -1;
	mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
	snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
	device_get_nameunit(dev));

	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
	device_get_nameunit(dev));
	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
	t4_add_adapter(sc);

	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
	TAILQ_INIT(&sc->sfl);
	callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);

	mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);

	sc->policy = NULL;
	rw_init(&sc->policy_lock, "connection offload policy");

	callout_init(&sc->ktls_tick, 1);

	#ifdef TCP_OFFLOAD
	TASK_INIT(&sc->async_event_task, 0, t4_async_event, sc);
	#endif

	rc = t4_map_bars_0_and_4(sc);
	if (rc != 0)
	goto done; /* error message displayed already */

	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));

	/* Prepare the adapter for operation. */
	buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO \| M_WAITOK);
	rc = -t4_prep_adapter(sc, buf);
	free(buf, M_CXGBE);
	if (rc != 0) {
	device_printf(dev, "failed to prepare adapter: %d.\n", rc);
	goto done;
	}

	/*
	* This is the real PF# to which we're attaching. Works from within PCI
	* passthrough environments too, where pci_get_function() could return a
	* different PF# depending on the passthrough configuration. We need to
	* use the real PF# in all our communication with the firmware.
	*/
	j = t4_read_reg(sc, A_PL_WHOAMI);
	sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
	sc->mbox = sc->pf;

	t4_init_devnames(sc);
	if (sc->names == NULL) {
	rc = ENOTSUP;
	goto done; /* error message displayed already */
	}

	/*
	* Do this really early, with the memory windows set up even before the
	* character device. The userland tool's register i/o and mem read
	* will work even in "recovery mode".
	*/
	setup_memwin(sc);
	if (t4_init_devlog_params(sc, 0) == 0)
	fixup_devlog_params(sc);
	make_dev_args_init(&mda);
	mda.mda_devsw = &t4_cdevsw;
	mda.mda_uid = UID_ROOT;
	mda.mda_gid = GID_WHEEL;
	mda.mda_mode = 0600;
	mda.mda_si_drv1 = sc;
	rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
	if (rc != 0)
	device_printf(dev, "failed to create nexus char device: %d.\n",
	rc);

	/* Go no further if recovery mode has been requested. */
	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
	device_printf(dev, "recovery mode.\n");
	goto done;
	}

	#if defined(__i386__)
	if ((cpu_feature & CPUID_CX8) == 0) {
	device_printf(dev, "64 bit atomics not available.\n");
	rc = ENOTSUP;
	goto done;
	}
	#endif

	/* Contact the firmware and try to become the master driver. */
	rc = contact_firmware(sc);
	if (rc != 0)
	goto done; /* error message displayed already */
	MPASS(sc->flags & FW_OK);

	rc = get_params__pre_init(sc);
	if (rc != 0)
	goto done; /* error message displayed already */

	if (sc->flags & MASTER_PF) {
	rc = partition_resources(sc);
	if (rc != 0)
	goto done; /* error message displayed already */
	t4_intr_clear(sc);
	}

	rc = get_params__post_init(sc);
	if (rc != 0)
	goto done; /* error message displayed already */

	rc = set_params__post_init(sc);
	if (rc != 0)
	goto done; /* error message displayed already */

	rc = t4_map_bar_2(sc);
	if (rc != 0)
	goto done; /* error message displayed already */

	rc = t4_create_dma_tag(sc);
	if (rc != 0)
	goto done; /* error message displayed already */

	/*
	* First pass over all the ports - allocate VIs and initialize some
	* basic parameters like mac address, port type, etc.
	*/
	for_each_port(sc, i) {
	struct port_info *pi;

	pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO \| M_WAITOK);
	sc->port[i] = pi;

	/* These must be set before t4_port_init */
	pi->adapter = sc;
	pi->port_id = i;
	/*
	* XXX: vi[0] is special so we can't delay this allocation until
	* pi->nvi's final value is known.
	*/
	pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE,
	M_ZERO \| M_WAITOK);

	/*
	* Allocate the "main" VI and initialize parameters
	* like mac addr.
	*/
	rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
	if (rc != 0) {
	device_printf(dev, "unable to initialize port %d: %d\n",
	i, rc);
	free(pi->vi, M_CXGBE);
	free(pi, M_CXGBE);
	sc->port[i] = NULL;
	goto done;
	}

	snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
	device_get_nameunit(dev), i);
	mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
	sc->chan_map[pi->tx_chan] = i;

	/* All VIs on this port share this media. */
	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
	cxgbe_media_status);

	PORT_LOCK(pi);
	init_link_config(pi);
	fixup_link_config(pi);
	build_medialist(pi);
	if (fixed_ifmedia(pi))
	pi->flags \|= FIXED_IFMEDIA;
	PORT_UNLOCK(pi);

	pi->dev = device_add_child(dev, sc->names->ifnet_name,
	t4_ifnet_unit(sc, pi));
	if (pi->dev == NULL) {
	device_printf(dev,
	"failed to add device for port %d.\n", i);
	rc = ENXIO;
	goto done;
	}
	pi->vi[0].dev = pi->dev;
	device_set_softc(pi->dev, pi);
	}

	/*
	* Interrupt type, # of interrupts, # of rx/tx queues, etc.
	*/
	nports = sc->params.nports;
	rc = cfg_itype_and_nqueues(sc, &iaq);
	if (rc != 0)
	goto done; /* error message displayed already */

	num_vis = iaq.num_vis;
	sc->intr_type = iaq.intr_type;
	sc->intr_count = iaq.nirq;

	s = &sc->sge;
	s->nrxq = nports * iaq.nrxq;
	s->ntxq = nports * iaq.ntxq;
	if (num_vis > 1) {
	s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi;
	s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi;
	}
	s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
	s->neq += nports; /* ctrl queues: 1 per port */
	s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	if (is_offload(sc) \|\| is_ethoffload(sc)) {
	s->nofldtxq = nports * iaq.nofldtxq;
	if (num_vis > 1)
	s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi;
	s->neq += s->nofldtxq;

	s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
	M_CXGBE, M_ZERO \| M_WAITOK);
	}
	#endif
	#ifdef TCP_OFFLOAD
	if (is_offload(sc)) {
	s->nofldrxq = nports * iaq.nofldrxq;
	if (num_vis > 1)
	s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi;
	s->neq += s->nofldrxq; /* free list */
	s->niq += s->nofldrxq;

	s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
	M_CXGBE, M_ZERO \| M_WAITOK);
	}
	#endif
	#ifdef DEV_NETMAP
	s->nnmrxq = 0;
	s->nnmtxq = 0;
	if (t4_native_netmap & NN_MAIN_VI) {
	s->nnmrxq += nports * iaq.nnmrxq;
	s->nnmtxq += nports * iaq.nnmtxq;
	}
	if (num_vis > 1 && t4_native_netmap & NN_EXTRA_VI) {
	s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi;
	s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi;
	}
	s->neq += s->nnmtxq + s->nnmrxq;
	s->niq += s->nnmrxq;

	s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
	M_CXGBE, M_ZERO \| M_WAITOK);
	s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
	M_CXGBE, M_ZERO \| M_WAITOK);
	#endif

	s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE,
	M_ZERO \| M_WAITOK);
	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
	M_ZERO \| M_WAITOK);
	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
	M_ZERO \| M_WAITOK);
	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
	M_ZERO \| M_WAITOK);
	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
	M_ZERO \| M_WAITOK);

	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
	M_ZERO \| M_WAITOK);

	t4_init_l2t(sc, M_WAITOK);
	t4_init_smt(sc, M_WAITOK);
	t4_init_tx_sched(sc);
	t4_init_atid_table(sc);
	#ifdef RATELIMIT
	t4_init_etid_table(sc);
	#endif
	#ifdef INET6
	t4_init_clip_table(sc);
	#endif
	if (sc->vres.key.size != 0)
	sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
	sc->vres.key.size, 32, 0, M_FIRSTFIT \| M_WAITOK);

	/*
	* Second pass over the ports. This time we know the number of rx and
	* tx queues that each port should get.
	*/
	rqidx = tqidx = 0;
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	ofld_tqidx = 0;
	#endif
	#ifdef TCP_OFFLOAD
	ofld_rqidx = 0;
	#endif
	#ifdef DEV_NETMAP
	nm_rqidx = nm_tqidx = 0;
	#endif
	for_each_port(sc, i) {
	struct port_info *pi = sc->port[i];
	struct vi_info *vi;

	if (pi == NULL)
	continue;

	pi->nvi = num_vis;
	for_each_vi(pi, j, vi) {
	vi->pi = pi;
	vi->adapter = sc;
	vi->qsize_rxq = t4_qsize_rxq;
	vi->qsize_txq = t4_qsize_txq;

	vi->first_rxq = rqidx;
	vi->first_txq = tqidx;
	vi->tmr_idx = t4_tmr_idx;
	vi->pktc_idx = t4_pktc_idx;
	vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi;
	vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi;

	rqidx += vi->nrxq;
	tqidx += vi->ntxq;

	if (j == 0 && vi->ntxq > 1)
	vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0;
	else
	vi->rsrv_noflowq = 0;

	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	vi->first_ofld_txq = ofld_tqidx;
	vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi;
	ofld_tqidx += vi->nofldtxq;
	#endif
	#ifdef TCP_OFFLOAD
	vi->ofld_tmr_idx = t4_tmr_idx_ofld;
	vi->ofld_pktc_idx = t4_pktc_idx_ofld;
	vi->first_ofld_rxq = ofld_rqidx;
	vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi;

	ofld_rqidx += vi->nofldrxq;
	#endif
	#ifdef DEV_NETMAP
	vi->first_nm_rxq = nm_rqidx;
	vi->first_nm_txq = nm_tqidx;
	if (j == 0) {
	vi->nnmrxq = iaq.nnmrxq;
	vi->nnmtxq = iaq.nnmtxq;
	} else {
	vi->nnmrxq = iaq.nnmrxq_vi;
	vi->nnmtxq = iaq.nnmtxq_vi;
	}
	nm_rqidx += vi->nnmrxq;
	nm_tqidx += vi->nnmtxq;
	#endif
	}
	}

	rc = t4_setup_intr_handlers(sc);
	if (rc != 0) {
	device_printf(dev,
	"failed to setup interrupt handlers: %d\n", rc);
	goto done;
	}

	rc = bus_generic_probe(dev);
	if (rc != 0) {
	device_printf(dev, "failed to probe child drivers: %d\n", rc);
	goto done;
	}

	/*
	* Ensure thread-safe mailbox access (in debug builds).
	*
	* So far this was the only thread accessing the mailbox but various
	* ifnets and sysctls are about to be created and their handlers/ioctls
	* will access the mailbox from different threads.
	*/
	sc->flags \|= CHK_MBOX_ACCESS;

	rc = bus_generic_attach(dev);
	if (rc != 0) {
	device_printf(dev,
	"failed to attach all child ports: %d\n", rc);
	goto done;
	}

	device_printf(dev,
	"PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
	sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
	sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
	(sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
	sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);

	t4_set_desc(sc);

	notify_siblings(dev, 0);

	done:
	if (rc != 0 && sc->cdev) {
	/* cdev was created and so cxgbetool works; recover that way. */
	device_printf(dev,
	"error during attach, adapter is now in recovery mode.\n");
	rc = 0;
	}

	if (rc != 0)
	t4_detach_common(dev);
	else
	t4_sysctls(sc);

	return (rc);
	}

	static int
	t4_child_location_str(device_t bus, device_t dev, char *buf, size_t buflen)
	{
	struct adapter *sc;
	struct port_info *pi;
	int i;

	sc = device_get_softc(bus);
	buf[0] = '\0';
	for_each_port(sc, i) {
	pi = sc->port[i];
	if (pi != NULL && pi->dev == dev) {
	snprintf(buf, buflen, "port=%d", pi->port_id);
	break;
	}
	}
	return (0);
	}

	static int
	t4_ready(device_t dev)
	{
	struct adapter *sc;

	sc = device_get_softc(dev);
	if (sc->flags & FW_OK)
	return (0);
	return (ENXIO);
	}

	static int
	t4_read_port_device(device_t dev, int port, device_t *child)
	{
	struct adapter *sc;
	struct port_info *pi;

	sc = device_get_softc(dev);
	if (port < 0 \|\| port >= MAX_NPORTS)
	return (EINVAL);
	pi = sc->port[port];
	if (pi == NULL \|\| pi->dev == NULL)
	return (ENXIO);
	*child = pi->dev;
	return (0);
	}

	static int
	notify_siblings(device_t dev, int detaching)
	{
	device_t sibling;
	int error, i;

	error = 0;
	for (i = 0; i < PCI_FUNCMAX; i++) {
	if (i == pci_get_function(dev))
	continue;
	sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
	pci_get_slot(dev), i);
	if (sibling == NULL \|\| !device_is_attached(sibling))
	continue;
	if (detaching)
	error = T4_DETACH_CHILD(sibling);
	else
	(void)T4_ATTACH_CHILD(sibling);
	if (error)
	break;
	}
	return (error);
	}

	/*
	* Idempotent
	*/
	static int
	t4_detach(device_t dev)
	{
	struct adapter *sc;
	int rc;

	sc = device_get_softc(dev);

	rc = notify_siblings(dev, 1);
	if (rc) {
	device_printf(dev,
	"failed to detach sibling devices: %d\n", rc);
	return (rc);
	}

	return (t4_detach_common(dev));
	}

	int
	t4_detach_common(device_t dev)
	{
	struct adapter *sc;
	struct port_info *pi;
	int i, rc;

	sc = device_get_softc(dev);

	if (sc->cdev) {
	destroy_dev(sc->cdev);
	sc->cdev = NULL;
	}

	sx_xlock(&t4_list_lock);
	SLIST_REMOVE(&t4_list, sc, adapter, link);
	sx_xunlock(&t4_list_lock);

	sc->flags &= ~CHK_MBOX_ACCESS;
	if (sc->flags & FULL_INIT_DONE) {
	if (!(sc->flags & IS_VF))
	t4_intr_disable(sc);
	}

	if (device_is_attached(dev)) {
	rc = bus_generic_detach(dev);
	if (rc) {
	device_printf(dev,
	"failed to detach child devices: %d\n", rc);
	return (rc);
	}
	}

	#ifdef TCP_OFFLOAD
	taskqueue_drain(taskqueue_thread, &sc->async_event_task);
	#endif

	for (i = 0; i < sc->intr_count; i++)
	t4_free_irq(sc, &sc->irq[i]);

	if ((sc->flags & (IS_VF \| FW_OK)) == FW_OK)
	t4_free_tx_sched(sc);

	for (i = 0; i < MAX_NPORTS; i++) {
	pi = sc->port[i];
	if (pi) {
	t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
	if (pi->dev)
	device_delete_child(dev, pi->dev);

	mtx_destroy(&pi->pi_lock);
	free(pi->vi, M_CXGBE);
	free(pi, M_CXGBE);
	}
	}

	device_delete_children(dev);

	if (sc->flags & FULL_INIT_DONE)
	adapter_full_uninit(sc);

	if ((sc->flags & (IS_VF \| FW_OK)) == FW_OK)
	t4_fw_bye(sc, sc->mbox);

	if (sc->intr_type == INTR_MSI \|\| sc->intr_type == INTR_MSIX)
	pci_release_msi(dev);

	if (sc->regs_res)
	bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
	sc->regs_res);

	if (sc->udbs_res)
	bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
	sc->udbs_res);

	if (sc->msix_res)
	bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
	sc->msix_res);

	if (sc->l2t)
	t4_free_l2t(sc->l2t);
	if (sc->smt)
	t4_free_smt(sc->smt);
	t4_free_atid_table(sc);
	#ifdef RATELIMIT
	t4_free_etid_table(sc);
	#endif
	if (sc->key_map)
	vmem_destroy(sc->key_map);
	#ifdef INET6
	t4_destroy_clip_table(sc);
	#endif

	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	free(sc->sge.ofld_txq, M_CXGBE);
	#endif
	#ifdef TCP_OFFLOAD
	free(sc->sge.ofld_rxq, M_CXGBE);
	#endif
	#ifdef DEV_NETMAP
	free(sc->sge.nm_rxq, M_CXGBE);
	free(sc->sge.nm_txq, M_CXGBE);
	#endif
	free(sc->irq, M_CXGBE);
	free(sc->sge.rxq, M_CXGBE);
	free(sc->sge.txq, M_CXGBE);
	free(sc->sge.ctrlq, M_CXGBE);
	free(sc->sge.iqmap, M_CXGBE);
	free(sc->sge.eqmap, M_CXGBE);
	free(sc->tids.ftid_tab, M_CXGBE);
	free(sc->tids.hpftid_tab, M_CXGBE);
	free_hftid_hash(&sc->tids);
	free(sc->tids.tid_tab, M_CXGBE);
	free(sc->tt.tls_rx_ports, M_CXGBE);
	t4_destroy_dma_tag(sc);

	callout_drain(&sc->ktls_tick);
	callout_drain(&sc->sfl_callout);
	if (mtx_initialized(&sc->tids.ftid_lock)) {
	mtx_destroy(&sc->tids.ftid_lock);
	cv_destroy(&sc->tids.ftid_cv);
	}
	if (mtx_initialized(&sc->tids.atid_lock))
	mtx_destroy(&sc->tids.atid_lock);
	if (mtx_initialized(&sc->ifp_lock))
	mtx_destroy(&sc->ifp_lock);

	if (rw_initialized(&sc->policy_lock)) {
	rw_destroy(&sc->policy_lock);
	#ifdef TCP_OFFLOAD
	if (sc->policy != NULL)
	free_offload_policy(sc->policy);
	#endif
	}

	for (i = 0; i < NUM_MEMWIN; i++) {
	struct memwin *mw = &sc->memwin[i];

	if (rw_initialized(&mw->mw_lock))
	rw_destroy(&mw->mw_lock);
	}

	mtx_destroy(&sc->sfl_lock);
	mtx_destroy(&sc->reg_lock);
	mtx_destroy(&sc->sc_lock);

	bzero(sc, sizeof(*sc));

	return (0);
	}

	static int
	cxgbe_probe(device_t dev)
	{
	char buf[128];
	struct port_info *pi = device_get_softc(dev);

	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
	device_set_desc_copy(dev, buf);

	return (BUS_PROBE_DEFAULT);
	}

	#define T4_CAP (IFCAP_VLAN_HWTAGGING \| IFCAP_VLAN_MTU \| IFCAP_HWCSUM \| \
	IFCAP_VLAN_HWCSUM \| IFCAP_TSO \| IFCAP_JUMBO_MTU \| IFCAP_LRO \| \
	IFCAP_VLAN_HWTSO \| IFCAP_LINKSTATE \| IFCAP_HWCSUM_IPV6 \| IFCAP_HWSTATS \| \
	IFCAP_HWRXTSTMP \| IFCAP_NOMAP)
	#define T4_CAP_ENABLE (T4_CAP)

	static int
	cxgbe_vi_attach(device_t dev, struct vi_info *vi)
	{
	struct ifnet *ifp;
	struct sbuf *sb;
	struct pfil_head_args pa;

	vi->xact_addr_filt = -1;
	callout_init(&vi->tick, 1);

	/* Allocate an ifnet and set it up */
	ifp = if_alloc_dev(IFT_ETHER, dev);
	if (ifp == NULL) {
	device_printf(dev, "Cannot allocate ifnet\n");
	return (ENOMEM);
	}
	vi->ifp = ifp;
	ifp->if_softc = vi;

	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
	ifp->if_flags = IFF_BROADCAST \| IFF_SIMPLEX \| IFF_MULTICAST;

	ifp->if_init = cxgbe_init;
	ifp->if_ioctl = cxgbe_ioctl;
	ifp->if_transmit = cxgbe_transmit;
	ifp->if_qflush = cxgbe_qflush;
	ifp->if_get_counter = cxgbe_get_counter;
	#if defined(KERN_TLS) \|\| defined(RATELIMIT)
	ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc;
	ifp->if_snd_tag_modify = cxgbe_snd_tag_modify;
	ifp->if_snd_tag_query = cxgbe_snd_tag_query;
	ifp->if_snd_tag_free = cxgbe_snd_tag_free;
	#endif
	#ifdef RATELIMIT
	ifp->if_ratelimit_query = cxgbe_ratelimit_query;
	#endif

	ifp->if_capabilities = T4_CAP;
	ifp->if_capenable = T4_CAP_ENABLE;
	#ifdef TCP_OFFLOAD
	if (vi->nofldrxq != 0 && (vi->adapter->flags & KERN_TLS_OK) == 0)
	ifp->if_capabilities \|= IFCAP_TOE;
	#endif
	#ifdef RATELIMIT
	if (is_ethoffload(vi->adapter) && vi->nofldtxq != 0) {
	ifp->if_capabilities \|= IFCAP_TXRTLMT;
	ifp->if_capenable \|= IFCAP_TXRTLMT;
	}
	#endif
	ifp->if_hwassist = CSUM_TCP \| CSUM_UDP \| CSUM_IP \| CSUM_TSO \|
	CSUM_UDP_IPV6 \| CSUM_TCP_IPV6;

	ifp->if_hw_tsomax = IP_MAXPACKET;
	ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_TSO;
	#ifdef RATELIMIT
	if (is_ethoffload(vi->adapter) && vi->nofldtxq != 0)
	ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_EO_TSO;
	#endif
	ifp->if_hw_tsomaxsegsize = 65536;
	#ifdef KERN_TLS
	if (vi->adapter->flags & KERN_TLS_OK) {
	ifp->if_capabilities \|= IFCAP_TXTLS;
	ifp->if_capenable \|= IFCAP_TXTLS;
	}
	#endif

	ether_ifattach(ifp, vi->hw_addr);
	#ifdef DEV_NETMAP
	if (vi->nnmrxq != 0)
	cxgbe_nm_attach(vi);
	#endif
	sb = sbuf_new_auto();
	sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	switch (ifp->if_capabilities & (IFCAP_TOE \| IFCAP_TXRTLMT)) {
	case IFCAP_TOE:
	sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq);
	break;
	case IFCAP_TOE \| IFCAP_TXRTLMT:
	sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq);
	break;
	case IFCAP_TXRTLMT:
	sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq);
	break;
	}
	#endif
	#ifdef TCP_OFFLOAD
	if (ifp->if_capabilities & IFCAP_TOE)
	sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq);
	#endif
	#ifdef DEV_NETMAP
	if (ifp->if_capabilities & IFCAP_NETMAP)
	sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
	vi->nnmtxq, vi->nnmrxq);
	#endif
	sbuf_finish(sb);
	device_printf(dev, "%s\n", sbuf_data(sb));
	sbuf_delete(sb);

	vi_sysctls(vi);

	pa.pa_version = PFIL_VERSION;
	pa.pa_flags = PFIL_IN;
	pa.pa_type = PFIL_TYPE_ETHERNET;
	pa.pa_headname = ifp->if_xname;
	vi->pfil = pfil_head_register(&pa);

	return (0);
	}

	static int
	cxgbe_attach(device_t dev)
	{
	struct port_info *pi = device_get_softc(dev);
	struct adapter *sc = pi->adapter;
	struct vi_info *vi;
	int i, rc;

	callout_init_mtx(&pi->tick, &pi->pi_lock, 0);

	rc = cxgbe_vi_attach(dev, &pi->vi[0]);
	if (rc)
	return (rc);

	for_each_vi(pi, i, vi) {
	if (i == 0)
	continue;
	vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1);
	if (vi->dev == NULL) {
	device_printf(dev, "failed to add VI %d\n", i);
	continue;
	}
	device_set_softc(vi->dev, vi);
	}

	cxgbe_sysctls(pi);

	bus_generic_attach(dev);

	return (0);
	}

	static void
	cxgbe_vi_detach(struct vi_info *vi)
	{
	struct ifnet *ifp = vi->ifp;

	if (vi->pfil != NULL) {
	pfil_head_unregister(vi->pfil);
	vi->pfil = NULL;
	}

	ether_ifdetach(ifp);

	/* Let detach proceed even if these fail. */
	#ifdef DEV_NETMAP
	if (ifp->if_capabilities & IFCAP_NETMAP)
	cxgbe_nm_detach(vi);
	#endif
	cxgbe_uninit_synchronized(vi);
	callout_drain(&vi->tick);
	vi_full_uninit(vi);

	if_free(vi->ifp);
	vi->ifp = NULL;
	}

	static int
	cxgbe_detach(device_t dev)
	{
	struct port_info *pi = device_get_softc(dev);
	struct adapter *sc = pi->adapter;
	int rc;

	/* Detach the extra VIs first. */
	rc = bus_generic_detach(dev);
	if (rc)
	return (rc);
	device_delete_children(dev);

	doom_vi(sc, &pi->vi[0]);

	if (pi->flags & HAS_TRACEQ) {
	sc->traceq = -1; /* cloner should not create ifnet */
	t4_tracer_port_detach(sc);
	}

	cxgbe_vi_detach(&pi->vi[0]);
	callout_drain(&pi->tick);
	ifmedia_removeall(&pi->media);

	end_synchronized_op(sc, 0);

	return (0);
	}

	static void
	cxgbe_init(void *arg)
	{
	struct vi_info *vi = arg;
	struct adapter *sc = vi->adapter;

	if (begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4init") != 0)
	return;
	cxgbe_init_synchronized(vi);
	end_synchronized_op(sc, 0);
	}

	static int
	cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
	{
	int rc = 0, mtu, flags;
	struct vi_info *vi = ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	struct ifreq ifr = (struct ifreq )data;
	uint32_t mask;

	switch (cmd) {
	case SIOCSIFMTU:
	mtu = ifr->ifr_mtu;
	if (mtu < ETHERMIN \|\| mtu > MAX_MTU)
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4mtu");
	if (rc)
	return (rc);
	ifp->if_mtu = mtu;
	if (vi->flags & VI_INIT_DONE) {
	t4_update_fl_bufsize(ifp);
	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
	rc = update_mac_settings(ifp, XGMAC_MTU);
	}
	end_synchronized_op(sc, 0);
	break;

	case SIOCSIFFLAGS:
	rc = begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4flg");
	if (rc)
	return (rc);

	if (ifp->if_flags & IFF_UP) {
	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
	flags = vi->if_flags;
	if ((ifp->if_flags ^ flags) &
	(IFF_PROMISC \| IFF_ALLMULTI)) {
	rc = update_mac_settings(ifp,
	XGMAC_PROMISC \| XGMAC_ALLMULTI);
	}
	} else {
	rc = cxgbe_init_synchronized(vi);
	}
	vi->if_flags = ifp->if_flags;
	} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
	rc = cxgbe_uninit_synchronized(vi);
	}
	end_synchronized_op(sc, 0);
	break;

	case SIOCADDMULTI:
	case SIOCDELMULTI:
	rc = begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4multi");
	if (rc)
	return (rc);
	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
	rc = update_mac_settings(ifp, XGMAC_MCADDRS);
	end_synchronized_op(sc, 0);
	break;

	case SIOCSIFCAP:
	rc = begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4cap");
	if (rc)
	return (rc);

	mask = ifr->ifr_reqcap ^ ifp->if_capenable;
	if (mask & IFCAP_TXCSUM) {
	ifp->if_capenable ^= IFCAP_TXCSUM;
	ifp->if_hwassist ^= (CSUM_TCP \| CSUM_UDP \| CSUM_IP);

	if (IFCAP_TSO4 & ifp->if_capenable &&
	!(IFCAP_TXCSUM & ifp->if_capenable)) {
	mask &= ~IFCAP_TSO4;
	ifp->if_capenable &= ~IFCAP_TSO4;
	if_printf(ifp,
	"tso4 disabled due to -txcsum.\n");
	}
	}
	if (mask & IFCAP_TXCSUM_IPV6) {
	ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
	ifp->if_hwassist ^= (CSUM_UDP_IPV6 \| CSUM_TCP_IPV6);

	if (IFCAP_TSO6 & ifp->if_capenable &&
	!(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
	mask &= ~IFCAP_TSO6;
	ifp->if_capenable &= ~IFCAP_TSO6;
	if_printf(ifp,
	"tso6 disabled due to -txcsum6.\n");
	}
	}
	if (mask & IFCAP_RXCSUM)
	ifp->if_capenable ^= IFCAP_RXCSUM;
	if (mask & IFCAP_RXCSUM_IPV6)
	ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;

	/*
	* Note that we leave CSUM_TSO alone (it is always set). The
	* kernel takes both IFCAP_TSOx and CSUM_TSO into account before
	* sending a TSO request our way, so it's sufficient to toggle
	* IFCAP_TSOx only.
	*/
	if (mask & IFCAP_TSO4) {
	if (!(IFCAP_TSO4 & ifp->if_capenable) &&
	!(IFCAP_TXCSUM & ifp->if_capenable)) {
	if_printf(ifp, "enable txcsum first.\n");
	rc = EAGAIN;
	goto fail;
	}
	ifp->if_capenable ^= IFCAP_TSO4;
	}
	if (mask & IFCAP_TSO6) {
	if (!(IFCAP_TSO6 & ifp->if_capenable) &&
	!(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
	if_printf(ifp, "enable txcsum6 first.\n");
	rc = EAGAIN;
	goto fail;
	}
	ifp->if_capenable ^= IFCAP_TSO6;
	}
	if (mask & IFCAP_LRO) {
	#if defined(INET) \|\| defined(INET6)
	int i;
	struct sge_rxq *rxq;

	ifp->if_capenable ^= IFCAP_LRO;
	for_each_rxq(vi, i, rxq) {
	if (ifp->if_capenable & IFCAP_LRO)
	rxq->iq.flags \|= IQ_LRO_ENABLED;
	else
	rxq->iq.flags &= ~IQ_LRO_ENABLED;
	}
	#endif
	}
	#ifdef TCP_OFFLOAD
	if (mask & IFCAP_TOE) {
	int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;

	rc = toe_capability(vi, enable);
	if (rc != 0)
	goto fail;

	ifp->if_capenable ^= mask;
	}
	#endif
	if (mask & IFCAP_VLAN_HWTAGGING) {
	ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
	rc = update_mac_settings(ifp, XGMAC_VLANEX);
	}
	if (mask & IFCAP_VLAN_MTU) {
	ifp->if_capenable ^= IFCAP_VLAN_MTU;

	/* Need to find out how to disable auto-mtu-inflation */
	}
	if (mask & IFCAP_VLAN_HWTSO)
	ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
	if (mask & IFCAP_VLAN_HWCSUM)
	ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
	#ifdef RATELIMIT
	if (mask & IFCAP_TXRTLMT)
	ifp->if_capenable ^= IFCAP_TXRTLMT;
	#endif
	if (mask & IFCAP_HWRXTSTMP) {
	int i;
	struct sge_rxq *rxq;

	ifp->if_capenable ^= IFCAP_HWRXTSTMP;
	for_each_rxq(vi, i, rxq) {
	if (ifp->if_capenable & IFCAP_HWRXTSTMP)
	rxq->iq.flags \|= IQ_RX_TIMESTAMP;
	else
	rxq->iq.flags &= ~IQ_RX_TIMESTAMP;
	}
	}
	if (mask & IFCAP_NOMAP)
	ifp->if_capenable ^= IFCAP_NOMAP;

	#ifdef KERN_TLS
	if (mask & IFCAP_TXTLS)
	ifp->if_capenable ^= (mask & IFCAP_TXTLS);
	#endif

	#ifdef VLAN_CAPABILITIES
	VLAN_CAPABILITIES(ifp);
	#endif
	fail:
	end_synchronized_op(sc, 0);
	break;

	case SIOCSIFMEDIA:
	case SIOCGIFMEDIA:
	case SIOCGIFXMEDIA:
	ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
	break;

	case SIOCGI2C: {
	struct ifi2creq i2c;

	rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
	if (rc != 0)
	break;
	if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
	rc = EPERM;
	break;
	}
	if (i2c.len > sizeof(i2c.data)) {
	rc = EINVAL;
	break;
	}
	rc = begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4i2c");
	if (rc)
	return (rc);
	rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
	i2c.offset, i2c.len, &i2c.data[0]);
	end_synchronized_op(sc, 0);
	if (rc == 0)
	rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
	break;
	}

	default:
	rc = ether_ioctl(ifp, cmd, data);
	}

	return (rc);
	}

	static int
	cxgbe_transmit(struct ifnet ifp, struct mbuf m)
	{
	struct vi_info *vi = ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	struct sge_txq *txq;
	#ifdef RATELIMIT
	struct cxgbe_snd_tag *cst;
	#endif
	void *items[1];
	int rc;

	M_ASSERTPKTHDR(m);
	MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
	#if defined(KERN_TLS) \|\| defined(RATELIMIT)
	if (m->m_pkthdr.csum_flags & CSUM_SND_TAG)
	MPASS(m->m_pkthdr.snd_tag->ifp == ifp);
	#endif

	if (__predict_false(pi->link_cfg.link_ok == false)) {
	m_freem(m);
	return (ENETDOWN);
	}

	rc = parse_pkt(sc, &m);
	if (__predict_false(rc != 0)) {
	MPASS(m == NULL); /* was freed already */
	atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
	return (rc);
	}
	#ifdef RATELIMIT
	if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) {
	cst = mst_to_cst(m->m_pkthdr.snd_tag);
	if (cst->type == IF_SND_TAG_TYPE_RATE_LIMIT)
	return (ethofld_transmit(ifp, m));
	}
	#endif

	/* Select a txq. */
	txq = &sc->sge.txq[vi->first_txq];
	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
	txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
	vi->rsrv_noflowq);

	items[0] = m;
	rc = mp_ring_enqueue(txq->r, items, 1, 256);
	if (__predict_false(rc != 0))
	m_freem(m);

	return (rc);
	}

	static void
	cxgbe_qflush(struct ifnet *ifp)
	{
	struct vi_info *vi = ifp->if_softc;
	struct sge_txq *txq;
	int i;

	/* queues do not exist if !VI_INIT_DONE. */
	if (vi->flags & VI_INIT_DONE) {
	for_each_txq(vi, i, txq) {
	TXQ_LOCK(txq);
	txq->eq.flags \|= EQ_QFLUSH;
	TXQ_UNLOCK(txq);
	while (!mp_ring_is_idle(txq->r)) {
	mp_ring_check_drainage(txq->r, 4096);
	pause("qflush", 1);
	}
	TXQ_LOCK(txq);
	txq->eq.flags &= ~EQ_QFLUSH;
	TXQ_UNLOCK(txq);
	}
	}
	if_qflush(ifp);
	}

	static uint64_t
	vi_get_counter(struct ifnet *ifp, ift_counter c)
	{
	struct vi_info *vi = ifp->if_softc;
	struct fw_vi_stats_vf *s = &vi->stats;

	vi_refresh_stats(vi->adapter, vi);

	switch (c) {
	case IFCOUNTER_IPACKETS:
	return (s->rx_bcast_frames + s->rx_mcast_frames +
	s->rx_ucast_frames);
	case IFCOUNTER_IERRORS:
	return (s->rx_err_frames);
	case IFCOUNTER_OPACKETS:
	return (s->tx_bcast_frames + s->tx_mcast_frames +
	s->tx_ucast_frames + s->tx_offload_frames);
	case IFCOUNTER_OERRORS:
	return (s->tx_drop_frames);
	case IFCOUNTER_IBYTES:
	return (s->rx_bcast_bytes + s->rx_mcast_bytes +
	s->rx_ucast_bytes);
	case IFCOUNTER_OBYTES:
	return (s->tx_bcast_bytes + s->tx_mcast_bytes +
	s->tx_ucast_bytes + s->tx_offload_bytes);
	case IFCOUNTER_IMCASTS:
	return (s->rx_mcast_frames);
	case IFCOUNTER_OMCASTS:
	return (s->tx_mcast_frames);
	case IFCOUNTER_OQDROPS: {
	uint64_t drops;

	drops = 0;
	if (vi->flags & VI_INIT_DONE) {
	int i;
	struct sge_txq *txq;

	for_each_txq(vi, i, txq)
	drops += counter_u64_fetch(txq->r->dropped);
	}

	return (drops);

	}

	default:
	return (if_get_counter_default(ifp, c));
	}
	}

	uint64_t
	cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
	{
	struct vi_info *vi = ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	struct port_stats *s = &pi->stats;

	if (pi->nvi > 1 \|\| sc->flags & IS_VF)
	return (vi_get_counter(ifp, c));

	cxgbe_refresh_stats(sc, pi);

	switch (c) {
	case IFCOUNTER_IPACKETS:
	return (s->rx_frames);

	case IFCOUNTER_IERRORS:
	return (s->rx_jabber + s->rx_runt + s->rx_too_long +
	s->rx_fcs_err + s->rx_len_err);

	case IFCOUNTER_OPACKETS:
	return (s->tx_frames);

	case IFCOUNTER_OERRORS:
	return (s->tx_error_frames);

	case IFCOUNTER_IBYTES:
	return (s->rx_octets);

	case IFCOUNTER_OBYTES:
	return (s->tx_octets);

	case IFCOUNTER_IMCASTS:
	return (s->rx_mcast_frames);

	case IFCOUNTER_OMCASTS:
	return (s->tx_mcast_frames);

	case IFCOUNTER_IQDROPS:
	return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
	s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
	s->rx_trunc3 + pi->tnl_cong_drops);

	case IFCOUNTER_OQDROPS: {
	uint64_t drops;

	drops = s->tx_drop;
	if (vi->flags & VI_INIT_DONE) {
	int i;
	struct sge_txq *txq;

	for_each_txq(vi, i, txq)
	drops += counter_u64_fetch(txq->r->dropped);
	}

	return (drops);

	}

	default:
	return (if_get_counter_default(ifp, c));
	}
	}

	#if defined(KERN_TLS) \|\| defined(RATELIMIT)
	void
	cxgbe_snd_tag_init(struct cxgbe_snd_tag cst, struct ifnet ifp, int type)
	{

	m_snd_tag_init(&cst->com, ifp);
	cst->type = type;
	}

	static int
	cxgbe_snd_tag_alloc(struct ifnet ifp, union if_snd_tag_alloc_params params,
	struct m_snd_tag **pt)
	{
	int error;

	switch (params->hdr.type) {
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_RATE_LIMIT:
	error = cxgbe_rate_tag_alloc(ifp, params, pt);
	break;
	#endif
	#ifdef KERN_TLS
	case IF_SND_TAG_TYPE_TLS:
	error = cxgbe_tls_tag_alloc(ifp, params, pt);
	break;
	#endif
	default:
	error = EOPNOTSUPP;
	}
	if (error == 0)
	MPASS(mst_to_cst(*pt)->type == params->hdr.type);
	return (error);
	}

	static int
	cxgbe_snd_tag_modify(struct m_snd_tag *mst,
	union if_snd_tag_modify_params *params)
	{
	struct cxgbe_snd_tag *cst;

	cst = mst_to_cst(mst);
	switch (cst->type) {
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_RATE_LIMIT:
	return (cxgbe_rate_tag_modify(mst, params));
	#endif
	default:
	return (EOPNOTSUPP);
	}
	}

	static int
	cxgbe_snd_tag_query(struct m_snd_tag *mst,
	union if_snd_tag_query_params *params)
	{
	struct cxgbe_snd_tag *cst;

	cst = mst_to_cst(mst);
	switch (cst->type) {
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_RATE_LIMIT:
	return (cxgbe_rate_tag_query(mst, params));
	#endif
	default:
	return (EOPNOTSUPP);
	}
	}

	static void
	cxgbe_snd_tag_free(struct m_snd_tag *mst)
	{
	struct cxgbe_snd_tag *cst;

	cst = mst_to_cst(mst);
	switch (cst->type) {
	#ifdef RATELIMIT
	case IF_SND_TAG_TYPE_RATE_LIMIT:
	cxgbe_rate_tag_free(mst);
	return;
	#endif
	#ifdef KERN_TLS
	case IF_SND_TAG_TYPE_TLS:
	cxgbe_tls_tag_free(mst);
	return;
	#endif
	default:
	panic("shouldn't get here");
	}
	}
	#endif

	/*
	* The kernel picks a media from the list we had provided but we still validate
	* the requeste.
	*/
	int
	cxgbe_media_change(struct ifnet *ifp)
	{
	struct vi_info *vi = ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct ifmedia *ifm = &pi->media;
	struct link_config *lc = &pi->link_cfg;
	struct adapter *sc = pi->adapter;
	int rc;

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4mec");
	if (rc != 0)
	return (rc);
	PORT_LOCK(pi);
	if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) {
	/* ifconfig .. media autoselect */
	if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
	rc = ENOTSUP; /* AN not supported by transceiver */
	goto done;
	}
	lc->requested_aneg = AUTONEG_ENABLE;
	lc->requested_speed = 0;
	lc->requested_fc \|= PAUSE_AUTONEG;
	} else {
	lc->requested_aneg = AUTONEG_DISABLE;
	lc->requested_speed =
	ifmedia_baudrate(ifm->ifm_media) / 1000000;
	lc->requested_fc = 0;
	if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE)
	lc->requested_fc \|= PAUSE_RX;
	if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)
	lc->requested_fc \|= PAUSE_TX;
	}
	if (pi->up_vis > 0) {
	fixup_link_config(pi);
	rc = apply_link_config(pi);
	}
	done:
	PORT_UNLOCK(pi);
	end_synchronized_op(sc, 0);
	return (rc);
	}

	/*
	* Base media word (without ETHER, pause, link active, etc.) for the port at the
	* given speed.
	*/
	static int
	port_mword(struct port_info *pi, uint32_t speed)
	{

	MPASS(speed & M_FW_PORT_CAP32_SPEED);
	MPASS(powerof2(speed));

	switch(pi->port_type) {
	case FW_PORT_TYPE_BT_SGMII:
	case FW_PORT_TYPE_BT_XFI:
	case FW_PORT_TYPE_BT_XAUI:
	/* BaseT */
	switch (speed) {
	case FW_PORT_CAP32_SPEED_100M:
	return (IFM_100_T);
	case FW_PORT_CAP32_SPEED_1G:
	return (IFM_1000_T);
	case FW_PORT_CAP32_SPEED_10G:
	return (IFM_10G_T);
	}
	break;
	case FW_PORT_TYPE_KX4:
	if (speed == FW_PORT_CAP32_SPEED_10G)
	return (IFM_10G_KX4);
	break;
	case FW_PORT_TYPE_CX4:
	if (speed == FW_PORT_CAP32_SPEED_10G)
	return (IFM_10G_CX4);
	break;
	case FW_PORT_TYPE_KX:
	if (speed == FW_PORT_CAP32_SPEED_1G)
	return (IFM_1000_KX);
	break;
	case FW_PORT_TYPE_KR:
	case FW_PORT_TYPE_BP_AP:
	case FW_PORT_TYPE_BP4_AP:
	case FW_PORT_TYPE_BP40_BA:
	case FW_PORT_TYPE_KR4_100G:
	case FW_PORT_TYPE_KR_SFP28:
	case FW_PORT_TYPE_KR_XLAUI:
	switch (speed) {
	case FW_PORT_CAP32_SPEED_1G:
	return (IFM_1000_KX);
	case FW_PORT_CAP32_SPEED_10G:
	return (IFM_10G_KR);
	case FW_PORT_CAP32_SPEED_25G:
	return (IFM_25G_KR);
	case FW_PORT_CAP32_SPEED_40G:
	return (IFM_40G_KR4);
	case FW_PORT_CAP32_SPEED_50G:
	return (IFM_50G_KR2);
	case FW_PORT_CAP32_SPEED_100G:
	return (IFM_100G_KR4);
	}
	break;
	case FW_PORT_TYPE_FIBER_XFI:
	case FW_PORT_TYPE_FIBER_XAUI:
	case FW_PORT_TYPE_SFP:
	case FW_PORT_TYPE_QSFP_10G:
	case FW_PORT_TYPE_QSA:
	case FW_PORT_TYPE_QSFP:
	case FW_PORT_TYPE_CR4_QSFP:
	case FW_PORT_TYPE_CR_QSFP:
	case FW_PORT_TYPE_CR2_QSFP:
	case FW_PORT_TYPE_SFP28:
	/* Pluggable transceiver */
	switch (pi->mod_type) {
	case FW_PORT_MOD_TYPE_LR:
	switch (speed) {
	case FW_PORT_CAP32_SPEED_1G:
	return (IFM_1000_LX);
	case FW_PORT_CAP32_SPEED_10G:
	return (IFM_10G_LR);
	case FW_PORT_CAP32_SPEED_25G:
	return (IFM_25G_LR);
	case FW_PORT_CAP32_SPEED_40G:
	return (IFM_40G_LR4);
	case FW_PORT_CAP32_SPEED_50G:
	return (IFM_50G_LR2);
	case FW_PORT_CAP32_SPEED_100G:
	return (IFM_100G_LR4);
	}
	break;
	case FW_PORT_MOD_TYPE_SR:
	switch (speed) {
	case FW_PORT_CAP32_SPEED_1G:
	return (IFM_1000_SX);
	case FW_PORT_CAP32_SPEED_10G:
	return (IFM_10G_SR);
	case FW_PORT_CAP32_SPEED_25G:
	return (IFM_25G_SR);
	case FW_PORT_CAP32_SPEED_40G:
	return (IFM_40G_SR4);
	case FW_PORT_CAP32_SPEED_50G:
	return (IFM_50G_SR2);
	case FW_PORT_CAP32_SPEED_100G:
	return (IFM_100G_SR4);
	}
	break;
	case FW_PORT_MOD_TYPE_ER:
	if (speed == FW_PORT_CAP32_SPEED_10G)
	return (IFM_10G_ER);
	break;
	case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
	case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
	switch (speed) {
	case FW_PORT_CAP32_SPEED_1G:
	return (IFM_1000_CX);
	case FW_PORT_CAP32_SPEED_10G:
	return (IFM_10G_TWINAX);
	case FW_PORT_CAP32_SPEED_25G:
	return (IFM_25G_CR);
	case FW_PORT_CAP32_SPEED_40G:
	return (IFM_40G_CR4);
	case FW_PORT_CAP32_SPEED_50G:
	return (IFM_50G_CR2);
	case FW_PORT_CAP32_SPEED_100G:
	return (IFM_100G_CR4);
	}
	break;
	case FW_PORT_MOD_TYPE_LRM:
	if (speed == FW_PORT_CAP32_SPEED_10G)
	return (IFM_10G_LRM);
	break;
	case FW_PORT_MOD_TYPE_NA:
	MPASS(0); /* Not pluggable? */
	/* fall throough */
	case FW_PORT_MOD_TYPE_ERROR:
	case FW_PORT_MOD_TYPE_UNKNOWN:
	case FW_PORT_MOD_TYPE_NOTSUPPORTED:
	break;
	case FW_PORT_MOD_TYPE_NONE:
	return (IFM_NONE);
	}
	break;
	case FW_PORT_TYPE_NONE:
	return (IFM_NONE);
	}

	return (IFM_UNKNOWN);
	}

	void
	cxgbe_media_status(struct ifnet ifp, struct ifmediareq ifmr)
	{
	struct vi_info *vi = ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	struct link_config *lc = &pi->link_cfg;

	if (begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4med") != 0)
	return;
	PORT_LOCK(pi);

	if (pi->up_vis == 0) {
	/*
	* If all the interfaces are administratively down the firmware
	* does not report transceiver changes. Refresh port info here
	* so that ifconfig displays accurate ifmedia at all times.
	* This is the only reason we have a synchronized op in this
	* function. Just PORT_LOCK would have been enough otherwise.
	*/
	t4_update_port_info(pi);
	build_medialist(pi);
	}

	/* ifm_status */
	ifmr->ifm_status = IFM_AVALID;
	if (lc->link_ok == false)
	goto done;
	ifmr->ifm_status \|= IFM_ACTIVE;

	/* ifm_active */
	ifmr->ifm_active = IFM_ETHER \| IFM_FDX;
	ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE \| IFM_ETH_RXPAUSE);
	if (lc->fc & PAUSE_RX)
	ifmr->ifm_active \|= IFM_ETH_RXPAUSE;
	if (lc->fc & PAUSE_TX)
	ifmr->ifm_active \|= IFM_ETH_TXPAUSE;
	ifmr->ifm_active \|= port_mword(pi, speed_to_fwcap(lc->speed));
	done:
	PORT_UNLOCK(pi);
	end_synchronized_op(sc, 0);
	}

	static int
	vcxgbe_probe(device_t dev)
	{
	char buf[128];
	struct vi_info *vi = device_get_softc(dev);

	snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
	vi - vi->pi->vi);
	device_set_desc_copy(dev, buf);

	return (BUS_PROBE_DEFAULT);
	}

	static int
	alloc_extra_vi(struct adapter sc, struct port_info pi, struct vi_info *vi)
	{
	int func, index, rc;
	uint32_t param, val;

	ASSERT_SYNCHRONIZED_OP(sc);

	index = vi - pi->vi;
	MPASS(index > 0); /* This function deals with _extra_ VIs only */
	KASSERT(index < nitems(vi_mac_funcs),
	("%s: VI %s doesn't have a MAC func", __func__,
	device_get_nameunit(vi->dev)));
	func = vi_mac_funcs[index];
	rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
	vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0);
	if (rc < 0) {
	device_printf(vi->dev, "failed to allocate virtual interface %d"
	"for port %d: %d\n", index, pi->port_id, -rc);
	return (-rc);
	}
	vi->viid = rc;

	if (vi->rss_size == 1) {
	/*
	* This VI didn't get a slice of the RSS table. Reduce the
	* number of VIs being created (hw.cxgbe.num_vis) or modify the
	* configuration file (nvi, rssnvi for this PF) if this is a
	* problem.
	*/
	device_printf(vi->dev, "RSS table not available.\n");
	vi->rss_base = 0xffff;

	return (0);
	}

	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \|
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) \|
	V_FW_PARAMS_PARAM_YZ(vi->viid);
	rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	if (rc)
	vi->rss_base = 0xffff;
	else {
	MPASS((val >> 16) == vi->rss_size);
	vi->rss_base = val & 0xffff;
	}

	return (0);
	}

	static int
	vcxgbe_attach(device_t dev)
	{
	struct vi_info *vi;
	struct port_info *pi;
	struct adapter *sc;
	int rc;

	vi = device_get_softc(dev);
	pi = vi->pi;
	sc = pi->adapter;

	rc = begin_synchronized_op(sc, vi, SLEEP_OK \| INTR_OK, "t4via");
	if (rc)
	return (rc);
	rc = alloc_extra_vi(sc, pi, vi);
	end_synchronized_op(sc, 0);
	if (rc)
	return (rc);

	rc = cxgbe_vi_attach(dev, vi);
	if (rc) {
	t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
	return (rc);
	}
	return (0);
	}

	static int
	vcxgbe_detach(device_t dev)
	{
	struct vi_info *vi;
	struct adapter *sc;

	vi = device_get_softc(dev);
	sc = vi->adapter;

	doom_vi(sc, vi);

	cxgbe_vi_detach(vi);
	t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);

	end_synchronized_op(sc, 0);

	return (0);
	}

	static struct callout fatal_callout;

	static void
	delayed_panic(void *arg)
	{
	struct adapter *sc = arg;

	panic("%s: panic on fatal error", device_get_nameunit(sc->dev));
	}

	void
	t4_fatal_err(struct adapter *sc, bool fw_error)
	{

	t4_shutdown_adapter(sc);
	log(LOG_ALERT, "%s: encountered fatal error, adapter stopped.\n",
	device_get_nameunit(sc->dev));
	if (fw_error) {
	ASSERT_SYNCHRONIZED_OP(sc);
	sc->flags \|= ADAP_ERR;
	} else {
	ADAPTER_LOCK(sc);
	sc->flags \|= ADAP_ERR;
	ADAPTER_UNLOCK(sc);
	}
	#ifdef TCP_OFFLOAD
	taskqueue_enqueue(taskqueue_thread, &sc->async_event_task);
	#endif

	if (t4_panic_on_fatal_err) {
	log(LOG_ALERT, "%s: panic on fatal error after 30s",
	device_get_nameunit(sc->dev));
	callout_reset(&fatal_callout, hz * 30, delayed_panic, sc);
	}
	}

	void
	t4_add_adapter(struct adapter *sc)
	{
	sx_xlock(&t4_list_lock);
	SLIST_INSERT_HEAD(&t4_list, sc, link);
	sx_xunlock(&t4_list_lock);
	}

	int
	t4_map_bars_0_and_4(struct adapter *sc)
	{
	sc->regs_rid = PCIR_BAR(0);
	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
	&sc->regs_rid, RF_ACTIVE);
	if (sc->regs_res == NULL) {
	device_printf(sc->dev, "cannot map registers.\n");
	return (ENXIO);
	}
	sc->bt = rman_get_bustag(sc->regs_res);
	sc->bh = rman_get_bushandle(sc->regs_res);
	sc->mmio_len = rman_get_size(sc->regs_res);
	setbit(&sc->doorbells, DOORBELL_KDB);

	sc->msix_rid = PCIR_BAR(4);
	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
	&sc->msix_rid, RF_ACTIVE);
	if (sc->msix_res == NULL) {
	device_printf(sc->dev, "cannot map MSI-X BAR.\n");
	return (ENXIO);
	}

	return (0);
	}

	int
	t4_map_bar_2(struct adapter *sc)
	{

	/*
	* T4: only iWARP driver uses the userspace doorbells. There is no need
	* to map it if RDMA is disabled.
	*/
	if (is_t4(sc) && sc->rdmacaps == 0)
	return (0);

	sc->udbs_rid = PCIR_BAR(2);
	sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
	&sc->udbs_rid, RF_ACTIVE);
	if (sc->udbs_res == NULL) {
	device_printf(sc->dev, "cannot map doorbell BAR.\n");
	return (ENXIO);
	}
	sc->udbs_base = rman_get_virtual(sc->udbs_res);

	if (chip_id(sc) >= CHELSIO_T5) {
	setbit(&sc->doorbells, DOORBELL_UDB);
	#if defined(__i386__) \|\| defined(__amd64__)
	if (t5_write_combine) {
	int rc, mode;

	/*
	* Enable write combining on BAR2. This is the
	* userspace doorbell BAR and is split into 128B
	* (UDBS_SEG_SIZE) doorbell regions, each associated
	* with an egress queue. The first 64B has the doorbell
	* and the second 64B can be used to submit a tx work
	* request with an implicit doorbell.
	*/

	rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
	rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
	if (rc == 0) {
	clrbit(&sc->doorbells, DOORBELL_UDB);
	setbit(&sc->doorbells, DOORBELL_WCWR);
	setbit(&sc->doorbells, DOORBELL_UDBWC);
	} else {
	device_printf(sc->dev,
	"couldn't enable write combining: %d\n",
	rc);
	}

	mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0);
	t4_write_reg(sc, A_SGE_STAT_CFG,
	V_STATSOURCE_T5(7) \| mode);
	}
	#endif
	}
	sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0;

	return (0);
	}

	struct memwin_init {
	uint32_t base;
	uint32_t aperture;
	};

	static const struct memwin_init t4_memwin[NUM_MEMWIN] = {
	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
	{ MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
	};

	static const struct memwin_init t5_memwin[NUM_MEMWIN] = {
	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
	};

	static void
	setup_memwin(struct adapter *sc)
	{
	const struct memwin_init *mw_init;
	struct memwin *mw;
	int i;
	uint32_t bar0;

	if (is_t4(sc)) {
	/*
	* Read low 32b of bar0 indirectly via the hardware backdoor
	* mechanism. Works from within PCI passthrough environments
	* too, where rman_get_start() can return a different value. We
	* need to program the T4 memory window decoders with the actual
	* addresses that will be coming across the PCIe link.
	*/
	bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
	bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;

	mw_init = &t4_memwin[0];
	} else {
	/* T5+ use the relative offset inside the PCIe BAR */
	bar0 = 0;

	mw_init = &t5_memwin[0];
	}

	for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
	rw_init(&mw->mw_lock, "memory window access");
	mw->mw_base = mw_init->base;
	mw->mw_aperture = mw_init->aperture;
	mw->mw_curpos = 0;
	t4_write_reg(sc,
	PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
	(mw->mw_base + bar0) \| V_BIR(0) \|
	V_WINDOW(ilog2(mw->mw_aperture) - 10));
	rw_wlock(&mw->mw_lock);
	position_memwin(sc, i, 0);
	rw_wunlock(&mw->mw_lock);
	}

	/* flush */
	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
	}

	/*
	* Positions the memory window at the given address in the card's address space.
	* There are some alignment requirements and the actual position may be at an
	* address prior to the requested address. mw->mw_curpos always has the actual
	* position of the window.
	*/
	static void
	position_memwin(struct adapter *sc, int idx, uint32_t addr)
	{
	struct memwin *mw;
	uint32_t pf;
	uint32_t reg;

	MPASS(idx >= 0 && idx < NUM_MEMWIN);
	mw = &sc->memwin[idx];
	rw_assert(&mw->mw_lock, RA_WLOCKED);

	if (is_t4(sc)) {
	pf = 0;
	mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */
	} else {
	pf = V_PFNUM(sc->pf);
	mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */
	}
	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx);
	t4_write_reg(sc, reg, mw->mw_curpos \| pf);
	t4_read_reg(sc, reg); /* flush */
	}

	int
	rw_via_memwin(struct adapter sc, int idx, uint32_t addr, uint32_t val,
	int len, int rw)
	{
	struct memwin *mw;
	uint32_t mw_end, v;

	MPASS(idx >= 0 && idx < NUM_MEMWIN);

	/* Memory can only be accessed in naturally aligned 4 byte units */
	if (addr & 3 \|\| len & 3 \|\| len <= 0)
	return (EINVAL);

	mw = &sc->memwin[idx];
	while (len > 0) {
	rw_rlock(&mw->mw_lock);
	mw_end = mw->mw_curpos + mw->mw_aperture;
	if (addr >= mw_end \|\| addr < mw->mw_curpos) {
	/* Will need to reposition the window */
	if (!rw_try_upgrade(&mw->mw_lock)) {
	rw_runlock(&mw->mw_lock);
	rw_wlock(&mw->mw_lock);
	}
	rw_assert(&mw->mw_lock, RA_WLOCKED);
	position_memwin(sc, idx, addr);
	rw_downgrade(&mw->mw_lock);
	mw_end = mw->mw_curpos + mw->mw_aperture;
	}
	rw_assert(&mw->mw_lock, RA_RLOCKED);
	while (addr < mw_end && len > 0) {
	if (rw == 0) {
	v = t4_read_reg(sc, mw->mw_base + addr -
	mw->mw_curpos);
	*val++ = le32toh(v);
	} else {
	v = *val++;
	t4_write_reg(sc, mw->mw_base + addr -
	mw->mw_curpos, htole32(v));
	}
	addr += 4;
	len -= 4;
	}
	rw_runlock(&mw->mw_lock);
	}

	return (0);
	}

	static void
	t4_init_atid_table(struct adapter *sc)
	{
	struct tid_info *t;
	int i;

	t = &sc->tids;
	if (t->natids == 0)
	return;

	MPASS(t->atid_tab == NULL);

	t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE,
	M_ZERO \| M_WAITOK);
	mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
	t->afree = t->atid_tab;
	t->atids_in_use = 0;
	for (i = 1; i < t->natids; i++)
	t->atid_tab[i - 1].next = &t->atid_tab[i];
	t->atid_tab[t->natids - 1].next = NULL;
	}

	static void
	t4_free_atid_table(struct adapter *sc)
	{
	struct tid_info *t;

	t = &sc->tids;

	KASSERT(t->atids_in_use == 0,
	("%s: %d atids still in use.", __func__, t->atids_in_use));

	if (mtx_initialized(&t->atid_lock))
	mtx_destroy(&t->atid_lock);
	free(t->atid_tab, M_CXGBE);
	t->atid_tab = NULL;
	}

	int
	alloc_atid(struct adapter sc, void ctx)
	{
	struct tid_info *t = &sc->tids;
	int atid = -1;

	mtx_lock(&t->atid_lock);
	if (t->afree) {
	union aopen_entry *p = t->afree;

	atid = p - t->atid_tab;
	MPASS(atid <= M_TID_TID);
	t->afree = p->next;
	p->data = ctx;
	t->atids_in_use++;
	}
	mtx_unlock(&t->atid_lock);
	return (atid);
	}

	void *
	lookup_atid(struct adapter *sc, int atid)
	{
	struct tid_info *t = &sc->tids;

	return (t->atid_tab[atid].data);
	}

	void
	free_atid(struct adapter *sc, int atid)
	{
	struct tid_info *t = &sc->tids;
	union aopen_entry *p = &t->atid_tab[atid];

	mtx_lock(&t->atid_lock);
	p->next = t->afree;
	t->afree = p;
	t->atids_in_use--;
	mtx_unlock(&t->atid_lock);
	}

	static void
	queue_tid_release(struct adapter *sc, int tid)
	{

	CXGBE_UNIMPLEMENTED("deferred tid release");
	}

	void
	release_tid(struct adapter sc, int tid, struct sge_wrq ctrlq)
	{
	struct wrqe *wr;
	struct cpl_tid_release *req;

	wr = alloc_wrqe(sizeof(*req), ctrlq);
	if (wr == NULL) {
	queue_tid_release(sc, tid); /* defer */
	return;
	}
	req = wrtod(wr);

	INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid);

	t4_wrq_tx(sc, wr);
	}

	static int
	t4_range_cmp(const void a, const void b)
	{
	return ((const struct t4_range *)a)->start -
	((const struct t4_range *)b)->start;
	}

	/*
	* Verify that the memory range specified by the addr/len pair is valid within
	* the card's address space.
	*/
	static int
	validate_mem_range(struct adapter *sc, uint32_t addr, uint32_t len)
	{
	struct t4_range mem_ranges[4], r, next;
	uint32_t em, addr_len;
	int i, n, remaining;

	/* Memory can only be accessed in naturally aligned 4 byte units */
	if (addr & 3 \|\| len & 3 \|\| len == 0)
	return (EINVAL);

	/* Enabled memories */
	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);

	r = &mem_ranges[0];
	n = 0;
	bzero(r, sizeof(mem_ranges));
	if (em & F_EDRAM0_ENABLE) {
	addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
	r->size = G_EDRAM0_SIZE(addr_len) << 20;
	if (r->size > 0) {
	r->start = G_EDRAM0_BASE(addr_len) << 20;
	if (addr >= r->start &&
	addr + len <= r->start + r->size)
	return (0);
	r++;
	n++;
	}
	}
	if (em & F_EDRAM1_ENABLE) {
	addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
	r->size = G_EDRAM1_SIZE(addr_len) << 20;
	if (r->size > 0) {
	r->start = G_EDRAM1_BASE(addr_len) << 20;
	if (addr >= r->start &&
	addr + len <= r->start + r->size)
	return (0);
	r++;
	n++;
	}
	}
	if (em & F_EXT_MEM_ENABLE) {
	addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
	r->size = G_EXT_MEM_SIZE(addr_len) << 20;
	if (r->size > 0) {
	r->start = G_EXT_MEM_BASE(addr_len) << 20;
	if (addr >= r->start &&
	addr + len <= r->start + r->size)
	return (0);
	r++;
	n++;
	}
	}
	if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) {
	addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
	r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
	if (r->size > 0) {
	r->start = G_EXT_MEM1_BASE(addr_len) << 20;
	if (addr >= r->start &&
	addr + len <= r->start + r->size)
	return (0);
	r++;
	n++;
	}
	}
	MPASS(n <= nitems(mem_ranges));

	if (n > 1) {
	/* Sort and merge the ranges. */
	qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp);

	/* Start from index 0 and examine the next n - 1 entries. */
	r = &mem_ranges[0];
	for (remaining = n - 1; remaining > 0; remaining--, r++) {

	MPASS(r->size > 0); /* r is a valid entry. */
	next = r + 1;
	MPASS(next->size > 0); /* and so is the next one. */

	while (r->start + r->size >= next->start) {
	/* Merge the next one into the current entry. */
	r->size = max(r->start + r->size,
	next->start + next->size) - r->start;
	n--; /* One fewer entry in total. */
	if (--remaining == 0)
	goto done; /* short circuit */
	next++;
	}
	if (next != r + 1) {
	/*
	* Some entries were merged into r and next
	* points to the first valid entry that couldn't
	* be merged.
	*/
	MPASS(next->size > 0); /* must be valid */
	memcpy(r + 1, next, remaining * sizeof(*r));
	#ifdef INVARIANTS
	/*
	* This so that the foo->size assertion in the
	* next iteration of the loop do the right
	* thing for entries that were pulled up and are
	* no longer valid.
	*/
	MPASS(n < nitems(mem_ranges));
	bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
	sizeof(struct t4_range));
	#endif
	}
	}
	done:
	/* Done merging the ranges. */
	MPASS(n > 0);
	r = &mem_ranges[0];
	for (i = 0; i < n; i++, r++) {
	if (addr >= r->start &&
	addr + len <= r->start + r->size)
	return (0);
	}
	}

	return (EFAULT);
	}

	static int
	fwmtype_to_hwmtype(int mtype)
	{

	switch (mtype) {
	case FW_MEMTYPE_EDC0:
	return (MEM_EDC0);
	case FW_MEMTYPE_EDC1:
	return (MEM_EDC1);
	case FW_MEMTYPE_EXTMEM:
	return (MEM_MC0);
	case FW_MEMTYPE_EXTMEM1:
	return (MEM_MC1);
	default:
	panic("%s: cannot translate fw mtype %d.", __func__, mtype);
	}
	}

	/*
	* Verify that the memory range specified by the memtype/offset/len pair is
	* valid and lies entirely within the memtype specified. The global address of
	* the start of the range is returned in addr.
	*/
	static int
	validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, uint32_t len,
	uint32_t *addr)
	{
	uint32_t em, addr_len, maddr;

	/* Memory can only be accessed in naturally aligned 4 byte units */
	if (off & 3 \|\| len & 3 \|\| len == 0)
	return (EINVAL);

	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
	switch (fwmtype_to_hwmtype(mtype)) {
	case MEM_EDC0:
	if (!(em & F_EDRAM0_ENABLE))
	return (EINVAL);
	addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
	maddr = G_EDRAM0_BASE(addr_len) << 20;
	break;
	case MEM_EDC1:
	if (!(em & F_EDRAM1_ENABLE))
	return (EINVAL);
	addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
	maddr = G_EDRAM1_BASE(addr_len) << 20;
	break;
	case MEM_MC:
	if (!(em & F_EXT_MEM_ENABLE))
	return (EINVAL);
	addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
	maddr = G_EXT_MEM_BASE(addr_len) << 20;
	break;
	case MEM_MC1:
	if (!is_t5(sc) \|\| !(em & F_EXT_MEM1_ENABLE))
	return (EINVAL);
	addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
	maddr = G_EXT_MEM1_BASE(addr_len) << 20;
	break;
	default:
	return (EINVAL);
	}

	addr = maddr + off; / global address */
	return (validate_mem_range(sc, *addr, len));
	}

	static int
	fixup_devlog_params(struct adapter *sc)
	{
	struct devlog_params *dparams = &sc->params.devlog;
	int rc;

	rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
	dparams->size, &dparams->addr);

	return (rc);
	}

	static void
	update_nirq(struct intrs_and_queues *iaq, int nports)
	{

	iaq->nirq = T4_EXTRA_INTR;
	iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq);
	iaq->nirq += nports * iaq->nofldrxq;
	iaq->nirq += nports * (iaq->num_vis - 1) *
	max(iaq->nrxq_vi, iaq->nnmrxq_vi);
	iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi;
	}

	/*
	* Adjust requirements to fit the number of interrupts available.
	*/
	static void
	calculate_iaq(struct adapter sc, struct intrs_and_queues iaq, int itype,
	int navail)
	{
	int old_nirq;
	const int nports = sc->params.nports;

	MPASS(nports > 0);
	MPASS(navail > 0);

	bzero(iaq, sizeof(*iaq));
	iaq->intr_type = itype;
	iaq->num_vis = t4_num_vis;
	iaq->ntxq = t4_ntxq;
	iaq->ntxq_vi = t4_ntxq_vi;
	iaq->nrxq = t4_nrxq;
	iaq->nrxq_vi = t4_nrxq_vi;
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	if (is_offload(sc) \|\| is_ethoffload(sc)) {
	iaq->nofldtxq = t4_nofldtxq;
	iaq->nofldtxq_vi = t4_nofldtxq_vi;
	}
	#endif
	#ifdef TCP_OFFLOAD
	if (is_offload(sc)) {
	iaq->nofldrxq = t4_nofldrxq;
	iaq->nofldrxq_vi = t4_nofldrxq_vi;
	}
	#endif
	#ifdef DEV_NETMAP
	if (t4_native_netmap & NN_MAIN_VI) {
	iaq->nnmtxq = t4_nnmtxq;
	iaq->nnmrxq = t4_nnmrxq;
	}
	if (t4_native_netmap & NN_EXTRA_VI) {
	iaq->nnmtxq_vi = t4_nnmtxq_vi;
	iaq->nnmrxq_vi = t4_nnmrxq_vi;
	}
	#endif

	update_nirq(iaq, nports);
	if (iaq->nirq <= navail &&
	(itype != INTR_MSI \|\| powerof2(iaq->nirq))) {
	/*
	* This is the normal case -- there are enough interrupts for
	* everything.
	*/
	goto done;
	}

	/*
	* If extra VIs have been configured try reducing their count and see if
	* that works.
	*/
	while (iaq->num_vis > 1) {
	iaq->num_vis--;
	update_nirq(iaq, nports);
	if (iaq->nirq <= navail &&
	(itype != INTR_MSI \|\| powerof2(iaq->nirq))) {
	device_printf(sc->dev, "virtual interfaces per port "
	"reduced to %d from %d. nrxq=%u, nofldrxq=%u, "
	"nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u. "
	"itype %d, navail %u, nirq %d.\n",
	iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq,
	iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi,
	itype, navail, iaq->nirq);
	goto done;
	}
	}

	/*
	* Extra VIs will not be created. Log a message if they were requested.
	*/
	MPASS(iaq->num_vis == 1);
	iaq->ntxq_vi = iaq->nrxq_vi = 0;
	iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
	iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
	if (iaq->num_vis != t4_num_vis) {
	device_printf(sc->dev, "extra virtual interfaces disabled. "
	"nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, "
	"nnmrxq_vi=%u. itype %d, navail %u, nirq %d.\n",
	iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi,
	iaq->nnmrxq_vi, itype, navail, iaq->nirq);
	}

	/*
	* Keep reducing the number of NIC rx queues to the next lower power of
	* 2 (for even RSS distribution) and halving the TOE rx queues and see
	* if that works.
	*/
	do {
	if (iaq->nrxq > 1) {
	do {
	iaq->nrxq--;
	} while (!powerof2(iaq->nrxq));
	if (iaq->nnmrxq > iaq->nrxq)
	iaq->nnmrxq = iaq->nrxq;
	}
	if (iaq->nofldrxq > 1)
	iaq->nofldrxq >>= 1;

	old_nirq = iaq->nirq;
	update_nirq(iaq, nports);
	if (iaq->nirq <= navail &&
	(itype != INTR_MSI \|\| powerof2(iaq->nirq))) {
	device_printf(sc->dev, "running with reduced number of "
	"rx queues because of shortage of interrupts. "
	"nrxq=%u, nofldrxq=%u. "
	"itype %d, navail %u, nirq %d.\n", iaq->nrxq,
	iaq->nofldrxq, itype, navail, iaq->nirq);
	goto done;
	}
	} while (old_nirq != iaq->nirq);

	/* One interrupt for everything. Ugh. */
	device_printf(sc->dev, "running with minimal number of queues. "
	"itype %d, navail %u.\n", itype, navail);
	iaq->nirq = 1;
	iaq->nrxq = 1;
	iaq->ntxq = 1;
	if (iaq->nofldrxq > 0) {
	iaq->nofldrxq = 1;
	iaq->nofldtxq = 1;
	}
	iaq->nnmtxq = 0;
	iaq->nnmrxq = 0;
	done:
	MPASS(iaq->num_vis > 0);
	if (iaq->num_vis > 1) {
	MPASS(iaq->nrxq_vi > 0);
	MPASS(iaq->ntxq_vi > 0);
	}
	MPASS(iaq->nirq > 0);
	MPASS(iaq->nrxq > 0);
	MPASS(iaq->ntxq > 0);
	if (itype == INTR_MSI) {
	MPASS(powerof2(iaq->nirq));
	}
	}

	static int
	cfg_itype_and_nqueues(struct adapter sc, struct intrs_and_queues iaq)
	{
	int rc, itype, navail, nalloc;

	for (itype = INTR_MSIX; itype; itype >>= 1) {

	if ((itype & t4_intr_types) == 0)
	continue; /* not allowed */

	if (itype == INTR_MSIX)
	navail = pci_msix_count(sc->dev);
	else if (itype == INTR_MSI)
	navail = pci_msi_count(sc->dev);
	else
	navail = 1;
	restart:
	if (navail == 0)
	continue;

	calculate_iaq(sc, iaq, itype, navail);
	nalloc = iaq->nirq;
	rc = 0;
	if (itype == INTR_MSIX)
	rc = pci_alloc_msix(sc->dev, &nalloc);
	else if (itype == INTR_MSI)
	rc = pci_alloc_msi(sc->dev, &nalloc);

	if (rc == 0 && nalloc > 0) {
	if (nalloc == iaq->nirq)
	return (0);

	/*
	* Didn't get the number requested. Use whatever number
	* the kernel is willing to allocate.
	*/
	device_printf(sc->dev, "fewer vectors than requested, "
	"type=%d, req=%d, rcvd=%d; will downshift req.\n",
	itype, iaq->nirq, nalloc);
	pci_release_msi(sc->dev);
	navail = nalloc;
	goto restart;
	}

	device_printf(sc->dev,
	"failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
	itype, rc, iaq->nirq, nalloc);
	}

	device_printf(sc->dev,
	"failed to find a usable interrupt type. "
	"allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
	pci_msix_count(sc->dev), pci_msi_count(sc->dev));

	return (ENXIO);
	}

	#define FW_VERSION(chip) ( \
	V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) \| \
	V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) \| \
	V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) \| \
	V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
	#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)

	/* Just enough of fw_hdr to cover all version info. */
	struct fw_h {
	__u8 ver;
	__u8 chip;
	__be16 len512;
	__be32 fw_ver;
	__be32 tp_microcode_ver;
	__u8 intfver_nic;
	__u8 intfver_vnic;
	__u8 intfver_ofld;
	__u8 intfver_ri;
	__u8 intfver_iscsipdu;
	__u8 intfver_iscsi;
	__u8 intfver_fcoepdu;
	__u8 intfver_fcoe;
	};
	/* Spot check a couple of fields. */
	CTASSERT(offsetof(struct fw_h, fw_ver) == offsetof(struct fw_hdr, fw_ver));
	CTASSERT(offsetof(struct fw_h, intfver_nic) == offsetof(struct fw_hdr, intfver_nic));
	CTASSERT(offsetof(struct fw_h, intfver_fcoe) == offsetof(struct fw_hdr, intfver_fcoe));

	struct fw_info {
	uint8_t chip;
	char *kld_name;
	char *fw_mod_name;
	struct fw_h fw_h;
	} fw_info[] = {
	{
	.chip = CHELSIO_T4,
	.kld_name = "t4fw_cfg",
	.fw_mod_name = "t4fw",
	.fw_h = {
	.chip = FW_HDR_CHIP_T4,
	.fw_ver = htobe32(FW_VERSION(T4)),
	.intfver_nic = FW_INTFVER(T4, NIC),
	.intfver_vnic = FW_INTFVER(T4, VNIC),
	.intfver_ofld = FW_INTFVER(T4, OFLD),
	.intfver_ri = FW_INTFVER(T4, RI),
	.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
	.intfver_iscsi = FW_INTFVER(T4, ISCSI),
	.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
	.intfver_fcoe = FW_INTFVER(T4, FCOE),
	},
	}, {
	.chip = CHELSIO_T5,
	.kld_name = "t5fw_cfg",
	.fw_mod_name = "t5fw",
	.fw_h = {
	.chip = FW_HDR_CHIP_T5,
	.fw_ver = htobe32(FW_VERSION(T5)),
	.intfver_nic = FW_INTFVER(T5, NIC),
	.intfver_vnic = FW_INTFVER(T5, VNIC),
	.intfver_ofld = FW_INTFVER(T5, OFLD),
	.intfver_ri = FW_INTFVER(T5, RI),
	.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
	.intfver_iscsi = FW_INTFVER(T5, ISCSI),
	.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
	.intfver_fcoe = FW_INTFVER(T5, FCOE),
	},
	}, {
	.chip = CHELSIO_T6,
	.kld_name = "t6fw_cfg",
	.fw_mod_name = "t6fw",
	.fw_h = {
	.chip = FW_HDR_CHIP_T6,
	.fw_ver = htobe32(FW_VERSION(T6)),
	.intfver_nic = FW_INTFVER(T6, NIC),
	.intfver_vnic = FW_INTFVER(T6, VNIC),
	.intfver_ofld = FW_INTFVER(T6, OFLD),
	.intfver_ri = FW_INTFVER(T6, RI),
	.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
	.intfver_iscsi = FW_INTFVER(T6, ISCSI),
	.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
	.intfver_fcoe = FW_INTFVER(T6, FCOE),
	},
	}
	};

	static struct fw_info *
	find_fw_info(int chip)
	{
	int i;

	for (i = 0; i < nitems(fw_info); i++) {
	if (fw_info[i].chip == chip)
	return (&fw_info[i]);
	}
	return (NULL);
	}

	/*
	* Is the given firmware API compatible with the one the driver was compiled
	* with?
	*/
	static int
	fw_compatible(const struct fw_h hdr1, const struct fw_h hdr2)
	{

	/* short circuit if it's the exact same firmware version */
	if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
	return (1);

	/*
	* XXX: Is this too conservative? Perhaps I should limit this to the
	* features that are supported in the driver.
	*/
	#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
	if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
	SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
	SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
	return (1);
	#undef SAME_INTF

	return (0);
	}

	static int
	load_fw_module(struct adapter sc, const struct firmware *dcfg,
	const struct firmware **fw)
	{
	struct fw_info *fw_info;

	*dcfg = NULL;
	if (fw != NULL)
	*fw = NULL;

	fw_info = find_fw_info(chip_id(sc));
	if (fw_info == NULL) {
	device_printf(sc->dev,
	"unable to look up firmware information for chip %d.\n",
	chip_id(sc));
	return (EINVAL);
	}

	*dcfg = firmware_get(fw_info->kld_name);
	if (*dcfg != NULL) {
	if (fw != NULL)
	*fw = firmware_get(fw_info->fw_mod_name);
	return (0);
	}

	return (ENOENT);
	}

	static void
	unload_fw_module(struct adapter sc, const struct firmware dcfg,
	const struct firmware *fw)
	{

	if (fw != NULL)
	firmware_put(fw, FIRMWARE_UNLOAD);
	if (dcfg != NULL)
	firmware_put(dcfg, FIRMWARE_UNLOAD);
	}

	/*
	* Return values:
	* 0 means no firmware install attempted.
	* ERESTART means a firmware install was attempted and was successful.
	* +ve errno means a firmware install was attempted but failed.
	*/
	static int
	install_kld_firmware(struct adapter sc, struct fw_h card_fw,
	const struct fw_h drv_fw, const char reason, int *already)
	{
	const struct firmware cfg, fw;
	const uint32_t c = be32toh(card_fw->fw_ver);
	uint32_t d, k;
	int rc, fw_install;
	struct fw_h bundled_fw;
	bool load_attempted;

	cfg = fw = NULL;
	load_attempted = false;
	fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install;

	memcpy(&bundled_fw, drv_fw, sizeof(bundled_fw));
	if (t4_fw_install < 0) {
	rc = load_fw_module(sc, &cfg, &fw);
	if (rc != 0 \|\| fw == NULL) {
	device_printf(sc->dev,
	"failed to load firmware module: %d. cfg %p, fw %p;"
	" will use compiled-in firmware version for"
	"hw.cxgbe.fw_install checks.\n",
	rc, cfg, fw);
	} else {
	memcpy(&bundled_fw, fw->data, sizeof(bundled_fw));
	}
	load_attempted = true;
	}
	d = be32toh(bundled_fw.fw_ver);

	if (reason != NULL)
	goto install;

	if ((sc->flags & FW_OK) == 0) {

	if (c == 0xffffffff) {
	reason = "missing";
	goto install;
	}

	rc = 0;
	goto done;
	}

	if (!fw_compatible(card_fw, &bundled_fw)) {
	reason = "incompatible or unusable";
	goto install;
	}

	if (d > c) {
	reason = "older than the version bundled with this driver";
	goto install;
	}

	if (fw_install == 2 && d != c) {
	reason = "different than the version bundled with this driver";
	goto install;
	}

	/* No reason to do anything to the firmware already on the card. */
	rc = 0;
	goto done;

	install:
	rc = 0;
	if ((*already)++)
	goto done;

	if (fw_install == 0) {
	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
	"but the driver is prohibited from installing a firmware "
	"on the card.\n",
	G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
	G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);

	goto done;
	}

	/*
	* We'll attempt to install a firmware. Load the module first (if it
	* hasn't been loaded already).
	*/
	if (!load_attempted) {
	rc = load_fw_module(sc, &cfg, &fw);
	if (rc != 0 \|\| fw == NULL) {
	device_printf(sc->dev,
	"failed to load firmware module: %d. cfg %p, fw %p\n",
	rc, cfg, fw);
	/* carry on */
	}
	}
	if (fw == NULL) {
	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
	"but the driver cannot take corrective action because it "
	"is unable to load the firmware module.\n",
	G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
	G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
	rc = sc->flags & FW_OK ? 0 : ENOENT;
	goto done;
	}
	k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver);
	if (k != d) {
	MPASS(t4_fw_install > 0);
	device_printf(sc->dev,
	"firmware in KLD (%u.%u.%u.%u) is not what the driver was "
	"expecting (%u.%u.%u.%u) and will not be used.\n",
	G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
	G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k),
	G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
	G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));
	rc = sc->flags & FW_OK ? 0 : EINVAL;
	goto done;
	}

	device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
	"installing firmware %u.%u.%u.%u on card.\n",
	G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
	G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
	G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
	G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));

	rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
	if (rc != 0) {
	device_printf(sc->dev, "failed to install firmware: %d\n", rc);
	} else {
	/* Installed successfully, update the cached header too. */
	rc = ERESTART;
	memcpy(card_fw, fw->data, sizeof(*card_fw));
	}
	done:
	unload_fw_module(sc, cfg, fw);

	return (rc);
	}

	/*
	* Establish contact with the firmware and attempt to become the master driver.
	*
	* A firmware will be installed to the card if needed (if the driver is allowed
	* to do so).
	*/
	static int
	contact_firmware(struct adapter *sc)
	{
	int rc, already = 0;
	enum dev_state state;
	struct fw_info *fw_info;
	struct fw_hdr card_fw; / fw on the card */
	const struct fw_h *drv_fw;

	fw_info = find_fw_info(chip_id(sc));
	if (fw_info == NULL) {
	device_printf(sc->dev,
	"unable to look up firmware information for chip %d.\n",
	chip_id(sc));
	return (EINVAL);
	}
	drv_fw = &fw_info->fw_h;

	/* Read the header of the firmware on the card */
	card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO \| M_WAITOK);
	restart:
	rc = -t4_get_fw_hdr(sc, card_fw);
	if (rc != 0) {
	device_printf(sc->dev,
	"unable to read firmware header from card's flash: %d\n",
	rc);
	goto done;
	}

	rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, NULL,
	&already);
	if (rc == ERESTART)
	goto restart;
	if (rc != 0)
	goto done;

	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
	if (rc < 0 \|\| state == DEV_STATE_ERR) {
	rc = -rc;
	device_printf(sc->dev,
	"failed to connect to the firmware: %d, %d. "
	"PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
	#if 0
	if (install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw,
	"not responding properly to HELLO", &already) == ERESTART)
	goto restart;
	#endif
	goto done;
	}
	MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT);
	sc->flags \|= FW_OK; /* The firmware responded to the FW_HELLO. */

	if (rc == sc->pf) {
	sc->flags \|= MASTER_PF;
	rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw,
	NULL, &already);
	if (rc == ERESTART)
	rc = 0;
	else if (rc != 0)
	goto done;
	} else if (state == DEV_STATE_UNINIT) {
	/*
	* We didn't get to be the master so we definitely won't be
	* configuring the chip. It's a bug if someone else hasn't
	* configured it already.
	*/
	device_printf(sc->dev, "couldn't be master(%d), "
	"device not already initialized either(%d). "
	"PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
	rc = EPROTO;
	goto done;
	} else {
	/*
	* Some other PF is the master and has configured the chip.
	* This is allowed but untested.
	*/
	device_printf(sc->dev, "PF%d is master, device state %d. "
	"PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
	snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc);
	sc->cfcsum = 0;
	rc = 0;
	}
	done:
	if (rc != 0 && sc->flags & FW_OK) {
	t4_fw_bye(sc, sc->mbox);
	sc->flags &= ~FW_OK;
	}
	free(card_fw, M_CXGBE);
	return (rc);
	}

	static int
	copy_cfg_file_to_card(struct adapter sc, char cfg_file,
	uint32_t mtype, uint32_t moff)
	{
	struct fw_info *fw_info;
	const struct firmware dcfg, rcfg = NULL;
	const uint32_t *cfdata;
	uint32_t cflen, addr;
	int rc;

	load_fw_module(sc, &dcfg, NULL);

	/* Card specific interpretation of "default". */
	if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
	if (pci_get_device(sc->dev) == 0x440a)
	snprintf(cfg_file, sizeof(t4_cfg_file), UWIRE_CF);
	if (is_fpga(sc))
	snprintf(cfg_file, sizeof(t4_cfg_file), FPGA_CF);
	}

	if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
	if (dcfg == NULL) {
	device_printf(sc->dev,
	"KLD with default config is not available.\n");
	rc = ENOENT;
	goto done;
	}
	cfdata = dcfg->data;
	cflen = dcfg->datasize & ~3;
	} else {
	char s[32];

	fw_info = find_fw_info(chip_id(sc));
	if (fw_info == NULL) {
	device_printf(sc->dev,
	"unable to look up firmware information for chip %d.\n",
	chip_id(sc));
	rc = EINVAL;
	goto done;
	}
	snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file);

	rcfg = firmware_get(s);
	if (rcfg == NULL) {
	device_printf(sc->dev,
	"unable to load module \"%s\" for configuration "
	"profile \"%s\".\n", s, cfg_file);
	rc = ENOENT;
	goto done;
	}
	cfdata = rcfg->data;
	cflen = rcfg->datasize & ~3;
	}

	if (cflen > FLASH_CFG_MAX_SIZE) {
	device_printf(sc->dev,
	"config file too long (%d, max allowed is %d).\n",
	cflen, FLASH_CFG_MAX_SIZE);
	rc = EINVAL;
	goto done;
	}

	rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
	if (rc != 0) {
	device_printf(sc->dev,
	"%s: addr (%d/0x%x) or len %d is not valid: %d.\n",
	__func__, mtype, moff, cflen, rc);
	rc = EINVAL;
	goto done;
	}
	write_via_memwin(sc, 2, addr, cfdata, cflen);
	done:
	if (rcfg != NULL)
	firmware_put(rcfg, FIRMWARE_UNLOAD);
	unload_fw_module(sc, dcfg, NULL);
	return (rc);
	}

	struct caps_allowed {
	uint16_t nbmcaps;
	uint16_t linkcaps;
	uint16_t switchcaps;
	uint16_t niccaps;
	uint16_t toecaps;
	uint16_t rdmacaps;
	uint16_t cryptocaps;
	uint16_t iscsicaps;
	uint16_t fcoecaps;
	};

	#define FW_PARAM_DEV(param) \
	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \| \
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
	#define FW_PARAM_PFVF(param) \
	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) \| \
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))

	/*
	* Provide a configuration profile to the firmware and have it initialize the
	* chip accordingly. This may involve uploading a configuration file to the
	* card.
	*/
	static int
	apply_cfg_and_initialize(struct adapter sc, char cfg_file,
	const struct caps_allowed *caps_allowed)
	{
	int rc;
	struct fw_caps_config_cmd caps;
	uint32_t mtype, moff, finicsum, cfcsum, param, val;

	rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE \| F_PIORST);
	if (rc != 0) {
	device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
	return (rc);
	}

	bzero(&caps, sizeof(caps));
	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) \|
	F_FW_CMD_REQUEST \| F_FW_CMD_READ);
	if (strncmp(cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0) {
	mtype = 0;
	moff = 0;
	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
	} else if (strncmp(cfg_file, FLASH_CF, sizeof(t4_cfg_file)) == 0) {
	mtype = FW_MEMTYPE_FLASH;
	moff = t4_flash_cfg_addr(sc);
	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID \|
	V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) \|
	V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) \|
	FW_LEN16(caps));
	} else {
	/*
	* Ask the firmware where it wants us to upload the config file.
	*/
	param = FW_PARAM_DEV(CF);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	if (rc != 0) {
	/* No support for config file? Shouldn't happen. */
	device_printf(sc->dev,
	"failed to query config file location: %d.\n", rc);
	goto done;
	}
	mtype = G_FW_PARAMS_PARAM_Y(val);
	moff = G_FW_PARAMS_PARAM_Z(val) << 16;
	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID \|
	V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) \|
	V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) \|
	FW_LEN16(caps));

	rc = copy_cfg_file_to_card(sc, cfg_file, mtype, moff);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to upload config file to card: %d.\n", rc);
	goto done;
	}
	}
	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
	if (rc != 0) {
	device_printf(sc->dev, "failed to pre-process config file: %d "
	"(mtype %d, moff 0x%x).\n", rc, mtype, moff);
	goto done;
	}

	finicsum = be32toh(caps.finicsum);
	cfcsum = be32toh(caps.cfcsum); /* actual */
	if (finicsum != cfcsum) {
	device_printf(sc->dev,
	"WARNING: config file checksum mismatch: %08x %08x\n",
	finicsum, cfcsum);
	}
	sc->cfcsum = cfcsum;
	snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file);

	/*
	* Let the firmware know what features will (not) be used so it can tune
	* things accordingly.
	*/
	#define LIMIT_CAPS(x) do { \
	caps.x##caps &= htobe16(caps_allowed->x##caps); \
	} while (0)
	LIMIT_CAPS(nbm);
	LIMIT_CAPS(link);
	LIMIT_CAPS(switch);
	LIMIT_CAPS(nic);
	LIMIT_CAPS(toe);
	LIMIT_CAPS(rdma);
	LIMIT_CAPS(crypto);
	LIMIT_CAPS(iscsi);
	LIMIT_CAPS(fcoe);
	#undef LIMIT_CAPS
	if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) {
	/*
	* TOE and hashfilters are mutually exclusive. It is a config
	* file or firmware bug if both are reported as available. Try
	* to cope with the situation in non-debug builds by disabling
	* TOE.
	*/
	MPASS(caps.toecaps == 0);

	caps.toecaps = 0;
	caps.rdmacaps = 0;
	caps.iscsicaps = 0;
	}

	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) \|
	F_FW_CMD_REQUEST \| F_FW_CMD_WRITE);
	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to process config file: %d.\n", rc);
	goto done;
	}

	t4_tweak_chip_settings(sc);
	set_params__pre_init(sc);

	/* get basic stuff going */
	rc = -t4_fw_initialize(sc, sc->mbox);
	if (rc != 0) {
	device_printf(sc->dev, "fw_initialize failed: %d.\n", rc);
	goto done;
	}
	done:
	return (rc);
	}

	/*
	* Partition chip resources for use between various PFs, VFs, etc.
	*/
	static int
	partition_resources(struct adapter *sc)
	{
	char cfg_file[sizeof(t4_cfg_file)];
	struct caps_allowed caps_allowed;
	int rc;
	bool fallback;

	/* Only the master driver gets to configure the chip resources. */
	MPASS(sc->flags & MASTER_PF);

	#define COPY_CAPS(x) do { \
	caps_allowed.x##caps = t4_##x##caps_allowed; \
	} while (0)
	bzero(&caps_allowed, sizeof(caps_allowed));
	COPY_CAPS(nbm);
	COPY_CAPS(link);
	COPY_CAPS(switch);
	COPY_CAPS(nic);
	COPY_CAPS(toe);
	COPY_CAPS(rdma);
	COPY_CAPS(crypto);
	COPY_CAPS(iscsi);
	COPY_CAPS(fcoe);
	fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true;
	snprintf(cfg_file, sizeof(cfg_file), "%s", t4_cfg_file);
	retry:
	rc = apply_cfg_and_initialize(sc, cfg_file, &caps_allowed);
	if (rc != 0 && fallback) {
	device_printf(sc->dev,
	"failed (%d) to configure card with \"%s\" profile, "
	"will fall back to a basic configuration and retry.\n",
	rc, cfg_file);
	snprintf(cfg_file, sizeof(cfg_file), "%s", BUILTIN_CF);
	bzero(&caps_allowed, sizeof(caps_allowed));
	COPY_CAPS(switch);
	caps_allowed.niccaps = FW_CAPS_CONFIG_NIC;
	fallback = false;
	goto retry;
	}
	#undef COPY_CAPS
	return (rc);
	}

	/*
	* Retrieve parameters that are needed (or nice to have) very early.
	*/
	static int
	get_params__pre_init(struct adapter *sc)
	{
	int rc;
	uint32_t param[2], val[2];

	t4_get_version_info(sc);

	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
	G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
	G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
	G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
	G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));

	snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
	G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
	G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
	G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
	G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));

	snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
	G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
	G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
	G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
	G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));

	snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
	G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
	G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
	G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
	G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));

	param[0] = FW_PARAM_DEV(PORTVEC);
	param[1] = FW_PARAM_DEV(CCLK);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query parameters (pre_init): %d.\n", rc);
	return (rc);
	}

	sc->params.portvec = val[0];
	sc->params.nports = bitcount32(val[0]);
	sc->params.vpd.cclk = val[1];

	/* Read device log parameters. */
	rc = -t4_init_devlog_params(sc, 1);
	if (rc == 0)
	fixup_devlog_params(sc);
	else {
	device_printf(sc->dev,
	"failed to get devlog parameters: %d.\n", rc);
	rc = 0; /* devlog isn't critical for device operation */
	}

	return (rc);
	}

	/*
	* Any params that need to be set before FW_INITIALIZE.
	*/
	static int
	set_params__pre_init(struct adapter *sc)
	{
	int rc = 0;
	uint32_t param, val;

	if (chip_id(sc) >= CHELSIO_T6) {
	param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT);
	val = 1;
	rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	/* firmwares < 1.20.1.0 do not have this param. */
	if (rc == FW_EINVAL &&
	sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) {
	rc = 0;
	}
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to enable high priority filters :%d.\n",
	rc);
	}
	}

	/* Enable opaque VIIDs with firmwares that support it. */
	param = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN);
	val = 1;
	rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	if (rc == 0 && val == 1)
	sc->params.viid_smt_extn_support = true;
	else
	sc->params.viid_smt_extn_support = false;

	return (rc);
	}

	/*
	* Retrieve various parameters that are of interest to the driver. The device
	* has been initialized by the firmware at this point.
	*/
	static int
	get_params__post_init(struct adapter *sc)
	{
	int rc;
	uint32_t param[7], val[7];
	struct fw_caps_config_cmd caps;

	param[0] = FW_PARAM_PFVF(IQFLINT_START);
	param[1] = FW_PARAM_PFVF(EQ_START);
	param[2] = FW_PARAM_PFVF(FILTER_START);
	param[3] = FW_PARAM_PFVF(FILTER_END);
	param[4] = FW_PARAM_PFVF(L2T_START);
	param[5] = FW_PARAM_PFVF(L2T_END);
	param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \|
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) \|
	V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query parameters (post_init): %d.\n", rc);
	return (rc);
	}

	sc->sge.iq_start = val[0];
	sc->sge.eq_start = val[1];
	if ((int)val[3] > (int)val[2]) {
	sc->tids.ftid_base = val[2];
	sc->tids.ftid_end = val[3];
	sc->tids.nftids = val[3] - val[2] + 1;
	}
	sc->vres.l2t.start = val[4];
	sc->vres.l2t.size = val[5] - val[4] + 1;
	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
	("%s: L2 table size (%u) larger than expected (%u)",
	__func__, sc->vres.l2t.size, L2T_SIZE));
	sc->params.core_vdd = val[6];

	if (chip_id(sc) >= CHELSIO_T6) {

	sc->tids.tid_base = t4_read_reg(sc,
	A_LE_DB_ACTIVE_TABLE_START_INDEX);

	param[0] = FW_PARAM_PFVF(HPFILTER_START);
	param[1] = FW_PARAM_PFVF(HPFILTER_END);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query hpfilter parameters: %d.\n", rc);
	return (rc);
	}
	if ((int)val[1] > (int)val[0]) {
	sc->tids.hpftid_base = val[0];
	sc->tids.hpftid_end = val[1];
	sc->tids.nhpftids = val[1] - val[0] + 1;

	/*
	* These should go off if the layout changes and the
	* driver needs to catch up.
	*/
	MPASS(sc->tids.hpftid_base == 0);
	MPASS(sc->tids.tid_base == sc->tids.nhpftids);
	}
	}

	/*
	* MPSBGMAP is queried separately because only recent firmwares support
	* it as a parameter and we don't want the compound query above to fail
	* on older firmwares.
	*/
	param[0] = FW_PARAM_DEV(MPSBGMAP);
	val[0] = 0;
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
	if (rc == 0)
	sc->params.mps_bg_map = val[0];
	else
	sc->params.mps_bg_map = 0;

	/*
	* Determine whether the firmware supports the filter2 work request.
	* This is queried separately for the same reason as MPSBGMAP above.
	*/
	param[0] = FW_PARAM_DEV(FILTER2_WR);
	val[0] = 0;
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
	if (rc == 0)
	sc->params.filter2_wr_support = val[0] != 0;
	else
	sc->params.filter2_wr_support = 0;

	/*
	* Find out whether we're allowed to use the ULPTX MEMWRITE DSGL.
	* This is queried separately for the same reason as other params above.
	*/
	param[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
	val[0] = 0;
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
	if (rc == 0)
	sc->params.ulptx_memwrite_dsgl = val[0] != 0;
	else
	sc->params.ulptx_memwrite_dsgl = false;

	/* FW_RI_FR_NSMR_TPTE_WR support */
	param[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
	if (rc == 0)
	sc->params.fr_nsmr_tpte_wr_support = val[0] != 0;
	else
	sc->params.fr_nsmr_tpte_wr_support = false;

	param[0] = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
	if (rc == 0)
	sc->params.max_pkts_per_eth_tx_pkts_wr = val[0];
	else
	sc->params.max_pkts_per_eth_tx_pkts_wr = 15;

	/* get capabilites */
	bzero(&caps, sizeof(caps));
	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) \|
	F_FW_CMD_REQUEST \| F_FW_CMD_READ);
	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to get card capabilities: %d.\n", rc);
	return (rc);
	}

	#define READ_CAPS(x) do { \
	sc->x = htobe16(caps.x); \
	} while (0)
	READ_CAPS(nbmcaps);
	READ_CAPS(linkcaps);
	READ_CAPS(switchcaps);
	READ_CAPS(niccaps);
	READ_CAPS(toecaps);
	READ_CAPS(rdmacaps);
	READ_CAPS(cryptocaps);
	READ_CAPS(iscsicaps);
	READ_CAPS(fcoecaps);

	if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) {
	MPASS(chip_id(sc) > CHELSIO_T4);
	MPASS(sc->toecaps == 0);
	sc->toecaps = 0;

	param[0] = FW_PARAM_DEV(NTID);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query HASHFILTER parameters: %d.\n", rc);
	return (rc);
	}
	sc->tids.ntids = val[0];
	if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
	MPASS(sc->tids.ntids >= sc->tids.nhpftids);
	sc->tids.ntids -= sc->tids.nhpftids;
	}
	sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
	sc->params.hash_filter = 1;
	}
	if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
	param[0] = FW_PARAM_PFVF(ETHOFLD_START);
	param[1] = FW_PARAM_PFVF(ETHOFLD_END);
	param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query NIC parameters: %d.\n", rc);
	return (rc);
	}
	if ((int)val[1] > (int)val[0]) {
	sc->tids.etid_base = val[0];
	sc->tids.etid_end = val[1];
	sc->tids.netids = val[1] - val[0] + 1;
	sc->params.eo_wr_cred = val[2];
	sc->params.ethoffload = 1;
	}
	}
	if (sc->toecaps) {
	/* query offload-related parameters */
	param[0] = FW_PARAM_DEV(NTID);
	param[1] = FW_PARAM_PFVF(SERVER_START);
	param[2] = FW_PARAM_PFVF(SERVER_END);
	param[3] = FW_PARAM_PFVF(TDDP_START);
	param[4] = FW_PARAM_PFVF(TDDP_END);
	param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query TOE parameters: %d.\n", rc);
	return (rc);
	}
	sc->tids.ntids = val[0];
	if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
	MPASS(sc->tids.ntids >= sc->tids.nhpftids);
	sc->tids.ntids -= sc->tids.nhpftids;
	}
	sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
	if ((int)val[2] > (int)val[1]) {
	sc->tids.stid_base = val[1];
	sc->tids.nstids = val[2] - val[1] + 1;
	}
	sc->vres.ddp.start = val[3];
	sc->vres.ddp.size = val[4] - val[3] + 1;
	sc->params.ofldq_wr_cred = val[5];
	sc->params.offload = 1;
	} else {
	/*
	* The firmware attempts memfree TOE configuration for -SO cards
	* and will report toecaps=0 if it runs out of resources (this
	* depends on the config file). It may not report 0 for other
	* capabilities dependent on the TOE in this case. Set them to
	* 0 here so that the driver doesn't bother tracking resources
	* that will never be used.
	*/
	sc->iscsicaps = 0;
	sc->rdmacaps = 0;
	}
	if (sc->rdmacaps) {
	param[0] = FW_PARAM_PFVF(STAG_START);
	param[1] = FW_PARAM_PFVF(STAG_END);
	param[2] = FW_PARAM_PFVF(RQ_START);
	param[3] = FW_PARAM_PFVF(RQ_END);
	param[4] = FW_PARAM_PFVF(PBL_START);
	param[5] = FW_PARAM_PFVF(PBL_END);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query RDMA parameters(1): %d.\n", rc);
	return (rc);
	}
	sc->vres.stag.start = val[0];
	sc->vres.stag.size = val[1] - val[0] + 1;
	sc->vres.rq.start = val[2];
	sc->vres.rq.size = val[3] - val[2] + 1;
	sc->vres.pbl.start = val[4];
	sc->vres.pbl.size = val[5] - val[4] + 1;

	param[0] = FW_PARAM_PFVF(SQRQ_START);
	param[1] = FW_PARAM_PFVF(SQRQ_END);
	param[2] = FW_PARAM_PFVF(CQ_START);
	param[3] = FW_PARAM_PFVF(CQ_END);
	param[4] = FW_PARAM_PFVF(OCQ_START);
	param[5] = FW_PARAM_PFVF(OCQ_END);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query RDMA parameters(2): %d.\n", rc);
	return (rc);
	}
	sc->vres.qp.start = val[0];
	sc->vres.qp.size = val[1] - val[0] + 1;
	sc->vres.cq.start = val[2];
	sc->vres.cq.size = val[3] - val[2] + 1;
	sc->vres.ocq.start = val[4];
	sc->vres.ocq.size = val[5] - val[4] + 1;

	param[0] = FW_PARAM_PFVF(SRQ_START);
	param[1] = FW_PARAM_PFVF(SRQ_END);
	param[2] = FW_PARAM_DEV(MAXORDIRD_QP);
	param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query RDMA parameters(3): %d.\n", rc);
	return (rc);
	}
	sc->vres.srq.start = val[0];
	sc->vres.srq.size = val[1] - val[0] + 1;
	sc->params.max_ordird_qp = val[2];
	sc->params.max_ird_adapter = val[3];
	}
	if (sc->iscsicaps) {
	param[0] = FW_PARAM_PFVF(ISCSI_START);
	param[1] = FW_PARAM_PFVF(ISCSI_END);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query iSCSI parameters: %d.\n", rc);
	return (rc);
	}
	sc->vres.iscsi.start = val[0];
	sc->vres.iscsi.size = val[1] - val[0] + 1;
	}
	if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
	param[0] = FW_PARAM_PFVF(TLS_START);
	param[1] = FW_PARAM_PFVF(TLS_END);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to query TLS parameters: %d.\n", rc);
	return (rc);
	}
	sc->vres.key.start = val[0];
	sc->vres.key.size = val[1] - val[0] + 1;
	}

	t4_init_sge_params(sc);

	/*
	* We've got the params we wanted to query via the firmware. Now grab
	* some others directly from the chip.
	*/
	rc = t4_read_chip_settings(sc);

	return (rc);
	}

	#ifdef KERN_TLS
	static void
	ktls_tick(void *arg)
	{
	struct adapter *sc;
	uint32_t tstamp;

	sc = arg;

	tstamp = tcp_ts_getticks();
	t4_write_reg(sc, A_TP_SYNC_TIME_HI, tstamp >> 1);
	t4_write_reg(sc, A_TP_SYNC_TIME_LO, tstamp << 31);

	callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK);
	}

	static void
	t4_enable_kern_tls(struct adapter *sc)
	{
	uint32_t m, v;

	m = F_ENABLECBYP;
	v = F_ENABLECBYP;
	t4_set_reg_field(sc, A_TP_PARA_REG6, m, v);

	m = F_CPL_FLAGS_UPDATE_EN \| F_SEQ_UPDATE_EN;
	v = F_CPL_FLAGS_UPDATE_EN \| F_SEQ_UPDATE_EN;
	t4_set_reg_field(sc, A_ULP_TX_CONFIG, m, v);

	m = F_NICMODE;
	v = F_NICMODE;
	t4_set_reg_field(sc, A_TP_IN_CONFIG, m, v);

	m = F_LOOKUPEVERYPKT;
	v = 0;
	t4_set_reg_field(sc, A_TP_INGRESS_CONFIG, m, v);

	m = F_TXDEFERENABLE \| F_DISABLEWINDOWPSH \| F_DISABLESEPPSHFLAG;
	v = F_DISABLEWINDOWPSH;
	t4_set_reg_field(sc, A_TP_PC_CONFIG, m, v);

	m = V_TIMESTAMPRESOLUTION(M_TIMESTAMPRESOLUTION);
	v = V_TIMESTAMPRESOLUTION(0x1f);
	t4_set_reg_field(sc, A_TP_TIMER_RESOLUTION, m, v);

	sc->flags \|= KERN_TLS_OK;

	sc->tlst.inline_keys = t4_tls_inline_keys;
	sc->tlst.combo_wrs = t4_tls_combo_wrs;
	}
	#endif

	static int
	set_params__post_init(struct adapter *sc)
	{
	uint32_t param, val;
	#ifdef TCP_OFFLOAD
	int i, v, shift;
	#endif

	/* ask for encapsulated CPLs */
	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
	val = 1;
	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);

	/* Enable 32b port caps if the firmware supports it. */
	param = FW_PARAM_PFVF(PORT_CAPS32);
	val = 1;
	if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val) == 0)
	sc->params.port_caps32 = 1;

	/* Let filter + maskhash steer to a part of the VI's RSS region. */
	val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
	t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER),
	V_MASKFILTER(val - 1));

	#ifdef TCP_OFFLOAD
	/*
	* Override the TOE timers with user provided tunables. This is not the
	* recommended way to change the timers (the firmware config file is) so
	* these tunables are not documented.
	*
	* All the timer tunables are in microseconds.
	*/
	if (t4_toe_keepalive_idle != 0) {
	v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle);
	v &= M_KEEPALIVEIDLE;
	t4_set_reg_field(sc, A_TP_KEEP_IDLE,
	V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v));
	}
	if (t4_toe_keepalive_interval != 0) {
	v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval);
	v &= M_KEEPALIVEINTVL;
	t4_set_reg_field(sc, A_TP_KEEP_INTVL,
	V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v));
	}
	if (t4_toe_keepalive_count != 0) {
	v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2;
	t4_set_reg_field(sc, A_TP_SHIFT_CNT,
	V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) \|
	V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2),
	V_KEEPALIVEMAXR1(1) \| V_KEEPALIVEMAXR2(v));
	}
	if (t4_toe_rexmt_min != 0) {
	v = us_to_tcp_ticks(sc, t4_toe_rexmt_min);
	v &= M_RXTMIN;
	t4_set_reg_field(sc, A_TP_RXT_MIN,
	V_RXTMIN(M_RXTMIN), V_RXTMIN(v));
	}
	if (t4_toe_rexmt_max != 0) {
	v = us_to_tcp_ticks(sc, t4_toe_rexmt_max);
	v &= M_RXTMAX;
	t4_set_reg_field(sc, A_TP_RXT_MAX,
	V_RXTMAX(M_RXTMAX), V_RXTMAX(v));
	}
	if (t4_toe_rexmt_count != 0) {
	v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2;
	t4_set_reg_field(sc, A_TP_SHIFT_CNT,
	V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) \|
	V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2),
	V_RXTSHIFTMAXR1(1) \| V_RXTSHIFTMAXR2(v));
	}
	for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) {
	if (t4_toe_rexmt_backoff[i] != -1) {
	v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0;
	shift = (i & 3) << 3;
	t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3),
	M_TIMERBACKOFFINDEX0 << shift, v << shift);
	}
	}
	#endif

	#ifdef KERN_TLS
	if (t4_kern_tls != 0 && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS &&
	sc->toecaps & FW_CAPS_CONFIG_TOE)
	t4_enable_kern_tls(sc);
	#endif
	return (0);
	}

	#undef FW_PARAM_PFVF
	#undef FW_PARAM_DEV

	static void
	t4_set_desc(struct adapter *sc)
	{
	char buf[128];
	struct adapter_params *p = &sc->params;

	snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id);

	device_set_desc_copy(sc->dev, buf);
	}

	static inline void
	ifmedia_add4(struct ifmedia *ifm, int m)
	{

	ifmedia_add(ifm, m, 0, NULL);
	ifmedia_add(ifm, m \| IFM_ETH_TXPAUSE, 0, NULL);
	ifmedia_add(ifm, m \| IFM_ETH_RXPAUSE, 0, NULL);
	ifmedia_add(ifm, m \| IFM_ETH_TXPAUSE \| IFM_ETH_RXPAUSE, 0, NULL);
	}

	/*
	* This is the selected media, which is not quite the same as the active media.
	* The media line in ifconfig is "media: Ethernet selected (active)" if selected
	* and active are not the same, and "media: Ethernet selected" otherwise.
	*/
	static void
	set_current_media(struct port_info *pi)
	{
	struct link_config *lc;
	struct ifmedia *ifm;
	int mword;
	u_int speed;

	PORT_LOCK_ASSERT_OWNED(pi);

	/* Leave current media alone if it's already set to IFM_NONE. */
	ifm = &pi->media;
	if (ifm->ifm_cur != NULL &&
	IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE)
	return;

	lc = &pi->link_cfg;
	if (lc->requested_aneg != AUTONEG_DISABLE &&
	lc->pcaps & FW_PORT_CAP32_ANEG) {
	ifmedia_set(ifm, IFM_ETHER \| IFM_AUTO);
	return;
	}
	mword = IFM_ETHER \| IFM_FDX;
	if (lc->requested_fc & PAUSE_TX)
	mword \|= IFM_ETH_TXPAUSE;
	if (lc->requested_fc & PAUSE_RX)
	mword \|= IFM_ETH_RXPAUSE;
	if (lc->requested_speed == 0)
	speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */
	else
	speed = lc->requested_speed;
	mword \|= port_mword(pi, speed_to_fwcap(speed));
	ifmedia_set(ifm, mword);
	}

	/*
	* Returns true if the ifmedia list for the port cannot change.
	*/
	static bool
	fixed_ifmedia(struct port_info *pi)
	{

	return (pi->port_type == FW_PORT_TYPE_BT_SGMII \|\|
	pi->port_type == FW_PORT_TYPE_BT_XFI \|\|
	pi->port_type == FW_PORT_TYPE_BT_XAUI \|\|
	pi->port_type == FW_PORT_TYPE_KX4 \|\|
	pi->port_type == FW_PORT_TYPE_KX \|\|
	pi->port_type == FW_PORT_TYPE_KR \|\|
	pi->port_type == FW_PORT_TYPE_BP_AP \|\|
	pi->port_type == FW_PORT_TYPE_BP4_AP \|\|
	pi->port_type == FW_PORT_TYPE_BP40_BA \|\|
	pi->port_type == FW_PORT_TYPE_KR4_100G \|\|
	pi->port_type == FW_PORT_TYPE_KR_SFP28 \|\|
	pi->port_type == FW_PORT_TYPE_KR_XLAUI);
	}

	static void
	build_medialist(struct port_info *pi)
	{
	uint32_t ss, speed;
	int unknown, mword, bit;
	struct link_config *lc;
	struct ifmedia *ifm;

	PORT_LOCK_ASSERT_OWNED(pi);

	if (pi->flags & FIXED_IFMEDIA)
	return;

	/*
	* Rebuild the ifmedia list.
	*/
	ifm = &pi->media;
	ifmedia_removeall(ifm);
	lc = &pi->link_cfg;
	ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */
	if (__predict_false(ss == 0)) { /* not supposed to happen. */
	MPASS(ss != 0);
	no_media:
	MPASS(LIST_EMPTY(&ifm->ifm_list));
	ifmedia_add(ifm, IFM_ETHER \| IFM_NONE, 0, NULL);
	ifmedia_set(ifm, IFM_ETHER \| IFM_NONE);
	return;
	}

	unknown = 0;
	for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) {
	speed = 1 << bit;
	MPASS(speed & M_FW_PORT_CAP32_SPEED);
	if (ss & speed) {
	mword = port_mword(pi, speed);
	if (mword == IFM_NONE) {
	goto no_media;
	} else if (mword == IFM_UNKNOWN)
	unknown++;
	else
	ifmedia_add4(ifm, IFM_ETHER \| IFM_FDX \| mword);
	}
	}
	if (unknown > 0) /* Add one unknown for all unknown media types. */
	ifmedia_add4(ifm, IFM_ETHER \| IFM_FDX \| IFM_UNKNOWN);
	if (lc->pcaps & FW_PORT_CAP32_ANEG)
	ifmedia_add(ifm, IFM_ETHER \| IFM_AUTO, 0, NULL);

	set_current_media(pi);
	}

	/*
	* Initialize the requested fields in the link config based on driver tunables.
	*/
	static void
	init_link_config(struct port_info *pi)
	{
	struct link_config *lc = &pi->link_cfg;

	PORT_LOCK_ASSERT_OWNED(pi);

	lc->requested_speed = 0;

	if (t4_autoneg == 0)
	lc->requested_aneg = AUTONEG_DISABLE;
	else if (t4_autoneg == 1)
	lc->requested_aneg = AUTONEG_ENABLE;
	else
	lc->requested_aneg = AUTONEG_AUTO;

	lc->requested_fc = t4_pause_settings & (PAUSE_TX \| PAUSE_RX \|
	PAUSE_AUTONEG);

	if (t4_fec & FEC_AUTO)
	lc->requested_fec = FEC_AUTO;
	else if (t4_fec == 0)
	lc->requested_fec = FEC_NONE;
	else {
	/* -1 is handled by the FEC_AUTO block above and not here. */
	lc->requested_fec = t4_fec &
	(FEC_RS \| FEC_BASER_RS \| FEC_NONE \| FEC_MODULE);
	if (lc->requested_fec == 0)
	lc->requested_fec = FEC_AUTO;
	}
	}

	/*
	* Makes sure that all requested settings comply with what's supported by the
	* port. Returns the number of settings that were invalid and had to be fixed.
	*/
	static int
	fixup_link_config(struct port_info *pi)
	{
	int n = 0;
	struct link_config *lc = &pi->link_cfg;
	uint32_t fwspeed;

	PORT_LOCK_ASSERT_OWNED(pi);

	/* Speed (when not autonegotiating) */
	if (lc->requested_speed != 0) {
	fwspeed = speed_to_fwcap(lc->requested_speed);
	if ((fwspeed & lc->pcaps) == 0) {
	n++;
	lc->requested_speed = 0;
	}
	}

	/* Link autonegotiation */
	MPASS(lc->requested_aneg == AUTONEG_ENABLE \|\|
	lc->requested_aneg == AUTONEG_DISABLE \|\|
	lc->requested_aneg == AUTONEG_AUTO);
	if (lc->requested_aneg == AUTONEG_ENABLE &&
	!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
	n++;
	lc->requested_aneg = AUTONEG_AUTO;
	}

	/* Flow control */
	MPASS((lc->requested_fc & ~(PAUSE_TX \| PAUSE_RX \| PAUSE_AUTONEG)) == 0);
	if (lc->requested_fc & PAUSE_TX &&
	!(lc->pcaps & FW_PORT_CAP32_FC_TX)) {
	n++;
	lc->requested_fc &= ~PAUSE_TX;
	}
	if (lc->requested_fc & PAUSE_RX &&
	!(lc->pcaps & FW_PORT_CAP32_FC_RX)) {
	n++;
	lc->requested_fc &= ~PAUSE_RX;
	}
	if (!(lc->requested_fc & PAUSE_AUTONEG) &&
	!(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) {
	n++;
	lc->requested_fc \|= PAUSE_AUTONEG;
	}

	/* FEC */
	if ((lc->requested_fec & FEC_RS &&
	!(lc->pcaps & FW_PORT_CAP32_FEC_RS)) \|\|
	(lc->requested_fec & FEC_BASER_RS &&
	!(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) {
	n++;
	lc->requested_fec = FEC_AUTO;
	}

	return (n);
	}

	/*
	* Apply the requested L1 settings, which are expected to be valid, to the
	* hardware.
	*/
	static int
	apply_link_config(struct port_info *pi)
	{
	struct adapter *sc = pi->adapter;
	struct link_config *lc = &pi->link_cfg;
	int rc;

	#ifdef INVARIANTS
	ASSERT_SYNCHRONIZED_OP(sc);
	PORT_LOCK_ASSERT_OWNED(pi);

	if (lc->requested_aneg == AUTONEG_ENABLE)
	MPASS(lc->pcaps & FW_PORT_CAP32_ANEG);
	if (!(lc->requested_fc & PAUSE_AUTONEG))
	MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE);
	if (lc->requested_fc & PAUSE_TX)
	MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX);
	if (lc->requested_fc & PAUSE_RX)
	MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX);
	if (lc->requested_fec & FEC_RS)
	MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS);
	if (lc->requested_fec & FEC_BASER_RS)
	MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS);
	#endif
	rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
	if (rc != 0) {
	/* Don't complain if the VF driver gets back an EPERM. */
	if (!(sc->flags & IS_VF) \|\| rc != FW_EPERM)
	device_printf(pi->dev, "l1cfg failed: %d\n", rc);
	} else {
	/*
	* An L1_CFG will almost always result in a link-change event if
	* the link is up, and the driver will refresh the actual
	* fec/fc/etc. when the notification is processed. If the link
	* is down then the actual settings are meaningless.
	*
	* This takes care of the case where a change in the L1 settings
	* may not result in a notification.
	*/
	if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG))
	lc->fc = lc->requested_fc & (PAUSE_TX \| PAUSE_RX);
	}
	return (rc);
	}

	#define FW_MAC_EXACT_CHUNK 7
	struct mcaddr_ctx {
	struct ifnet *ifp;
	const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
	uint64_t hash;
	int i;
	int del;
	int rc;
	};

	static u_int
	add_maddr(void arg, struct sockaddr_dl sdl, u_int cnt)
	{
	struct mcaddr_ctx *ctx = arg;
	struct vi_info *vi = ctx->ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;

	if (ctx->rc < 0)
	return (0);

	ctx->mcaddr[ctx->i] = LLADDR(sdl);
	MPASS(ETHER_IS_MULTICAST(ctx->mcaddr[ctx->i]));
	ctx->i++;

	if (ctx->i == FW_MAC_EXACT_CHUNK) {
	ctx->rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx->del,
	ctx->i, ctx->mcaddr, NULL, &ctx->hash, 0);
	if (ctx->rc < 0) {
	int j;

	for (j = 0; j < ctx->i; j++) {
	if_printf(ctx->ifp,
	"failed to add mc address"
	" %02x:%02x:%02x:"
	"%02x:%02x:%02x rc=%d\n",
	ctx->mcaddr[j][0], ctx->mcaddr[j][1],
	ctx->mcaddr[j][2], ctx->mcaddr[j][3],
	ctx->mcaddr[j][4], ctx->mcaddr[j][5],
	-ctx->rc);
	}
	return (0);
	}
	ctx->del = 0;
	ctx->i = 0;
	}

	return (1);
	}

	/*
	* Program the port's XGMAC based on parameters in ifnet. The caller also
	* indicates which parameters should be programmed (the rest are left alone).
	*/
	int
	update_mac_settings(struct ifnet *ifp, int flags)
	{
	int rc = 0;
	struct vi_info *vi = ifp->if_softc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;

	ASSERT_SYNCHRONIZED_OP(sc);
	KASSERT(flags, ("%s: not told what to update.", __func__));

	if (flags & XGMAC_MTU)
	mtu = ifp->if_mtu;

	if (flags & XGMAC_PROMISC)
	promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;

	if (flags & XGMAC_ALLMULTI)
	allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;

	if (flags & XGMAC_VLANEX)
	vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;

	if (flags & (XGMAC_MTU\|XGMAC_PROMISC\|XGMAC_ALLMULTI\|XGMAC_VLANEX)) {
	rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
	allmulti, 1, vlanex, false);
	if (rc) {
	if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
	rc);
	return (rc);
	}
	}

	if (flags & XGMAC_UCADDR) {
	uint8_t ucaddr[ETHER_ADDR_LEN];

	bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
	rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
	ucaddr, true, &vi->smt_idx);
	if (rc < 0) {
	rc = -rc;
	if_printf(ifp, "change_mac failed: %d\n", rc);
	return (rc);
	} else {
	vi->xact_addr_filt = rc;
	rc = 0;
	}
	}

	if (flags & XGMAC_MCADDRS) {
	struct epoch_tracker et;
	struct mcaddr_ctx ctx;
	int j;

	ctx.ifp = ifp;
	ctx.hash = 0;
	ctx.i = 0;
	ctx.del = 1;
	ctx.rc = 0;
	/*
	* Unlike other drivers, we accumulate list of pointers into
	* interface address lists and we need to keep it safe even
	* after if_foreach_llmaddr() returns, thus we must enter the
	* network epoch.
	*/
	NET_EPOCH_ENTER(et);
	if_foreach_llmaddr(ifp, add_maddr, &ctx);
	if (ctx.rc < 0) {
	NET_EPOCH_EXIT(et);
	rc = -ctx.rc;
	return (rc);
	}
	if (ctx.i > 0) {
	rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
	ctx.del, ctx.i, ctx.mcaddr, NULL, &ctx.hash, 0);
	NET_EPOCH_EXIT(et);
	if (rc < 0) {
	rc = -rc;
	for (j = 0; j < ctx.i; j++) {
	if_printf(ifp,
	"failed to add mc address"
	" %02x:%02x:%02x:"
	"%02x:%02x:%02x rc=%d\n",
	ctx.mcaddr[j][0], ctx.mcaddr[j][1],
	ctx.mcaddr[j][2], ctx.mcaddr[j][3],
	ctx.mcaddr[j][4], ctx.mcaddr[j][5],
	rc);
	}
	return (rc);
	}
	} else
	NET_EPOCH_EXIT(et);

	rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, ctx.hash, 0);
	if (rc != 0)
	if_printf(ifp, "failed to set mc address hash: %d", rc);
	}

	return (rc);
	}

	/*
	* {begin\|end}_synchronized_op must be called from the same thread.
	*/
	int
	begin_synchronized_op(struct adapter sc, struct vi_info vi, int flags,
	char *wmesg)
	{
	int rc, pri;

	#ifdef WITNESS
	/* the caller thinks it's ok to sleep, but is it really? */
	if (flags & SLEEP_OK)
	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
	"begin_synchronized_op");
	#endif

	if (INTR_OK)
	pri = PCATCH;
	else
	pri = 0;

	ADAPTER_LOCK(sc);
	for (;;) {

	if (vi && IS_DOOMED(vi)) {
	rc = ENXIO;
	goto done;
	}

	if (!IS_BUSY(sc)) {
	rc = 0;
	break;
	}

	if (!(flags & SLEEP_OK)) {
	rc = EBUSY;
	goto done;
	}

	if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
	rc = EINTR;
	goto done;
	}
	}

	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
	SET_BUSY(sc);
	#ifdef INVARIANTS
	sc->last_op = wmesg;
	sc->last_op_thr = curthread;
	sc->last_op_flags = flags;
	#endif

	done:
	if (!(flags & HOLD_LOCK) \|\| rc)
	ADAPTER_UNLOCK(sc);

	return (rc);
	}

	/*
	* Tell if_ioctl and if_init that the VI is going away. This is
	* special variant of begin_synchronized_op and must be paired with a
	* call to end_synchronized_op.
	*/
	void
	doom_vi(struct adapter sc, struct vi_info vi)
	{

	ADAPTER_LOCK(sc);
	SET_DOOMED(vi);
	wakeup(&sc->flags);
	while (IS_BUSY(sc))
	mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
	SET_BUSY(sc);
	#ifdef INVARIANTS
	sc->last_op = "t4detach";
	sc->last_op_thr = curthread;
	sc->last_op_flags = 0;
	#endif
	ADAPTER_UNLOCK(sc);
	}

	/*
	* {begin\|end}_synchronized_op must be called from the same thread.
	*/
	void
	end_synchronized_op(struct adapter *sc, int flags)
	{

	if (flags & LOCK_HELD)
	ADAPTER_LOCK_ASSERT_OWNED(sc);
	else
	ADAPTER_LOCK(sc);

	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
	CLR_BUSY(sc);
	wakeup(&sc->flags);
	ADAPTER_UNLOCK(sc);
	}

	static int
	cxgbe_init_synchronized(struct vi_info *vi)
	{
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	struct ifnet *ifp = vi->ifp;
	int rc = 0, i;
	struct sge_txq *txq;

	ASSERT_SYNCHRONIZED_OP(sc);

	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
	return (0); /* already running */

	if (!(sc->flags & FULL_INIT_DONE) &&
	((rc = adapter_full_init(sc)) != 0))
	return (rc); /* error message displayed already */

	if (!(vi->flags & VI_INIT_DONE) &&
	((rc = vi_full_init(vi)) != 0))
	return (rc); /* error message displayed already */

	rc = update_mac_settings(ifp, XGMAC_ALL);
	if (rc)
	goto done; /* error message displayed already */

	PORT_LOCK(pi);
	if (pi->up_vis == 0) {
	t4_update_port_info(pi);
	fixup_link_config(pi);
	build_medialist(pi);
	apply_link_config(pi);
	}

	rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
	if (rc != 0) {
	if_printf(ifp, "enable_vi failed: %d\n", rc);
	PORT_UNLOCK(pi);
	goto done;
	}

	/*
	* Can't fail from this point onwards. Review cxgbe_uninit_synchronized
	* if this changes.
	*/

	for_each_txq(vi, i, txq) {
	TXQ_LOCK(txq);
	txq->eq.flags \|= EQ_ENABLED;
	TXQ_UNLOCK(txq);
	}

	/*
	* The first iq of the first port to come up is used for tracing.
	*/
	if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
	sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
	t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
	A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) \|
	V_QUEUENUMBER(sc->traceq));
	pi->flags \|= HAS_TRACEQ;
	}

	/* all ok */
	pi->up_vis++;
	ifp->if_drv_flags \|= IFF_DRV_RUNNING;

	if (pi->nvi > 1 \|\| sc->flags & IS_VF)
	callout_reset(&vi->tick, hz, vi_tick, vi);
	else
	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
	if (pi->link_cfg.link_ok)
	t4_os_link_changed(pi);
	PORT_UNLOCK(pi);
	done:
	if (rc != 0)
	cxgbe_uninit_synchronized(vi);

	return (rc);
	}

	/*
	* Idempotent.
	*/
	static int
	cxgbe_uninit_synchronized(struct vi_info *vi)
	{
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	struct ifnet *ifp = vi->ifp;
	int rc, i;
	struct sge_txq *txq;

	ASSERT_SYNCHRONIZED_OP(sc);

	if (!(vi->flags & VI_INIT_DONE)) {
	if (__predict_false(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
	KASSERT(0, ("uninited VI is running"));
	if_printf(ifp, "uninited VI with running ifnet. "
	"vi->flags 0x%016lx, if_flags 0x%08x, "
	"if_drv_flags 0x%08x\n", vi->flags, ifp->if_flags,
	ifp->if_drv_flags);
	}
	return (0);
	}

	/*
	* Disable the VI so that all its data in either direction is discarded
	* by the MPS. Leave everything else (the queues, interrupts, and 1Hz
	* tick) intact as the TP can deliver negative advice or data that it's
	* holding in its RAM (for an offloaded connection) even after the VI is
	* disabled.
	*/
	rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
	if (rc) {
	if_printf(ifp, "disable_vi failed: %d\n", rc);
	return (rc);
	}

	for_each_txq(vi, i, txq) {
	TXQ_LOCK(txq);
	txq->eq.flags &= ~EQ_ENABLED;
	TXQ_UNLOCK(txq);
	}

	PORT_LOCK(pi);
	if (pi->nvi > 1 \|\| sc->flags & IS_VF)
	callout_stop(&vi->tick);
	else
	callout_stop(&pi->tick);
	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
	PORT_UNLOCK(pi);
	return (0);
	}
	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
	pi->up_vis--;
	if (pi->up_vis > 0) {
	PORT_UNLOCK(pi);
	return (0);
	}

	pi->link_cfg.link_ok = false;
	pi->link_cfg.speed = 0;
	pi->link_cfg.link_down_rc = 255;
	t4_os_link_changed(pi);
	PORT_UNLOCK(pi);

	return (0);
	}

	/*
	* It is ok for this function to fail midway and return right away. t4_detach
	* will walk the entire sc->irq list and clean up whatever is valid.
	*/
	int
	t4_setup_intr_handlers(struct adapter *sc)
	{
	int rc, rid, p, q, v;
	char s[8];
	struct irq *irq;
	struct port_info *pi;
	struct vi_info *vi;
	struct sge *sge = &sc->sge;
	struct sge_rxq *rxq;
	#ifdef TCP_OFFLOAD
	struct sge_ofld_rxq *ofld_rxq;
	#endif
	#ifdef DEV_NETMAP
	struct sge_nm_rxq *nm_rxq;
	#endif
	#ifdef RSS
	int nbuckets = rss_getnumbuckets();
	#endif

	/*
	* Setup interrupts.
	*/
	irq = &sc->irq[0];
	rid = sc->intr_type == INTR_INTX ? 0 : 1;
	if (forwarding_intr_to_fwq(sc))
	return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));

	/* Multiple interrupts. */
	if (sc->flags & IS_VF)
	KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
	("%s: too few intr.", __func__));
	else
	KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
	("%s: too few intr.", __func__));

	/* The first one is always error intr on PFs */
	if (!(sc->flags & IS_VF)) {
	rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
	if (rc != 0)
	return (rc);
	irq++;
	rid++;
	}

	/* The second one is always the firmware event queue (first on VFs) */
	rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
	if (rc != 0)
	return (rc);
	irq++;
	rid++;

	for_each_port(sc, p) {
	pi = sc->port[p];
	for_each_vi(pi, v, vi) {
	vi->first_intr = rid - 1;

	if (vi->nnmrxq > 0) {
	int n = max(vi->nrxq, vi->nnmrxq);

	rxq = &sge->rxq[vi->first_rxq];
	#ifdef DEV_NETMAP
	nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
	#endif
	for (q = 0; q < n; q++) {
	snprintf(s, sizeof(s), "%x%c%x", p,
	'a' + v, q);
	if (q < vi->nrxq)
	irq->rxq = rxq++;
	#ifdef DEV_NETMAP
	if (q < vi->nnmrxq)
	irq->nm_rxq = nm_rxq++;

	if (irq->nm_rxq != NULL &&
	irq->rxq == NULL) {
	/* Netmap rx only */
	rc = t4_alloc_irq(sc, irq, rid,
	t4_nm_intr, irq->nm_rxq, s);
	}
	if (irq->nm_rxq != NULL &&
	irq->rxq != NULL) {
	/* NIC and Netmap rx */
	rc = t4_alloc_irq(sc, irq, rid,
	t4_vi_intr, irq, s);
	}
	#endif
	if (irq->rxq != NULL &&
	irq->nm_rxq == NULL) {
	/* NIC rx only */
	rc = t4_alloc_irq(sc, irq, rid,
	t4_intr, irq->rxq, s);
	}
	if (rc != 0)
	return (rc);
	#ifdef RSS
	if (q < vi->nrxq) {
	bus_bind_intr(sc->dev, irq->res,
	rss_getcpu(q % nbuckets));
	}
	#endif
	irq++;
	rid++;
	vi->nintr++;
	}
	} else {
	for_each_rxq(vi, q, rxq) {
	snprintf(s, sizeof(s), "%x%c%x", p,
	'a' + v, q);
	rc = t4_alloc_irq(sc, irq, rid,
	t4_intr, rxq, s);
	if (rc != 0)
	return (rc);
	#ifdef RSS
	bus_bind_intr(sc->dev, irq->res,
	rss_getcpu(q % nbuckets));
	#endif
	irq++;
	rid++;
	vi->nintr++;
	}
	}
	#ifdef TCP_OFFLOAD
	for_each_ofld_rxq(vi, q, ofld_rxq) {
	snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q);
	rc = t4_alloc_irq(sc, irq, rid, t4_intr,
	ofld_rxq, s);
	if (rc != 0)
	return (rc);
	irq++;
	rid++;
	vi->nintr++;
	}
	#endif
	}
	}
	MPASS(irq == &sc->irq[sc->intr_count]);

	return (0);
	}

	int
	adapter_full_init(struct adapter *sc)
	{
	int rc, i;
	#ifdef RSS
	uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
	uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
	#endif

	ASSERT_SYNCHRONIZED_OP(sc);
	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
	("%s: FULL_INIT_DONE already", __func__));

	/*
	* queues that belong to the adapter (not any particular port).
	*/
	rc = t4_setup_adapter_queues(sc);
	if (rc != 0)
	goto done;

	for (i = 0; i < nitems(sc->tq); i++) {
	sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
	taskqueue_thread_enqueue, &sc->tq[i]);
	if (sc->tq[i] == NULL) {
	device_printf(sc->dev,
	"failed to allocate task queue %d\n", i);
	rc = ENOMEM;
	goto done;
	}
	taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
	device_get_nameunit(sc->dev), i);
	}
	#ifdef RSS
	MPASS(RSS_KEYSIZE == 40);
	rss_getkey((void *)&raw_rss_key[0]);
	for (i = 0; i < nitems(rss_key); i++) {
	rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
	}
	t4_write_rss_key(sc, &rss_key[0], -1, 1);
	#endif

	if (!(sc->flags & IS_VF))
	t4_intr_enable(sc);
	#ifdef KERN_TLS
	if (sc->flags & KERN_TLS_OK)
	callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc,
	C_HARDCLOCK);
	#endif
	sc->flags \|= FULL_INIT_DONE;
	done:
	if (rc != 0)
	adapter_full_uninit(sc);

	return (rc);
	}

	int
	adapter_full_uninit(struct adapter *sc)
	{
	int i;

	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);

	t4_teardown_adapter_queues(sc);

	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
	taskqueue_free(sc->tq[i]);
	sc->tq[i] = NULL;
	}

	sc->flags &= ~FULL_INIT_DONE;

	return (0);
	}

	#ifdef RSS
	#define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 \| \
	RSS_HASHTYPE_RSS_TCP_IPV4 \| RSS_HASHTYPE_RSS_IPV6 \| \
	RSS_HASHTYPE_RSS_TCP_IPV6 \| RSS_HASHTYPE_RSS_UDP_IPV4 \| \
	RSS_HASHTYPE_RSS_UDP_IPV6)

	/* Translates kernel hash types to hardware. */
	static int
	hashconfig_to_hashen(int hashconfig)
	{
	int hashen = 0;

	if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
	hashen \|= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
	if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
	hashen \|= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
	if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
	hashen \|= F_FW_RSS_VI_CONFIG_CMD_UDPEN \|
	F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
	}
	if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
	hashen \|= F_FW_RSS_VI_CONFIG_CMD_UDPEN \|
	F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
	}
	if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
	hashen \|= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
	if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
	hashen \|= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;

	return (hashen);
	}

	/* Translates hardware hash types to kernel. */
	static int
	hashen_to_hashconfig(int hashen)
	{
	int hashconfig = 0;

	if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
	/*
	* If UDP hashing was enabled it must have been enabled for
	* either IPv4 or IPv6 (inclusive or). Enabling UDP without
	* enabling any 4-tuple hash is nonsense configuration.
	*/
	MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN \|
	F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));

	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
	hashconfig \|= RSS_HASHTYPE_RSS_UDP_IPV4;
	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
	hashconfig \|= RSS_HASHTYPE_RSS_UDP_IPV6;
	}
	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
	hashconfig \|= RSS_HASHTYPE_RSS_TCP_IPV4;
	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
	hashconfig \|= RSS_HASHTYPE_RSS_TCP_IPV6;
	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
	hashconfig \|= RSS_HASHTYPE_RSS_IPV4;
	if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
	hashconfig \|= RSS_HASHTYPE_RSS_IPV6;

	return (hashconfig);
	}
	#endif

	int
	vi_full_init(struct vi_info *vi)
	{
	struct adapter *sc = vi->adapter;
	struct ifnet *ifp = vi->ifp;
	uint16_t *rss;
	struct sge_rxq *rxq;
	int rc, i, j;
	#ifdef RSS
	int nbuckets = rss_getnumbuckets();
	int hashconfig = rss_gethashconfig();
	int extra;
	#endif

	ASSERT_SYNCHRONIZED_OP(sc);
	KASSERT((vi->flags & VI_INIT_DONE) == 0,
	("%s: VI_INIT_DONE already", __func__));

	sysctl_ctx_init(&vi->ctx);
	vi->flags \|= VI_SYSCTL_CTX;

	/*
	* Allocate tx/rx/fl queues for this VI.
	*/
	rc = t4_setup_vi_queues(vi);
	if (rc != 0)
	goto done; /* error message displayed already */

	/*
	* Setup RSS for this VI. Save a copy of the RSS table for later use.
	*/
	if (vi->nrxq > vi->rss_size) {
	if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
	"some queues will never receive traffic.\n", vi->nrxq,
	vi->rss_size);
	} else if (vi->rss_size % vi->nrxq) {
	if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
	"expect uneven traffic distribution.\n", vi->nrxq,
	vi->rss_size);
	}
	#ifdef RSS
	if (vi->nrxq != nbuckets) {
	if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
	"performance will be impacted.\n", vi->nrxq, nbuckets);
	}
	#endif
	rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO \| M_WAITOK);
	for (i = 0; i < vi->rss_size;) {
	#ifdef RSS
	j = rss_get_indirection_to_bucket(i);
	j %= vi->nrxq;
	rxq = &sc->sge.rxq[vi->first_rxq + j];
	rss[i++] = rxq->iq.abs_id;
	#else
	for_each_rxq(vi, j, rxq) {
	rss[i++] = rxq->iq.abs_id;
	if (i == vi->rss_size)
	break;
	}
	#endif
	}

	rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
	vi->rss_size);
	if (rc != 0) {
	free(rss, M_CXGBE);
	if_printf(ifp, "rss_config failed: %d\n", rc);
	goto done;
	}

	#ifdef RSS
	vi->hashen = hashconfig_to_hashen(hashconfig);

	/*
	* We may have had to enable some hashes even though the global config
	* wants them disabled. This is a potential problem that must be
	* reported to the user.
	*/
	extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig;

	/*
	* If we consider only the supported hash types, then the enabled hashes
	* are a superset of the requested hashes. In other words, there cannot
	* be any supported hash that was requested but not enabled, but there
	* can be hashes that were not requested but had to be enabled.
	*/
	extra &= SUPPORTED_RSS_HASHTYPES;
	MPASS((extra & hashconfig) == 0);

	if (extra) {
	if_printf(ifp,
	"global RSS config (0x%x) cannot be accommodated.\n",
	hashconfig);
	}
	if (extra & RSS_HASHTYPE_RSS_IPV4)
	if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
	if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
	if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
	if (extra & RSS_HASHTYPE_RSS_IPV6)
	if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
	if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
	if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
	if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
	if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
	if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
	if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
	#else
	vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN \|
	F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN \|
	F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN \|
	F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN \| F_FW_RSS_VI_CONFIG_CMD_UDPEN;
	#endif
	rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, rss[0], 0, 0);
	if (rc != 0) {
	free(rss, M_CXGBE);
	if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
	goto done;
	}

	vi->rss = rss;
	vi->flags \|= VI_INIT_DONE;
	done:
	if (rc != 0)
	vi_full_uninit(vi);

	return (rc);
	}

	/*
	* Idempotent.
	*/
	int
	vi_full_uninit(struct vi_info *vi)
	{
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;
	int i;
	struct sge_rxq *rxq;
	struct sge_txq *txq;
	#ifdef TCP_OFFLOAD
	struct sge_ofld_rxq *ofld_rxq;
	#endif
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	struct sge_wrq *ofld_txq;
	#endif

	if (vi->flags & VI_INIT_DONE) {

	/* Need to quiesce queues. */

	/* XXX: Only for the first VI? */
	if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF))
	quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);

	for_each_txq(vi, i, txq) {
	quiesce_txq(sc, txq);
	}

	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	for_each_ofld_txq(vi, i, ofld_txq) {
	quiesce_wrq(sc, ofld_txq);
	}
	#endif

	for_each_rxq(vi, i, rxq) {
	quiesce_iq(sc, &rxq->iq);
	quiesce_fl(sc, &rxq->fl);
	}

	#ifdef TCP_OFFLOAD
	for_each_ofld_rxq(vi, i, ofld_rxq) {
	quiesce_iq(sc, &ofld_rxq->iq);
	quiesce_fl(sc, &ofld_rxq->fl);
	}
	#endif
	free(vi->rss, M_CXGBE);
	free(vi->nm_rss, M_CXGBE);
	}

	t4_teardown_vi_queues(vi);
	vi->flags &= ~VI_INIT_DONE;

	return (0);
	}

	static void
	quiesce_txq(struct adapter sc, struct sge_txq txq)
	{
	struct sge_eq *eq = &txq->eq;
	struct sge_qstat spg = (void )&eq->desc[eq->sidx];

	(void) sc; /* unused */

	#ifdef INVARIANTS
	TXQ_LOCK(txq);
	MPASS((eq->flags & EQ_ENABLED) == 0);
	TXQ_UNLOCK(txq);
	#endif

	/* Wait for the mp_ring to empty. */
	while (!mp_ring_is_idle(txq->r)) {
	mp_ring_check_drainage(txq->r, 4096);
	pause("rquiesce", 1);
	}

	/* Then wait for the hardware to finish. */
	while (spg->cidx != htobe16(eq->pidx))
	pause("equiesce", 1);

	/* Finally, wait for the driver to reclaim all descriptors. */
	while (eq->cidx != eq->pidx)
	pause("dquiesce", 1);
	}

	static void
	quiesce_wrq(struct adapter sc, struct sge_wrq wrq)
	{

	/* XXXTX */
	}

	static void
	quiesce_iq(struct adapter sc, struct sge_iq iq)
	{
	(void) sc; /* unused */

	/* Synchronize with the interrupt handler */
	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
	pause("iqfree", 1);
	}

	static void
	quiesce_fl(struct adapter sc, struct sge_fl fl)
	{
	mtx_lock(&sc->sfl_lock);
	FL_LOCK(fl);
	fl->flags \|= FL_DOOMED;
	FL_UNLOCK(fl);
	callout_stop(&sc->sfl_callout);
	mtx_unlock(&sc->sfl_lock);

	KASSERT((fl->flags & FL_STARVING) == 0,
	("%s: still starving", __func__));
	}

	static int
	t4_alloc_irq(struct adapter sc, struct irq irq, int rid,
	driver_intr_t handler, void arg, char *name)
	{
	int rc;

	irq->rid = rid;
	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
	RF_SHAREABLE \| RF_ACTIVE);
	if (irq->res == NULL) {
	device_printf(sc->dev,
	"failed to allocate IRQ for rid %d, name %s.\n", rid, name);
	return (ENOMEM);
	}

	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE \| INTR_TYPE_NET,
	NULL, handler, arg, &irq->tag);
	if (rc != 0) {
	device_printf(sc->dev,
	"failed to setup interrupt for rid %d, name %s: %d\n",
	rid, name, rc);
	} else if (name)
	bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);

	return (rc);
	}

	static int
	t4_free_irq(struct adapter sc, struct irq irq)
	{
	if (irq->tag)
	bus_teardown_intr(sc->dev, irq->res, irq->tag);
	if (irq->res)
	bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);

	bzero(irq, sizeof(*irq));

	return (0);
	}

	static void
	get_regs(struct adapter sc, struct t4_regdump regs, uint8_t *buf)
	{

	regs->version = chip_id(sc) \| chip_rev(sc) << 10;
	t4_get_regs(sc, buf, regs->len);
	}

	#define A_PL_INDIR_CMD 0x1f8

	#define S_PL_AUTOINC 31
	#define M_PL_AUTOINC 0x1U
	#define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
	#define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)

	#define S_PL_VFID 20
	#define M_PL_VFID 0xffU
	#define V_PL_VFID(x) ((x) << S_PL_VFID)
	#define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID)

	#define S_PL_ADDR 0
	#define M_PL_ADDR 0xfffffU
	#define V_PL_ADDR(x) ((x) << S_PL_ADDR)
	#define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR)

	#define A_PL_INDIR_DATA 0x1fc

	static uint64_t
	read_vf_stat(struct adapter *sc, u_int vin, int reg)
	{
	u32 stats[2];

	mtx_assert(&sc->reg_lock, MA_OWNED);
	if (sc->flags & IS_VF) {
	stats[0] = t4_read_reg(sc, VF_MPS_REG(reg));
	stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4));
	} else {
	t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) \|
	V_PL_VFID(vin) \| V_PL_ADDR(VF_MPS_REG(reg)));
	stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
	stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
	}
	return (((uint64_t)stats[1]) << 32 \| stats[0]);
	}

	static void
	t4_get_vi_stats(struct adapter sc, u_int vin, struct fw_vi_stats_vf stats)
	{

	#define GET_STAT(name) \
	read_vf_stat(sc, vin, A_MPS_VF_STAT_##name##_L)

	stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES);
	stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES);
	stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES);
	stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES);
	stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES);
	stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES);
	stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES);
	stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES);
	stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
	stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES);
	stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES);
	stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES);
	stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES);
	stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES);
	stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES);
	stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES);

	#undef GET_STAT
	}

	static void
	t4_clr_vi_stats(struct adapter *sc, u_int vin)
	{
	int reg;

	t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) \| V_PL_VFID(vin) \|
	V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
	for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
	reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
	t4_write_reg(sc, A_PL_INDIR_DATA, 0);
	}

	static void
	vi_refresh_stats(struct adapter sc, struct vi_info vi)
	{
	struct timeval tv;
	const struct timeval interval = {0, 250000}; /* 250ms */

	if (!(vi->flags & VI_INIT_DONE))
	return;

	getmicrotime(&tv);
	timevalsub(&tv, &interval);
	if (timevalcmp(&tv, &vi->last_refreshed, <))
	return;

	mtx_lock(&sc->reg_lock);
	t4_get_vi_stats(sc, vi->vin, &vi->stats);
	getmicrotime(&vi->last_refreshed);
	mtx_unlock(&sc->reg_lock);
	}

	static void
	cxgbe_refresh_stats(struct adapter sc, struct port_info pi)
	{
	u_int i, v, tnl_cong_drops, chan_map;
	struct timeval tv;
	const struct timeval interval = {0, 250000}; /* 250ms */

	getmicrotime(&tv);
	timevalsub(&tv, &interval);
	if (timevalcmp(&tv, &pi->last_refreshed, <))
	return;

	tnl_cong_drops = 0;
	t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
	chan_map = pi->rx_e_chan_map;
	while (chan_map) {
	i = ffs(chan_map) - 1;
	mtx_lock(&sc->reg_lock);
	t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1,
	A_TP_MIB_TNL_CNG_DROP_0 + i);
	mtx_unlock(&sc->reg_lock);
	tnl_cong_drops += v;
	chan_map &= ~(1 << i);
	}
	pi->tnl_cong_drops = tnl_cong_drops;
	getmicrotime(&pi->last_refreshed);
	}

	static void
	cxgbe_tick(void *arg)
	{
	struct port_info *pi = arg;
	struct adapter *sc = pi->adapter;

	PORT_LOCK_ASSERT_OWNED(pi);
	cxgbe_refresh_stats(sc, pi);

	callout_schedule(&pi->tick, hz);
	}

	void
	vi_tick(void *arg)
	{
	struct vi_info *vi = arg;
	struct adapter *sc = vi->adapter;

	vi_refresh_stats(sc, vi);

	callout_schedule(&vi->tick, hz);
	}

	/*
	* Should match fw_caps_config_<foo> enums in t4fw_interface.h
	*/
	static char *caps_decoder[] = {
	"\20\001IPMI\002NCSI", /* 0: NBM */
	"\20\001PPP\002QFC\003DCBX", /* 1: link */
	"\20\001INGRESS\002EGRESS", /* 2: switch */
	"\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */
	"\006HASHFILTER\007ETHOFLD",
	"\20\001TOE", /* 4: TOE */
	"\20\001RDDP\002RDMAC", /* 5: RDMA */
	"\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */
	"\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD"
	"\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD"
	"\007T10DIF"
	"\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD",
	"\20\001LOOKASIDE\002TLSKEYS", /* 7: Crypto */
	"\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */
	"\004PO_INITIATOR\005PO_TARGET",
	};

	void
	t4_sysctls(struct adapter *sc)
	{
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *oid;
	struct sysctl_oid_list children, c0;
	static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};

	ctx = device_get_sysctl_ctx(sc->dev);

	/*
	* dev.t4nex.X.
	*/
	oid = device_get_sysctl_tree(sc->dev);
	c0 = children = SYSCTL_CHILDREN(oid);

	sc->sc_do_rxcopy = 1;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
	&sc->sc_do_rxcopy, 1, "Do RX copy of small frames");

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
	sc->params.nports, "# of ports");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, doorbells,
	(uintptr_t)&sc->doorbells, sysctl_bitfield_8b, "A",
	"available doorbells");

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
	sc->params.vpd.cclk, "core clock frequency (in KHz)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val),
	sysctl_int_array, "A", "interrupt holdoff timer values (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val),
	sysctl_int_array, "A", "interrupt holdoff packet counter values");

	t4_sge_sysctls(sc, ctx, children);

	sc->lro_timeout = 100;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
	&sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
	&sc->debug_flags, 0, "flags to enable runtime debugging");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
	CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
	CTLFLAG_RD, sc->fw_version, 0, "firmware version");

	if (sc->flags & IS_VF)
	return;

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
	NULL, chip_rev(sc), "chip hardware revision");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn",
	CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn",
	CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec",
	CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version",
	CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na",
	CTLFLAG_RD, sc->params.vpd.na, 0, "network address");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD,
	sc->er_version, 0, "expansion ROM version");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD,
	sc->bs_version, 0, "bootstrap firmware version");

	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD,
	NULL, sc->params.scfg_vers, "serial config version");

	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD,
	NULL, sc->params.vpd_vers, "VPD version");

	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
	CTLFLAG_RD, sc->cfg_file, 0, "configuration file");

	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
	sc->cfcsum, "config file checksum");

	#define SYSCTL_CAP(name, n, text) \
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, caps_decoder[n], \
	(uintptr_t)&sc->name, sysctl_bitfield_16b, "A", \
	"available " text " capabilities")

	SYSCTL_CAP(nbmcaps, 0, "NBM");
	SYSCTL_CAP(linkcaps, 1, "link");
	SYSCTL_CAP(switchcaps, 2, "switch");
	SYSCTL_CAP(niccaps, 3, "NIC");
	SYSCTL_CAP(toecaps, 4, "TCP offload");
	SYSCTL_CAP(rdmacaps, 5, "RDMA");
	SYSCTL_CAP(iscsicaps, 6, "iSCSI");
	SYSCTL_CAP(cryptocaps, 7, "crypto");
	SYSCTL_CAP(fcoecaps, 8, "FCoE");
	#undef SYSCTL_CAP

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
	NULL, sc->tids.nftids, "number of filters");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_temperature, "I", "chip temperature (in Celsius)");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset_sensor",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_reset_sensor, "I", "reset the chip's temperature sensor.");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "loadavg",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_loadavg, "A",
	"microprocessor load averages (debug firmwares only)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "core_vdd",
	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0, sysctl_vdd,
	"I", "core Vdd (in mV)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, LOCAL_CPUS,
	sysctl_cpus, "A", "local CPUs");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, INTR_CPUS,
	sysctl_cpus, "A", "preferred CPUs for interrupts");

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "swintr", CTLFLAG_RW,
	&sc->swintr, 0, "software triggered interrupts");

	/*
	* dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload.
	*/
	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
	CTLFLAG_RD \| CTLFLAG_SKIP \| CTLFLAG_MPSAFE, NULL,
	"logs and miscellaneous information");
	children = SYSCTL_CHILDREN(oid);

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cctrl, "A", "congestion control");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 1,
	sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 2,
	sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 3,
	sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 4,
	sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 5,
	sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cim_la, "A", "CIM logic analyzer");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cim_ma_la, "A", "CIM MA logic analyzer");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	0 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	1 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	2 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	3 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	4 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	5 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");

	if (chip_id(sc) > CHELSIO_T4) {
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	6 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A",
	"CIM OBQ 6 (SGE0-RX)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	7 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A",
	"CIM OBQ 7 (SGE1-RX)");
	}

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cim_qcfg, "A", "CIM queue configuration");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_cpl_stats, "A", "CPL statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_ddp_stats, "A", "non-TCP DDP statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_devlog, "A", "firmware's device log");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_fcoe_stats, "A", "FCoE statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_hw_sched, "A", "hardware scheduler ");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_l2t, "A", "hardware L2 table");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_smt, "A", "hardware source MAC table");

	#ifdef INET6
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "clip",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_clip, "A", "active CLIP table entries");
	#endif

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_lb_stats, "A", "loopback statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_meminfo, "A", "memory regions");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6,
	"A", "MPS TCAM entries");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_path_mtus, "A", "path MTUs");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_pm_stats, "A", "PM statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_rdma_stats, "A", "RDMA statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tcp_stats, "A", "TCP statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tids, "A", "TID information");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tp_err_stats, "A", "TP error statistics");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tp_la_mask, "I", "TP logic analyzer event capture mask");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tp_la, "A", "TP logic analyzer");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tx_rate, "A", "Tx rate");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_ulprx_la, "A", "ULPRX logic analyzer");

	if (chip_id(sc) >= CHELSIO_T5) {
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_wcwr_stats, "A", "write combined work requests");
	}

	#ifdef KERN_TLS
	if (sc->flags & KERN_TLS_OK) {
	/*
	* dev.t4nex.0.tls.
	*/
	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "tls",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "KERN_TLS parameters");
	children = SYSCTL_CHILDREN(oid);

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "inline_keys",
	CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS "
	"keys in work requests (1) or attempt to store TLS keys "
	"in card memory.");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "combo_wrs",
	CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to combine "
	"TCB field updates with TLS record work requests.");
	}
	#endif

	#ifdef TCP_OFFLOAD
	if (is_offload(sc)) {
	int i;
	char s[4];

	/*
	* dev.t4nex.X.toe.
	*/
	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "TOE parameters");
	children = SYSCTL_CHILDREN(oid);

	sc->tt.cong_algorithm = -1;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm",
	CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control "
	"(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, "
	"3 = highspeed)");

	sc->tt.sndbuf = -1;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
	&sc->tt.sndbuf, 0, "hardware send buffer");

	sc->tt.ddp = 0;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp",
	CTLFLAG_RW \| CTLFLAG_SKIP, &sc->tt.ddp, 0, "");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_zcopy", CTLFLAG_RW,
	&sc->tt.ddp, 0, "Enable zero-copy aio_read(2)");

	sc->tt.rx_coalesce = -1;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
	CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");

	sc->tt.tls = 0;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tls", CTLFLAG_RW,
	&sc->tt.tls, 0, "Inline TLS allowed");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_ports",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tls_rx_ports, "I",
	"TCP ports that use inline TLS+TOE RX");

	sc->tt.tx_align = -1;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
	CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");

	sc->tt.tx_zcopy = 0;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy",
	CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
	"Enable zero-copy aio_write(2)");

	sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO,
	"cop_managed_offloading", CTLFLAG_RW,
	&sc->tt.cop_managed_offloading, 0,
	"COP (Connection Offload Policy) controls all TOE offload");

	sc->tt.autorcvbuf_inc = 16 * 1024;
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "autorcvbuf_inc",
	CTLFLAG_RW, &sc->tt.autorcvbuf_inc, 0,
	"autorcvbuf increment");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tp_tick, "A", "TP timer tick (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 1,
	sysctl_tp_tick, "A", "TCP timestamp tick (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 2,
	sysctl_tp_tick, "A", "DACK tick (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer",
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	sysctl_tp_dack_timer, "IU", "DACK timer (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_RXT_MIN, sysctl_tp_timer, "LU",
	"Minimum retransmit interval (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_RXT_MAX, sysctl_tp_timer, "LU",
	"Maximum retransmit interval (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_PERS_MIN, sysctl_tp_timer, "LU",
	"Persist timer min (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_PERS_MAX, sysctl_tp_timer, "LU",
	"Persist timer max (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_KEEP_IDLE, sysctl_tp_timer, "LU",
	"Keepalive idle timer (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_KEEP_INTVL, sysctl_tp_timer, "LU",
	"Keepalive interval timer (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_INIT_SRTT, sysctl_tp_timer, "LU", "Initial SRTT (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer",
	CTLTYPE_ULONG \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	A_TP_FINWAIT2_TIMER, sysctl_tp_timer, "LU",
	"FINWAIT2 timer (us)");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count",
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	S_SYNSHIFTMAX, sysctl_tp_shift_cnt, "IU",
	"Number of SYN retransmissions before abort");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count",
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	S_RXTSHIFTMAXR2, sysctl_tp_shift_cnt, "IU",
	"Number of retransmissions before abort");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count",
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	S_KEEPALIVEMAXR2, sysctl_tp_shift_cnt, "IU",
	"Number of keepalive probes before abort");

	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"TOE retransmit backoffs");
	children = SYSCTL_CHILDREN(oid);
	for (i = 0; i < 16; i++) {
	snprintf(s, sizeof(s), "%u", i);
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s,
	CTLTYPE_UINT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	i, sysctl_tp_backoff, "IU",
	"TOE retransmit backoff");
	}
	}
	#endif
	}

	void
	vi_sysctls(struct vi_info *vi)
	{
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *oid;
	struct sysctl_oid_list *children;

	ctx = device_get_sysctl_ctx(vi->dev);

	/*
	* dev.v?(cxgbe\|cxl).X.
	*/
	oid = device_get_sysctl_tree(vi->dev);
	children = SYSCTL_CHILDREN(oid);

	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
	vi->viid, "VI identifer");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
	&vi->nrxq, 0, "# of rx queues");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
	&vi->ntxq, 0, "# of tx queues");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
	&vi->first_rxq, 0, "index of first rx queue");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
	&vi->first_txq, 0, "index of first tx queue");
	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL,
	vi->rss_base, "start of RSS indirection table");
	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL,
	vi->rss_size, "size of RSS indirection table");

	if (IS_MAIN_VI(vi)) {
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_noflowq, "IU",
	"Reserve queue 0 for non-flowid packets");
	}

	#ifdef TCP_OFFLOAD
	if (vi->nofldrxq != 0) {
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
	&vi->nofldrxq, 0,
	"# of rx queues for offloaded TCP connections");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
	CTLFLAG_RD, &vi->first_ofld_rxq, 0,
	"index of first TOE rx queue");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_holdoff_tmr_idx_ofld, "I",
	"holdoff timer index for TOE queues");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_holdoff_pktc_idx_ofld, "I",
	"holdoff packet counter index for TOE queues");
	}
	#endif
	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	if (vi->nofldtxq != 0) {
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
	&vi->nofldtxq, 0,
	"# of tx queues for TOE/ETHOFLD");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
	CTLFLAG_RD, &vi->first_ofld_txq, 0,
	"index of first TOE/ETHOFLD tx queue");
	}
	#endif
	#ifdef DEV_NETMAP
	if (vi->nnmrxq != 0) {
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
	&vi->nnmrxq, 0, "# of netmap rx queues");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
	&vi->nnmtxq, 0, "# of netmap tx queues");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
	CTLFLAG_RD, &vi->first_nm_rxq, 0,
	"index of first netmap rx queue");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
	CTLFLAG_RD, &vi->first_nm_txq, 0,
	"index of first netmap tx queue");
	}
	#endif

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_holdoff_tmr_idx, "I", "holdoff timer index");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_holdoff_pktc_idx, "I", "holdoff packet counter index");

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_qsize_rxq, "I", "rx queue size");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, vi, 0,
	sysctl_qsize_txq, "I", "tx queue size");
	}

	static void
	cxgbe_sysctls(struct port_info *pi)
	{
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *oid;
	struct sysctl_oid_list children, children2;
	struct adapter *sc = pi->adapter;
	int i;
	char name[16];
	static char *tc_flags = {"\20\1USER\2SYNC\3ASYNC\4ERR"};

	ctx = device_get_sysctl_ctx(pi->dev);

	/*
	* dev.cxgbe.X.
	*/
	oid = device_get_sysctl_tree(pi->dev);
	children = SYSCTL_CHILDREN(oid);

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, pi, 0,
	sysctl_linkdnrc, "A", "reason why link is down");
	if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, pi, 0,
	sysctl_btphy, "I", "PHY temperature (in Celsius)");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
	CTLTYPE_INT \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, pi, 1,
	sysctl_btphy, "I", "PHY firmware version");
	}

	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, pi, 0,
	sysctl_pause_settings, "A",
	"PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fec",
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, pi, 0,
	sysctl_fec, "A",
	"FECs to use (bit 0 = RS, 1 = FC, 2 = none, 5 = auto, 6 = module)");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "module_fec",
	CTLTYPE_STRING \| CTLFLAG_NEEDGIANT, pi, 0, sysctl_module_fec, "A",
	"FEC recommended by the cable/transceiver");
	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg",
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, pi, 0,
	sysctl_autoneg, "I",
	"autonegotiation (-1 = not supported)");

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcaps", CTLFLAG_RD,
	&pi->link_cfg.pcaps, 0, "port capabilities");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "acaps", CTLFLAG_RD,
	&pi->link_cfg.acaps, 0, "advertised capabilities");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lpacaps", CTLFLAG_RD,
	&pi->link_cfg.lpacaps, 0, "link partner advertised capabilities");

	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL,
	port_top_speed(pi), "max speed (in Gbps)");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL,
	pi->mps_bg_map, "MPS buffer group map");
	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD,
	NULL, pi->rx_e_chan_map, "TP rx e-channel map");

	if (sc->flags & IS_VF)
	return;

	/*
	* dev.(cxgbe\|cxl).X.tc.
	*/
	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"Tx scheduler traffic classes (cl_rl)");
	children2 = SYSCTL_CHILDREN(oid);
	SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize",
	CTLFLAG_RW, &pi->sched_params->pktsize, 0,
	"pktsize for per-flow cl-rl (0 means up to the driver )");
	SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize",
	CTLFLAG_RW, &pi->sched_params->burstsize, 0,
	"burstsize for per-flow cl-rl (0 means up to the driver)");
	for (i = 0; i < sc->chip_params->nsched_cls; i++) {
	struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];

	snprintf(name, sizeof(name), "%d", i);
	children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx,
	SYSCTL_CHILDREN(oid), OID_AUTO, name,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "traffic class"));
	SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, tc_flags,
	(uintptr_t)&tc->flags, sysctl_bitfield_8b, "A", "flags");
	SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount",
	CTLFLAG_RD, &tc->refcount, 0, "references to this class");
	SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc,
	(pi->port_id << 16) \| i, sysctl_tc_params, "A",
	"traffic class parameters");
	}

	/*
	* dev.cxgbe.X.stats.
	*/
	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "port statistics");
	children = SYSCTL_CHILDREN(oid);
	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
	&pi->tx_parse_error, 0,
	"# of tx packets with invalid length or # of segments");

	#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
	CTLTYPE_U64 \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, reg, \
	sysctl_handle_t4_reg64, "QU", desc)

	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
	"# of tx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));

	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
	"# of frames received with bad FCS",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
	"# of frames received with length error",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
	"# of rx frames in this range",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
	PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));

	#undef SYSCTL_ADD_T4_REG64

	#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
	&pi->stats.name, desc)

	/* We get these from port_stats and they may be stale by up to 1s */
	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
	"# drops due to buffer-group 0 overflows");
	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
	"# drops due to buffer-group 1 overflows");
	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
	"# drops due to buffer-group 2 overflows");
	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
	"# drops due to buffer-group 3 overflows");
	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
	"# of buffer-group 0 truncated packets");
	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
	"# of buffer-group 1 truncated packets");
	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
	"# of buffer-group 2 truncated packets");
	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
	"# of buffer-group 3 truncated packets");

	#undef SYSCTL_ADD_T4_PORTSTAT

	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_toe_tls_records",
	CTLFLAG_RD, &pi->tx_toe_tls_records,
	"# of TOE TLS records transmitted");
	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_toe_tls_octets",
	CTLFLAG_RD, &pi->tx_toe_tls_octets,
	"# of payload octets in transmitted TOE TLS records");
	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_toe_tls_records",
	CTLFLAG_RD, &pi->rx_toe_tls_records,
	"# of TOE TLS records received");
	SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_toe_tls_octets",
	CTLFLAG_RD, &pi->rx_toe_tls_octets,
	"# of payload octets in received TOE TLS records");
	}

	static int
	sysctl_int_array(SYSCTL_HANDLER_ARGS)
	{
	int rc, *i, space = 0;
	struct sbuf sb;

	sbuf_new_for_sysctl(&sb, NULL, 64, req);
	for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
	if (space)
	sbuf_printf(&sb, " ");
	sbuf_printf(&sb, "%d", *i);
	space = 1;
	}
	rc = sbuf_finish(&sb);
	sbuf_delete(&sb);
	return (rc);
	}

	static int
	sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS)
	{
	int rc;
	struct sbuf *sb;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return(rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	if (sb == NULL)
	return (ENOMEM);

	sbuf_printf(sb, "%b", (uint8_t )(uintptr_t)arg2, (char *)arg1);
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS)
	{
	int rc;
	struct sbuf *sb;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return(rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	if (sb == NULL)
	return (ENOMEM);

	sbuf_printf(sb, "%b", (uint16_t )(uintptr_t)arg2, (char *)arg1);
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_btphy(SYSCTL_HANDLER_ARGS)
	{
	struct port_info *pi = arg1;
	int op = arg2;
	struct adapter *sc = pi->adapter;
	u_int v;
	int rc;

	rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK \| INTR_OK, "t4btt");
	if (rc)
	return (rc);
	/* XXX: magic numbers */
	rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
	&v);
	end_synchronized_op(sc, 0);
	if (rc)
	return (rc);
	if (op == 0)
	v /= 256;

	rc = sysctl_handle_int(oidp, &v, 0, req);
	return (rc);
	}

	static int
	sysctl_noflowq(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	int rc, val;

	val = vi->rsrv_noflowq;
	rc = sysctl_handle_int(oidp, &val, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if ((val >= 1) && (vi->ntxq > 1))
	vi->rsrv_noflowq = 1;
	else
	vi->rsrv_noflowq = 0;

	return (rc);
	}

	static int
	sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	struct adapter *sc = vi->adapter;
	int idx, rc, i;
	struct sge_rxq *rxq;
	uint8_t v;

	idx = vi->tmr_idx;

	rc = sysctl_handle_int(oidp, &idx, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if (idx < 0 \|\| idx >= SGE_NTIMERS)
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, HOLD_LOCK \| SLEEP_OK \| INTR_OK,
	"t4tmr");
	if (rc)
	return (rc);

	v = V_QINTR_TIMER_IDX(idx) \| V_QINTR_CNT_EN(vi->pktc_idx != -1);
	for_each_rxq(vi, i, rxq) {
	#ifdef atomic_store_rel_8
	atomic_store_rel_8(&rxq->iq.intr_params, v);
	#else
	rxq->iq.intr_params = v;
	#endif
	}
	vi->tmr_idx = idx;

	end_synchronized_op(sc, LOCK_HELD);
	return (0);
	}

	static int
	sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	struct adapter *sc = vi->adapter;
	int idx, rc;

	idx = vi->pktc_idx;

	rc = sysctl_handle_int(oidp, &idx, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if (idx < -1 \|\| idx >= SGE_NCOUNTERS)
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, HOLD_LOCK \| SLEEP_OK \| INTR_OK,
	"t4pktc");
	if (rc)
	return (rc);

	if (vi->flags & VI_INIT_DONE)
	rc = EBUSY; /* cannot be changed once the queues are created */
	else
	vi->pktc_idx = idx;

	end_synchronized_op(sc, LOCK_HELD);
	return (rc);
	}

	static int
	sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	struct adapter *sc = vi->adapter;
	int qsize, rc;

	qsize = vi->qsize_rxq;

	rc = sysctl_handle_int(oidp, &qsize, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if (qsize < 128 \|\| (qsize & 7))
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, HOLD_LOCK \| SLEEP_OK \| INTR_OK,
	"t4rxqs");
	if (rc)
	return (rc);

	if (vi->flags & VI_INIT_DONE)
	rc = EBUSY; /* cannot be changed once the queues are created */
	else
	vi->qsize_rxq = qsize;

	end_synchronized_op(sc, LOCK_HELD);
	return (rc);
	}

	static int
	sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	struct adapter *sc = vi->adapter;
	int qsize, rc;

	qsize = vi->qsize_txq;

	rc = sysctl_handle_int(oidp, &qsize, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if (qsize < 128 \|\| qsize > 65536)
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, HOLD_LOCK \| SLEEP_OK \| INTR_OK,
	"t4txqs");
	if (rc)
	return (rc);

	if (vi->flags & VI_INIT_DONE)
	rc = EBUSY; /* cannot be changed once the queues are created */
	else
	vi->qsize_txq = qsize;

	end_synchronized_op(sc, LOCK_HELD);
	return (rc);
	}

	static int
	sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
	{
	struct port_info *pi = arg1;
	struct adapter *sc = pi->adapter;
	struct link_config *lc = &pi->link_cfg;
	int rc;

	if (req->newptr == NULL) {
	struct sbuf *sb;
	static char *bits = "\20\1RX\2TX\3AUTO";

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return(rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	if (sb == NULL)
	return (ENOMEM);

	if (lc->link_ok) {
	sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX \| PAUSE_RX)) \|
	(lc->requested_fc & PAUSE_AUTONEG), bits);
	} else {
	sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX \|
	PAUSE_RX \| PAUSE_AUTONEG), bits);
	}
	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	} else {
	char s[2];
	int n;

	s[0] = '0' + (lc->requested_fc & (PAUSE_TX \| PAUSE_RX \|
	PAUSE_AUTONEG));
	s[1] = 0;

	rc = sysctl_handle_string(oidp, s, sizeof(s), req);
	if (rc != 0)
	return(rc);

	if (s[1] != 0)
	return (EINVAL);
	if (s[0] < '0' \|\| s[0] > '9')
	return (EINVAL); /* not a number */
	n = s[0] - '0';
	if (n & ~(PAUSE_TX \| PAUSE_RX \| PAUSE_AUTONEG))
	return (EINVAL); /* some other bit is set too */

	rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK \| INTR_OK,
	"t4PAUSE");
	if (rc)
	return (rc);
	PORT_LOCK(pi);
	lc->requested_fc = n;
	fixup_link_config(pi);
	if (pi->up_vis > 0)
	rc = apply_link_config(pi);
	set_current_media(pi);
	PORT_UNLOCK(pi);
	end_synchronized_op(sc, 0);
	}

	return (rc);
	}

	static int
	sysctl_fec(SYSCTL_HANDLER_ARGS)
	{
	struct port_info *pi = arg1;
	struct adapter *sc = pi->adapter;
	struct link_config *lc = &pi->link_cfg;
	int rc;
	int8_t old;

	if (req->newptr == NULL) {
	struct sbuf *sb;
	static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2"
	"\5RSVD3\6auto\7module";

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return(rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	if (sb == NULL)
	return (ENOMEM);

	/*
	* Display the requested_fec when the link is down -- the actual
	* FEC makes sense only when the link is up.
	*/
	if (lc->link_ok) {
	sbuf_printf(sb, "%b", (lc->fec & M_FW_PORT_CAP32_FEC) \|
	(lc->requested_fec & (FEC_AUTO \| FEC_MODULE)),
	bits);
	} else {
	sbuf_printf(sb, "%b", lc->requested_fec, bits);
	}
	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	} else {
	char s[8];
	int n;

	snprintf(s, sizeof(s), "%d",
	lc->requested_fec == FEC_AUTO ? -1 :
	lc->requested_fec & (M_FW_PORT_CAP32_FEC \| FEC_MODULE));

	rc = sysctl_handle_string(oidp, s, sizeof(s), req);
	if (rc != 0)
	return(rc);

	n = strtol(&s[0], NULL, 0);
	if (n < 0 \|\| n & FEC_AUTO)
	n = FEC_AUTO;
	else if (n & ~(M_FW_PORT_CAP32_FEC \| FEC_MODULE))
	return (EINVAL);/* some other bit is set too */

	rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK \| INTR_OK,
	"t4fec");
	if (rc)
	return (rc);
	PORT_LOCK(pi);
	old = lc->requested_fec;
	if (n == FEC_AUTO)
	lc->requested_fec = FEC_AUTO;
	else if (n == 0 \|\| n == FEC_NONE)
	lc->requested_fec = FEC_NONE;
	else {
	if ((lc->pcaps \|
	V_FW_PORT_CAP32_FEC(n & M_FW_PORT_CAP32_FEC)) !=
	lc->pcaps) {
	rc = ENOTSUP;
	goto done;
	}
	lc->requested_fec = n & (M_FW_PORT_CAP32_FEC \|
	FEC_MODULE);
	}
	fixup_link_config(pi);
	if (pi->up_vis > 0) {
	rc = apply_link_config(pi);
	if (rc != 0) {
	lc->requested_fec = old;
	if (rc == FW_EPROTO)
	rc = ENOTSUP;
	}
	}
	done:
	PORT_UNLOCK(pi);
	end_synchronized_op(sc, 0);
	}

	return (rc);
	}

	static int
	sysctl_module_fec(SYSCTL_HANDLER_ARGS)
	{
	struct port_info *pi = arg1;
	struct adapter *sc = pi->adapter;
	struct link_config *lc = &pi->link_cfg;
	int rc;
	int8_t fec;
	struct sbuf *sb;
	static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2\5RSVD3";

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	if (sb == NULL)
	return (ENOMEM);

	if (begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4mfec") != 0)
	return (EBUSY);
	PORT_LOCK(pi);
	if (pi->up_vis == 0) {
	/*
	* If all the interfaces are administratively down the firmware
	* does not report transceiver changes. Refresh port info here.
	* This is the only reason we have a synchronized op in this
	* function. Just PORT_LOCK would have been enough otherwise.
	*/
	t4_update_port_info(pi);
	}

	fec = lc->fec_hint;
	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE \|\|
	!fec_supported(lc->pcaps)) {
	sbuf_printf(sb, "n/a");
	} else {
	if (fec == 0)
	fec = FEC_NONE;
	sbuf_printf(sb, "%b", fec & M_FW_PORT_CAP32_FEC, bits);
	}
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	PORT_UNLOCK(pi);
	end_synchronized_op(sc, 0);

	return (rc);
	}

	static int
	sysctl_autoneg(SYSCTL_HANDLER_ARGS)
	{
	struct port_info *pi = arg1;
	struct adapter *sc = pi->adapter;
	struct link_config *lc = &pi->link_cfg;
	int rc, val;

	if (lc->pcaps & FW_PORT_CAP32_ANEG)
	val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1;
	else
	val = -1;
	rc = sysctl_handle_int(oidp, &val, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);
	if (val == 0)
	val = AUTONEG_DISABLE;
	else if (val == 1)
	val = AUTONEG_ENABLE;
	else
	val = AUTONEG_AUTO;

	rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK \| INTR_OK,
	"t4aneg");
	if (rc)
	return (rc);
	PORT_LOCK(pi);
	if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
	rc = ENOTSUP;
	goto done;
	}
	lc->requested_aneg = val;
	fixup_link_config(pi);
	if (pi->up_vis > 0)
	rc = apply_link_config(pi);
	set_current_media(pi);
	done:
	PORT_UNLOCK(pi);
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static int
	sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int reg = arg2;
	uint64_t val;

	val = t4_read_reg64(sc, reg);

	return (sysctl_handle_64(oidp, &val, 0, req));
	}

	static int
	sysctl_temperature(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int rc, t;
	uint32_t param, val;

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4temp");
	if (rc)
	return (rc);
	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \|
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) \|
	V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	end_synchronized_op(sc, 0);
	if (rc)
	return (rc);

	/* unknown is returned as 0 but we display -1 in that case */
	t = val == 0 ? -1 : val;

	rc = sysctl_handle_int(oidp, &t, 0, req);
	return (rc);
	}

	static int
	sysctl_vdd(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int rc;
	uint32_t param, val;

	if (sc->params.core_vdd == 0) {
	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK,
	"t4vdd");
	if (rc)
	return (rc);
	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \|
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) \|
	V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	end_synchronized_op(sc, 0);
	if (rc)
	return (rc);
	sc->params.core_vdd = val;
	}

	return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req));
	}

	static int
	sysctl_reset_sensor(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int rc, v;
	uint32_t param, val;

	v = sc->sensor_resets;
	rc = sysctl_handle_int(oidp, &v, 0, req);
	if (rc != 0 \|\| req->newptr == NULL \|\| v <= 0)
	return (rc);

	if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) \|\|
	chip_id(sc) < CHELSIO_T5)
	return (ENOTSUP);

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4srst");
	if (rc)
	return (rc);
	param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \|
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) \|
	V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_RESET_TMP_SENSOR));
	val = 1;
	rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	end_synchronized_op(sc, 0);
	if (rc == 0)
	sc->sensor_resets++;
	return (rc);
	}

	static int
	sysctl_loadavg(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	uint32_t param, val;

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4lavg");
	if (rc)
	return (rc);
	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) \|
	V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD);
	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
	end_synchronized_op(sc, 0);
	if (rc)
	return (rc);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	if (val == 0xffffffff) {
	/* Only debug and custom firmwares report load averages. */
	sbuf_printf(sb, "not available");
	} else {
	sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff,
	(val >> 16) & 0xff);
	}
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_cctrl(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i;
	uint16_t incr[NMTUS][NCCTRL_WIN];
	static const char *dec_fac[] = {
	"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
	"0.9375"
	};

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	t4_read_cong_tbl(sc, incr);

	for (i = 0; i < NCCTRL_WIN; ++i) {
	sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
	incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
	incr[5][i], incr[6][i], incr[7][i]);
	sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
	incr[8][i], incr[9][i], incr[10][i], incr[11][i],
	incr[12][i], incr[13][i], incr[14][i], incr[15][i],
	sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
	"TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
	"ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
	"SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
	};

	static int
	sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i, n, qid = arg2;
	uint32_t buf, p;
	char *qtype;
	u_int cim_num_obq = sc->chip_params->cim_num_obq;

	KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
	("%s: bad qid %d\n", __func__, qid));

	if (qid < CIM_NUM_IBQ) {
	/* inbound queue */
	qtype = "IBQ";
	n = 4 * CIM_IBQ_SIZE;
	buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO \| M_WAITOK);
	rc = t4_read_cim_ibq(sc, qid, buf, n);
	} else {
	/* outbound queue */
	qtype = "OBQ";
	qid -= CIM_NUM_IBQ;
	n = 4 * cim_num_obq * CIM_OBQ_SIZE;
	buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO \| M_WAITOK);
	rc = t4_read_cim_obq(sc, qid, buf, n);
	}

	if (rc < 0) {
	rc = -rc;
	goto done;
	}
	n = rc * sizeof(uint32_t); /* rc has # of words actually read */

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	goto done;

	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
	if (sb == NULL) {
	rc = ENOMEM;
	goto done;
	}

	sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
	for (i = 0, p = buf; i < n; i += 16, p += 4)
	sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
	p[2], p[3]);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	done:
	free(buf, M_CXGBE);
	return (rc);
	}

	static void
	sbuf_cim_la4(struct adapter sc, struct sbuf sb, uint32_t *buf, uint32_t cfg)
	{
	uint32_t *p;

	sbuf_printf(sb, "Status Data PC%s",
	cfg & F_UPDBGLACAPTPCONLY ? "" :
	" LS0Stat LS0Addr LS0Data");

	for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
	if (cfg & F_UPDBGLACAPTPCONLY) {
	sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff,
	p[6], p[7]);
	sbuf_printf(sb, "\n %02x %02x%06x %02x%06x",
	(p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
	p[4] & 0xff, p[5] >> 8);
	sbuf_printf(sb, "\n %02x %x%07x %x%07x",
	(p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
	p[1] & 0xf, p[2] >> 4);
	} else {
	sbuf_printf(sb,
	"\n %02x %x%07x %x%07x %08x %08x "
	"%08x%08x%08x%08x",
	(p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
	p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
	p[6], p[7]);
	}
	}
	}

	static void
	sbuf_cim_la6(struct adapter sc, struct sbuf sb, uint32_t *buf, uint32_t cfg)
	{
	uint32_t *p;

	sbuf_printf(sb, "Status Inst Data PC%s",
	cfg & F_UPDBGLACAPTPCONLY ? "" :
	" LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data");

	for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
	if (cfg & F_UPDBGLACAPTPCONLY) {
	sbuf_printf(sb, "\n %02x %08x %08x %08x",
	p[3] & 0xff, p[2], p[1], p[0]);
	sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x",
	(p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8,
	p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8);
	sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x",
	(p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16,
	p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff,
	p[6] >> 16);
	} else {
	sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x "
	"%08x %08x %08x %08x %08x %08x",
	(p[9] >> 16) & 0xff,
	p[9] & 0xffff, p[8] >> 16,
	p[8] & 0xffff, p[7] >> 16,
	p[7] & 0xffff, p[6] >> 16,
	p[2], p[1], p[0], p[5], p[4], p[3]);
	}
	}
	}

	static int
	sbuf_cim_la(struct adapter sc, struct sbuf sb, int flags)
	{
	uint32_t cfg, *buf;
	int rc;

	rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
	if (rc != 0)
	return (rc);

	MPASS(flags == M_WAITOK \|\| flags == M_NOWAIT);
	buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
	M_ZERO \| flags);
	if (buf == NULL)
	return (ENOMEM);

	rc = -t4_cim_read_la(sc, buf, NULL);
	if (rc != 0)
	goto done;
	if (chip_id(sc) < CHELSIO_T6)
	sbuf_cim_la4(sc, sb, buf, cfg);
	else
	sbuf_cim_la6(sc, sb, buf, cfg);

	done:
	free(buf, M_CXGBE);
	return (rc);
	}

	static int
	sysctl_cim_la(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);
	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	rc = sbuf_cim_la(sc, sb, M_WAITOK);
	if (rc == 0)
	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	return (rc);
	}

	bool
	t4_os_dump_cimla(struct adapter *sc, int arg, bool verbose)
	{
	struct sbuf sb;
	int rc;

	if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb)
	return (false);
	rc = sbuf_cim_la(sc, &sb, M_NOWAIT);
	if (rc == 0) {
	rc = sbuf_finish(&sb);
	if (rc == 0) {
	log(LOG_DEBUG, "%s: CIM LA dump follows.\n%s",
	device_get_nameunit(sc->dev), sbuf_data(&sb));
	}
	}
	sbuf_delete(&sb);
	return (false);
	}

	static int
	sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	u_int i;
	struct sbuf *sb;
	uint32_t buf, p;
	int rc;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
	M_ZERO \| M_WAITOK);

	t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
	p = buf;

	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
	sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
	p[1], p[0]);
	}

	sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD");
	for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
	sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u",
	(p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
	(p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
	(p[1] >> 2) \| ((p[2] & 3) << 30),
	(p[0] >> 2) \| ((p[1] & 3) << 30), (p[0] >> 1) & 1,
	p[0] & 1);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	free(buf, M_CXGBE);
	return (rc);
	}

	static int
	sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	u_int i;
	struct sbuf *sb;
	uint32_t buf, p;
	int rc;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
	M_ZERO \| M_WAITOK);

	t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
	p = buf;

	sbuf_printf(sb, "Cntl ID DataBE Addr Data");
	for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
	sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x",
	(p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
	p[4], p[3], p[2], p[1], p[0]);
	}

	sbuf_printf(sb, "\n\nCntl ID Data");
	for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
	sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x",
	(p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	free(buf, M_CXGBE);
	return (rc);
	}

	static int
	sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i;
	uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
	uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
	uint16_t thres[CIM_NUM_IBQ];
	uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
	uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
	u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;

	cim_num_obq = sc->chip_params->cim_num_obq;
	if (is_t4(sc)) {
	ibq_rdaddr = A_UP_IBQ_0_RDADDR;
	obq_rdaddr = A_UP_OBQ_0_REALADDR;
	} else {
	ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
	obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
	}
	nq = CIM_NUM_IBQ + cim_num_obq;

	rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
	if (rc == 0)
	rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
	if (rc != 0)
	return (rc);

	t4_read_cimq_cfg(sc, base, size, thres);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
	if (sb == NULL)
	return (ENOMEM);

	sbuf_printf(sb,
	" Queue Base Size Thres RdPtr WrPtr SOP EOP Avail");

	for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
	sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u",
	qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
	G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
	G_QUEREMFLITS(p[2]) * 16);
	for ( ; i < nq; i++, p += 4, wr += 2)
	sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i],
	base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
	wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
	G_QUEREMFLITS(p[2]) * 16);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tp_cpl_stats stats;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	mtx_lock(&sc->reg_lock);
	t4_tp_get_cpl_stats(sc, &stats, 0);
	mtx_unlock(&sc->reg_lock);

	if (sc->chip_params->nchan > 2) {
	sbuf_printf(sb, " channel 0 channel 1"
	" channel 2 channel 3");
	sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u",
	stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
	sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u",
	stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
	} else {
	sbuf_printf(sb, " channel 0 channel 1");
	sbuf_printf(sb, "\nCPL requests: %10u %10u",
	stats.req[0], stats.req[1]);
	sbuf_printf(sb, "\nCPL responses: %10u %10u",
	stats.rsp[0], stats.rsp[1]);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tp_usm_stats stats;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return(rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	t4_get_usm_stats(sc, &stats, 1);

	sbuf_printf(sb, "Frames: %u\n", stats.frames);
	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
	sbuf_printf(sb, "Drops: %u", stats.drops);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static const char * const devlog_level_strings[] = {
	[FW_DEVLOG_LEVEL_EMERG] = "EMERG",
	[FW_DEVLOG_LEVEL_CRIT] = "CRIT",
	[FW_DEVLOG_LEVEL_ERR] = "ERR",
	[FW_DEVLOG_LEVEL_NOTICE] = "NOTICE",
	[FW_DEVLOG_LEVEL_INFO] = "INFO",
	[FW_DEVLOG_LEVEL_DEBUG] = "DEBUG"
	};

	static const char * const devlog_facility_strings[] = {
	[FW_DEVLOG_FACILITY_CORE] = "CORE",
	[FW_DEVLOG_FACILITY_CF] = "CF",
	[FW_DEVLOG_FACILITY_SCHED] = "SCHED",
	[FW_DEVLOG_FACILITY_TIMER] = "TIMER",
	[FW_DEVLOG_FACILITY_RES] = "RES",
	[FW_DEVLOG_FACILITY_HW] = "HW",
	[FW_DEVLOG_FACILITY_FLR] = "FLR",
	[FW_DEVLOG_FACILITY_DMAQ] = "DMAQ",
	[FW_DEVLOG_FACILITY_PHY] = "PHY",
	[FW_DEVLOG_FACILITY_MAC] = "MAC",
	[FW_DEVLOG_FACILITY_PORT] = "PORT",
	[FW_DEVLOG_FACILITY_VI] = "VI",
	[FW_DEVLOG_FACILITY_FILTER] = "FILTER",
	[FW_DEVLOG_FACILITY_ACL] = "ACL",
	[FW_DEVLOG_FACILITY_TM] = "TM",
	[FW_DEVLOG_FACILITY_QFC] = "QFC",
	[FW_DEVLOG_FACILITY_DCB] = "DCB",
	[FW_DEVLOG_FACILITY_ETH] = "ETH",
	[FW_DEVLOG_FACILITY_OFLD] = "OFLD",
	[FW_DEVLOG_FACILITY_RI] = "RI",
	[FW_DEVLOG_FACILITY_ISCSI] = "ISCSI",
	[FW_DEVLOG_FACILITY_FCOE] = "FCOE",
	[FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI",
	[FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE",
	[FW_DEVLOG_FACILITY_CHNET] = "CHNET",
	};

	static int
	sbuf_devlog(struct adapter sc, struct sbuf sb, int flags)
	{
	int i, j, rc, nentries, first = 0;
	struct devlog_params *dparams = &sc->params.devlog;
	struct fw_devlog_e buf, e;
	uint64_t ftstamp = UINT64_MAX;

	if (dparams->addr == 0)
	return (ENXIO);

	MPASS(flags == M_WAITOK \|\| flags == M_NOWAIT);
	buf = malloc(dparams->size, M_CXGBE, M_ZERO \| flags);
	if (buf == NULL)
	return (ENOMEM);

	rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size);
	if (rc != 0)
	goto done;

	nentries = dparams->size / sizeof(struct fw_devlog_e);
	for (i = 0; i < nentries; i++) {
	e = &buf[i];

	if (e->timestamp == 0)
	break; /* end */

	e->timestamp = be64toh(e->timestamp);
	e->seqno = be32toh(e->seqno);
	for (j = 0; j < 8; j++)
	e->params[j] = be32toh(e->params[j]);

	if (e->timestamp < ftstamp) {
	ftstamp = e->timestamp;
	first = i;
	}
	}

	if (buf[first].timestamp == 0)
	goto done; /* nothing in the log */

	sbuf_printf(sb, "%10s %15s %8s %8s %s\n",
	"Seq#", "Tstamp", "Level", "Facility", "Message");

	i = first;
	do {
	e = &buf[i];
	if (e->timestamp == 0)
	break; /* end */

	sbuf_printf(sb, "%10d %15ju %8s %8s ",
	e->seqno, e->timestamp,
	(e->level < nitems(devlog_level_strings) ?
	devlog_level_strings[e->level] : "UNKNOWN"),
	(e->facility < nitems(devlog_facility_strings) ?
	devlog_facility_strings[e->facility] : "UNKNOWN"));
	sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
	e->params[2], e->params[3], e->params[4],
	e->params[5], e->params[6], e->params[7]);

	if (++i == nentries)
	i = 0;
	} while (i != first);
	done:
	free(buf, M_CXGBE);
	return (rc);
	}

	static int
	sysctl_devlog(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int rc;
	struct sbuf *sb;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);
	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	rc = sbuf_devlog(sc, sb, M_WAITOK);
	if (rc == 0)
	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	return (rc);
	}

	void
	t4_os_dump_devlog(struct adapter *sc)
	{
	int rc;
	struct sbuf sb;

	if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb)
	return;
	rc = sbuf_devlog(sc, &sb, M_NOWAIT);
	if (rc == 0) {
	rc = sbuf_finish(&sb);
	if (rc == 0) {
	log(LOG_DEBUG, "%s: device log follows.\n%s",
	device_get_nameunit(sc->dev), sbuf_data(&sb));
	}
	}
	sbuf_delete(&sb);
	}

	static int
	sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tp_fcoe_stats stats[MAX_NCHAN];
	int i, nchan = sc->chip_params->nchan;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	for (i = 0; i < nchan; i++)
	t4_get_fcoe_stats(sc, i, &stats[i], 1);

	if (nchan > 2) {
	sbuf_printf(sb, " channel 0 channel 1"
	" channel 2 channel 3");
	sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju",
	stats[0].octets_ddp, stats[1].octets_ddp,
	stats[2].octets_ddp, stats[3].octets_ddp);
	sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u",
	stats[0].frames_ddp, stats[1].frames_ddp,
	stats[2].frames_ddp, stats[3].frames_ddp);
	sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u",
	stats[0].frames_drop, stats[1].frames_drop,
	stats[2].frames_drop, stats[3].frames_drop);
	} else {
	sbuf_printf(sb, " channel 0 channel 1");
	sbuf_printf(sb, "\noctetsDDP: %16ju %16ju",
	stats[0].octets_ddp, stats[1].octets_ddp);
	sbuf_printf(sb, "\nframesDDP: %16u %16u",
	stats[0].frames_ddp, stats[1].frames_ddp);
	sbuf_printf(sb, "\nframesDrop: %16u %16u",
	stats[0].frames_drop, stats[1].frames_drop);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i;
	unsigned int map, kbps, ipg, mode;
	unsigned int pace_tab[NTX_SCHED];

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
	t4_read_pace_tbl(sc, pace_tab);

	sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) "
	"Class IPG (0.1 ns) Flow IPG (us)");

	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
	t4_get_tx_sched(sc, i, &kbps, &ipg, 1);
	sbuf_printf(sb, "\n %u %-5s %u ", i,
	(mode & (1 << i)) ? "flow" : "class", map & 3);
	if (kbps)
	sbuf_printf(sb, "%9u ", kbps);
	else
	sbuf_printf(sb, " disabled ");

	if (ipg)
	sbuf_printf(sb, "%13u ", ipg);
	else
	sbuf_printf(sb, " disabled ");

	if (pace_tab[i])
	sbuf_printf(sb, "%10u", pace_tab[i]);
	else
	sbuf_printf(sb, " disabled");
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i, j;
	uint64_t p0, p1;
	struct lb_port_stats s[2];
	static const char *stat_name[] = {
	"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
	"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
	"Frames128To255:", "Frames256To511:", "Frames512To1023:",
	"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
	"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
	"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
	"BG2FramesTrunc:", "BG3FramesTrunc:"
	};

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	memset(s, 0, sizeof(s));

	for (i = 0; i < sc->chip_params->nchan; i += 2) {
	t4_get_lb_stats(sc, i, &s[0]);
	t4_get_lb_stats(sc, i + 1, &s[1]);

	p0 = &s[0].octets;
	p1 = &s[1].octets;
	sbuf_printf(sb, "%s Loopback %u"
	" Loopback %u", i == 0 ? "" : "\n", i, i + 1);

	for (j = 0; j < nitems(stat_name); j++)
	sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
	p0++, p1++);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
	{
	int rc = 0;
	struct port_info *pi = arg1;
	struct link_config *lc = &pi->link_cfg;
	struct sbuf *sb;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return(rc);
	sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
	if (sb == NULL)
	return (ENOMEM);

	if (lc->link_ok \|\| lc->link_down_rc == 255)
	sbuf_printf(sb, "n/a");
	else
	sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc));

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	struct mem_desc {
	unsigned int base;
	unsigned int limit;
	unsigned int idx;
	};

	static int
	mem_desc_cmp(const void a, const void b)
	{
	return ((const struct mem_desc *)a)->base -
	((const struct mem_desc *)b)->base;
	}

	static void
	mem_region_show(struct sbuf sb, const char name, unsigned int from,
	unsigned int to)
	{
	unsigned int size;

	if (from == to)
	return;

	size = to - from + 1;
	if (size == 0)
	return;

	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
	}

	static int
	sysctl_meminfo(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i, n;
	uint32_t lo, hi, used, alloc;
	static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
	static const char *region[] = {
	"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
	"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
	"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
	"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
	"RQUDP region:", "PBL region:", "TXPBL region:",
	"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
	"On-chip queues:", "TLS keys:",
	};
	struct mem_desc avail[4];
	struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */
	struct mem_desc *md = mem;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	for (i = 0; i < nitems(mem); i++) {
	mem[i].limit = 0;
	mem[i].idx = i;
	}

	/* Find and sort the populated memory ranges */
	i = 0;
	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
	if (lo & F_EDRAM0_ENABLE) {
	hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
	avail[i].base = G_EDRAM0_BASE(hi) << 20;
	avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
	avail[i].idx = 0;
	i++;
	}
	if (lo & F_EDRAM1_ENABLE) {
	hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
	avail[i].base = G_EDRAM1_BASE(hi) << 20;
	avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
	avail[i].idx = 1;
	i++;
	}
	if (lo & F_EXT_MEM_ENABLE) {
	hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
	avail[i].base = G_EXT_MEM_BASE(hi) << 20;
	avail[i].limit = avail[i].base +
	(G_EXT_MEM_SIZE(hi) << 20);
	avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */
	i++;
	}
	if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) {
	hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
	avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
	avail[i].limit = avail[i].base +
	(G_EXT_MEM1_SIZE(hi) << 20);
	avail[i].idx = 4;
	i++;
	}
	if (!i) /* no memory available */
	return 0;
	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);

	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);

	/* the next few have explicit upper bounds */
	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
	md->limit = md->base - 1 +
	t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
	G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
	md++;

	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
	md->limit = md->base - 1 +
	t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
	G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
	md++;

	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
	if (chip_id(sc) <= CHELSIO_T5)
	md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
	else
	md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
	md->limit = 0;
	} else {
	md->base = 0;
	md->idx = nitems(region); /* hide it */
	}
	md++;

	#define ulp_region(reg) \
	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)

	ulp_region(RX_ISCSI);
	ulp_region(RX_TDDP);
	ulp_region(TX_TPT);
	ulp_region(RX_STAG);
	ulp_region(RX_RQ);
	ulp_region(RX_RQUDP);
	ulp_region(RX_PBL);
	ulp_region(TX_PBL);
	#undef ulp_region

	md->base = 0;
	md->idx = nitems(region);
	if (!is_t4(sc)) {
	uint32_t size = 0;
	uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2);
	uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE);

	if (is_t5(sc)) {
	if (sge_ctrl & F_VFIFO_ENABLE)
	size = G_DBVFIFO_SIZE(fifo_size);
	} else
	size = G_T6_DBVFIFO_SIZE(fifo_size);

	if (size) {
	md->base = G_BASEADDR(t4_read_reg(sc,
	A_SGE_DBVFIFO_BADDR));
	md->limit = md->base + (size << 2) - 1;
	}
	}
	md++;

	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
	md->limit = 0;
	md++;
	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
	md->limit = 0;
	md++;

	md->base = sc->vres.ocq.start;
	if (sc->vres.ocq.size)
	md->limit = md->base + sc->vres.ocq.size - 1;
	else
	md->idx = nitems(region); /* hide it */
	md++;

	md->base = sc->vres.key.start;
	if (sc->vres.key.size)
	md->limit = md->base + sc->vres.key.size - 1;
	else
	md->idx = nitems(region); /* hide it */
	md++;

	/* add any address-space holes, there can be up to 3 */
	for (n = 0; n < i - 1; n++)
	if (avail[n].limit < avail[n + 1].base)
	(md++)->base = avail[n].limit;
	if (avail[n].limit)
	(md++)->base = avail[n].limit;

	n = md - mem;
	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);

	for (lo = 0; lo < i; lo++)
	mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
	avail[lo].limit - 1);

	sbuf_printf(sb, "\n");
	for (i = 0; i < n; i++) {
	if (mem[i].idx >= nitems(region))
	continue; /* skip holes */
	if (!mem[i].limit)
	mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
	mem_region_show(sb, region[mem[i].idx], mem[i].base,
	mem[i].limit);
	}

	sbuf_printf(sb, "\n");
	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
	mem_region_show(sb, "uP RAM:", lo, hi);

	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
	mem_region_show(sb, "uP Extmem2:", lo, hi);

	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
	G_PMRXMAXPAGE(lo),
	t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
	(lo & F_PMRXNUMCHN) ? 2 : 1);

	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
	G_PMTXMAXPAGE(lo),
	hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
	hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
	sbuf_printf(sb, "%u p-structs\n",
	t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));

	for (i = 0; i < 4; i++) {
	if (chip_id(sc) > CHELSIO_T5)
	lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4);
	else
	lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
	if (is_t5(sc)) {
	used = G_T5_USED(lo);
	alloc = G_T5_ALLOC(lo);
	} else {
	used = G_USED(lo);
	alloc = G_ALLOC(lo);
	}
	/* For T6 these are MAC buffer groups */
	sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
	i, used, alloc);
	}
	for (i = 0; i < sc->chip_params->nchan; i++) {
	if (chip_id(sc) > CHELSIO_T5)
	lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4);
	else
	lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
	if (is_t5(sc)) {
	used = G_T5_USED(lo);
	alloc = G_T5_ALLOC(lo);
	} else {
	used = G_USED(lo);
	alloc = G_ALLOC(lo);
	}
	/* For T6 these are MAC buffer groups */
	sbuf_printf(sb,
	"\nLoopback %d using %u pages out of %u allocated",
	i, used, alloc);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static inline void
	tcamxy2valmask(uint64_t x, uint64_t y, uint8_t addr, uint64_t mask)
	{
	*mask = x \| y;
	y = htobe64(y);
	memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
	}

	static int
	sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i;

	MPASS(chip_id(sc) <= CHELSIO_T5);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	sbuf_printf(sb,
	"Idx Ethernet address Mask Vld Ports PF"
	" VF Replication P0 P1 P2 P3 ML");
	for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
	uint64_t tcamx, tcamy, mask;
	uint32_t cls_lo, cls_hi;
	uint8_t addr[ETHER_ADDR_LEN];

	tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
	tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
	if (tcamx & tcamy)
	continue;
	tcamxy2valmask(tcamx, tcamy, addr, &mask);
	cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
	cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
	sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
	" %c %#x%4u%4d", i, addr[0], addr[1], addr[2],
	addr[3], addr[4], addr[5], (uintmax_t)mask,
	(cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
	G_PORTMAP(cls_hi), G_PF(cls_lo),
	(cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);

	if (cls_lo & F_REPLICATE) {
	struct fw_ldst_cmd ldst_cmd;

	memset(&ldst_cmd, 0, sizeof(ldst_cmd));
	ldst_cmd.op_to_addrspace =
	htobe32(V_FW_CMD_OP(FW_LDST_CMD) \|
	F_FW_CMD_REQUEST \| F_FW_CMD_READ \|
	V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
	ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
	ldst_cmd.u.mps.rplc.fid_idx =
	htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) \|
	V_FW_LDST_CMD_IDX(i));

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK,
	"t4mps");
	if (rc)
	break;
	rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
	sizeof(ldst_cmd), &ldst_cmd);
	end_synchronized_op(sc, 0);

	if (rc != 0) {
	sbuf_printf(sb, "%36d", rc);
	rc = 0;
	} else {
	sbuf_printf(sb, " %08x %08x %08x %08x",
	be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
	be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
	be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
	be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
	}
	} else
	sbuf_printf(sb, "%36s", "");

	sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
	G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
	G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
	}

	if (rc)
	(void) sbuf_finish(sb);
	else
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i;

	MPASS(chip_id(sc) > CHELSIO_T5);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask"
	" IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF"
	" Replication"
	" P0 P1 P2 P3 ML\n");

	for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
	uint8_t dip_hit, vlan_vld, lookup_type, port_num;
	uint16_t ivlan;
	uint64_t tcamx, tcamy, val, mask;
	uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy;
	uint8_t addr[ETHER_ADDR_LEN];

	ctl = V_CTLREQID(1) \| V_CTLCMDTYPE(0) \| V_CTLXYBITSEL(0);
	if (i < 256)
	ctl \|= V_CTLTCAMINDEX(i) \| V_CTLTCAMSEL(0);
	else
	ctl \|= V_CTLTCAMINDEX(i - 256) \| V_CTLTCAMSEL(1);
	t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
	val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
	tcamy = G_DMACH(val) << 32;
	tcamy \|= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
	data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
	lookup_type = G_DATALKPTYPE(data2);
	port_num = G_DATAPORTNUM(data2);
	if (lookup_type && lookup_type != M_DATALKPTYPE) {
	/* Inner header VNI */
	vniy = ((data2 & F_DATAVIDH2) << 23) \|
	(G_DATAVIDH1(data2) << 16) \| G_VIDL(val);
	dip_hit = data2 & F_DATADIPHIT;
	vlan_vld = 0;
	} else {
	vniy = 0;
	dip_hit = 0;
	vlan_vld = data2 & F_DATAVIDH2;
	ivlan = G_VIDL(val);
	}

	ctl \|= V_CTLXYBITSEL(1);
	t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
	val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
	tcamx = G_DMACH(val) << 32;
	tcamx \|= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
	data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
	if (lookup_type && lookup_type != M_DATALKPTYPE) {
	/* Inner header VNI mask */
	vnix = ((data2 & F_DATAVIDH2) << 23) \|
	(G_DATAVIDH1(data2) << 16) \| G_VIDL(val);
	} else
	vnix = 0;

	if (tcamx & tcamy)
	continue;
	tcamxy2valmask(tcamx, tcamy, addr, &mask);

	cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
	cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));

	if (lookup_type && lookup_type != M_DATALKPTYPE) {
	sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
	"%012jx %06x %06x - - %3c"
	" 'I' %4x %3c %#x%4u%4d", i, addr[0],
	addr[1], addr[2], addr[3], addr[4], addr[5],
	(uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N',
	port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
	G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
	cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
	} else {
	sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
	"%012jx - - ", i, addr[0], addr[1],
	addr[2], addr[3], addr[4], addr[5],
	(uintmax_t)mask);

	if (vlan_vld)
	sbuf_printf(sb, "%4u Y ", ivlan);
	else
	sbuf_printf(sb, " - N ");

	sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d",
	lookup_type ? 'I' : 'O', port_num,
	cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
	G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
	cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
	}


	if (cls_lo & F_T6_REPLICATE) {
	struct fw_ldst_cmd ldst_cmd;

	memset(&ldst_cmd, 0, sizeof(ldst_cmd));
	ldst_cmd.op_to_addrspace =
	htobe32(V_FW_CMD_OP(FW_LDST_CMD) \|
	F_FW_CMD_REQUEST \| F_FW_CMD_READ \|
	V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
	ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
	ldst_cmd.u.mps.rplc.fid_idx =
	htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) \|
	V_FW_LDST_CMD_IDX(i));

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK,
	"t6mps");
	if (rc)
	break;
	rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
	sizeof(ldst_cmd), &ldst_cmd);
	end_synchronized_op(sc, 0);

	if (rc != 0) {
	sbuf_printf(sb, "%72d", rc);
	rc = 0;
	} else {
	sbuf_printf(sb, " %08x %08x %08x %08x"
	" %08x %08x %08x %08x",
	be32toh(ldst_cmd.u.mps.rplc.rplc255_224),
	be32toh(ldst_cmd.u.mps.rplc.rplc223_192),
	be32toh(ldst_cmd.u.mps.rplc.rplc191_160),
	be32toh(ldst_cmd.u.mps.rplc.rplc159_128),
	be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
	be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
	be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
	be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
	}
	} else
	sbuf_printf(sb, "%72s", "");

	sbuf_printf(sb, "%4u%3u%3u%3u %#x",
	G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo),
	G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo),
	(cls_lo >> S_T6_MULTILISTEN0) & 0xf);
	}

	if (rc)
	(void) sbuf_finish(sb);
	else
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	uint16_t mtus[NMTUS];

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	t4_read_mtu_tbl(sc, mtus, NULL);

	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
	mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
	mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
	mtus[14], mtus[15]);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, i;
	uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS];
	uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS];
	static const char *tx_stats[MAX_PM_NSTATS] = {
	"Read:", "Write bypass:", "Write mem:", "Bypass + mem:",
	"Tx FIFO wait", NULL, "Tx latency"
	};
	static const char *rx_stats[MAX_PM_NSTATS] = {
	"Read:", "Write bypass:", "Write mem:", "Flush:",
	"Rx FIFO wait", NULL, "Rx latency"
	};

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
	t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);

	sbuf_printf(sb, " Tx pcmds Tx bytes");
	for (i = 0; i < 4; i++) {
	sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
	tx_cyc[i]);
	}

	sbuf_printf(sb, "\n Rx pcmds Rx bytes");
	for (i = 0; i < 4; i++) {
	sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
	rx_cyc[i]);
	}

	if (chip_id(sc) > CHELSIO_T5) {
	sbuf_printf(sb,
	"\n Total wait Total occupancy");
	sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
	tx_cyc[i]);
	sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
	rx_cyc[i]);

	i += 2;
	MPASS(i < nitems(tx_stats));

	sbuf_printf(sb,
	"\n Reads Total wait");
	sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
	tx_cyc[i]);
	sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
	rx_cyc[i]);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tp_rdma_stats stats;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	mtx_lock(&sc->reg_lock);
	t4_tp_get_rdma_stats(sc, &stats, 0);
	mtx_unlock(&sc->reg_lock);

	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tp_tcp_stats v4, v6;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	mtx_lock(&sc->reg_lock);
	t4_tp_get_tcp_stats(sc, &v4, &v6, 0);
	mtx_unlock(&sc->reg_lock);

	sbuf_printf(sb,
	" IP IPv6\n");
	sbuf_printf(sb, "OutRsts: %20u %20u\n",
	v4.tcp_out_rsts, v6.tcp_out_rsts);
	sbuf_printf(sb, "InSegs: %20ju %20ju\n",
	v4.tcp_in_segs, v6.tcp_in_segs);
	sbuf_printf(sb, "OutSegs: %20ju %20ju\n",
	v4.tcp_out_segs, v6.tcp_out_segs);
	sbuf_printf(sb, "RetransSegs: %20ju %20ju",
	v4.tcp_retrans_segs, v6.tcp_retrans_segs);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_tids(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tid_info *t = &sc->tids;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	if (t->natids) {
	sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
	t->atids_in_use);
	}

	if (t->nhpftids) {
	sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
	t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
	}

	if (t->ntids) {
	sbuf_printf(sb, "TID range: ");
	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
	uint32_t b, hb;

	if (chip_id(sc) <= CHELSIO_T5) {
	b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
	hb = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
	} else {
	b = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX);
	hb = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE);
	}

	if (b)
	sbuf_printf(sb, "%u-%u, ", t->tid_base, b - 1);
	sbuf_printf(sb, "%u-%u", hb, t->ntids - 1);
	} else
	sbuf_printf(sb, "%u-%u", t->tid_base, t->ntids - 1);
	sbuf_printf(sb, ", in use: %u\n",
	atomic_load_acq_int(&t->tids_in_use));
	}

	if (t->nstids) {
	sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
	t->stid_base + t->nstids - 1, t->stids_in_use);
	}

	if (t->nftids) {
	sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
	t->ftid_end, t->ftids_in_use);
	}

	if (t->netids) {
	sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base,
	t->etid_base + t->netids - 1, t->etids_in_use);
	}

	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
	t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
	t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	struct tp_err_stats stats;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	mtx_lock(&sc->reg_lock);
	t4_tp_get_err_stats(sc, &stats, 0);
	mtx_unlock(&sc->reg_lock);

	if (sc->chip_params->nchan > 2) {
	sbuf_printf(sb, " channel 0 channel 1"
	" channel 2 channel 3\n");
	sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n",
	stats.mac_in_errs[0], stats.mac_in_errs[1],
	stats.mac_in_errs[2], stats.mac_in_errs[3]);
	sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n",
	stats.hdr_in_errs[0], stats.hdr_in_errs[1],
	stats.hdr_in_errs[2], stats.hdr_in_errs[3]);
	sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n",
	stats.tcp_in_errs[0], stats.tcp_in_errs[1],
	stats.tcp_in_errs[2], stats.tcp_in_errs[3]);
	sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n",
	stats.tcp6_in_errs[0], stats.tcp6_in_errs[1],
	stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]);
	sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n",
	stats.tnl_cong_drops[0], stats.tnl_cong_drops[1],
	stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]);
	sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n",
	stats.tnl_tx_drops[0], stats.tnl_tx_drops[1],
	stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]);
	sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n",
	stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1],
	stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]);
	sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n",
	stats.ofld_chan_drops[0], stats.ofld_chan_drops[1],
	stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]);
	} else {
	sbuf_printf(sb, " channel 0 channel 1\n");
	sbuf_printf(sb, "macInErrs: %10u %10u\n",
	stats.mac_in_errs[0], stats.mac_in_errs[1]);
	sbuf_printf(sb, "hdrInErrs: %10u %10u\n",
	stats.hdr_in_errs[0], stats.hdr_in_errs[1]);
	sbuf_printf(sb, "tcpInErrs: %10u %10u\n",
	stats.tcp_in_errs[0], stats.tcp_in_errs[1]);
	sbuf_printf(sb, "tcp6InErrs: %10u %10u\n",
	stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]);
	sbuf_printf(sb, "tnlCongDrops: %10u %10u\n",
	stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]);
	sbuf_printf(sb, "tnlTxDrops: %10u %10u\n",
	stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]);
	sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n",
	stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]);
	sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n",
	stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]);
	}

	sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u",
	stats.ofld_no_neigh, stats.ofld_cong_defer);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct tp_params *tpp = &sc->params.tp;
	u_int mask;
	int rc;

	mask = tpp->la_mask >> 16;
	rc = sysctl_handle_int(oidp, &mask, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);
	if (mask > 0xffff)
	return (EINVAL);
	tpp->la_mask = mask << 16;
	t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask);

	return (0);
	}

	struct field_desc {
	const char *name;
	u_int start;
	u_int width;
	};

	static void
	field_desc_show(struct sbuf sb, uint64_t v, const struct field_desc f)
	{
	char buf[32];
	int line_size = 0;

	while (f->name) {
	uint64_t mask = (1ULL << f->width) - 1;
	int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
	((uintmax_t)v >> f->start) & mask);

	if (line_size + len >= 79) {
	line_size = 8;
	sbuf_printf(sb, "\n ");
	}
	sbuf_printf(sb, "%s ", buf);
	line_size += len + 1;
	f++;
	}
	sbuf_printf(sb, "\n");
	}

	static const struct field_desc tp_la0[] = {
	{ "RcfOpCodeOut", 60, 4 },
	{ "State", 56, 4 },
	{ "WcfState", 52, 4 },
	{ "RcfOpcSrcOut", 50, 2 },
	{ "CRxError", 49, 1 },
	{ "ERxError", 48, 1 },
	{ "SanityFailed", 47, 1 },
	{ "SpuriousMsg", 46, 1 },
	{ "FlushInputMsg", 45, 1 },
	{ "FlushInputCpl", 44, 1 },
	{ "RssUpBit", 43, 1 },
	{ "RssFilterHit", 42, 1 },
	{ "Tid", 32, 10 },
	{ "InitTcb", 31, 1 },
	{ "LineNumber", 24, 7 },
	{ "Emsg", 23, 1 },
	{ "EdataOut", 22, 1 },
	{ "Cmsg", 21, 1 },
	{ "CdataOut", 20, 1 },
	{ "EreadPdu", 19, 1 },
	{ "CreadPdu", 18, 1 },
	{ "TunnelPkt", 17, 1 },
	{ "RcfPeerFin", 16, 1 },
	{ "RcfReasonOut", 12, 4 },
	{ "TxCchannel", 10, 2 },
	{ "RcfTxChannel", 8, 2 },
	{ "RxEchannel", 6, 2 },
	{ "RcfRxChannel", 5, 1 },
	{ "RcfDataOutSrdy", 4, 1 },
	{ "RxDvld", 3, 1 },
	{ "RxOoDvld", 2, 1 },
	{ "RxCongestion", 1, 1 },
	{ "TxCongestion", 0, 1 },
	{ NULL }
	};

	static const struct field_desc tp_la1[] = {
	{ "CplCmdIn", 56, 8 },
	{ "CplCmdOut", 48, 8 },
	{ "ESynOut", 47, 1 },
	{ "EAckOut", 46, 1 },
	{ "EFinOut", 45, 1 },
	{ "ERstOut", 44, 1 },
	{ "SynIn", 43, 1 },
	{ "AckIn", 42, 1 },
	{ "FinIn", 41, 1 },
	{ "RstIn", 40, 1 },
	{ "DataIn", 39, 1 },
	{ "DataInVld", 38, 1 },
	{ "PadIn", 37, 1 },
	{ "RxBufEmpty", 36, 1 },
	{ "RxDdp", 35, 1 },
	{ "RxFbCongestion", 34, 1 },
	{ "TxFbCongestion", 33, 1 },
	{ "TxPktSumSrdy", 32, 1 },
	{ "RcfUlpType", 28, 4 },
	{ "Eread", 27, 1 },
	{ "Ebypass", 26, 1 },
	{ "Esave", 25, 1 },
	{ "Static0", 24, 1 },
	{ "Cread", 23, 1 },
	{ "Cbypass", 22, 1 },
	{ "Csave", 21, 1 },
	{ "CPktOut", 20, 1 },
	{ "RxPagePoolFull", 18, 2 },
	{ "RxLpbkPkt", 17, 1 },
	{ "TxLpbkPkt", 16, 1 },
	{ "RxVfValid", 15, 1 },
	{ "SynLearned", 14, 1 },
	{ "SetDelEntry", 13, 1 },
	{ "SetInvEntry", 12, 1 },
	{ "CpcmdDvld", 11, 1 },
	{ "CpcmdSave", 10, 1 },
	{ "RxPstructsFull", 8, 2 },
	{ "EpcmdDvld", 7, 1 },
	{ "EpcmdFlush", 6, 1 },
	{ "EpcmdTrimPrefix", 5, 1 },
	{ "EpcmdTrimPostfix", 4, 1 },
	{ "ERssIp4Pkt", 3, 1 },
	{ "ERssIp6Pkt", 2, 1 },
	{ "ERssTcpUdpPkt", 1, 1 },
	{ "ERssFceFipPkt", 0, 1 },
	{ NULL }
	};

	static const struct field_desc tp_la2[] = {
	{ "CplCmdIn", 56, 8 },
	{ "MpsVfVld", 55, 1 },
	{ "MpsPf", 52, 3 },
	{ "MpsVf", 44, 8 },
	{ "SynIn", 43, 1 },
	{ "AckIn", 42, 1 },
	{ "FinIn", 41, 1 },
	{ "RstIn", 40, 1 },
	{ "DataIn", 39, 1 },
	{ "DataInVld", 38, 1 },
	{ "PadIn", 37, 1 },
	{ "RxBufEmpty", 36, 1 },
	{ "RxDdp", 35, 1 },
	{ "RxFbCongestion", 34, 1 },
	{ "TxFbCongestion", 33, 1 },
	{ "TxPktSumSrdy", 32, 1 },
	{ "RcfUlpType", 28, 4 },
	{ "Eread", 27, 1 },
	{ "Ebypass", 26, 1 },
	{ "Esave", 25, 1 },
	{ "Static0", 24, 1 },
	{ "Cread", 23, 1 },
	{ "Cbypass", 22, 1 },
	{ "Csave", 21, 1 },
	{ "CPktOut", 20, 1 },
	{ "RxPagePoolFull", 18, 2 },
	{ "RxLpbkPkt", 17, 1 },
	{ "TxLpbkPkt", 16, 1 },
	{ "RxVfValid", 15, 1 },
	{ "SynLearned", 14, 1 },
	{ "SetDelEntry", 13, 1 },
	{ "SetInvEntry", 12, 1 },
	{ "CpcmdDvld", 11, 1 },
	{ "CpcmdSave", 10, 1 },
	{ "RxPstructsFull", 8, 2 },
	{ "EpcmdDvld", 7, 1 },
	{ "EpcmdFlush", 6, 1 },
	{ "EpcmdTrimPrefix", 5, 1 },
	{ "EpcmdTrimPostfix", 4, 1 },
	{ "ERssIp4Pkt", 3, 1 },
	{ "ERssIp6Pkt", 2, 1 },
	{ "ERssTcpUdpPkt", 1, 1 },
	{ "ERssFceFipPkt", 0, 1 },
	{ NULL }
	};

	static void
	tp_la_show(struct sbuf sb, uint64_t p, int idx)
	{

	field_desc_show(sb, *p, tp_la0);
	}

	static void
	tp_la_show2(struct sbuf sb, uint64_t p, int idx)
	{

	if (idx)
	sbuf_printf(sb, "\n");
	field_desc_show(sb, p[0], tp_la0);
	if (idx < (TPLA_SIZE / 2 - 1) \|\| p[1] != ~0ULL)
	field_desc_show(sb, p[1], tp_la0);
	}

	static void
	tp_la_show3(struct sbuf sb, uint64_t p, int idx)
	{

	if (idx)
	sbuf_printf(sb, "\n");
	field_desc_show(sb, p[0], tp_la0);
	if (idx < (TPLA_SIZE / 2 - 1) \|\| p[1] != ~0ULL)
	field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
	}

	static int
	sysctl_tp_la(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	uint64_t buf, p;
	int rc;
	u_int i, inc;
	void (show_func)(struct sbuf , uint64_t *, int);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO \| M_WAITOK);

	t4_tp_read_la(sc, buf, NULL);
	p = buf;

	switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
	case 2:
	inc = 2;
	show_func = tp_la_show2;
	break;
	case 3:
	inc = 2;
	show_func = tp_la_show3;
	break;
	default:
	inc = 1;
	show_func = tp_la_show;
	}

	for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
	(*show_func)(sb, p, i);

	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	free(buf, M_CXGBE);
	return (rc);
	}

	static int
	sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc;
	u64 nrate[MAX_NCHAN], orate[MAX_NCHAN];

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
	return (ENOMEM);

	t4_get_chan_txrate(sc, nrate, orate);

	if (sc->chip_params->nchan > 2) {
	sbuf_printf(sb, " channel 0 channel 1"
	" channel 2 channel 3\n");
	sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n",
	nrate[0], nrate[1], nrate[2], nrate[3]);
	sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju",
	orate[0], orate[1], orate[2], orate[3]);
	} else {
	sbuf_printf(sb, " channel 0 channel 1\n");
	sbuf_printf(sb, "NIC B/s: %10ju %10ju\n",
	nrate[0], nrate[1]);
	sbuf_printf(sb, "Offload B/s: %10ju %10ju",
	orate[0], orate[1]);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	uint32_t buf, p;
	int rc, i;

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
	M_ZERO \| M_WAITOK);

	t4_ulprx_read_la(sc, buf);
	p = buf;

	sbuf_printf(sb, " Pcmd Type Message"
	" Data");
	for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
	sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x",
	p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
	}

	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	free(buf, M_CXGBE);
	return (rc);
	}

	static int
	sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	struct sbuf *sb;
	int rc, v;

	MPASS(chip_id(sc) >= CHELSIO_T5);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	v = t4_read_reg(sc, A_SGE_STAT_CFG);
	if (G_STATSOURCE_T5(v) == 7) {
	int mode;

	mode = is_t5(sc) ? G_STATMODE(v) : G_T6_STATMODE(v);
	if (mode == 0) {
	sbuf_printf(sb, "total %d, incomplete %d",
	t4_read_reg(sc, A_SGE_STAT_TOTAL),
	t4_read_reg(sc, A_SGE_STAT_MATCH));
	} else if (mode == 1) {
	sbuf_printf(sb, "total %d, data overflow %d",
	t4_read_reg(sc, A_SGE_STAT_TOTAL),
	t4_read_reg(sc, A_SGE_STAT_MATCH));
	} else {
	sbuf_printf(sb, "unknown mode %d", mode);
	}
	}
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	static int
	sysctl_cpus(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	enum cpu_sets op = arg2;
	cpuset_t cpuset;
	struct sbuf *sb;
	int i, rc;

	MPASS(op == LOCAL_CPUS \|\| op == INTR_CPUS);

	CPU_ZERO(&cpuset);
	rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset);
	if (rc != 0)
	return (rc);

	rc = sysctl_wire_old_buffer(req, 0);
	if (rc != 0)
	return (rc);

	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
	if (sb == NULL)
	return (ENOMEM);

	CPU_FOREACH(i)
	sbuf_printf(sb, "%d ", i);
	rc = sbuf_finish(sb);
	sbuf_delete(sb);

	return (rc);
	}

	#ifdef TCP_OFFLOAD
	static int
	sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int old_ports, new_ports;
	int i, new_count, rc;

	if (req->newptr == NULL && req->oldptr == NULL)
	return (SYSCTL_OUT(req, NULL, imax(sc->tt.num_tls_rx_ports, 1) *
	sizeof(sc->tt.tls_rx_ports[0])));

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4tlsrx");
	if (rc)
	return (rc);

	if (sc->tt.num_tls_rx_ports == 0) {
	i = -1;
	rc = SYSCTL_OUT(req, &i, sizeof(i));
	} else
	rc = SYSCTL_OUT(req, sc->tt.tls_rx_ports,
	sc->tt.num_tls_rx_ports * sizeof(sc->tt.tls_rx_ports[0]));
	if (rc == 0 && req->newptr != NULL) {
	new_count = req->newlen / sizeof(new_ports[0]);
	new_ports = malloc(new_count * sizeof(new_ports[0]), M_CXGBE,
	M_WAITOK);
	rc = SYSCTL_IN(req, new_ports, new_count *
	sizeof(new_ports[0]));
	if (rc)
	goto err;

	/* Allow setting to a single '-1' to clear the list. */
	if (new_count == 1 && new_ports[0] == -1) {
	ADAPTER_LOCK(sc);
	old_ports = sc->tt.tls_rx_ports;
	sc->tt.tls_rx_ports = NULL;
	sc->tt.num_tls_rx_ports = 0;
	ADAPTER_UNLOCK(sc);
	free(old_ports, M_CXGBE);
	} else {
	for (i = 0; i < new_count; i++) {
	if (new_ports[i] < 1 \|\|
	new_ports[i] > IPPORT_MAX) {
	rc = EINVAL;
	goto err;
	}
	}

	ADAPTER_LOCK(sc);
	old_ports = sc->tt.tls_rx_ports;
	sc->tt.tls_rx_ports = new_ports;
	sc->tt.num_tls_rx_ports = new_count;
	ADAPTER_UNLOCK(sc);
	free(old_ports, M_CXGBE);
	new_ports = NULL;
	}
	err:
	free(new_ports, M_CXGBE);
	}
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static void
	unit_conv(char *buf, size_t len, u_int val, u_int factor)
	{
	u_int rem = val % factor;

	if (rem == 0)
	snprintf(buf, len, "%u", val / factor);
	else {
	while (rem % 10 == 0)
	rem /= 10;
	snprintf(buf, len, "%u.%u", val / factor, rem);
	}
	}

	static int
	sysctl_tp_tick(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	char buf[16];
	u_int res, re;
	u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;

	res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
	switch (arg2) {
	case 0:
	/* timer_tick */
	re = G_TIMERRESOLUTION(res);
	break;
	case 1:
	/* TCP timestamp tick */
	re = G_TIMESTAMPRESOLUTION(res);
	break;
	case 2:
	/* DACK tick */
	re = G_DELAYEDACKRESOLUTION(res);
	break;
	default:
	return (EDOOFUS);
	}

	unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000);

	return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
	}

	static int
	sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	u_int res, dack_re, v;
	u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;

	res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
	dack_re = G_DELAYEDACKRESOLUTION(res);
	v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER);

	return (sysctl_handle_int(oidp, &v, 0, req));
	}

	static int
	sysctl_tp_timer(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int reg = arg2;
	u_int tre;
	u_long tp_tick_us, v;
	u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;

	MPASS(reg == A_TP_RXT_MIN \|\| reg == A_TP_RXT_MAX \|\|
	reg == A_TP_PERS_MIN \|\| reg == A_TP_PERS_MAX \|\|
	reg == A_TP_KEEP_IDLE \|\| reg == A_TP_KEEP_INTVL \|\|
	reg == A_TP_INIT_SRTT \|\| reg == A_TP_FINWAIT2_TIMER);

	tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION));
	tp_tick_us = (cclk_ps << tre) / 1000000;

	if (reg == A_TP_INIT_SRTT)
	v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg));
	else
	v = tp_tick_us * t4_read_reg(sc, reg);

	return (sysctl_handle_long(oidp, &v, 0, req));
	}

	/*
	* All fields in TP_SHIFT_CNT are 4b and the starting location of the field is
	* passed to this function.
	*/
	static int
	sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int idx = arg2;
	u_int v;

	MPASS(idx >= 0 && idx <= 24);

	v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf;

	return (sysctl_handle_int(oidp, &v, 0, req));
	}

	static int
	sysctl_tp_backoff(SYSCTL_HANDLER_ARGS)
	{
	struct adapter *sc = arg1;
	int idx = arg2;
	u_int shift, v, r;

	MPASS(idx >= 0 && idx < 16);

	r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3);
	shift = (idx & 3) << 3;
	v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0;

	return (sysctl_handle_int(oidp, &v, 0, req));
	}

	static int
	sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	struct adapter *sc = vi->adapter;
	int idx, rc, i;
	struct sge_ofld_rxq *ofld_rxq;
	uint8_t v;

	idx = vi->ofld_tmr_idx;

	rc = sysctl_handle_int(oidp, &idx, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if (idx < 0 \|\| idx >= SGE_NTIMERS)
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, HOLD_LOCK \| SLEEP_OK \| INTR_OK,
	"t4otmr");
	if (rc)
	return (rc);

	v = V_QINTR_TIMER_IDX(idx) \| V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1);
	for_each_ofld_rxq(vi, i, ofld_rxq) {
	#ifdef atomic_store_rel_8
	atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
	#else
	ofld_rxq->iq.intr_params = v;
	#endif
	}
	vi->ofld_tmr_idx = idx;

	end_synchronized_op(sc, LOCK_HELD);
	return (0);
	}

	static int
	sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS)
	{
	struct vi_info *vi = arg1;
	struct adapter *sc = vi->adapter;
	int idx, rc;

	idx = vi->ofld_pktc_idx;

	rc = sysctl_handle_int(oidp, &idx, 0, req);
	if (rc != 0 \|\| req->newptr == NULL)
	return (rc);

	if (idx < -1 \|\| idx >= SGE_NCOUNTERS)
	return (EINVAL);

	rc = begin_synchronized_op(sc, vi, HOLD_LOCK \| SLEEP_OK \| INTR_OK,
	"t4opktc");
	if (rc)
	return (rc);

	if (vi->flags & VI_INIT_DONE)
	rc = EBUSY; /* cannot be changed once the queues are created */
	else
	vi->ofld_pktc_idx = idx;

	end_synchronized_op(sc, LOCK_HELD);
	return (rc);
	}
	#endif

	static int
	get_sge_context(struct adapter sc, struct t4_sge_context cntxt)
	{
	int rc;

	if (cntxt->cid > M_CTXTQID)
	return (EINVAL);

	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
	cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
	return (EINVAL);

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4ctxt");
	if (rc)
	return (rc);

	if (sc->flags & FW_OK) {
	rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
	&cntxt->data[0]);
	if (rc == 0)
	goto done;
	}

	/*
	* Read via firmware failed or wasn't even attempted. Read directly via
	* the backdoor.
	*/
	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
	done:
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static int
	load_fw(struct adapter sc, struct t4_data fw)
	{
	int rc;
	uint8_t *fw_data;

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4ldfw");
	if (rc)
	return (rc);

	/*
	* The firmware, with the sole exception of the memory parity error
	* handler, runs from memory and not flash. It is almost always safe to
	* install a new firmware on a running system. Just set bit 1 in
	* hw.cxgbe.dflags or dev.<nexus>.<n>.dflags first.
	*/
	if (sc->flags & FULL_INIT_DONE &&
	(sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) {
	rc = EBUSY;
	goto done;
	}

	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
	- if (fw_data == NULL) {
	- rc = ENOMEM;
	- goto done;
	- }

	rc = copyin(fw->data, fw_data, fw->len);
	if (rc == 0)
	rc = -t4_load_fw(sc, fw_data, fw->len);

	free(fw_data, M_CXGBE);
	done:
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static int
	load_cfg(struct adapter sc, struct t4_data cfg)
	{
	int rc;
	uint8_t *cfg_data = NULL;

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4ldcf");
	if (rc)
	return (rc);

	if (cfg->len == 0) {
	/* clear */
	rc = -t4_load_cfg(sc, NULL, 0);
	goto done;
	}

	cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
	- if (cfg_data == NULL) {
	- rc = ENOMEM;
	- goto done;
	- }

	rc = copyin(cfg->data, cfg_data, cfg->len);
	if (rc == 0)
	rc = -t4_load_cfg(sc, cfg_data, cfg->len);

	free(cfg_data, M_CXGBE);
	done:
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static int
	load_boot(struct adapter sc, struct t4_bootrom br)
	{
	int rc;
	uint8_t *br_data = NULL;
	u_int offset;

	if (br->len > 1024 * 1024)
	return (EFBIG);

	if (br->pf_offset == 0) {
	/* pfidx */
	if (br->pfidx_addr > 7)
	return (EINVAL);
	offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr,
	A_PCIE_PF_EXPROM_OFST)));
	} else if (br->pf_offset == 1) {
	/* offset */
	offset = G_OFFSET(br->pfidx_addr);
	} else {
	return (EINVAL);
	}

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4ldbr");
	if (rc)
	return (rc);

	if (br->len == 0) {
	/* clear */
	rc = -t4_load_boot(sc, NULL, offset, 0);
	goto done;
	}

	br_data = malloc(br->len, M_CXGBE, M_WAITOK);
	- if (br_data == NULL) {
	- rc = ENOMEM;
	- goto done;
	- }

	rc = copyin(br->data, br_data, br->len);
	if (rc == 0)
	rc = -t4_load_boot(sc, br_data, offset, br->len);

	free(br_data, M_CXGBE);
	done:
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static int
	load_bootcfg(struct adapter sc, struct t4_data bc)
	{
	int rc;
	uint8_t *bc_data = NULL;

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4ldcf");
	if (rc)
	return (rc);

	if (bc->len == 0) {
	/* clear */
	rc = -t4_load_bootcfg(sc, NULL, 0);
	goto done;
	}

	bc_data = malloc(bc->len, M_CXGBE, M_WAITOK);
	- if (bc_data == NULL) {
	- rc = ENOMEM;
	- goto done;
	- }

	rc = copyin(bc->data, bc_data, bc->len);
	if (rc == 0)
	rc = -t4_load_bootcfg(sc, bc_data, bc->len);

	free(bc_data, M_CXGBE);
	done:
	end_synchronized_op(sc, 0);
	return (rc);
	}

	static int
	cudbg_dump(struct adapter sc, struct t4_cudbg_dump dump)
	{
	int rc;
	struct cudbg_init *cudbg;
	void handle, buf;

	/* buf is large, don't block if no memory is available */
	buf = malloc(dump->len, M_CXGBE, M_NOWAIT \| M_ZERO);
	if (buf == NULL)
	return (ENOMEM);

	handle = cudbg_alloc_handle();
	if (handle == NULL) {
	rc = ENOMEM;
	goto done;
	}

	cudbg = cudbg_get_init(handle);
	cudbg->adap = sc;
	cudbg->print = (cudbg_print_cb)printf;

	#ifndef notyet
	device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n",
	__func__, dump->wr_flash, dump->len, dump->data);
	#endif

	if (dump->wr_flash)
	cudbg->use_flash = 1;
	MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap));
	memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap));

	rc = cudbg_collect(handle, buf, &dump->len);
	if (rc != 0)
	goto done;

	rc = copyout(buf, dump->data, dump->len);
	done:
	cudbg_free_handle(handle);
	free(buf, M_CXGBE);
	return (rc);
	}

	static void
	free_offload_policy(struct t4_offload_policy *op)
	{
	struct offload_rule *r;
	int i;

	if (op == NULL)
	return;

	r = &op->rule[0];
	for (i = 0; i < op->nrules; i++, r++) {
	free(r->bpf_prog.bf_insns, M_CXGBE);
	}
	free(op->rule, M_CXGBE);
	free(op, M_CXGBE);
	}

	static int
	set_offload_policy(struct adapter sc, struct t4_offload_policy uop)
	{
	int i, rc, len;
	struct t4_offload_policy op, old;
	struct bpf_program *bf;
	const struct offload_settings *s;
	struct offload_rule *r;
	void *u;

	if (!is_offload(sc))
	return (ENODEV);

	if (uop->nrules == 0) {
	/* Delete installed policies. */
	op = NULL;
	goto set_policy;
	} else if (uop->nrules > 256) { /* arbitrary */
	return (E2BIG);
	}

	/* Copy userspace offload policy to kernel */
	op = malloc(sizeof(*op), M_CXGBE, M_ZERO \| M_WAITOK);
	op->nrules = uop->nrules;
	len = op->nrules * sizeof(struct offload_rule);
	op->rule = malloc(len, M_CXGBE, M_ZERO \| M_WAITOK);
	rc = copyin(uop->rule, op->rule, len);
	if (rc) {
	free(op->rule, M_CXGBE);
	free(op, M_CXGBE);
	return (rc);
	}

	r = &op->rule[0];
	for (i = 0; i < op->nrules; i++, r++) {

	/* Validate open_type */
	if (r->open_type != OPEN_TYPE_LISTEN &&
	r->open_type != OPEN_TYPE_ACTIVE &&
	r->open_type != OPEN_TYPE_PASSIVE &&
	r->open_type != OPEN_TYPE_DONTCARE) {
	error:
	/*
	* Rules 0 to i have malloc'd filters that need to be
	* freed. Rules i+1 to nrules have userspace pointers
	* and should be left alone.
	*/
	op->nrules = i;
	free_offload_policy(op);
	return (rc);
	}

	/* Validate settings */
	s = &r->settings;
	if ((s->offload != 0 && s->offload != 1) \|\|
	s->cong_algo < -1 \|\| s->cong_algo > CONG_ALG_HIGHSPEED \|\|
	s->sched_class < -1 \|\|
	s->sched_class >= sc->chip_params->nsched_cls) {
	rc = EINVAL;
	goto error;
	}

	bf = &r->bpf_prog;
	u = bf->bf_insns; /* userspace ptr */
	bf->bf_insns = NULL;
	if (bf->bf_len == 0) {
	/* legal, matches everything */
	continue;
	}
	len = bf->bf_len * sizeof(*bf->bf_insns);
	bf->bf_insns = malloc(len, M_CXGBE, M_ZERO \| M_WAITOK);
	rc = copyin(u, bf->bf_insns, len);
	if (rc != 0)
	goto error;

	if (!bpf_validate(bf->bf_insns, bf->bf_len)) {
	rc = EINVAL;
	goto error;
	}
	}
	set_policy:
	rw_wlock(&sc->policy_lock);
	old = sc->policy;
	sc->policy = op;
	rw_wunlock(&sc->policy_lock);
	free_offload_policy(old);

	return (0);
	}

	#define MAX_READ_BUF_SIZE (128 * 1024)
	static int
	read_card_mem(struct adapter sc, int win, struct t4_mem_range mr)
	{
	uint32_t addr, remaining, n;
	uint32_t *buf;
	int rc;
	uint8_t *dst;

	rc = validate_mem_range(sc, mr->addr, mr->len);
	if (rc != 0)
	return (rc);

	buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
	addr = mr->addr;
	remaining = mr->len;
	dst = (void *)mr->data;

	while (remaining) {
	n = min(remaining, MAX_READ_BUF_SIZE);
	read_via_memwin(sc, 2, addr, buf, n);

	rc = copyout(buf, dst, n);
	if (rc != 0)
	break;

	dst += n;
	remaining -= n;
	addr += n;
	}

	free(buf, M_CXGBE);
	return (rc);
	}
	#undef MAX_READ_BUF_SIZE

	static int
	read_i2c(struct adapter sc, struct t4_i2c_data i2cd)
	{
	int rc;

	if (i2cd->len == 0 \|\| i2cd->port_id >= sc->params.nports)
	return (EINVAL);

	if (i2cd->len > sizeof(i2cd->data))
	return (EFBIG);

	rc = begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4i2crd");
	if (rc)
	return (rc);
	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
	i2cd->offset, i2cd->len, &i2cd->data[0]);
	end_synchronized_op(sc, 0);

	return (rc);
	}

	static int
	clear_stats(struct adapter *sc, u_int port_id)
	{
	int i, v, chan_map;
	struct port_info *pi;
	struct vi_info *vi;
	struct sge_rxq *rxq;
	struct sge_txq *txq;
	struct sge_wrq *wrq;
	#ifdef TCP_OFFLOAD
	struct sge_ofld_rxq *ofld_rxq;
	#endif

	if (port_id >= sc->params.nports)
	return (EINVAL);
	pi = sc->port[port_id];
	if (pi == NULL)
	return (EIO);

	/* MAC stats */
	t4_clr_port_stats(sc, pi->tx_chan);
	pi->tx_parse_error = 0;
	pi->tnl_cong_drops = 0;
	mtx_lock(&sc->reg_lock);
	for_each_vi(pi, v, vi) {
	if (vi->flags & VI_INIT_DONE)
	t4_clr_vi_stats(sc, vi->vin);
	}
	chan_map = pi->rx_e_chan_map;
	v = 0; /* reuse */
	while (chan_map) {
	i = ffs(chan_map) - 1;
	t4_write_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
	1, A_TP_MIB_TNL_CNG_DROP_0 + i);
	chan_map &= ~(1 << i);
	}
	mtx_unlock(&sc->reg_lock);

	/*
	* Since this command accepts a port, clear stats for
	* all VIs on this port.
	*/
	for_each_vi(pi, v, vi) {
	if (vi->flags & VI_INIT_DONE) {

	for_each_rxq(vi, i, rxq) {
	#if defined(INET) \|\| defined(INET6)
	rxq->lro.lro_queued = 0;
	rxq->lro.lro_flushed = 0;
	#endif
	rxq->rxcsum = 0;
	rxq->vlan_extraction = 0;

	rxq->fl.cl_allocated = 0;
	rxq->fl.cl_recycled = 0;
	rxq->fl.cl_fast_recycled = 0;
	}

	for_each_txq(vi, i, txq) {
	txq->txcsum = 0;
	txq->tso_wrs = 0;
	txq->vlan_insertion = 0;
	txq->imm_wrs = 0;
	txq->sgl_wrs = 0;
	txq->txpkt_wrs = 0;
	txq->txpkts0_wrs = 0;
	txq->txpkts1_wrs = 0;
	txq->txpkts0_pkts = 0;
	txq->txpkts1_pkts = 0;
	txq->raw_wrs = 0;
	txq->kern_tls_records = 0;
	txq->kern_tls_short = 0;
	txq->kern_tls_partial = 0;
	txq->kern_tls_full = 0;
	txq->kern_tls_octets = 0;
	txq->kern_tls_waste = 0;
	txq->kern_tls_options = 0;
	txq->kern_tls_header = 0;
	txq->kern_tls_fin = 0;
	txq->kern_tls_fin_short = 0;
	txq->kern_tls_cbc = 0;
	txq->kern_tls_gcm = 0;
	mp_ring_reset_stats(txq->r);
	}

	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	for_each_ofld_txq(vi, i, wrq) {
	wrq->tx_wrs_direct = 0;
	wrq->tx_wrs_copied = 0;
	}
	#endif
	#ifdef TCP_OFFLOAD
	for_each_ofld_rxq(vi, i, ofld_rxq) {
	ofld_rxq->fl.cl_allocated = 0;
	ofld_rxq->fl.cl_recycled = 0;
	ofld_rxq->fl.cl_fast_recycled = 0;
	}
	#endif

	if (IS_MAIN_VI(vi)) {
	wrq = &sc->sge.ctrlq[pi->port_id];
	wrq->tx_wrs_direct = 0;
	wrq->tx_wrs_copied = 0;
	}
	}
	}

	return (0);
	}

	int
	t4_os_find_pci_capability(struct adapter *sc, int cap)
	{
	int i;

	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
	}

	int
	t4_os_pci_save_state(struct adapter *sc)
	{
	device_t dev;
	struct pci_devinfo *dinfo;

	dev = sc->dev;
	dinfo = device_get_ivars(dev);

	pci_cfg_save(dev, dinfo, 0);
	return (0);
	}

	int
	t4_os_pci_restore_state(struct adapter *sc)
	{
	device_t dev;
	struct pci_devinfo *dinfo;

	dev = sc->dev;
	dinfo = device_get_ivars(dev);

	pci_cfg_restore(dev, dinfo);
	return (0);
	}

	void
	t4_os_portmod_changed(struct port_info *pi)
	{
	struct adapter *sc = pi->adapter;
	struct vi_info *vi;
	struct ifnet *ifp;
	static const char *mod_str[] = {
	NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
	};

	KASSERT((pi->flags & FIXED_IFMEDIA) == 0,
	("%s: port_type %u", __func__, pi->port_type));

	vi = &pi->vi[0];
	if (begin_synchronized_op(sc, vi, HOLD_LOCK, "t4mod") == 0) {
	PORT_LOCK(pi);
	build_medialist(pi);
	if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) {
	fixup_link_config(pi);
	apply_link_config(pi);
	}
	PORT_UNLOCK(pi);
	end_synchronized_op(sc, LOCK_HELD);
	}

	ifp = vi->ifp;
	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
	if_printf(ifp, "transceiver unplugged.\n");
	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
	if_printf(ifp, "unknown transceiver inserted.\n");
	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
	if_printf(ifp, "unsupported transceiver inserted.\n");
	else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
	if_printf(ifp, "%dGbps %s transceiver inserted.\n",
	port_top_speed(pi), mod_str[pi->mod_type]);
	} else {
	if_printf(ifp, "transceiver (type %d) inserted.\n",
	pi->mod_type);
	}
	}

	void
	t4_os_link_changed(struct port_info *pi)
	{
	struct vi_info *vi;
	struct ifnet *ifp;
	struct link_config *lc;
	int v;

	PORT_LOCK_ASSERT_OWNED(pi);

	for_each_vi(pi, v, vi) {
	ifp = vi->ifp;
	if (ifp == NULL)
	continue;

	lc = &pi->link_cfg;
	if (lc->link_ok) {
	ifp->if_baudrate = IF_Mbps(lc->speed);
	if_link_state_change(ifp, LINK_STATE_UP);
	} else {
	if_link_state_change(ifp, LINK_STATE_DOWN);
	}
	}
	}

	void
	t4_iterate(void (func)(struct adapter , void ), void arg)
	{
	struct adapter *sc;

	sx_slock(&t4_list_lock);
	SLIST_FOREACH(sc, &t4_list, link) {
	/*
	* func should not make any assumptions about what state sc is
	* in - the only guarantee is that sc->sc_lock is a valid lock.
	*/
	func(sc, arg);
	}
	sx_sunlock(&t4_list_lock);
	}

	static int
	t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
	struct thread *td)
	{
	int rc;
	struct adapter *sc = dev->si_drv1;

	rc = priv_check(td, PRIV_DRIVER);
	if (rc != 0)
	return (rc);

	switch (cmd) {
	case CHELSIO_T4_GETREG: {
	struct t4_reg edata = (struct t4_reg )data;

	if ((edata->addr & 0x3) != 0 \|\| edata->addr >= sc->mmio_len)
	return (EFAULT);

	if (edata->size == 4)
	edata->val = t4_read_reg(sc, edata->addr);
	else if (edata->size == 8)
	edata->val = t4_read_reg64(sc, edata->addr);
	else
	return (EINVAL);

	break;
	}
	case CHELSIO_T4_SETREG: {
	struct t4_reg edata = (struct t4_reg )data;

	if ((edata->addr & 0x3) != 0 \|\| edata->addr >= sc->mmio_len)
	return (EFAULT);

	if (edata->size == 4) {
	if (edata->val & 0xffffffff00000000)
	return (EINVAL);
	t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
	} else if (edata->size == 8)
	t4_write_reg64(sc, edata->addr, edata->val);
	else
	return (EINVAL);
	break;
	}
	case CHELSIO_T4_REGDUMP: {
	struct t4_regdump regs = (struct t4_regdump )data;
	int reglen = t4_get_regs_len(sc);
	uint8_t *buf;

	if (regs->len < reglen) {
	regs->len = reglen; /* hint to the caller */
	return (ENOBUFS);
	}

	regs->len = reglen;
	buf = malloc(reglen, M_CXGBE, M_WAITOK \| M_ZERO);
	get_regs(sc, regs, buf);
	rc = copyout(buf, regs->data, reglen);
	free(buf, M_CXGBE);
	break;
	}
	case CHELSIO_T4_GET_FILTER_MODE:
	rc = get_filter_mode(sc, (uint32_t *)data);
	break;
	case CHELSIO_T4_SET_FILTER_MODE:
	rc = set_filter_mode(sc, (uint32_t )data);
	break;
	case CHELSIO_T4_GET_FILTER:
	rc = get_filter(sc, (struct t4_filter *)data);
	break;
	case CHELSIO_T4_SET_FILTER:
	rc = set_filter(sc, (struct t4_filter *)data);
	break;
	case CHELSIO_T4_DEL_FILTER:
	rc = del_filter(sc, (struct t4_filter *)data);
	break;
	case CHELSIO_T4_GET_SGE_CONTEXT:
	rc = get_sge_context(sc, (struct t4_sge_context *)data);
	break;
	case CHELSIO_T4_LOAD_FW:
	rc = load_fw(sc, (struct t4_data *)data);
	break;
	case CHELSIO_T4_GET_MEM:
	rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
	break;
	case CHELSIO_T4_GET_I2C:
	rc = read_i2c(sc, (struct t4_i2c_data *)data);
	break;
	case CHELSIO_T4_CLEAR_STATS:
	rc = clear_stats(sc, (uint32_t )data);
	break;
	case CHELSIO_T4_SCHED_CLASS:
	rc = t4_set_sched_class(sc, (struct t4_sched_params *)data);
	break;
	case CHELSIO_T4_SCHED_QUEUE:
	rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data);
	break;
	case CHELSIO_T4_GET_TRACER:
	rc = t4_get_tracer(sc, (struct t4_tracer *)data);
	break;
	case CHELSIO_T4_SET_TRACER:
	rc = t4_set_tracer(sc, (struct t4_tracer *)data);
	break;
	case CHELSIO_T4_LOAD_CFG:
	rc = load_cfg(sc, (struct t4_data *)data);
	break;
	case CHELSIO_T4_LOAD_BOOT:
	rc = load_boot(sc, (struct t4_bootrom *)data);
	break;
	case CHELSIO_T4_LOAD_BOOTCFG:
	rc = load_bootcfg(sc, (struct t4_data *)data);
	break;
	case CHELSIO_T4_CUDBG_DUMP:
	rc = cudbg_dump(sc, (struct t4_cudbg_dump *)data);
	break;
	case CHELSIO_T4_SET_OFLD_POLICY:
	rc = set_offload_policy(sc, (struct t4_offload_policy *)data);
	break;
	default:
	rc = ENOTTY;
	}

	return (rc);
	}

	#ifdef TCP_OFFLOAD
	static int
	toe_capability(struct vi_info *vi, int enable)
	{
	int rc;
	struct port_info *pi = vi->pi;
	struct adapter *sc = pi->adapter;

	ASSERT_SYNCHRONIZED_OP(sc);

	if (!is_offload(sc))
	return (ENODEV);

	if (enable) {
	if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) {
	/* TOE is already enabled. */
	return (0);
	}

	/*
	* We need the port's queues around so that we're able to send
	* and receive CPLs to/from the TOE even if the ifnet for this
	* port has never been UP'd administratively.
	*/
	if (!(vi->flags & VI_INIT_DONE)) {
	rc = vi_full_init(vi);
	if (rc)
	return (rc);
	}
	if (!(pi->vi[0].flags & VI_INIT_DONE)) {
	rc = vi_full_init(&pi->vi[0]);
	if (rc)
	return (rc);
	}

	if (isset(&sc->offload_map, pi->port_id)) {
	/* TOE is enabled on another VI of this port. */
	pi->uld_vis++;
	return (0);
	}

	if (!uld_active(sc, ULD_TOM)) {
	rc = t4_activate_uld(sc, ULD_TOM);
	if (rc == EAGAIN) {
	log(LOG_WARNING,
	"You must kldload t4_tom.ko before trying "
	"to enable TOE on a cxgbe interface.\n");
	}
	if (rc != 0)
	return (rc);
	KASSERT(sc->tom_softc != NULL,
	("%s: TOM activated but softc NULL", __func__));
	KASSERT(uld_active(sc, ULD_TOM),
	("%s: TOM activated but flag not set", __func__));
	}

	/* Activate iWARP and iSCSI too, if the modules are loaded. */
	if (!uld_active(sc, ULD_IWARP))
	(void) t4_activate_uld(sc, ULD_IWARP);
	if (!uld_active(sc, ULD_ISCSI))
	(void) t4_activate_uld(sc, ULD_ISCSI);

	pi->uld_vis++;
	setbit(&sc->offload_map, pi->port_id);
	} else {
	pi->uld_vis--;

	if (!isset(&sc->offload_map, pi->port_id) \|\| pi->uld_vis > 0)
	return (0);

	KASSERT(uld_active(sc, ULD_TOM),
	("%s: TOM never initialized?", __func__));
	clrbit(&sc->offload_map, pi->port_id);
	}

	return (0);
	}

	/*
	* Add an upper layer driver to the global list.
	*/
	int
	t4_register_uld(struct uld_info *ui)
	{
	int rc = 0;
	struct uld_info *u;

	sx_xlock(&t4_uld_list_lock);
	SLIST_FOREACH(u, &t4_uld_list, link) {
	if (u->uld_id == ui->uld_id) {
	rc = EEXIST;
	goto done;
	}
	}

	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
	ui->refcount = 0;
	done:
	sx_xunlock(&t4_uld_list_lock);
	return (rc);
	}

	int
	t4_unregister_uld(struct uld_info *ui)
	{
	int rc = EINVAL;
	struct uld_info *u;

	sx_xlock(&t4_uld_list_lock);

	SLIST_FOREACH(u, &t4_uld_list, link) {
	if (u == ui) {
	if (ui->refcount > 0) {
	rc = EBUSY;
	goto done;
	}

	SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
	rc = 0;
	goto done;
	}
	}
	done:
	sx_xunlock(&t4_uld_list_lock);
	return (rc);
	}

	int
	t4_activate_uld(struct adapter *sc, int id)
	{
	int rc;
	struct uld_info *ui;

	ASSERT_SYNCHRONIZED_OP(sc);

	if (id < 0 \|\| id > ULD_MAX)
	return (EINVAL);
	rc = EAGAIN; /* kldoad the module with this ULD and try again. */

	sx_slock(&t4_uld_list_lock);

	SLIST_FOREACH(ui, &t4_uld_list, link) {
	if (ui->uld_id == id) {
	if (!(sc->flags & FULL_INIT_DONE)) {
	rc = adapter_full_init(sc);
	if (rc != 0)
	break;
	}

	rc = ui->activate(sc);
	if (rc == 0) {
	setbit(&sc->active_ulds, id);
	ui->refcount++;
	}
	break;
	}
	}

	sx_sunlock(&t4_uld_list_lock);

	return (rc);
	}

	int
	t4_deactivate_uld(struct adapter *sc, int id)
	{
	int rc;
	struct uld_info *ui;

	ASSERT_SYNCHRONIZED_OP(sc);

	if (id < 0 \|\| id > ULD_MAX)
	return (EINVAL);
	rc = ENXIO;

	sx_slock(&t4_uld_list_lock);

	SLIST_FOREACH(ui, &t4_uld_list, link) {
	if (ui->uld_id == id) {
	rc = ui->deactivate(sc);
	if (rc == 0) {
	clrbit(&sc->active_ulds, id);
	ui->refcount--;
	}
	break;
	}
	}

	sx_sunlock(&t4_uld_list_lock);

	return (rc);
	}

	static void
	t4_async_event(void *arg, int n)
	{
	struct uld_info *ui;
	struct adapter sc = (struct adapter )arg;

	if (begin_synchronized_op(sc, NULL, SLEEP_OK \| INTR_OK, "t4async") != 0)
	return;
	sx_slock(&t4_uld_list_lock);
	SLIST_FOREACH(ui, &t4_uld_list, link) {
	if (ui->uld_id == ULD_IWARP) {
	ui->async_event(sc);
	break;
	}
	}
	sx_sunlock(&t4_uld_list_lock);
	end_synchronized_op(sc, 0);
	}

	int
	uld_active(struct adapter *sc, int uld_id)
	{

	MPASS(uld_id >= 0 && uld_id <= ULD_MAX);

	return (isset(&sc->active_ulds, uld_id));
	}
	#endif

	/*
	* t = ptr to tunable.
	* nc = number of CPUs.
	* c = compiled in default for that tunable.
	*/
	static void
	calculate_nqueues(int *t, int nc, const int c)
	{
	int nq;

	if (*t > 0)
	return;
	nq = t < 0 ? -t : c;
	*t = min(nc, nq);
	}

	/*
	* Come up with reasonable defaults for some of the tunables, provided they're
	* not set by the user (in which case we'll use the values as is).
	*/
	static void
	tweak_tunables(void)
	{
	int nc = mp_ncpus; /* our snapshot of the number of CPUs */

	if (t4_ntxq < 1) {
	#ifdef RSS
	t4_ntxq = rss_getnumbuckets();
	#else
	calculate_nqueues(&t4_ntxq, nc, NTXQ);
	#endif
	}

	calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI);

	if (t4_nrxq < 1) {
	#ifdef RSS
	t4_nrxq = rss_getnumbuckets();
	#else
	calculate_nqueues(&t4_nrxq, nc, NRXQ);
	#endif
	}

	calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI);

	#if defined(TCP_OFFLOAD) \|\| defined(RATELIMIT)
	calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ);
	calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI);
	#endif
	#ifdef TCP_OFFLOAD
	calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ);
	calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI);
	#endif

	#if defined(TCP_OFFLOAD) \|\| defined(KERN_TLS)
	if (t4_toecaps_allowed == -1)
	t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
	#else
	if (t4_toecaps_allowed == -1)
	t4_toecaps_allowed = 0;
	#endif

	#ifdef TCP_OFFLOAD
	if (t4_rdmacaps_allowed == -1) {
	t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP \|
	FW_CAPS_CONFIG_RDMA_RDMAC;
	}

	if (t4_iscsicaps_allowed == -1) {
	t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU \|
	FW_CAPS_CONFIG_ISCSI_TARGET_PDU \|
	FW_CAPS_CONFIG_ISCSI_T10DIF;
	}

	if (t4_tmr_idx_ofld < 0 \|\| t4_tmr_idx_ofld >= SGE_NTIMERS)
	t4_tmr_idx_ofld = TMR_IDX_OFLD;

	if (t4_pktc_idx_ofld < -1 \|\| t4_pktc_idx_ofld >= SGE_NCOUNTERS)
	t4_pktc_idx_ofld = PKTC_IDX_OFLD;
	#else
	if (t4_rdmacaps_allowed == -1)
	t4_rdmacaps_allowed = 0;

	if (t4_iscsicaps_allowed == -1)
	t4_iscsicaps_allowed = 0;
	#endif

	#ifdef DEV_NETMAP
	calculate_nqueues(&t4_nnmtxq, nc, NNMTXQ);
	calculate_nqueues(&t4_nnmrxq, nc, NNMRXQ);
	calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI);
	calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI);
	#endif

	if (t4_tmr_idx < 0 \|\| t4_tmr_idx >= SGE_NTIMERS)
	t4_tmr_idx = TMR_IDX;

	if (t4_pktc_idx < -1 \|\| t4_pktc_idx >= SGE_NCOUNTERS)
	t4_pktc_idx = PKTC_IDX;

	if (t4_qsize_txq < 128)
	t4_qsize_txq = 128;

	if (t4_qsize_rxq < 128)
	t4_qsize_rxq = 128;
	while (t4_qsize_rxq & 7)
	t4_qsize_rxq++;

	t4_intr_types &= INTR_MSIX \| INTR_MSI \| INTR_INTX;

	/*
	* Number of VIs to create per-port. The first VI is the "main" regular
	* VI for the port. The rest are additional virtual interfaces on the
	* same physical port. Note that the main VI does not have native
	* netmap support but the extra VIs do.
	*
	* Limit the number of VIs per port to the number of available
	* MAC addresses per port.
	*/
	if (t4_num_vis < 1)
	t4_num_vis = 1;
	if (t4_num_vis > nitems(vi_mac_funcs)) {
	t4_num_vis = nitems(vi_mac_funcs);
	printf("cxgbe: number of VIs limited to %d\n", t4_num_vis);
	}

	if (pcie_relaxed_ordering < 0 \|\| pcie_relaxed_ordering > 2) {
	pcie_relaxed_ordering = 1;
	#if defined(__i386__) \|\| defined(__amd64__)
	if (cpu_vendor_id == CPU_VENDOR_INTEL)
	pcie_relaxed_ordering = 0;
	#endif
	}
	}

	#ifdef DDB
	static void
	t4_dump_tcb(struct adapter *sc, int tid)
	{
	uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos;

	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2);
	save = t4_read_reg(sc, reg);
	base = sc->memwin[2].mw_base;

	/* Dump TCB for the tid */
	tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
	tcb_addr += tid * TCB_SIZE;

	if (is_t4(sc)) {
	pf = 0;
	win_pos = tcb_addr & ~0xf; /* start must be 16B aligned */
	} else {
	pf = V_PFNUM(sc->pf);
	win_pos = tcb_addr & ~0x7f; /* start must be 128B aligned */
	}
	t4_write_reg(sc, reg, win_pos \| pf);
	t4_read_reg(sc, reg);

	off = tcb_addr - win_pos;
	for (i = 0; i < 4; i++) {
	uint32_t buf[8];
	for (j = 0; j < 8; j++, off += 4)
	buf[j] = htonl(t4_read_reg(sc, base + off));

	db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n",
	buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
	buf[7]);
	}

	t4_write_reg(sc, reg, save);
	t4_read_reg(sc, reg);
	}

	static void
	t4_dump_devlog(struct adapter *sc)
	{
	struct devlog_params *dparams = &sc->params.devlog;
	struct fw_devlog_e e;
	int i, first, j, m, nentries, rc;
	uint64_t ftstamp = UINT64_MAX;

	if (dparams->start == 0) {
	db_printf("devlog params not valid\n");
	return;
	}

	nentries = dparams->size / sizeof(struct fw_devlog_e);
	m = fwmtype_to_hwmtype(dparams->memtype);

	/* Find the first entry. */
	first = -1;
	for (i = 0; i < nentries && !db_pager_quit; i++) {
	rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
	sizeof(e), (void *)&e);
	if (rc != 0)
	break;

	if (e.timestamp == 0)
	break;

	e.timestamp = be64toh(e.timestamp);
	if (e.timestamp < ftstamp) {
	ftstamp = e.timestamp;
	first = i;
	}
	}

	if (first == -1)
	return;

	i = first;
	do {
	rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
	sizeof(e), (void *)&e);
	if (rc != 0)
	return;

	if (e.timestamp == 0)
	return;

	e.timestamp = be64toh(e.timestamp);
	e.seqno = be32toh(e.seqno);
	for (j = 0; j < 8; j++)
	e.params[j] = be32toh(e.params[j]);

	db_printf("%10d %15ju %8s %8s ",
	e.seqno, e.timestamp,
	(e.level < nitems(devlog_level_strings) ?
	devlog_level_strings[e.level] : "UNKNOWN"),
	(e.facility < nitems(devlog_facility_strings) ?
	devlog_facility_strings[e.facility] : "UNKNOWN"));
	db_printf(e.fmt, e.params[0], e.params[1], e.params[2],
	e.params[3], e.params[4], e.params[5], e.params[6],
	e.params[7]);

	if (++i == nentries)
	i = 0;
	} while (i != first && !db_pager_quit);
	}

	static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table);
	_DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table);

	DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL)
	{
	device_t dev;
	int t;
	bool valid;

	valid = false;
	t = db_read_token();
	if (t == tIDENT) {
	dev = device_lookup_by_name(db_tok_string);
	valid = true;
	}
	db_skip_to_eol();
	if (!valid) {
	db_printf("usage: show t4 devlog <nexus>\n");
	return;
	}

	if (dev == NULL) {
	db_printf("device not found\n");
	return;
	}

	t4_dump_devlog(device_get_softc(dev));
	}

	DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL)
	{
	device_t dev;
	int radix, tid, t;
	bool valid;

	valid = false;
	radix = db_radix;
	db_radix = 10;
	t = db_read_token();
	if (t == tIDENT) {
	dev = device_lookup_by_name(db_tok_string);
	t = db_read_token();
	if (t == tNUMBER) {
	tid = db_tok_number;
	valid = true;
	}
	}
	db_radix = radix;
	db_skip_to_eol();
	if (!valid) {
	db_printf("usage: show t4 tcb <nexus> <tid>\n");
	return;
	}

	if (dev == NULL) {
	db_printf("device not found\n");
	return;
	}
	if (tid < 0) {
	db_printf("invalid tid\n");
	return;
	}

	t4_dump_tcb(device_get_softc(dev), tid);
	}
	#endif

	static struct sx mlu; /* mod load unload */
	SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");

	static int
	mod_event(module_t mod, int cmd, void *arg)
	{
	int rc = 0;
	static int loaded = 0;

	switch (cmd) {
	case MOD_LOAD:
	sx_xlock(&mlu);
	if (loaded++ == 0) {
	t4_sge_modload();
	t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
	t4_filter_rpl, CPL_COOKIE_FILTER);
	t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL,
	do_l2t_write_rpl, CPL_COOKIE_FILTER);
	t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL,
	t4_hashfilter_ao_rpl, CPL_COOKIE_HASHFILTER);
	t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
	t4_hashfilter_tcb_rpl, CPL_COOKIE_HASHFILTER);
	t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS,
	t4_del_hashfilter_rpl, CPL_COOKIE_HASHFILTER);
	t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
	t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
	t4_register_cpl_handler(CPL_SMT_WRITE_RPL,
	do_smt_write_rpl);
	sx_init(&t4_list_lock, "T4/T5 adapters");
	SLIST_INIT(&t4_list);
	callout_init(&fatal_callout, 1);
	#ifdef TCP_OFFLOAD
	sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
	SLIST_INIT(&t4_uld_list);
	#endif
	#ifdef INET6
	t4_clip_modload();
	#endif
	#ifdef KERN_TLS
	t6_ktls_modload();
	#endif
	t4_tracer_modload();
	tweak_tunables();
	}
	sx_xunlock(&mlu);
	break;

	case MOD_UNLOAD:
	sx_xlock(&mlu);
	if (--loaded == 0) {
	int tries;

	sx_slock(&t4_list_lock);
	if (!SLIST_EMPTY(&t4_list)) {
	rc = EBUSY;
	sx_sunlock(&t4_list_lock);
	goto done_unload;
	}
	#ifdef TCP_OFFLOAD
	sx_slock(&t4_uld_list_lock);
	if (!SLIST_EMPTY(&t4_uld_list)) {
	rc = EBUSY;
	sx_sunlock(&t4_uld_list_lock);
	sx_sunlock(&t4_list_lock);
	goto done_unload;
	}
	#endif
	tries = 0;
	while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
	uprintf("%ju clusters with custom free routine "
	"still is use.\n", t4_sge_extfree_refs());
	pause("t4unload", 2 * hz);
	}
	#ifdef TCP_OFFLOAD
	sx_sunlock(&t4_uld_list_lock);
	#endif
	sx_sunlock(&t4_list_lock);

	if (t4_sge_extfree_refs() == 0) {
	t4_tracer_modunload();
	#ifdef KERN_TLS
	t6_ktls_modunload();
	#endif
	#ifdef INET6
	t4_clip_modunload();
	#endif
	#ifdef TCP_OFFLOAD
	sx_destroy(&t4_uld_list_lock);
	#endif
	sx_destroy(&t4_list_lock);
	t4_sge_modunload();
	loaded = 0;
	} else {
	rc = EBUSY;
	loaded++; /* undo earlier decrement */
	}
	}
	done_unload:
	sx_xunlock(&mlu);
	break;
	}

	return (rc);
	}

	static devclass_t t4_devclass, t5_devclass, t6_devclass;
	static devclass_t cxgbe_devclass, cxl_devclass, cc_devclass;
	static devclass_t vcxgbe_devclass, vcxl_devclass, vcc_devclass;

	DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
	MODULE_VERSION(t4nex, 1);
	MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
	#ifdef DEV_NETMAP
	MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
	#endif /* DEV_NETMAP */

	DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
	MODULE_VERSION(t5nex, 1);
	MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
	#ifdef DEV_NETMAP
	MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
	#endif /* DEV_NETMAP */

	DRIVER_MODULE(t6nex, pci, t6_driver, t6_devclass, mod_event, 0);
	MODULE_VERSION(t6nex, 1);
	MODULE_DEPEND(t6nex, firmware, 1, 1, 1);
	#ifdef DEV_NETMAP
	MODULE_DEPEND(t6nex, netmap, 1, 1, 1);
	#endif /* DEV_NETMAP */

	DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
	MODULE_VERSION(cxgbe, 1);

	DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
	MODULE_VERSION(cxl, 1);

	DRIVER_MODULE(cc, t6nex, cc_driver, cc_devclass, 0, 0);
	MODULE_VERSION(cc, 1);

	DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0);
	MODULE_VERSION(vcxgbe, 1);

	DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0);
	MODULE_VERSION(vcxl, 1);

	DRIVER_MODULE(vcc, cc, vcc_driver, vcc_devclass, 0, 0);
	MODULE_VERSION(vcc, 1);
	diff --git a/sys/dev/cy/cy.c b/sys/dev/cy/cy.c
	deleted file mode 100644
	index 3d397ae775cc..000000000000
	--- a/sys/dev/cy/cy.c
	+++ /dev/null
	@@ -1,2242 +0,0 @@
	-/*-
	- * cyclades cyclom-y serial driver
	- * Andrew Herbert <andrew@werple.apana.org.au>, 17 August 1993
	- *
	- * SPDX-License-Identifier: BSD-3-Clause
	- *
	- * Copyright (c) 1993 Andrew Herbert.
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- * 3. The name Andrew Herbert may not be used to endorse or promote products
	- * derived from this software without specific prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR IMPLIED
	- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
	- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
	- * NO EVENT SHALL I BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
	- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
	- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
	- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
	- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
	- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-/*
	- * TODO:
	- * Atomic COR change.
	- * Consoles.
	- */
	-
	-/*
	- * Temporary compile-time configuration options.
	- */
	-#define RxFifoThreshold (CD1400_RX_FIFO_SIZE / 2)
	- /* Number of chars in the receiver FIFO before an
	- * an interrupt is generated. Should depend on
	- * line speed. Needs to be about 6 on a 486DX33
	- * for 4 active ports at 115200 bps. Why doesn't
	- * 10 work?
	- */
	-#define PollMode /* Use polling-based irq service routine, not the
	- * hardware svcack lines. Must be defined for
	- * Cyclom-16Y boards. Less efficient for Cyclom-8Ys,
	- * and stops 4 * 115200 bps from working.
	- */
	-#undef Smarts /* Enable slightly more CD1400 intelligence. Mainly
	- * the output CR/LF processing, plus we can avoid a
	- * few checks usually done in ttyinput().
	- *
	- * XXX not fully implemented, and not particularly
	- * worthwhile.
	- */
	-#undef CyDebug /* Include debugging code (not very expensive). */
	-
	-/* These will go away. */
	-#undef SOFT_CTS_OFLOW
	-#define SOFT_HOTCHAR
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/bus.h>
	-#include <sys/conf.h>
	-#include <sys/fcntl.h>
	-#include <sys/interrupt.h>
	-#include <sys/kernel.h>
	-#include <sys/lock.h>
	-#include <sys/malloc.h>
	-#include <sys/mutex.h>
	-#include <sys/serial.h>
	-#include <sys/syslog.h>
	-#include <sys/tty.h>
	-
	-#include <machine/psl.h>
	-
	-#include <dev/ic/cd1400.h>
	-
	-#include <dev/cy/cyreg.h>
	-#include <dev/cy/cyvar.h>
	-
	-#define NCY 10 /* KLUDGE */
	-
	-#define NPORTS (NCY * CY_MAX_PORTS)
	-
	-#define CY_MAX_PORTS (CD1400_NO_OF_CHANNELS * CY_MAX_CD1400s)
	-
	-/* We encode the cyclom unit number (cyu) in spare bits in the IVR's. */
	-#define CD1400_xIVR_CHAN_SHIFT 3
	-#define CD1400_xIVR_CHAN 0x1F
	-
	-/*
	- * ETC states. com->etc may also contain a hardware ETC command value,
	- * meaning that execution of that command is pending.
	- */
	-#define ETC_NONE 0 /* we depend on bzero() setting this */
	-#define ETC_BREAK_STARTING 1
	-#define ETC_BREAK_STARTED 2
	-#define ETC_BREAK_ENDING 3
	-#define ETC_BREAK_ENDED 4
	-
	-#define LOTS_OF_EVENTS 64 /* helps separate urgent events from input */
	-
	-/*
	- * com state bits.
	- * (CS_BUSY \| CS_TTGO) and (CS_BUSY \| CS_TTGO \| CS_ODEVREADY) must be higher
	- * than the other bits so that they can be tested as a group without masking
	- * off the low bits.
	- *
	- * The following com and tty flags correspond closely:
	- * CS_BUSY = TS_BUSY (maintained by cystart(), cypoll() and
	- * comstop())
	- * CS_TTGO = ~TS_TTSTOP (maintained by cyparam() and cystart())
	- * CS_CTS_OFLOW = CCTS_OFLOW (maintained by cyparam())
	- * CS_RTS_IFLOW = CRTS_IFLOW (maintained by cyparam())
	- * TS_FLUSH is not used.
	- * XXX I think TIOCSETA doesn't clear TS_TTSTOP when it clears IXON.
	- * XXX CS_FLOW should be CF_FLOW in com->flags (control flags not state).
	- */
	-#define CS_BUSY 0x80 /* output in progress */
	-#define CS_TTGO 0x40 /* output not stopped by XOFF */
	-#define CS_ODEVREADY 0x20 /* external device h/w ready (CTS) */
	-#define CS_CHECKMSR 1 /* check of MSR scheduled */
	-#define CS_CTS_OFLOW 2 /* use CTS output flow control */
	-#define CS_ODONE 4 /* output completed */
	-#define CS_RTS_IFLOW 8 /* use RTS input flow control */
	-#define CSE_ODONE 1 /* output transmitted */
	-
	-static char const * const error_desc[] = {
	-#define CE_OVERRUN 0
	- "silo overflow",
	-#define CE_INTERRUPT_BUF_OVERFLOW 1
	- "interrupt-level buffer overflow",
	-#define CE_TTY_BUF_OVERFLOW 2
	- "tty-level buffer overflow",
	-};
	-
	-#define CE_NTYPES 3
	-#define CE_RECORD(com, errnum) (++(com)->delta_error_counts[errnum])
	-
	-#ifdef SMP
	-#define COM_LOCK() mtx_lock_spin(&cy_lock)
	-#define COM_UNLOCK() mtx_unlock_spin(&cy_lock)
	-#else
	-#define COM_LOCK()
	-#define COM_UNLOCK()
	-#endif
	-
	-/* types. XXX - should be elsewhere */
	-typedef u_char bool_t; /* boolean */
	-
	-/* queue of linear buffers */
	-struct lbq {
	- u_char l_head; / next char to process */
	- u_char l_tail; / one past the last char to process */
	- struct lbq l_next; / next in queue */
	- bool_t l_queued; /* nonzero if queued */
	-};
	-
	-/* com device structure */
	-struct com_s {
	- u_char state; /* miscellaneous flag bits */
	- u_char etc; /* pending Embedded Transmit Command */
	- u_char extra_state; /* more flag bits, separate for order trick */
	- u_char gfrcr_image; /* copy of value read from GFRCR */
	- u_char mcr_dtr; /* MCR bit that is wired to DTR */
	- u_char mcr_image; /* copy of value written to MCR */
	- u_char mcr_rts; /* MCR bit that is wired to RTS */
	- int unit; /* unit number */
	-
	- /*
	- * The high level of the driver never reads status registers directly
	- * because there would be too many side effects to handle conveniently.
	- * Instead, it reads copies of the registers stored here by the
	- * interrupt handler.
	- */
	- u_char last_modem_status; /* last MSR read by intr handler */
	- u_char prev_modem_status; /* last MSR handled by high level */
	-
	- u_char ibuf; / start of input buffer */
	- u_char ibufend; / end of input buffer */
	- u_char ibufold; / old input buffer, to be freed */
	- u_char ihighwater; / threshold in input buffer */
	- u_char iptr; / next free spot in input buffer */
	- int ibufsize; /* size of ibuf (not include error bytes) */
	- int ierroff; /* offset of error bytes in ibuf */
	-
	- struct lbq obufq; /* head of queue of output buffers */
	- struct lbq obufs[2]; /* output buffers */
	-
	- int cy_align; /* index for register alignment */
	- cy_addr cy_iobase; /* base address of this port's cyclom */
	- cy_addr iobase; /* base address of this port's cd1400 */
	- int mcr_rts_reg; /* cd1400 reg number of reg holding mcr_rts */
	-
	- struct tty tp; / cross reference */
	-
	- u_long bytes_in; /* statistics */
	- u_long bytes_out;
	- u_int delta_error_counts[CE_NTYPES];
	- u_long error_counts[CE_NTYPES];
	-
	- u_int recv_exception; /* exception chars received */
	- u_int mdm; /* modem signal changes */
	-#ifdef CyDebug
	- u_int start_count; /* no. of calls to cystart() */
	- u_int start_real; /* no. of calls that did something */
	-#endif
	- u_char car; /* CD1400 CAR shadow (if first unit in cd) */
	- u_char channel_control;/* CD1400 CCR control command shadow */
	- u_char cor[3]; /* CD1400 COR1-3 shadows */
	- u_char intr_enable; /* CD1400 SRER shadow */
	-
	- /*
	- * Data area for output buffers. Someday we should build the output
	- * buffer queue without copying data.
	- */
	- u_char obuf1[256];
	- u_char obuf2[256];
	-};
	-
	-devclass_t cy_devclass;
	-char cy_driver_name[] = "cy";
	-
	-static void cd1400_channel_cmd(struct com_s *com, int cmd);
	-static void cd1400_channel_cmd_wait(struct com_s *com);
	-static void cd_etc(struct com_s *com, int etc);
	-static int cd_getreg(struct com_s *com, int reg);
	-static void cd_setreg(struct com_s *com, int reg, int val);
	-static void cyinput(struct com_s *com);
	-static int cyparam(struct tty tp, struct termios t);
	-static void cypoll(void *arg);
	-static void cysettimeout(void);
	-static int cysetwater(struct com_s *com, speed_t speed);
	-static int cyspeed(speed_t speed, u_long cy_clock, int *prescaler_io);
	-static void cystart(struct tty *tp);
	-static void comstop(struct tty *tp, int rw);
	-static timeout_t cywakeup;
	-static void disc_optim(struct tty tp, struct termios t,
	- struct com_s *com);
	-
	-static t_break_t cybreak;
	-static t_modem_t cymodem;
	-static t_open_t cyopen;
	-static t_close_t cyclose;
	-
	-#ifdef CyDebug
	-void cystatus(int unit);
	-#endif
	-
	-static struct mtx cy_lock;
	-static int cy_inited;
	-
	-/* table and macro for fast conversion from a unit number to its com struct */
	-static struct com_s *p_cy_addr[NPORTS];
	-#define cy_addr(unit) (p_cy_addr[unit])
	-
	-static u_int cy_events; /* input chars + weighted output completions */
	-static void *cy_fast_ih;
	-static void *cy_slow_ih;
	-static int cy_timeout;
	-static int cy_timeouts_until_log;
	-static struct callout_handle cy_timeout_handle
	- = CALLOUT_HANDLE_INITIALIZER(&cy_timeout_handle);
	-
	-#ifdef CyDebug
	-static u_int cd_inbs;
	-static u_int cy_inbs;
	-static u_int cd_outbs;
	-static u_int cy_outbs;
	-static u_int cy_svrr_probes;
	-static u_int cy_timeouts;
	-#endif
	-
	-static int cy_chip_offset[] = {
	- 0x0000, 0x0400, 0x0800, 0x0c00, 0x0200, 0x0600, 0x0a00, 0x0e00,
	-};
	-static int cy_nr_cd1400s[NCY];
	-static int cy_total_devices;
	-#undef RxFifoThreshold
	-static int volatile RxFifoThreshold = (CD1400_RX_FIFO_SIZE / 2);
	-
	-int
	-cy_units(cy_addr cy_iobase, int cy_align)
	-{
	- int cyu;
	- u_char firmware_version;
	- int i;
	- cy_addr iobase;
	-
	- for (cyu = 0; cyu < CY_MAX_CD1400s; ++cyu) {
	- iobase = cy_iobase + (cy_chip_offset[cyu] << cy_align);
	-
	- /* wait for chip to become ready for new command */
	- for (i = 0; i < 10; i++) {
	- DELAY(50);
	- if (!cd_inb(iobase, CD1400_CCR, cy_align))
	- break;
	- }
	-
	- /* clear the GFRCR register */
	- cd_outb(iobase, CD1400_GFRCR, cy_align, 0);
	-
	- /* issue a reset command */
	- cd_outb(iobase, CD1400_CCR, cy_align,
	- CD1400_CCR_CMDRESET \| CD1400_CCR_FULLRESET);
	-
	- /* XXX bogus initialization to avoid a gcc bug/warning. */
	- firmware_version = 0;
	-
	- /* wait for the CD1400 to initialize itself */
	- for (i = 0; i < 200; i++) {
	- DELAY(50);
	-
	- /* retrieve firmware version */
	- firmware_version = cd_inb(iobase, CD1400_GFRCR,
	- cy_align);
	- if ((firmware_version & 0xf0) == 0x40)
	- break;
	- }
	-
	- /*
	- * Anything in the 0x40-0x4F range is fine.
	- * If one CD1400 is bad then we don't support higher
	- * numbered good ones on this board.
	- */
	- if ((firmware_version & 0xf0) != 0x40)
	- break;
	- }
	- return (cyu);
	-}
	-
	-void *
	-cyattach_common(cy_addr cy_iobase, int cy_align)
	-{
	- int adapter;
	- int cyu;
	- u_char firmware_version;
	- cy_addr iobase;
	- int ncyu;
	- int unit;
	- struct tty *tp;
	-
	- while (cy_inited != 2)
	- if (atomic_cmpset_int(&cy_inited, 0, 1)) {
	- mtx_init(&cy_lock, cy_driver_name, NULL, MTX_SPIN);
	- atomic_store_rel_int(&cy_inited, 2);
	- }
	-
	- adapter = cy_total_devices;
	- if ((u_int)adapter >= NCY) {
	- printf(
	- "cy%d: can't attach adapter: insufficient cy devices configured\n",
	- adapter);
	- return (NULL);
	- }
	- ncyu = cy_units(cy_iobase, cy_align);
	- if (ncyu == 0)
	- return (NULL);
	- cy_nr_cd1400s[adapter] = ncyu;
	- cy_total_devices++;
	-
	- unit = adapter * CY_MAX_PORTS;
	- for (cyu = 0; cyu < ncyu; ++cyu) {
	- int cdu;
	-
	- iobase = (cy_addr) (cy_iobase
	- + (cy_chip_offset[cyu] << cy_align));
	- firmware_version = cd_inb(iobase, CD1400_GFRCR, cy_align);
	-
	- /* Set up a receive timeout period of than 1+ ms. */
	- cd_outb(iobase, CD1400_PPR, cy_align,
	- howmany(CY_CLOCK(firmware_version)
	- / CD1400_PPR_PRESCALER, 1000));
	-
	- for (cdu = 0; cdu < CD1400_NO_OF_CHANNELS; ++cdu, ++unit) {
	- struct com_s *com;
	- int s;
	-
	- com = malloc(sizeof *com, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (com == NULL)
	- break;
	- com->unit = unit;
	- com->gfrcr_image = firmware_version;
	- if (CY_RTS_DTR_SWAPPED(firmware_version)) {
	- com->mcr_dtr = CD1400_MSVR1_RTS;
	- com->mcr_rts = CD1400_MSVR2_DTR;
	- com->mcr_rts_reg = CD1400_MSVR2;
	- } else {
	- com->mcr_dtr = CD1400_MSVR2_DTR;
	- com->mcr_rts = CD1400_MSVR1_RTS;
	- com->mcr_rts_reg = CD1400_MSVR1;
	- }
	- com->obufs[0].l_head = com->obuf1;
	- com->obufs[1].l_head = com->obuf2;
	-
	- com->cy_align = cy_align;
	- com->cy_iobase = cy_iobase;
	- com->iobase = iobase;
	- com->car = ~CD1400_CAR_CHAN;
	-
	- tp = com->tp = ttyalloc();
	- tp->t_open = cyopen;
	- tp->t_close = cyclose;
	- tp->t_oproc = cystart;
	- tp->t_stop = comstop;
	- tp->t_param = cyparam;
	- tp->t_break = cybreak;
	- tp->t_modem = cymodem;
	- tp->t_sc = com;
	-
	- if (cysetwater(com, tp->t_init_in.c_ispeed) != 0) {
	- free(com, M_DEVBUF);
	- return (NULL);
	- }
	-
	- s = spltty();
	- cy_addr(unit) = com;
	- splx(s);
	-
	- if (cy_fast_ih == NULL) {
	- swi_add(&tty_intr_event, "cy", cypoll, NULL, SWI_TTY, 0,
	- &cy_fast_ih);
	- swi_add(&clk_intr_event, "cy", cypoll, NULL, SWI_CLOCK, 0,
	- &cy_slow_ih);
	- }
	- ttycreate(tp, TS_CALLOUT, "c%r%r",
	- adapter, unit % CY_MAX_PORTS);
	- }
	- }
	-
	- /* ensure an edge for the next interrupt */
	- cy_outb(cy_iobase, CY_CLEAR_INTR, cy_align, 0);
	-
	- return (cy_addr(adapter * CY_MAX_PORTS));
	-}
	-
	-static int
	-cyopen(struct tty tp, struct cdev dev)
	-{
	- struct com_s *com;
	- int s;
	-
	- com = tp->t_sc;
	- s = spltty();
	- /*
	- * We jump to this label after all non-interrupted sleeps to pick
	- * up any changes of the device state.
	- */
	-
	- /* Encode per-board unit in LIVR for access in intr routines. */
	- cd_setreg(com, CD1400_LIVR,
	- (com->unit & CD1400_xIVR_CHAN) << CD1400_xIVR_CHAN_SHIFT);
	-
	- /*
	- * Flush fifos. This requires a full channel reset which
	- * also disables the transmitter and receiver. Recover
	- * from this.
	- */
	- cd1400_channel_cmd(com,
	- CD1400_CCR_CMDRESET \| CD1400_CCR_CHANRESET);
	- cd1400_channel_cmd(com, com->channel_control);
	-
	- critical_enter();
	- COM_LOCK();
	- com->prev_modem_status = com->last_modem_status
	- = cd_getreg(com, CD1400_MSVR2);
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = CD1400_SRER_MDMCH \| CD1400_SRER_RXDATA);
	- COM_UNLOCK();
	- critical_exit();
	- cysettimeout();
	- return (0);
	-}
	-
	-
	-static void
	-cyclose(struct tty *tp)
	-{
	- cy_addr iobase;
	- struct com_s *com;
	- int s;
	- int unit;
	-
	- com = tp->t_sc;
	- unit = com->unit;
	- iobase = com->iobase;
	- s = spltty();
	- /* XXX */
	- critical_enter();
	- COM_LOCK();
	- com->etc = ETC_NONE;
	- cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC);
	- COM_UNLOCK();
	- critical_exit();
	- cd_etc(com, CD1400_ETC_STOPBREAK);
	- cd1400_channel_cmd(com, CD1400_CCR_CMDRESET \| CD1400_CCR_FTF);
	-
	- {
	- critical_enter();
	- COM_LOCK();
	- cd_setreg(com, CD1400_SRER, com->intr_enable = 0);
	- COM_UNLOCK();
	- critical_exit();
	- tp = com->tp;
	- if ((tp->t_cflag & HUPCL)
	- /*
	- * XXX we will miss any carrier drop between here and the
	- * next open. Perhaps we should watch DCD even when the
	- * port is closed; it is not sufficient to check it at
	- * the next open because it might go up and down while
	- * we're not watching.
	- */
	- \|\| (!tp->t_actout
	- && !(com->prev_modem_status & CD1400_MSVR2_CD)
	- && !(tp->t_init_in.c_cflag & CLOCAL))
	- \|\| !(tp->t_state & TS_ISOPEN)) {
	- (void)cymodem(tp, 0, SER_DTR);
	-
	- /* Disable receiver (leave transmitter enabled). */
	- com->channel_control = CD1400_CCR_CMDCHANCTL
	- \| CD1400_CCR_XMTEN
	- \| CD1400_CCR_RCVDIS;
	- cd1400_channel_cmd(com, com->channel_control);
	-
	- ttydtrwaitstart(tp);
	- }
	- }
	- tp->t_actout = FALSE;
	- wakeup(&tp->t_actout);
	- wakeup(TSA_CARR_ON(tp)); /* restart any wopeners */
	- splx(s);
	-}
	-
	-/*
	- * This function:
	- * a) needs to be called with COM_LOCK() held, and
	- * b) needs to return with COM_LOCK() held.
	- */
	-static void
	-cyinput(struct com_s *com)
	-{
	- u_char *buf;
	- int incc;
	- u_char line_status;
	- int recv_data;
	- struct tty *tp;
	-
	- buf = com->ibuf;
	- tp = com->tp;
	- if (!(tp->t_state & TS_ISOPEN)) {
	- cy_events -= (com->iptr - com->ibuf);
	- com->iptr = com->ibuf;
	- return;
	- }
	- if (tp->t_state & TS_CAN_BYPASS_L_RINT) {
	- /*
	- * Avoid the grotesquely inefficient lineswitch routine
	- * (ttyinput) in "raw" mode. It usually takes about 450
	- * instructions (that's without canonical processing or echo!).
	- * slinput is reasonably fast (usually 40 instructions plus
	- * call overhead).
	- */
	-
	- do {
	- /*
	- * This may look odd, but it is using save-and-enable
	- * semantics instead of the save-and-disable semantics
	- * that are used everywhere else.
	- */
	- COM_UNLOCK();
	- critical_exit();
	- incc = com->iptr - buf;
	- if (tp->t_rawq.c_cc + incc > tp->t_ihiwat
	- && (com->state & CS_RTS_IFLOW
	- \|\| tp->t_iflag & IXOFF)
	- && !(tp->t_state & TS_TBLOCK))
	- ttyblock(tp);
	- com->delta_error_counts[CE_TTY_BUF_OVERFLOW]
	- += b_to_q((char *)buf, incc, &tp->t_rawq);
	- buf += incc;
	- tk_nin += incc;
	- tk_rawcc += incc;
	- tp->t_rawcc += incc;
	- ttwakeup(tp);
	- if (tp->t_state & TS_TTSTOP
	- && (tp->t_iflag & IXANY
	- \|\| tp->t_cc[VSTART] == tp->t_cc[VSTOP])) {
	- tp->t_state &= ~TS_TTSTOP;
	- tp->t_lflag &= ~FLUSHO;
	- cystart(tp);
	- }
	- critical_enter();
	- COM_LOCK();
	- } while (buf < com->iptr);
	- } else {
	- do {
	- /*
	- * This may look odd, but it is using save-and-enable
	- * semantics instead of the save-and-disable semantics
	- * that are used everywhere else.
	- */
	- COM_UNLOCK();
	- critical_exit();
	- line_status = buf[com->ierroff];
	- recv_data = *buf++;
	- if (line_status
	- & (CD1400_RDSR_BREAK \| CD1400_RDSR_FE \| CD1400_RDSR_OE \| CD1400_RDSR_PE)) {
	- if (line_status & CD1400_RDSR_BREAK)
	- recv_data \|= TTY_BI;
	- if (line_status & CD1400_RDSR_FE)
	- recv_data \|= TTY_FE;
	- if (line_status & CD1400_RDSR_OE)
	- recv_data \|= TTY_OE;
	- if (line_status & CD1400_RDSR_PE)
	- recv_data \|= TTY_PE;
	- }
	- ttyld_rint(tp, recv_data);
	- critical_enter();
	- COM_LOCK();
	- } while (buf < com->iptr);
	- }
	- cy_events -= (com->iptr - com->ibuf);
	- com->iptr = com->ibuf;
	-
	- /*
	- * There is now room for another low-level buffer full of input,
	- * so enable RTS if it is now disabled and there is room in the
	- * high-level buffer.
	- */
	- if ((com->state & CS_RTS_IFLOW) && !(com->mcr_image & com->mcr_rts) &&
	- !(tp->t_state & TS_TBLOCK))
	- cd_setreg(com, com->mcr_rts_reg,
	- com->mcr_image \|= com->mcr_rts);
	-}
	-
	-int
	-cyintr(void *vcom)
	-{
	- struct com_s *basecom;
	- int baseu;
	- int cy_align;
	- cy_addr cy_iobase;
	- int cyu;
	- cy_addr iobase;
	- u_char status;
	- int unit;
	-
	- COM_LOCK(); /* XXX could this be placed down lower in the loop? */
	-
	- basecom = (struct com_s *)vcom;
	- baseu = basecom->unit;
	- cy_align = basecom->cy_align;
	- cy_iobase = basecom->cy_iobase;
	- unit = baseu / CY_MAX_PORTS;
	-
	- /* check each CD1400 in turn */
	- for (cyu = 0; cyu < cy_nr_cd1400s[unit]; ++cyu) {
	- iobase = (cy_addr) (cy_iobase
	- + (cy_chip_offset[cyu] << cy_align));
	- /* poll to see if it has any work */
	- status = cd_inb(iobase, CD1400_SVRR, cy_align);
	- if (status == 0)
	- continue; // XXX - FILTER_STRAY?
	-#ifdef CyDebug
	- ++cy_svrr_probes;
	-#endif
	- /* service requests as appropriate, giving priority to RX */
	- if (status & CD1400_SVRR_RXRDY) {
	- struct com_s *com;
	- u_int count;
	- u_char *ioptr;
	- u_char line_status;
	- u_char recv_data;
	- u_char serv_type;
	-#ifdef PollMode
	- u_char save_rir;
	-#endif
	-
	-#ifdef PollMode
	- save_rir = cd_inb(iobase, CD1400_RIR, cy_align);
	-
	- /* enter rx service */
	- cd_outb(iobase, CD1400_CAR, cy_align, save_rir);
	- cy_addr(baseu + cyu * CD1400_NO_OF_CHANNELS)->car
	- = save_rir & CD1400_CAR_CHAN;
	-
	- serv_type = cd_inb(iobase, CD1400_RIVR, cy_align);
	- com = cy_addr(baseu
	- + ((serv_type >> CD1400_xIVR_CHAN_SHIFT)
	- & CD1400_xIVR_CHAN));
	-#else
	- /* ack receive service */
	- serv_type = cy_inb(iobase, CY8_SVCACKR, cy_align);
	-
	- com = cy_addr(baseu +
	- + ((serv_type >> CD1400_xIVR_CHAN_SHIFT)
	- & CD1400_xIVR_CHAN));
	-#endif
	-
	- if (serv_type & CD1400_RIVR_EXCEPTION) {
	- ++com->recv_exception;
	- line_status = cd_inb(iobase, CD1400_RDSR, cy_align);
	- /* break/unnattached error bits or real input? */
	- recv_data = cd_inb(iobase, CD1400_RDSR, cy_align);
	-#ifndef SOFT_HOTCHAR
	- if (line_status & CD1400_RDSR_SPECIAL
	- && com->tp->t_hotchar != 0)
	- swi_sched(cy_fast_ih, 0);
	-
	-#endif
	-#if 1 /* XXX "intelligent" PFO error handling would break O error handling */
	- if (line_status & (CD1400_RDSR_PE\|CD1400_RDSR_FE\|CD1400_RDSR_BREAK)) {
	- /*
	- Don't store PE if IGNPAR and BI if IGNBRK,
	- this hack allows "raw" tty optimization
	- works even if IGN* is set.
	- */
	- if ( com->tp == NULL
	- \|\| !(com->tp->t_state & TS_ISOPEN)
	- \|\| ((line_status & (CD1400_RDSR_PE\|CD1400_RDSR_FE))
	- && (com->tp->t_iflag & IGNPAR))
	- \|\| ((line_status & CD1400_RDSR_BREAK)
	- && (com->tp->t_iflag & IGNBRK)))
	- goto cont;
	- if ( (line_status & (CD1400_RDSR_PE\|CD1400_RDSR_FE))
	- && (com->tp->t_state & TS_CAN_BYPASS_L_RINT)
	- && ((line_status & CD1400_RDSR_FE)
	- \|\| ((line_status & CD1400_RDSR_PE)
	- && (com->tp->t_iflag & INPCK))))
	- recv_data = 0;
	- }
	-#endif /* 1 */
	- ++com->bytes_in;
	-#ifdef SOFT_HOTCHAR
	- if (com->tp->t_hotchar != 0 && recv_data == com->tp->t_hotchar)
	- swi_sched(cy_fast_ih, 0);
	-#endif
	- ioptr = com->iptr;
	- if (ioptr >= com->ibufend)
	- CE_RECORD(com, CE_INTERRUPT_BUF_OVERFLOW);
	- else {
	- if (com->tp != NULL && com->tp->t_do_timestamp)
	- microtime(&com->tp->t_timestamp);
	- ++cy_events;
	- ioptr[0] = recv_data;
	- ioptr[com->ierroff] = line_status;
	- com->iptr = ++ioptr;
	- if (ioptr == com->ihighwater
	- && com->state & CS_RTS_IFLOW)
	- cd_outb(iobase, com->mcr_rts_reg,
	- cy_align,
	- com->mcr_image &=
	- ~com->mcr_rts);
	- if (line_status & CD1400_RDSR_OE)
	- CE_RECORD(com, CE_OVERRUN);
	- }
	- goto cont;
	- } else {
	- int ifree;
	-
	- count = cd_inb(iobase, CD1400_RDCR, cy_align);
	- if (!count)
	- goto cont;
	- com->bytes_in += count;
	- ioptr = com->iptr;
	- ifree = com->ibufend - ioptr;
	- if (count > ifree) {
	- count -= ifree;
	- cy_events += ifree;
	- if (ifree != 0) {
	- if (com->tp != NULL && com->tp->t_do_timestamp)
	- microtime(&com->tp->t_timestamp);
	- do {
	- recv_data = cd_inb(iobase,
	- CD1400_RDSR,
	- cy_align);
	-#ifdef SOFT_HOTCHAR
	- if (com->tp->t_hotchar != 0
	- && recv_data
	- == com->tp->t_hotchar)
	- swi_sched(cy_fast_ih,
	- 0);
	-#endif
	- ioptr[0] = recv_data;
	- ioptr[com->ierroff] = 0;
	- ++ioptr;
	- } while (--ifree != 0);
	- }
	- com->delta_error_counts
	- [CE_INTERRUPT_BUF_OVERFLOW] += count;
	- do {
	- recv_data = cd_inb(iobase, CD1400_RDSR,
	- cy_align);
	-#ifdef SOFT_HOTCHAR
	- if (com->tp->t_hotchar != 0
	- && recv_data == com->tp->t_hotchar)
	- swi_sched(cy_fast_ih, 0);
	-#endif
	- } while (--count != 0);
	- } else {
	- if (com->tp != NULL && com->tp->t_do_timestamp)
	- microtime(&com->tp->t_timestamp);
	- if (ioptr <= com->ihighwater
	- && ioptr + count > com->ihighwater
	- && com->state & CS_RTS_IFLOW)
	- cd_outb(iobase, com->mcr_rts_reg,
	- cy_align,
	- com->mcr_image
	- &= ~com->mcr_rts);
	- cy_events += count;
	- do {
	- recv_data = cd_inb(iobase, CD1400_RDSR,
	- cy_align);
	-#ifdef SOFT_HOTCHAR
	- if (com->tp->t_hotchar != 0
	- && recv_data == com->tp->t_hotchar)
	- swi_sched(cy_fast_ih, 0);
	-#endif
	- ioptr[0] = recv_data;
	- ioptr[com->ierroff] = 0;
	- ++ioptr;
	- } while (--count != 0);
	- }
	- com->iptr = ioptr;
	- }
	-cont:
	-
	- /* terminate service context */
	-#ifdef PollMode
	- cd_outb(iobase, CD1400_RIR, cy_align,
	- save_rir
	- & ~(CD1400_RIR_RDIREQ \| CD1400_RIR_RBUSY));
	-#else
	- cd_outb(iobase, CD1400_EOSRR, cy_align, 0);
	-#endif
	- }
	- if (status & CD1400_SVRR_MDMCH) {
	- struct com_s *com;
	- u_char modem_status;
	-#ifdef PollMode
	- u_char save_mir;
	-#else
	- u_char vector;
	-#endif
	-
	-#ifdef PollMode
	- save_mir = cd_inb(iobase, CD1400_MIR, cy_align);
	-
	- /* enter modem service */
	- cd_outb(iobase, CD1400_CAR, cy_align, save_mir);
	- cy_addr(baseu + cyu * CD1400_NO_OF_CHANNELS)->car
	- = save_mir & CD1400_CAR_CHAN;
	-
	- com = cy_addr(baseu + cyu * CD1400_NO_OF_CHANNELS
	- + (save_mir & CD1400_MIR_CHAN));
	-#else
	- /* ack modem service */
	- vector = cy_inb(iobase, CY8_SVCACKM, cy_align);
	-
	- com = cy_addr(baseu
	- + ((vector >> CD1400_xIVR_CHAN_SHIFT)
	- & CD1400_xIVR_CHAN));
	-#endif
	- ++com->mdm;
	- modem_status = cd_inb(iobase, CD1400_MSVR2, cy_align);
	- if (modem_status != com->last_modem_status) {
	- /*
	- * Schedule high level to handle DCD changes. Note
	- * that we don't use the delta bits anywhere. Some
	- * UARTs mess them up, and it's easy to remember the
	- * previous bits and calculate the delta.
	- */
	- com->last_modem_status = modem_status;
	- if (!(com->state & CS_CHECKMSR)) {
	- cy_events += LOTS_OF_EVENTS;
	- com->state \|= CS_CHECKMSR;
	- swi_sched(cy_fast_ih, 0);
	- }
	-
	-#ifdef SOFT_CTS_OFLOW
	- /* handle CTS change immediately for crisp flow ctl */
	- if (com->state & CS_CTS_OFLOW) {
	- if (modem_status & CD1400_MSVR2_CTS) {
	- com->state \|= CS_ODEVREADY;
	- if (com->state >= (CS_BUSY \| CS_TTGO
	- \| CS_ODEVREADY)
	- && !(com->intr_enable
	- & CD1400_SRER_TXRDY))
	- cd_outb(iobase, CD1400_SRER,
	- cy_align,
	- com->intr_enable
	- = com->intr_enable
	- & ~CD1400_SRER_TXMPTY
	- \| CD1400_SRER_TXRDY);
	- } else {
	- com->state &= ~CS_ODEVREADY;
	- if (com->intr_enable
	- & CD1400_SRER_TXRDY)
	- cd_outb(iobase, CD1400_SRER,
	- cy_align,
	- com->intr_enable
	- = com->intr_enable
	- & ~CD1400_SRER_TXRDY
	- \| CD1400_SRER_TXMPTY);
	- }
	- }
	-#endif
	- }
	-
	- /* terminate service context */
	-#ifdef PollMode
	- cd_outb(iobase, CD1400_MIR, cy_align,
	- save_mir
	- & ~(CD1400_MIR_RDIREQ \| CD1400_MIR_RBUSY));
	-#else
	- cd_outb(iobase, CD1400_EOSRR, cy_align, 0);
	-#endif
	- }
	- if (status & CD1400_SVRR_TXRDY) {
	- struct com_s *com;
	-#ifdef PollMode
	- u_char save_tir;
	-#else
	- u_char vector;
	-#endif
	-
	-#ifdef PollMode
	- save_tir = cd_inb(iobase, CD1400_TIR, cy_align);
	-
	- /* enter tx service */
	- cd_outb(iobase, CD1400_CAR, cy_align, save_tir);
	- cy_addr(baseu + cyu * CD1400_NO_OF_CHANNELS)->car
	- = save_tir & CD1400_CAR_CHAN;
	-
	- com = cy_addr(baseu
	- + cyu * CD1400_NO_OF_CHANNELS
	- + (save_tir & CD1400_TIR_CHAN));
	-#else
	- /* ack transmit service */
	- vector = cy_inb(iobase, CY8_SVCACKT, cy_align);
	-
	- com = cy_addr(baseu
	- + ((vector >> CD1400_xIVR_CHAN_SHIFT)
	- & CD1400_xIVR_CHAN));
	-#endif
	-
	- if (com->etc != ETC_NONE) {
	- if (com->intr_enable & CD1400_SRER_TXRDY) {
	- /*
	- * Here due to sloppy SRER_TXRDY
	- * enabling. Ignore. Come back when
	- * tx is empty.
	- */
	- cd_outb(iobase, CD1400_SRER, cy_align,
	- com->intr_enable
	- = (com->intr_enable
	- & ~CD1400_SRER_TXRDY)
	- \| CD1400_SRER_TXMPTY);
	- goto terminate_tx_service;
	- }
	- switch (com->etc) {
	- case CD1400_ETC_SENDBREAK:
	- case CD1400_ETC_STOPBREAK:
	- /*
	- * Start the command. Come back on
	- * next tx empty interrupt, hopefully
	- * after command has been executed.
	- */
	- cd_outb(iobase, CD1400_COR2, cy_align,
	- com->cor[1] \|= CD1400_COR2_ETC);
	- cd_outb(iobase, CD1400_TDR, cy_align,
	- CD1400_ETC_CMD);
	- cd_outb(iobase, CD1400_TDR, cy_align,
	- com->etc);
	- if (com->etc == CD1400_ETC_SENDBREAK)
	- com->etc = ETC_BREAK_STARTING;
	- else
	- com->etc = ETC_BREAK_ENDING;
	- goto terminate_tx_service;
	- case ETC_BREAK_STARTING:
	- /*
	- * BREAK is now on. Continue with
	- * SRER_TXMPTY processing, hopefully
	- * don't come back.
	- */
	- com->etc = ETC_BREAK_STARTED;
	- break;
	- case ETC_BREAK_STARTED:
	- /*
	- * Came back due to sloppy SRER_TXMPTY
	- * enabling. Hope again.
	- */
	- break;
	- case ETC_BREAK_ENDING:
	- /*
	- * BREAK is now off. Continue with
	- * SRER_TXMPTY processing and don't
	- * come back. The SWI handler will
	- * restart tx interrupts if necessary.
	- */
	- cd_outb(iobase, CD1400_COR2, cy_align,
	- com->cor[1]
	- &= ~CD1400_COR2_ETC);
	- com->etc = ETC_BREAK_ENDED;
	- if (!(com->state & CS_ODONE)) {
	- cy_events += LOTS_OF_EVENTS;
	- com->state \|= CS_ODONE;
	- swi_sched(cy_fast_ih, 0);
	- }
	- break;
	- case ETC_BREAK_ENDED:
	- /*
	- * Shouldn't get here. Hope again.
	- */
	- break;
	- }
	- }
	- if (com->intr_enable & CD1400_SRER_TXMPTY) {
	- if (!(com->extra_state & CSE_ODONE)) {
	- cy_events += LOTS_OF_EVENTS;
	- com->extra_state \|= CSE_ODONE;
	- swi_sched(cy_fast_ih, 0);
	- }
	- cd_outb(iobase, CD1400_SRER, cy_align,
	- com->intr_enable
	- &= ~CD1400_SRER_TXMPTY);
	- goto terminate_tx_service;
	- }
	- if (com->state >= (CS_BUSY \| CS_TTGO \| CS_ODEVREADY)) {
	- u_char *ioptr;
	- u_int ocount;
	-
	- ioptr = com->obufq.l_head;
	- ocount = com->obufq.l_tail - ioptr;
	- if (ocount > CD1400_TX_FIFO_SIZE)
	- ocount = CD1400_TX_FIFO_SIZE;
	- com->bytes_out += ocount;
	- do
	- cd_outb(iobase, CD1400_TDR, cy_align,
	- *ioptr++);
	- while (--ocount != 0);
	- com->obufq.l_head = ioptr;
	- if (ioptr >= com->obufq.l_tail) {
	- struct lbq *qp;
	-
	- qp = com->obufq.l_next;
	- qp->l_queued = FALSE;
	- qp = qp->l_next;
	- if (qp != NULL) {
	- com->obufq.l_head = qp->l_head;
	- com->obufq.l_tail = qp->l_tail;
	- com->obufq.l_next = qp;
	- } else {
	- /* output just completed */
	- com->state &= ~CS_BUSY;
	-
	- /*
	- * The setting of CSE_ODONE may be
	- * stale here. We currently only
	- * use it when CS_BUSY is set, and
	- * fixing it when we clear CS_BUSY
	- * is easiest.
	- */
	- if (com->extra_state & CSE_ODONE) {
	- cy_events -= LOTS_OF_EVENTS;
	- com->extra_state &= ~CSE_ODONE;
	- }
	-
	- cd_outb(iobase, CD1400_SRER, cy_align,
	- com->intr_enable
	- = (com->intr_enable
	- & ~CD1400_SRER_TXRDY)
	- \| CD1400_SRER_TXMPTY);
	- }
	- if (!(com->state & CS_ODONE)) {
	- cy_events += LOTS_OF_EVENTS;
	- com->state \|= CS_ODONE;
	-
	- /* handle at high level ASAP */
	- swi_sched(cy_fast_ih, 0);
	- }
	- }
	- }
	-
	- /* terminate service context */
	-terminate_tx_service:
	-#ifdef PollMode
	- cd_outb(iobase, CD1400_TIR, cy_align,
	- save_tir
	- & ~(CD1400_TIR_RDIREQ \| CD1400_TIR_RBUSY));
	-#else
	- cd_outb(iobase, CD1400_EOSRR, cy_align, 0);
	-#endif
	- }
	- }
	-
	- /* ensure an edge for the next interrupt */
	- cy_outb(cy_iobase, CY_CLEAR_INTR, cy_align, 0);
	-
	- swi_sched(cy_slow_ih, SWI_DELAY);
	-
	- COM_UNLOCK();
	- return (FILTER_HANDLED);
	-}
	-
	-static void
	-cybreak(struct tty *tp, int sig)
	-{
	- struct com_s *com;
	-
	- com = tp->t_sc;
	- if (sig)
	- cd_etc(com, CD1400_ETC_SENDBREAK);
	- else
	- cd_etc(com, CD1400_ETC_STOPBREAK);
	-}
	-
	-static void
	-cypoll(void *arg)
	-{
	- int unit;
	-
	-#ifdef CyDebug
	- ++cy_timeouts;
	-#endif
	- if (cy_events == 0)
	- return;
	-repeat:
	- for (unit = 0; unit < NPORTS; ++unit) {
	- struct com_s *com;
	- int incc;
	- struct tty *tp;
	-
	- com = cy_addr(unit);
	- if (com == NULL)
	- continue;
	- tp = com->tp;
	- if (tp == NULL) {
	- /*
	- * XXX forget any events related to closed devices
	- * (actually never opened devices) so that we don't
	- * loop.
	- */
	- critical_enter();
	- COM_LOCK();
	- incc = com->iptr - com->ibuf;
	- com->iptr = com->ibuf;
	- if (com->state & CS_CHECKMSR) {
	- incc += LOTS_OF_EVENTS;
	- com->state &= ~CS_CHECKMSR;
	- }
	- cy_events -= incc;
	- COM_UNLOCK();
	- critical_exit();
	- if (incc != 0)
	- log(LOG_DEBUG,
	- "cy%d: %d events for device with no tp\n",
	- unit, incc);
	- continue;
	- }
	- if (com->iptr != com->ibuf) {
	- critical_enter();
	- COM_LOCK();
	- cyinput(com);
	- COM_UNLOCK();
	- critical_exit();
	- }
	- if (com->state & CS_CHECKMSR) {
	- u_char delta_modem_status;
	-
	- critical_enter();
	- COM_LOCK();
	- cyinput(com);
	- delta_modem_status = com->last_modem_status
	- ^ com->prev_modem_status;
	- com->prev_modem_status = com->last_modem_status;
	- cy_events -= LOTS_OF_EVENTS;
	- com->state &= ~CS_CHECKMSR;
	- COM_UNLOCK();
	- critical_exit();
	- if (delta_modem_status & CD1400_MSVR2_CD)
	- ttyld_modem(tp,
	- com->prev_modem_status & CD1400_MSVR2_CD);
	- }
	- if (com->extra_state & CSE_ODONE) {
	- critical_enter();
	- COM_LOCK();
	- cy_events -= LOTS_OF_EVENTS;
	- com->extra_state &= ~CSE_ODONE;
	- COM_UNLOCK();
	- critical_exit();
	- if (!(com->state & CS_BUSY)) {
	- tp->t_state &= ~TS_BUSY;
	- ttwwakeup(com->tp);
	- }
	- if (com->etc != ETC_NONE) {
	- if (com->etc == ETC_BREAK_ENDED)
	- com->etc = ETC_NONE;
	- wakeup(&com->etc);
	- }
	- }
	- if (com->state & CS_ODONE) {
	- critical_enter();
	- COM_LOCK();
	- cy_events -= LOTS_OF_EVENTS;
	- com->state &= ~CS_ODONE;
	- COM_UNLOCK();
	- critical_exit();
	- ttyld_start(tp);
	- }
	- if (cy_events == 0)
	- break;
	- }
	- if (cy_events >= LOTS_OF_EVENTS)
	- goto repeat;
	-}
	-
	-static int
	-cyparam(struct tty tp, struct termios t)
	-{
	- int bits;
	- int cflag;
	- struct com_s *com;
	- u_char cor_change;
	- u_long cy_clock;
	- int idivisor;
	- int iflag;
	- int iprescaler;
	- int itimeout;
	- int odivisor;
	- int oprescaler;
	- u_char opt;
	- int s;
	-
	- com = tp->t_sc;
	-
	- /* check requested parameters */
	- cy_clock = CY_CLOCK(com->gfrcr_image);
	- idivisor = cyspeed(t->c_ispeed, cy_clock, &iprescaler);
	- if (idivisor <= 0)
	- return (EINVAL);
	- odivisor = cyspeed(t->c_ospeed != 0 ? t->c_ospeed : tp->t_ospeed,
	- cy_clock, &oprescaler);
	- if (odivisor <= 0)
	- return (EINVAL);
	-
	- /* parameters are OK, convert them to the com struct and the device */
	- s = spltty();
	- if (t->c_ospeed == 0)
	- (void)cymodem(tp, 0, SER_DTR);
	- else
	- (void)cymodem(tp, SER_DTR, 0);
	-
	- (void) cysetwater(com, t->c_ispeed);
	-
	- /* XXX we don't actually change the speed atomically. */
	-
	- cd_setreg(com, CD1400_RBPR, idivisor);
	- cd_setreg(com, CD1400_RCOR, iprescaler);
	- cd_setreg(com, CD1400_TBPR, odivisor);
	- cd_setreg(com, CD1400_TCOR, oprescaler);
	-
	- /*
	- * channel control
	- * receiver enable
	- * transmitter enable (always set)
	- */
	- cflag = t->c_cflag;
	- opt = CD1400_CCR_CMDCHANCTL \| CD1400_CCR_XMTEN
	- \| (cflag & CREAD ? CD1400_CCR_RCVEN : CD1400_CCR_RCVDIS);
	- if (opt != com->channel_control) {
	- com->channel_control = opt;
	- cd1400_channel_cmd(com, opt);
	- }
	-
	-#ifdef Smarts
	- /* set special chars */
	- /* XXX if one is _POSIX_VDISABLE, can't use some others */
	- if (t->c_cc[VSTOP] != _POSIX_VDISABLE)
	- cd_setreg(com, CD1400_SCHR1, t->c_cc[VSTOP]);
	- if (t->c_cc[VSTART] != _POSIX_VDISABLE)
	- cd_setreg(com, CD1400_SCHR2, t->c_cc[VSTART]);
	- if (t->c_cc[VINTR] != _POSIX_VDISABLE)
	- cd_setreg(com, CD1400_SCHR3, t->c_cc[VINTR]);
	- if (t->c_cc[VSUSP] != _POSIX_VDISABLE)
	- cd_setreg(com, CD1400_SCHR4, t->c_cc[VSUSP]);
	-#endif
	-
	- /*
	- * set channel option register 1 -
	- * parity mode
	- * stop bits
	- * char length
	- */
	- opt = 0;
	- /* parity */
	- if (cflag & PARENB) {
	- if (cflag & PARODD)
	- opt \|= CD1400_COR1_PARODD;
	- opt \|= CD1400_COR1_PARNORMAL;
	- }
	- iflag = t->c_iflag;
	- if (!(iflag & INPCK))
	- opt \|= CD1400_COR1_NOINPCK;
	- bits = 1 + 1;
	- /* stop bits */
	- if (cflag & CSTOPB) {
	- ++bits;
	- opt \|= CD1400_COR1_STOP2;
	- }
	- /* char length */
	- switch (cflag & CSIZE) {
	- case CS5:
	- bits += 5;
	- opt \|= CD1400_COR1_CS5;
	- break;
	- case CS6:
	- bits += 6;
	- opt \|= CD1400_COR1_CS6;
	- break;
	- case CS7:
	- bits += 7;
	- opt \|= CD1400_COR1_CS7;
	- break;
	- default:
	- bits += 8;
	- opt \|= CD1400_COR1_CS8;
	- break;
	- }
	- cor_change = 0;
	- if (opt != com->cor[0]) {
	- cor_change \|= CD1400_CCR_COR1;
	- cd_setreg(com, CD1400_COR1, com->cor[0] = opt);
	- }
	-
	- /*
	- * Set receive time-out period, normally to max(one char time, 5 ms).
	- */
	- itimeout = howmany(1000 * bits, t->c_ispeed);
	-#ifdef SOFT_HOTCHAR
	-#define MIN_RTP 1
	-#else
	-#define MIN_RTP 5
	-#endif
	- if (itimeout < MIN_RTP)
	- itimeout = MIN_RTP;
	- if (!(t->c_lflag & ICANON) && t->c_cc[VMIN] != 0 && t->c_cc[VTIME] != 0
	- && t->c_cc[VTIME] * 10 > itimeout)
	- itimeout = t->c_cc[VTIME] * 10;
	- if (itimeout > 255)
	- itimeout = 255;
	- cd_setreg(com, CD1400_RTPR, itimeout);
	-
	- /*
	- * set channel option register 2 -
	- * flow control
	- */
	- opt = 0;
	-#ifdef Smarts
	- if (iflag & IXANY)
	- opt \|= CD1400_COR2_IXANY;
	- if (iflag & IXOFF)
	- opt \|= CD1400_COR2_IXOFF;
	-#endif
	-#ifndef SOFT_CTS_OFLOW
	- if (cflag & CCTS_OFLOW)
	- opt \|= CD1400_COR2_CCTS_OFLOW;
	-#endif
	- critical_enter();
	- COM_LOCK();
	- if (opt != com->cor[1]) {
	- cor_change \|= CD1400_CCR_COR2;
	- cd_setreg(com, CD1400_COR2, com->cor[1] = opt);
	- }
	- COM_UNLOCK();
	- critical_exit();
	-
	- /*
	- * set channel option register 3 -
	- * receiver FIFO interrupt threshold
	- * flow control
	- */
	- opt = RxFifoThreshold;
	-#ifdef Smarts
	- if (t->c_lflag & ICANON)
	- opt \|= CD1400_COR3_SCD34; /* detect INTR & SUSP chars */
	- if (iflag & IXOFF)
	- /* detect and transparently handle START and STOP chars */
	- opt \|= CD1400_COR3_FCT \| CD1400_COR3_SCD12;
	-#endif
	- if (opt != com->cor[2]) {
	- cor_change \|= CD1400_CCR_COR3;
	- cd_setreg(com, CD1400_COR3, com->cor[2] = opt);
	- }
	-
	- /* notify the CD1400 if COR1-3 have changed */
	- if (cor_change)
	- cd1400_channel_cmd(com, CD1400_CCR_CMDCORCHG \| cor_change);
	-
	- /*
	- * set channel option register 4 -
	- * CR/NL processing
	- * break processing
	- * received exception processing
	- */
	- opt = 0;
	- if (iflag & IGNCR)
	- opt \|= CD1400_COR4_IGNCR;
	-#ifdef Smarts
	- /*
	- * we need a new ttyinput() for this, as we don't want to
	- * have ICRNL && INLCR being done in both layers, or to have
	- * synchronisation problems
	- */
	- if (iflag & ICRNL)
	- opt \|= CD1400_COR4_ICRNL;
	- if (iflag & INLCR)
	- opt \|= CD1400_COR4_INLCR;
	-#endif
	- if (iflag & IGNBRK)
	- opt \|= CD1400_COR4_IGNBRK \| CD1400_COR4_NOBRKINT;
	- /*
	- * The `-ignbrk -brkint parmrk' case is not handled by the hardware,
	- * so only tell the hardware about -brkint if -parmrk.
	- */
	- if (!(iflag & (BRKINT \| PARMRK)))
	- opt \|= CD1400_COR4_NOBRKINT;
	-#if 0
	- /* XXX using this "intelligence" breaks reporting of overruns. */
	- if (iflag & IGNPAR)
	- opt \|= CD1400_COR4_PFO_DISCARD;
	- else {
	- if (iflag & PARMRK)
	- opt \|= CD1400_COR4_PFO_ESC;
	- else
	- opt \|= CD1400_COR4_PFO_NUL;
	- }
	-#else
	- opt \|= CD1400_COR4_PFO_EXCEPTION;
	-#endif
	- cd_setreg(com, CD1400_COR4, opt);
	-
	- /*
	- * set channel option register 5 -
	- */
	- opt = 0;
	- if (iflag & ISTRIP)
	- opt \|= CD1400_COR5_ISTRIP;
	- if (t->c_iflag & IEXTEN)
	- /* enable LNEXT (e.g. ctrl-v quoting) handling */
	- opt \|= CD1400_COR5_LNEXT;
	-#ifdef Smarts
	- if (t->c_oflag & ONLCR)
	- opt \|= CD1400_COR5_ONLCR;
	- if (t->c_oflag & OCRNL)
	- opt \|= CD1400_COR5_OCRNL;
	-#endif
	- cd_setreg(com, CD1400_COR5, opt);
	-
	- /*
	- * We always generate modem status change interrupts for CD changes.
	- * Among other things, this is necessary to track TS_CARR_ON for
	- * pstat to print even when the driver doesn't care. CD changes
	- * should be rare so interrupts for them are not worth extra code to
	- * avoid. We avoid interrupts for other modem status changes (except
	- * for CTS changes when SOFT_CTS_OFLOW is configured) since this is
	- * simplest and best.
	- */
	-
	- /*
	- * set modem change option register 1
	- * generate modem interrupts on which 1 -> 0 input transitions
	- * also controls auto-DTR output flow-control, which we don't use
	- */
	- opt = CD1400_MCOR1_CDzd;
	-#ifdef SOFT_CTS_OFLOW
	- if (cflag & CCTS_OFLOW)
	- opt \|= CD1400_MCOR1_CTSzd;
	-#endif
	- cd_setreg(com, CD1400_MCOR1, opt);
	-
	- /*
	- * set modem change option register 2
	- * generate modem interrupts on specific 0 -> 1 input transitions
	- */
	- opt = CD1400_MCOR2_CDod;
	-#ifdef SOFT_CTS_OFLOW
	- if (cflag & CCTS_OFLOW)
	- opt \|= CD1400_MCOR2_CTSod;
	-#endif
	- cd_setreg(com, CD1400_MCOR2, opt);
	-
	- /*
	- * XXX should have done this long ago, but there is too much state
	- * to change all atomically.
	- */
	- critical_enter();
	- COM_LOCK();
	-
	- com->state &= ~CS_TTGO;
	- if (!(tp->t_state & TS_TTSTOP))
	- com->state \|= CS_TTGO;
	- if (cflag & CRTS_IFLOW) {
	- com->state \|= CS_RTS_IFLOW;
	- /*
	- * If CS_RTS_IFLOW just changed from off to on, the change
	- * needs to be propagated to CD1400_MSVR1_RTS. This isn't urgent,
	- * so do it later by calling cystart() instead of repeating
	- * a lot of code from cystart() here.
	- */
	- } else if (com->state & CS_RTS_IFLOW) {
	- com->state &= ~CS_RTS_IFLOW;
	- /*
	- * CS_RTS_IFLOW just changed from on to off. Force CD1400_MSVR1_RTS
	- * on here, since cystart() won't do it later.
	- */
	- cd_setreg(com, com->mcr_rts_reg,
	- com->mcr_image \|= com->mcr_rts);
	- }
	-
	- /*
	- * Set up state to handle output flow control.
	- * XXX - worth handling MDMBUF (DCD) flow control at the lowest level?
	- * Now has 10+ msec latency, while CTS flow has 50- usec latency.
	- */
	- com->state \|= CS_ODEVREADY;
	-#ifdef SOFT_CTS_OFLOW
	- com->state &= ~CS_CTS_OFLOW;
	- if (cflag & CCTS_OFLOW) {
	- com->state \|= CS_CTS_OFLOW;
	- if (!(com->last_modem_status & CD1400_MSVR2_CTS))
	- com->state &= ~CS_ODEVREADY;
	- }
	-#endif
	- /* XXX shouldn't call functions while intrs are disabled. */
	- disc_optim(tp, t, com);
	-#if 0
	- /*
	- * Recover from fiddling with CS_TTGO. We used to call cyintr1()
	- * unconditionally, but that defeated the careful discarding of
	- * stale input in cyopen().
	- */
	- if (com->state >= (CS_BUSY \| CS_TTGO))
	- cyintr1(com);
	-#endif
	- if (com->state >= (CS_BUSY \| CS_TTGO \| CS_ODEVREADY)) {
	- if (!(com->intr_enable & CD1400_SRER_TXRDY))
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable & ~CD1400_SRER_TXMPTY)
	- \| CD1400_SRER_TXRDY);
	- } else {
	- if (com->intr_enable & CD1400_SRER_TXRDY)
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable & ~CD1400_SRER_TXRDY)
	- \| CD1400_SRER_TXMPTY);
	- }
	-
	- COM_UNLOCK();
	- critical_exit();
	- splx(s);
	- cystart(tp);
	- if (com->ibufold != NULL) {
	- free(com->ibufold, M_DEVBUF);
	- com->ibufold = NULL;
	- }
	- return (0);
	-}
	-
	-static int
	-cysetwater(struct com_s *com, speed_t speed)
	-{
	- int cp4ticks;
	- u_char *ibuf;
	- int ibufsize;
	- struct tty *tp;
	-
	- /*
	- * Make the buffer size large enough to handle a softtty interrupt
	- * latency of about 2 ticks without loss of throughput or data
	- * (about 3 ticks if input flow control is not used or not honoured,
	- * but a bit less for CS5-CS7 modes).
	- */
	- cp4ticks = speed / 10 / hz * 4;
	- for (ibufsize = 128; ibufsize < cp4ticks;)
	- ibufsize <<= 1;
	- if (ibufsize == com->ibufsize) {
	- return (0);
	- }
	-
	- /*
	- * Allocate input buffer. The extra factor of 2 in the size is
	- * to allow for an error byte for each input byte.
	- */
	- ibuf = malloc(2 * ibufsize, M_DEVBUF, M_NOWAIT);
	- if (ibuf == NULL) {
	- return (ENOMEM);
	- }
	-
	- /* Initialize non-critical variables. */
	- com->ibufold = com->ibuf;
	- com->ibufsize = ibufsize;
	- tp = com->tp;
	- if (tp != NULL) {
	- tp->t_ififosize = 2 * ibufsize;
	- tp->t_ispeedwat = (speed_t)-1;
	- tp->t_ospeedwat = (speed_t)-1;
	- }
	-
	- /*
	- * Read current input buffer, if any. Continue with interrupts
	- * disabled.
	- */
	- critical_enter();
	- COM_LOCK();
	- if (com->iptr != com->ibuf)
	- cyinput(com);
	-
	- /*-
	- * Initialize critical variables, including input buffer watermarks.
	- * The external device is asked to stop sending when the buffer
	- * exactly reaches high water, or when the high level requests it.
	- * The high level is notified immediately (rather than at a later
	- * clock tick) when this watermark is reached.
	- * The buffer size is chosen so the watermark should almost never
	- * be reached.
	- * The low watermark is invisibly 0 since the buffer is always
	- * emptied all at once.
	- */
	- com->iptr = com->ibuf = ibuf;
	- com->ibufend = ibuf + ibufsize;
	- com->ierroff = ibufsize;
	- com->ihighwater = ibuf + 3 * ibufsize / 4;
	-
	- COM_UNLOCK();
	- critical_exit();
	- return (0);
	-}
	-
	-static void
	-cystart(struct tty *tp)
	-{
	- struct com_s *com;
	- int s;
	-#ifdef CyDebug
	- bool_t started;
	-#endif
	-
	- com = tp->t_sc;
	- s = spltty();
	-
	-#ifdef CyDebug
	- ++com->start_count;
	- started = FALSE;
	-#endif
	-
	- critical_enter();
	- COM_LOCK();
	- if (tp->t_state & TS_TTSTOP) {
	- com->state &= ~CS_TTGO;
	- if (com->intr_enable & CD1400_SRER_TXRDY)
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable & ~CD1400_SRER_TXRDY)
	- \| CD1400_SRER_TXMPTY);
	- } else {
	- com->state \|= CS_TTGO;
	- if (com->state >= (CS_BUSY \| CS_TTGO \| CS_ODEVREADY)
	- && !(com->intr_enable & CD1400_SRER_TXRDY))
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable & ~CD1400_SRER_TXMPTY)
	- \| CD1400_SRER_TXRDY);
	- }
	- if (tp->t_state & TS_TBLOCK) {
	- if (com->mcr_image & com->mcr_rts && com->state & CS_RTS_IFLOW)
	-#if 0
	- outb(com->modem_ctl_port, com->mcr_image &= ~CD1400_MSVR1_RTS);
	-#else
	- cd_setreg(com, com->mcr_rts_reg,
	- com->mcr_image &= ~com->mcr_rts);
	-#endif
	- } else {
	- if (!(com->mcr_image & com->mcr_rts)
	- && com->iptr < com->ihighwater
	- && com->state & CS_RTS_IFLOW)
	-#if 0
	- outb(com->modem_ctl_port, com->mcr_image \|= CD1400_MSVR1_RTS);
	-#else
	- cd_setreg(com, com->mcr_rts_reg,
	- com->mcr_image \|= com->mcr_rts);
	-#endif
	- }
	- COM_UNLOCK();
	- critical_exit();
	- if (tp->t_state & (TS_TIMEOUT \| TS_TTSTOP)) {
	- ttwwakeup(tp);
	- splx(s);
	- return;
	- }
	- if (tp->t_outq.c_cc != 0) {
	- struct lbq *qp;
	- struct lbq *next;
	-
	- if (!com->obufs[0].l_queued) {
	-#ifdef CyDebug
	- started = TRUE;
	-#endif
	- com->obufs[0].l_tail
	- = com->obuf1 + q_to_b(&tp->t_outq, com->obuf1,
	- sizeof com->obuf1);
	- com->obufs[0].l_next = NULL;
	- com->obufs[0].l_queued = TRUE;
	- critical_enter();
	- COM_LOCK();
	- if (com->state & CS_BUSY) {
	- qp = com->obufq.l_next;
	- while ((next = qp->l_next) != NULL)
	- qp = next;
	- qp->l_next = &com->obufs[0];
	- } else {
	- com->obufq.l_head = com->obufs[0].l_head;
	- com->obufq.l_tail = com->obufs[0].l_tail;
	- com->obufq.l_next = &com->obufs[0];
	- com->state \|= CS_BUSY;
	- if (com->state >= (CS_BUSY \| CS_TTGO
	- \| CS_ODEVREADY))
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable
	- & ~CD1400_SRER_TXMPTY)
	- \| CD1400_SRER_TXRDY);
	- }
	- COM_UNLOCK();
	- critical_exit();
	- }
	- if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) {
	-#ifdef CyDebug
	- started = TRUE;
	-#endif
	- com->obufs[1].l_tail
	- = com->obuf2 + q_to_b(&tp->t_outq, com->obuf2,
	- sizeof com->obuf2);
	- com->obufs[1].l_next = NULL;
	- com->obufs[1].l_queued = TRUE;
	- critical_enter();
	- COM_LOCK();
	- if (com->state & CS_BUSY) {
	- qp = com->obufq.l_next;
	- while ((next = qp->l_next) != NULL)
	- qp = next;
	- qp->l_next = &com->obufs[1];
	- } else {
	- com->obufq.l_head = com->obufs[1].l_head;
	- com->obufq.l_tail = com->obufs[1].l_tail;
	- com->obufq.l_next = &com->obufs[1];
	- com->state \|= CS_BUSY;
	- if (com->state >= (CS_BUSY \| CS_TTGO
	- \| CS_ODEVREADY))
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable
	- & ~CD1400_SRER_TXMPTY)
	- \| CD1400_SRER_TXRDY);
	- }
	- COM_UNLOCK();
	- critical_exit();
	- }
	- tp->t_state \|= TS_BUSY;
	- }
	-#ifdef CyDebug
	- if (started)
	- ++com->start_real;
	-#endif
	-#if 0
	- critical_enter();
	- COM_LOCK();
	- if (com->state >= (CS_BUSY \| CS_TTGO))
	- cyintr1(com); /* fake interrupt to start output */
	- COM_UNLOCK();
	- critical_exit();
	-#endif
	- ttwwakeup(tp);
	- splx(s);
	-}
	-
	-static void
	-comstop(struct tty *tp, int rw)
	-{
	- struct com_s *com;
	- bool_t wakeup_etc;
	-
	- com = tp->t_sc;
	- wakeup_etc = FALSE;
	- critical_enter();
	- COM_LOCK();
	- if (rw & FWRITE) {
	- com->obufs[0].l_queued = FALSE;
	- com->obufs[1].l_queued = FALSE;
	- if (com->extra_state & CSE_ODONE) {
	- cy_events -= LOTS_OF_EVENTS;
	- com->extra_state &= ~CSE_ODONE;
	- if (com->etc != ETC_NONE) {
	- if (com->etc == ETC_BREAK_ENDED)
	- com->etc = ETC_NONE;
	- wakeup_etc = TRUE;
	- }
	- }
	- com->tp->t_state &= ~TS_BUSY;
	- if (com->state & CS_ODONE)
	- cy_events -= LOTS_OF_EVENTS;
	- com->state &= ~(CS_ODONE \| CS_BUSY);
	- }
	- if (rw & FREAD) {
	- /* XXX no way to reset only input fifo. */
	- cy_events -= (com->iptr - com->ibuf);
	- com->iptr = com->ibuf;
	- }
	- COM_UNLOCK();
	- critical_exit();
	- if (wakeup_etc)
	- wakeup(&com->etc);
	- if (rw & FWRITE && com->etc == ETC_NONE)
	- cd1400_channel_cmd(com, CD1400_CCR_CMDRESET \| CD1400_CCR_FTF);
	- cystart(tp);
	-}
	-
	-static int
	-cymodem(struct tty *tp, int sigon, int sigoff)
	-{
	- struct com_s *com;
	- int mcr;
	- int msr;
	-
	- com = tp->t_sc;
	- if (sigon == 0 && sigoff == 0) {
	- sigon = 0;
	- mcr = com->mcr_image;
	- if (mcr & com->mcr_dtr)
	- sigon \|= SER_DTR;
	- if (mcr & com->mcr_rts)
	- /* XXX wired on for Cyclom-8Ys */
	- sigon \|= SER_RTS;
	-
	- /*
	- * We must read the modem status from the hardware because
	- * we don't generate modem status change interrupts for all
	- * changes, so com->prev_modem_status is not guaranteed to
	- * be up to date. This is safe, unlike for sio, because
	- * reading the status register doesn't clear pending modem
	- * status change interrupts.
	- */
	- msr = cd_getreg(com, CD1400_MSVR2);
	-
	- if (msr & CD1400_MSVR2_CTS)
	- sigon \|= SER_CTS;
	- if (msr & CD1400_MSVR2_CD)
	- sigon \|= SER_DCD;
	- if (msr & CD1400_MSVR2_DSR)
	- sigon \|= SER_DSR;
	- if (msr & CD1400_MSVR2_RI)
	- /* XXX not connected except for Cyclom-16Y? */
	- sigon \|= SER_RI;
	- return (sigon);
	- }
	- mcr = com->mcr_image;
	- if (sigon & SER_DTR)
	- mcr \|= com->mcr_dtr;
	- if (sigoff & SER_DTR)
	- mcr &= ~com->mcr_dtr;
	- if (sigon & SER_RTS)
	- mcr \|= com->mcr_rts;
	- if (sigoff & SER_RTS)
	- mcr &= ~com->mcr_rts;
	- critical_enter();
	- COM_LOCK();
	- com->mcr_image = mcr;
	- cd_setreg(com, CD1400_MSVR1, mcr);
	- cd_setreg(com, CD1400_MSVR2, mcr);
	- COM_UNLOCK();
	- critical_exit();
	- return (0);
	-}
	-
	-static void
	-cysettimeout()
	-{
	- struct com_s *com;
	- bool_t someopen;
	- int unit;
	-
	- /*
	- * Set our timeout period to 1 second if no polled devices are open.
	- * Otherwise set it to max(1/200, 1/hz).
	- * Enable timeouts iff some device is open.
	- */
	- untimeout(cywakeup, (void *)NULL, cy_timeout_handle);
	- cy_timeout = hz;
	- someopen = FALSE;
	- for (unit = 0; unit < NPORTS; ++unit) {
	- com = cy_addr(unit);
	- if (com != NULL && com->tp != NULL
	- && com->tp->t_state & TS_ISOPEN) {
	- someopen = TRUE;
	- }
	- }
	- if (someopen) {
	- cy_timeouts_until_log = hz / cy_timeout;
	- cy_timeout_handle = timeout(cywakeup, (void *)NULL,
	- cy_timeout);
	- } else {
	- /* Flush error messages, if any. */
	- cy_timeouts_until_log = 1;
	- cywakeup((void *)NULL);
	- untimeout(cywakeup, (void *)NULL, cy_timeout_handle);
	- }
	-}
	-
	-static void
	-cywakeup(void *chan)
	-{
	- struct com_s *com;
	- int unit;
	-
	- cy_timeout_handle = timeout(cywakeup, (void *)NULL, cy_timeout);
	-
	- /*
	- * Check for and log errors, but not too often.
	- */
	- if (--cy_timeouts_until_log > 0)
	- return;
	- cy_timeouts_until_log = hz / cy_timeout;
	- for (unit = 0; unit < NPORTS; ++unit) {
	- int errnum;
	-
	- com = cy_addr(unit);
	- if (com == NULL)
	- continue;
	- for (errnum = 0; errnum < CE_NTYPES; ++errnum) {
	- u_int delta;
	- u_long total;
	-
	- critical_enter();
	- COM_LOCK();
	- delta = com->delta_error_counts[errnum];
	- com->delta_error_counts[errnum] = 0;
	- COM_UNLOCK();
	- critical_exit();
	- if (delta == 0)
	- continue;
	- total = com->error_counts[errnum] += delta;
	- log(LOG_ERR, "cy%d: %u more %s%s (total %lu)\n",
	- unit, delta, error_desc[errnum],
	- delta == 1 ? "" : "s", total);
	- }
	- }
	-}
	-
	-static void
	-disc_optim(struct tty tp, struct termios t, struct com_s *com)
	-{
	-#ifndef SOFT_HOTCHAR
	- u_char opt;
	-#endif
	-
	- ttyldoptim(tp);
	-#ifndef SOFT_HOTCHAR
	- opt = com->cor[2] & ~CD1400_COR3_SCD34;
	- if (com->tp->t_hotchar != 0) {
	- cd_setreg(com, CD1400_SCHR3, com->tp->t_hotchar);
	- cd_setreg(com, CD1400_SCHR4, com->tp->t_hotchar);
	- opt \|= CD1400_COR3_SCD34;
	- }
	- if (opt != com->cor[2]) {
	- cd_setreg(com, CD1400_COR3, com->cor[2] = opt);
	- cd1400_channel_cmd(com, CD1400_CCR_CMDCORCHG \| CD1400_CCR_COR3);
	- }
	-#endif
	-}
	-
	-#ifdef Smarts
	-/* standard line discipline input routine */
	-int
	-cyinput(int c, struct tty *tp)
	-{
	- /* XXX duplicate ttyinput(), but without the IXOFF/IXON/ISTRIP/IPARMRK
	- * bits, as they are done by the CD1400. Hardly worth the effort,
	- * given that high-throughput session are raw anyhow.
	- */
	-}
	-#endif /* Smarts */
	-
	-static int
	-cyspeed(speed_t speed, u_long cy_clock, int *prescaler_io)
	-{
	- int actual;
	- int error;
	- int divider;
	- int prescaler;
	- int prescaler_unit;
	-
	- if (speed == 0)
	- return (0);
	- if (speed < 0 \|\| speed > 150000)
	- return (-1);
	-
	- /* determine which prescaler to use */
	- for (prescaler_unit = 4, prescaler = 2048; prescaler_unit;
	- prescaler_unit--, prescaler >>= 2) {
	- if (cy_clock / prescaler / speed > 63)
	- break;
	- }
	-
	- divider = (cy_clock / prescaler * 2 / speed + 1) / 2; /* round off */
	- if (divider > 255)
	- divider = 255;
	- actual = cy_clock/prescaler/divider;
	-
	- /* 10 times error in percent: */
	- error = ((actual - (long)speed) * 2000 / (long)speed + 1) / 2;
	-
	- /* 3.0% max error tolerance */
	- if (error < -30 \|\| error > 30)
	- return (-1);
	-
	- *prescaler_io = prescaler_unit;
	- return (divider);
	-}
	-
	-static void
	-cd1400_channel_cmd(struct com_s *com, int cmd)
	-{
	- cd1400_channel_cmd_wait(com);
	- cd_setreg(com, CD1400_CCR, cmd);
	- cd1400_channel_cmd_wait(com);
	-}
	-
	-static void
	-cd1400_channel_cmd_wait(struct com_s *com)
	-{
	- struct timeval start;
	- struct timeval tv;
	- long usec;
	-
	- if (cd_getreg(com, CD1400_CCR) == 0)
	- return;
	- microtime(&start);
	- for (;;) {
	- if (cd_getreg(com, CD1400_CCR) == 0)
	- return;
	- microtime(&tv);
	- usec = 1000000 * (tv.tv_sec - start.tv_sec) +
	- tv.tv_usec - start.tv_usec;
	- if (usec >= 5000) {
	- log(LOG_ERR,
	- "cy%d: channel command timeout (%ld usec)\n",
	- com->unit, usec);
	- return;
	- }
	- }
	-}
	-
	-static void
	-cd_etc(struct com_s *com, int etc)
	-{
	-
	- /*
	- * We can't change the hardware's ETC state while there are any
	- * characters in the tx fifo, since those characters would be
	- * interpreted as commands! Unputting characters from the fifo
	- * is difficult, so we wait up to 12 character times for the fifo
	- * to drain. The command will be delayed for up to 2 character
	- * times for the tx to become empty. Unputting characters from
	- * the tx holding and shift registers is impossible, so we wait
	- * for the tx to become empty so that the command is sure to be
	- * executed soon after we issue it.
	- */
	- critical_enter();
	- COM_LOCK();
	- if (com->etc == etc)
	- goto wait;
	- if ((etc == CD1400_ETC_SENDBREAK
	- && (com->etc == ETC_BREAK_STARTING
	- \|\| com->etc == ETC_BREAK_STARTED))
	- \|\| (etc == CD1400_ETC_STOPBREAK
	- && (com->etc == ETC_BREAK_ENDING \|\| com->etc == ETC_BREAK_ENDED
	- \|\| com->etc == ETC_NONE))) {
	- COM_UNLOCK();
	- critical_exit();
	- return;
	- }
	- com->etc = etc;
	- cd_setreg(com, CD1400_SRER,
	- com->intr_enable
	- = (com->intr_enable & ~CD1400_SRER_TXRDY) \| CD1400_SRER_TXMPTY);
	-wait:
	- COM_UNLOCK();
	- critical_exit();
	- while (com->etc == etc
	- && tsleep(&com->etc, TTIPRI \| PCATCH, "cyetc", 0) == 0)
	- continue;
	-}
	-
	-static int
	-cd_getreg(struct com_s *com, int reg)
	-{
	- struct com_s *basecom;
	- u_char car;
	- int cy_align;
	- cy_addr iobase;
	-#ifdef SMP
	- int need_unlock;
	-#endif
	- int val;
	-
	- basecom = cy_addr(com->unit & ~(CD1400_NO_OF_CHANNELS - 1));
	- car = com->unit & CD1400_CAR_CHAN;
	- cy_align = com->cy_align;
	- iobase = com->iobase;
	- critical_enter();
	-#ifdef SMP
	- need_unlock = 0;
	- if (!mtx_owned(&cy_lock)) {
	- COM_LOCK();
	- need_unlock = 1;
	- }
	-#endif
	- if (basecom->car != car)
	- cd_outb(iobase, CD1400_CAR, cy_align, basecom->car = car);
	- val = cd_inb(iobase, reg, cy_align);
	-#ifdef SMP
	- if (need_unlock)
	- COM_UNLOCK();
	-#endif
	- critical_exit();
	- return (val);
	-}
	-
	-static void
	-cd_setreg(struct com_s *com, int reg, int val)
	-{
	- struct com_s *basecom;
	- u_char car;
	- int cy_align;
	- cy_addr iobase;
	-#ifdef SMP
	- int need_unlock;
	-#endif
	-
	- basecom = cy_addr(com->unit & ~(CD1400_NO_OF_CHANNELS - 1));
	- car = com->unit & CD1400_CAR_CHAN;
	- cy_align = com->cy_align;
	- iobase = com->iobase;
	- critical_enter();
	-#ifdef SMP
	- need_unlock = 0;
	- if (!mtx_owned(&cy_lock)) {
	- COM_LOCK();
	- need_unlock = 1;
	- }
	-#endif
	- if (basecom->car != car)
	- cd_outb(iobase, CD1400_CAR, cy_align, basecom->car = car);
	- cd_outb(iobase, reg, cy_align, val);
	-#ifdef SMP
	- if (need_unlock)
	- COM_UNLOCK();
	-#endif
	- critical_exit();
	-}
	-
	-#ifdef CyDebug
	-/* useful in ddb */
	-void
	-cystatus(int unit)
	-{
	- struct com_s *com;
	- cy_addr iobase;
	- u_int ocount;
	- struct tty *tp;
	-
	- com = cy_addr(unit);
	- printf("info for channel %d\n", unit);
	- printf("------------------\n");
	- printf("total cyclom service probes:\t%d\n", cy_svrr_probes);
	- printf("calls to upper layer:\t\t%d\n", cy_timeouts);
	- if (com == NULL)
	- return;
	- iobase = com->iobase;
	- printf("\n");
	- printf("cd1400 base address:\\tt%p\n", iobase);
	- printf("saved channel_control:\t\t0x%02x\n", com->channel_control);
	- printf("saved cor1-3:\t\t\t0x%02x 0x%02x 0x%02x\n",
	- com->cor[0], com->cor[1], com->cor[2]);
	- printf("service request enable reg:\t0x%02x (0x%02x cached)\n",
	- cd_getreg(com, CD1400_SRER), com->intr_enable);
	- printf("service request register:\t0x%02x\n",
	- cd_inb(iobase, CD1400_SVRR, com->cy_align));
	- printf("modem status:\t\t\t0x%02x (0x%02x cached)\n",
	- cd_getreg(com, CD1400_MSVR2), com->prev_modem_status);
	- printf("rx/tx/mdm interrupt registers:\t0x%02x 0x%02x 0x%02x\n",
	- cd_inb(iobase, CD1400_RIR, com->cy_align),
	- cd_inb(iobase, CD1400_TIR, com->cy_align),
	- cd_inb(iobase, CD1400_MIR, com->cy_align));
	- printf("\n");
	- printf("com state:\t\t\t0x%02x\n", com->state);
	- printf("calls to cystart():\t\t%d (%d useful)\n",
	- com->start_count, com->start_real);
	- printf("rx buffer chars free:\t\t%d\n", com->iptr - com->ibuf);
	- ocount = 0;
	- if (com->obufs[0].l_queued)
	- ocount += com->obufs[0].l_tail - com->obufs[0].l_head;
	- if (com->obufs[1].l_queued)
	- ocount += com->obufs[1].l_tail - com->obufs[1].l_head;
	- printf("tx buffer chars:\t\t%u\n", ocount);
	- printf("received chars:\t\t\t%d\n", com->bytes_in);
	- printf("received exceptions:\t\t%d\n", com->recv_exception);
	- printf("modem signal deltas:\t\t%d\n", com->mdm);
	- printf("transmitted chars:\t\t%d\n", com->bytes_out);
	- printf("\n");
	- tp = com->tp;
	- if (tp != NULL) {
	- printf("tty state:\t\t\t0x%08x\n", tp->t_state);
	- printf(
	- "upper layer queue lengths:\t%d raw, %d canon, %d output\n",
	- tp->t_rawq.c_cc, tp->t_canq.c_cc, tp->t_outq.c_cc);
	- } else
	- printf("tty state:\t\t\tclosed\n");
	-}
	-#endif /* CyDebug */
	diff --git a/sys/dev/cy/cy_isa.c b/sys/dev/cy/cy_isa.c
	deleted file mode 100644
	index a53da4fc33e6..000000000000
	--- a/sys/dev/cy/cy_isa.c
	+++ /dev/null
	@@ -1,152 +0,0 @@
	-/*-
	- * cyclades cyclom-y serial driver
	- * Andrew Herbert <andrew@werple.apana.org.au>, 17 August 1993
	- *
	- * SPDX-License-Identifier: BSD-3-Clause
	- *
	- * Copyright (c) 1993 Andrew Herbert.
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- * 3. The name Andrew Herbert may not be used to endorse or promote products
	- * derived from this software without specific prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR IMPLIED
	- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
	- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
	- * NO EVENT SHALL I BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
	- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
	- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
	- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
	- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
	- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	- */
	-
	-/*
	- * Cyclades Y ISA serial interface driver
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/bus.h>
	-#include <sys/kernel.h>
	-#include <sys/module.h>
	-
	-#include <machine/bus.h>
	-#include <sys/rman.h>
	-#include <machine/resource.h>
	-
	-#include <isa/isavar.h>
	-
	-#include <dev/cy/cyreg.h>
	-#include <dev/cy/cyvar.h>
	-
	-static int cy_isa_attach(device_t dev);
	-static int cy_isa_probe(device_t dev);
	-
	-static device_method_t cy_isa_methods[] = {
	- /* Device interface. */
	- DEVMETHOD(device_probe, cy_isa_probe),
	- DEVMETHOD(device_attach, cy_isa_attach),
	-
	- { 0, 0 }
	-};
	-
	-static driver_t cy_isa_driver = {
	- cy_driver_name,
	- cy_isa_methods,
	- 0,
	-};
	-
	-DRIVER_MODULE(cy, isa, cy_isa_driver, cy_devclass, 0, 0);
	-
	-static int
	-cy_isa_probe(device_t dev)
	-{
	- struct resource *mem_res;
	- cy_addr iobase;
	- int error, mem_rid;
	-
	- if (isa_get_logicalid(dev) != 0) /* skip PnP probes */
	- return (ENXIO);
	-
	- mem_rid = 0;
	- mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &mem_rid,
	- RF_ACTIVE);
	- if (mem_res == NULL) {
	- device_printf(dev, "ioport resource allocation failed\n");
	- return (ENXIO);
	- }
	- iobase = rman_get_virtual(mem_res);
	-
	- /* Cyclom-16Y hardware reset (Cyclom-8Ys don't care) */
	- cy_inb(iobase, CY16_RESET, 0); /* XXX? */
	- DELAY(500); /* wait for the board to get its act together */
	-
	- /* this is needed to get the board out of reset */
	- cy_outb(iobase, CY_CLEAR_INTR, 0, 0);
	- DELAY(500);
	-
	- error = (cy_units(iobase, 0) == 0 ? ENXIO : 0);
	- bus_release_resource(dev, SYS_RES_MEMORY, mem_rid, mem_res);
	- return (error);
	-}
	-
	-static int
	-cy_isa_attach(device_t dev)
	-{
	- struct resource irq_res, mem_res;
	- void irq_cookie, vaddr, *vsc;
	- int irq_rid, mem_rid;
	-
	- irq_res = NULL;
	- mem_res = NULL;
	-
	- mem_rid = 0;
	- mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &mem_rid,
	- RF_ACTIVE);
	- if (mem_res == NULL) {
	- device_printf(dev, "memory resource allocation failed\n");
	- goto fail;
	- }
	- vaddr = rman_get_virtual(mem_res);
	-
	- vsc = cyattach_common(vaddr, 0);
	- if (vsc == NULL) {
	- device_printf(dev, "no ports found!\n");
	- goto fail;
	- }
	-
	- irq_rid = 0;
	- irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq_rid,
	- RF_SHAREABLE \| RF_ACTIVE);
	- if (irq_res == NULL) {
	- device_printf(dev, "interrupt resource allocation failed\n");
	- goto fail;
	- }
	- if (bus_setup_intr(dev, irq_res, INTR_TYPE_TTY,
	- cyintr, NULL, vsc, &irq_cookie) != 0) {
	- device_printf(dev, "interrupt setup failed\n");
	- goto fail;
	- }
	-
	- return (0);
	-
	-fail:
	- if (irq_res != NULL)
	- bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
	- if (mem_res != NULL)
	- bus_release_resource(dev, SYS_RES_MEMORY, mem_rid, mem_res);
	- return (ENXIO);
	-}
	diff --git a/sys/dev/cy/cy_pci.c b/sys/dev/cy/cy_pci.c
	deleted file mode 100644
	index 4bd833e27b19..000000000000
	--- a/sys/dev/cy/cy_pci.c
	+++ /dev/null
	@@ -1,194 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 1996, David Greenman
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice unmodified, this list of conditions, and the following
	- * disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- */
	-
	-/*
	- * Cyclades Y PCI serial interface driver
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include "opt_cy_pci_fastintr.h"
	-
	-#include <sys/param.h>
	-#include <sys/bus.h>
	-#include <sys/kernel.h>
	-#include <sys/module.h>
	-
	-#include <machine/bus.h>
	-#include <sys/rman.h>
	-#include <machine/resource.h>
	-
	-#include <dev/pci/pcivar.h>
	-
	-#include <dev/cy/cyvar.h>
	-
	-#define CY_PCI_BASE_ADDR0 0x10
	-#define CY_PCI_BASE_ADDR1 0x14
	-#define CY_PCI_BASE_ADDR2 0x18
	-
	-#define CY_PLX_9050_ICS 0x4c
	-#define CY_PLX_9060_ICS 0x68
	-#define CY_PLX_9050_ICS_IENABLE 0x040
	-#define CY_PLX_9050_ICS_LOCAL_IENABLE 0x001
	-#define CY_PLX_9050_ICS_LOCAL_IPOLARITY 0x002
	-#define CY_PLX_9060_ICS_IENABLE 0x100
	-#define CY_PLX_9060_ICS_LOCAL_IENABLE 0x800
	-
	-/* Cyclom-Y Custom Register for PLX ID. */
	-#define PLX_VER 0x3400
	-#define PLX_9050 0x0b
	-#define PLX_9060 0x0c
	-#define PLX_9080 0x0d
	-
	-static int cy_pci_attach(device_t dev);
	-static int cy_pci_probe(device_t dev);
	-
	-static device_method_t cy_pci_methods[] = {
	- /* Device interface. */
	- DEVMETHOD(device_probe, cy_pci_probe),
	- DEVMETHOD(device_attach, cy_pci_attach),
	-
	- { 0, 0 }
	-};
	-
	-static driver_t cy_pci_driver = {
	- cy_driver_name,
	- cy_pci_methods,
	- 0,
	-};
	-
	-DRIVER_MODULE(cy, pci, cy_pci_driver, cy_devclass, 0, 0);
	-MODULE_DEPEND(cy, pci, 1, 1, 1);
	-
	-static int
	-cy_pci_probe(dev)
	- device_t dev;
	-{
	- u_int32_t device_id;
	-
	- device_id = pci_get_devid(dev);
	- device_id &= ~0x00060000;
	- if (device_id != 0x0100120e && device_id != 0x0101120e)
	- return (ENXIO);
	- device_set_desc(dev, "Cyclades Cyclom-Y Serial Adapter");
	- return (BUS_PROBE_DEFAULT);
	-}
	-
	-static int
	-cy_pci_attach(dev)
	- device_t dev;
	-{
	- struct resource ioport_res, irq_res, *mem_res;
	- void irq_cookie, vaddr, *vsc;
	- u_int32_t ioport;
	- int irq_setup, ioport_rid, irq_rid, mem_rid;
	- u_char plx_ver;
	-
	- ioport_res = NULL;
	- irq_res = NULL;
	- mem_res = NULL;
	-
	- ioport_rid = CY_PCI_BASE_ADDR1;
	- ioport_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &ioport_rid,
	- RF_ACTIVE);
	- if (ioport_res == NULL) {
	- device_printf(dev, "ioport resource allocation failed\n");
	- goto fail;
	- }
	- ioport = rman_get_start(ioport_res);
	-
	- mem_rid = CY_PCI_BASE_ADDR2;
	- mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &mem_rid,
	- RF_ACTIVE);
	- if (mem_res == NULL) {
	- device_printf(dev, "memory resource allocation failed\n");
	- goto fail;
	- }
	- vaddr = rman_get_virtual(mem_res);
	-
	- vsc = cyattach_common(vaddr, 1);
	- if (vsc == NULL) {
	- device_printf(dev, "no ports found!\n");
	- goto fail;
	- }
	-
	- irq_rid = 0;
	- irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq_rid,
	- RF_SHAREABLE \| RF_ACTIVE);
	- if (irq_res == NULL) {
	- device_printf(dev, "interrupt resource allocation failed\n");
	- goto fail;
	- }
	-#ifdef CY_PCI_FASTINTR
	- irq_setup = bus_setup_intr(dev, irq_res, INTR_TYPE_TTY,
	- cyintr, NULL, vsc, &irq_cookie);
	-#else
	- irq_setup = ENXIO;
	-#endif
	- if (irq_setup != 0)
	- irq_setup = bus_setup_intr(dev, irq_res, INTR_TYPE_TTY,
	- NULL, (driver_intr_t *)cyintr, vsc, &irq_cookie);
	- if (irq_setup != 0) {
	- device_printf(dev, "interrupt setup failed\n");
	- goto fail;
	- }
	-
	- /*
	- * Enable the "local" interrupt input to generate a
	- * PCI interrupt.
	- */
	- plx_ver = ((u_char )vaddr + PLX_VER) & 0x0f;
	- switch (plx_ver) {
	- case PLX_9050:
	- outw(ioport + CY_PLX_9050_ICS,
	- CY_PLX_9050_ICS_IENABLE \| CY_PLX_9050_ICS_LOCAL_IENABLE \|
	- CY_PLX_9050_ICS_LOCAL_IPOLARITY);
	- break;
	- case PLX_9060:
	- case PLX_9080:
	- default: /* Old board, use PLX_9060 values. */
	- outw(ioport + CY_PLX_9060_ICS,
	- inw(ioport + CY_PLX_9060_ICS) \| CY_PLX_9060_ICS_IENABLE \|
	- CY_PLX_9060_ICS_LOCAL_IENABLE);
	- break;
	- }
	-
	- return (0);
	-
	-fail:
	- if (ioport_res != NULL)
	- bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid,
	- ioport_res);
	- if (irq_res != NULL)
	- bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
	- if (mem_res != NULL)
	- bus_release_resource(dev, SYS_RES_MEMORY, mem_rid, mem_res);
	- return (ENXIO);
	-}
	diff --git a/sys/dev/cy/cyreg.h b/sys/dev/cy/cyreg.h
	deleted file mode 100644
	index bb29064b471e..000000000000
	--- a/sys/dev/cy/cyreg.h
	+++ /dev/null
	@@ -1,77 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-3-Clause
	- *
	- * Copyright (c) 1995 Bruce Evans.
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- * 3. Neither the name of the author nor the names of contributors
	- * may be used to endorse or promote products derived from this software
	- * without specific prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-/*
	- * Definitions for Cyclades Cyclom-Y serial boards.
	- */
	-
	-/*
	- * Cyclades register offsets. These are physical offsets for ISA boards
	- * and physical offsets divided by 2 for PCI boards.
	- */
	-#define CY8_SVCACKR 0x100 /* (r) */
	-#define CY8_SVCACKT 0x200 /* (r) */
	-#define CY8_SVCACKM 0x300 /* (r) */
	-#define CY16_RESET 0x1400 /* (r) */
	-#define CY_CLEAR_INTR 0x1800 /* intr ack address (w) */
	-
	-#define CY_MAX_CD1400s 8 /* for Cyclom-32Y */
	-
	-#define CY_CLOCK(version) ((version) >= 0x48 ? 60000000 : 25000000)
	-#define CY_RTS_DTR_SWAPPED(version) ((version) >= 0x48)
	-
	-/*
	- * The `cd' macros are for access to cd1400 registers. The `cy' macros
	- * are for access to Cyclades registers. Both sets of macros scale the
	- * register number to get an offset, but the scales are different for
	- * mostly historical reasons.
	- */
	-#ifdef CyDebug
	-#define cd_inb(iobase, reg, cy_align) \
	- (++cd_inbs, ((iobase) + (2 (reg) << (cy_align))))
	-#define cy_inb(iobase, reg, cy_align) \
	- (++cy_inbs, *((iobase) + ((reg) << (cy_align))))
	-#define cd_outb(iobase, reg, cy_align, val) \
	- (++cd_outbs, (void)(((iobase) + (2 (reg) << (cy_align))) = (val)))
	-#define cy_outb(iobase, reg, cy_align, val) \
	- (++cy_outbs, (void)(*((iobase) + ((reg) << (cy_align))) = (val)))
	-#else
	-#define cd_inb(iobase, reg, cy_align) \
	- (((iobase) + (2 (reg) << (cy_align))))
	-#define cy_inb(iobase, reg, cy_align) \
	- (*((iobase) + ((reg) << (cy_align))))
	-#define cd_outb(iobase, reg, cy_align, val) \
	- ((void)(((iobase) + (2 (reg) << (cy_align))) = (val)))
	-#define cy_outb(iobase, reg, cy_align, val) \
	- ((void)(*((iobase) + ((reg) << (cy_align))) = (val)))
	-#endif
	diff --git a/sys/dev/cy/cyvar.h b/sys/dev/cy/cyvar.h
	deleted file mode 100644
	index 0289ecba4f3e..000000000000
	--- a/sys/dev/cy/cyvar.h
	+++ /dev/null
	@@ -1,38 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 2004 Bruce D. Evans
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-typedef u_char volatile *cy_addr;
	-
	-extern devclass_t cy_devclass;
	-extern char cy_driver_name[];
	-
	-void *cyattach_common(cy_addr cy_iobase, int cy_align);
	-driver_filter_t cyintr;
	-int cy_units(cy_addr cy_iobase, int cy_align);
	diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
	index df7400f73528..d343bdced22d 100644
	--- a/sys/dev/hwpmc/hwpmc_mod.c
	+++ b/sys/dev/hwpmc/hwpmc_mod.c
	@@ -1,5969 +1,5969 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2003-2008 Joseph Koshy
	* Copyright (c) 2007 The FreeBSD Foundation
	* Copyright (c) 2018 Matthew Macy
	* All rights reserved.
	*
	* Portions of this software were developed by A. Joseph Koshy under
	* sponsorship from the FreeBSD Foundation and Google, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/domainset.h>
	#include <sys/eventhandler.h>
	#include <sys/jail.h>
	#include <sys/kernel.h>
	#include <sys/kthread.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/mount.h>
	#include <sys/mutex.h>
	#include <sys/pmc.h>
	#include <sys/pmckern.h>
	#include <sys/pmclog.h>
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/queue.h>
	#include <sys/resourcevar.h>
	#include <sys/rwlock.h>
	#include <sys/sched.h>
	#include <sys/signalvar.h>
	#include <sys/smp.h>
	#include <sys/sx.h>
	#include <sys/sysctl.h>
	#include <sys/sysent.h>
	#include <sys/syslog.h>
	#include <sys/taskqueue.h>
	#include <sys/vnode.h>

	#include <sys/linker.h> /* needs to be after <sys/malloc.h> */

	#include <machine/atomic.h>
	#include <machine/md_var.h>

	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/pmap.h>
	#include <vm/vm_map.h>
	#include <vm/vm_object.h>

	#include "hwpmc_soft.h"

	#define PMC_EPOCH_ENTER() struct epoch_tracker pmc_et; epoch_enter_preempt(global_epoch_preempt, &pmc_et)
	#define PMC_EPOCH_EXIT() epoch_exit_preempt(global_epoch_preempt, &pmc_et)

	/*
	* Types
	*/

	enum pmc_flags {
	PMC_FLAG_NONE = 0x00, /* do nothing */
	PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */
	PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
	PMC_FLAG_NOWAIT = 0x04, /* do not wait for mallocs */
	};

	/*
	* The offset in sysent where the syscall is allocated.
	*/

	static int pmc_syscall_num = NO_SYSCALL;
	struct pmc_cpu *pmc_pcpu; / per-cpu state */
	pmc_value_t pmc_pcpu_saved; / saved PMC values: CSW handling */

	#define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]

	struct mtx_pool *pmc_mtxpool;
	static int pmc_pmcdisp; / PMC row dispositions */

	#define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0)
	#define PMC_ROW_DISP_IS_THREAD(R) (pmc_pmcdisp[(R)] > 0)
	#define PMC_ROW_DISP_IS_STANDALONE(R) (pmc_pmcdisp[(R)] < 0)

	#define PMC_MARK_ROW_FREE(R) do { \
	pmc_pmcdisp[(R)] = 0; \
	} while (0)

	#define PMC_MARK_ROW_STANDALONE(R) do { \
	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
	__LINE__)); \
	atomic_add_int(&pmc_pmcdisp[(R)], -1); \
	KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()), \
	("[pmc,%d] row disposition error", __LINE__)); \
	} while (0)

	#define PMC_UNMARK_ROW_STANDALONE(R) do { \
	atomic_add_int(&pmc_pmcdisp[(R)], 1); \
	KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
	__LINE__)); \
	} while (0)

	#define PMC_MARK_ROW_THREAD(R) do { \
	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
	__LINE__)); \
	atomic_add_int(&pmc_pmcdisp[(R)], 1); \
	} while (0)

	#define PMC_UNMARK_ROW_THREAD(R) do { \
	atomic_add_int(&pmc_pmcdisp[(R)], -1); \
	KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
	__LINE__)); \
	} while (0)


	/* various event handlers */
	static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag,
	pmc_kld_unload_tag;

	/* Module statistics */
	struct pmc_driverstats pmc_stats;


	/* Machine/processor dependent operations */
	static struct pmc_mdep *md;

	/*
	* Hash tables mapping owner processes and target threads to PMCs.
	*/

	struct mtx pmc_processhash_mtx; /* spin mutex */
	static u_long pmc_processhashmask;
	static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash;

	/*
	* Hash table of PMC owner descriptors. This table is protected by
	* the shared PMC "sx" lock.
	*/

	static u_long pmc_ownerhashmask;
	static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash;

	/*
	* List of PMC owners with system-wide sampling PMCs.
	*/

	static CK_LIST_HEAD(, pmc_owner) pmc_ss_owners;

	/*
	* List of free thread entries. This is protected by the spin
	* mutex.
	*/
	static struct mtx pmc_threadfreelist_mtx; /* spin mutex */
	static LIST_HEAD(, pmc_thread) pmc_threadfreelist;
	static int pmc_threadfreelist_entries=0;
	#define THREADENTRY_SIZE \
	(sizeof(struct pmc_thread) + (md->pmd_npmc * sizeof(struct pmc_threadpmcstate)))

	/*
	* Task to free thread descriptors
	*/
	static struct task free_task;

	/*
	* A map of row indices to classdep structures.
	*/
	static struct pmc_classdep **pmc_rowindex_to_classdep;

	/*
	* Prototypes
	*/

	#ifdef HWPMC_DEBUG
	static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
	static int pmc_debugflags_parse(char newstr, char fence);
	#endif

	static int load(struct module module, int cmd, void arg);
	static int pmc_add_sample(ring_type_t ring, struct pmc pm, struct trapframe tf);
	static void pmc_add_thread_descriptors_from_proc(struct proc *p,
	struct pmc_process *pp);
	static int pmc_attach_process(struct proc p, struct pmc pm);
	static struct pmc *pmc_allocate_pmc_descriptor(void);
	static struct pmc_owner pmc_allocate_owner_descriptor(struct proc p);
	static int pmc_attach_one_process(struct proc p, struct pmc pm);
	static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
	int cpu);
	static int pmc_can_attach(struct pmc pm, struct proc p);
	static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
	static void pmc_cleanup(void);
	static int pmc_detach_process(struct proc p, struct pmc pm);
	static int pmc_detach_one_process(struct proc p, struct pmc pm,
	int flags);
	static void pmc_destroy_owner_descriptor(struct pmc_owner *po);
	static void pmc_destroy_pmc_descriptor(struct pmc *pm);
	static void pmc_destroy_process_descriptor(struct pmc_process *pp);
	static struct pmc_owner pmc_find_owner_descriptor(struct proc p);
	static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
	static struct pmc pmc_find_pmc_descriptor_in_process(struct pmc_owner po,
	pmc_id_t pmc);
	static struct pmc_process pmc_find_process_descriptor(struct proc p,
	uint32_t mode);
	static struct pmc_thread pmc_find_thread_descriptor(struct pmc_process pp,
	struct thread *td, uint32_t mode);
	static void pmc_force_context_switch(void);
	static void pmc_link_target_process(struct pmc *pm,
	struct pmc_process *pp);
	static void pmc_log_all_process_mappings(struct pmc_owner *po);
	static void pmc_log_kernel_mappings(struct pmc *pm);
	static void pmc_log_process_mappings(struct pmc_owner po, struct proc p);
	static void pmc_maybe_remove_owner(struct pmc_owner *po);
	static void pmc_process_csw_in(struct thread *td);
	static void pmc_process_csw_out(struct thread *td);
	static void pmc_process_exit(void arg, struct proc p);
	static void pmc_process_fork(void arg, struct proc p1,
	struct proc *p2, int n);
	static void pmc_process_samples(int cpu, ring_type_t soft);
	static void pmc_release_pmc_descriptor(struct pmc *pmc);
	static void pmc_process_thread_add(struct thread *td);
	static void pmc_process_thread_delete(struct thread *td);
	static void pmc_process_thread_userret(struct thread *td);
	static void pmc_remove_owner(struct pmc_owner *po);
	static void pmc_remove_process_descriptor(struct pmc_process *pp);
	static void pmc_restore_cpu_binding(struct pmc_binding *pb);
	static void pmc_save_cpu_binding(struct pmc_binding *pb);
	static void pmc_select_cpu(int cpu);
	static int pmc_start(struct pmc *pm);
	static int pmc_stop(struct pmc *pm);
	static int pmc_syscall_handler(struct thread td, void syscall_args);
	static struct pmc_thread *pmc_thread_descriptor_pool_alloc(void);
	static void pmc_thread_descriptor_pool_drain(void);
	static void pmc_thread_descriptor_pool_free(struct pmc_thread *pt);
	static void pmc_unlink_target_process(struct pmc *pmc,
	struct pmc_process *pp);
	static int generic_switch_in(struct pmc_cpu pc, struct pmc_process pp);
	static int generic_switch_out(struct pmc_cpu pc, struct pmc_process pp);
	static struct pmc_mdep *pmc_generic_cpu_initialize(void);
	static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
	static void pmc_post_callchain_callback(void);
	static void pmc_process_threadcreate(struct thread *td);
	static void pmc_process_threadexit(struct thread *td);
	static void pmc_process_proccreate(struct proc *p);
	static void pmc_process_allproc(struct pmc *pm);

	/*
	* Kernel tunables and sysctl(8) interface.
	*/

	SYSCTL_DECL(_kern_hwpmc);
	SYSCTL_NODE(_kern_hwpmc, OID_AUTO, stats, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"HWPMC stats");


	/* Stats. */
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_ignored, CTLFLAG_RW,
	&pmc_stats.pm_intr_ignored, "# of interrupts ignored");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_processed, CTLFLAG_RW,
	&pmc_stats.pm_intr_processed, "# of interrupts processed");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, intr_bufferfull, CTLFLAG_RW,
	&pmc_stats.pm_intr_bufferfull, "# of interrupts where buffer was full");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscalls, CTLFLAG_RW,
	&pmc_stats.pm_syscalls, "# of syscalls");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, syscall_errors, CTLFLAG_RW,
	&pmc_stats.pm_syscall_errors, "# of syscall_errors");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests, CTLFLAG_RW,
	&pmc_stats.pm_buffer_requests, "# of buffer requests");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, buffer_requests_failed, CTLFLAG_RW,
	&pmc_stats.pm_buffer_requests_failed, "# of buffer requests which failed");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, log_sweeps, CTLFLAG_RW,
	&pmc_stats.pm_log_sweeps, "# of ?");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, merges, CTLFLAG_RW,
	&pmc_stats.pm_merges, "# of times kernel stack was found for user trace");
	SYSCTL_COUNTER_U64(_kern_hwpmc_stats, OID_AUTO, overwrites, CTLFLAG_RW,
	&pmc_stats.pm_overwrites, "# of times a sample was overwritten before being logged");

	static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
	SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN,
	&pmc_callchaindepth, 0, "depth of call chain records");

	-char pmc_cpuid[64];
	+char pmc_cpuid[PMC_CPUID_LEN];
	SYSCTL_STRING(_kern_hwpmc, OID_AUTO, cpuid, CTLFLAG_RD,
	pmc_cpuid, 0, "cpu version string");
	#ifdef HWPMC_DEBUG
	struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
	char pmc_debugstr[PMC_DEBUG_STRSIZE];
	TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
	sizeof(pmc_debugstr));
	SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
	CTLTYPE_STRING \| CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_NEEDGIANT,
	0, 0, pmc_debugflags_sysctl_handler, "A",
	"debug flags");
	#endif


	/*
	* kern.hwpmc.hashrows -- determines the number of rows in the
	* of the hash table used to look up threads
	*/

	static int pmc_hashsize = PMC_HASH_SIZE;
	SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN,
	&pmc_hashsize, 0, "rows in hash tables");

	/*
	* kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
	*/

	static int pmc_nsamples = PMC_NSAMPLES;
	SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN,
	&pmc_nsamples, 0, "number of PC samples per CPU");

	static uint64_t pmc_sample_mask = PMC_NSAMPLES-1;

	/*
	* kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
	*/

	static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
	SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN,
	&pmc_mtxpool_size, 0, "size of spin mutex pool");


	/*
	* kern.hwpmc.threadfreelist_entries -- number of free entries
	*/

	SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_entries, CTLFLAG_RD,
	&pmc_threadfreelist_entries, 0, "number of avalable thread entries");


	/*
	* kern.hwpmc.threadfreelist_max -- maximum number of free entries
	*/

	static int pmc_threadfreelist_max = PMC_THREADLIST_MAX;
	SYSCTL_INT(_kern_hwpmc, OID_AUTO, threadfreelist_max, CTLFLAG_RW,
	&pmc_threadfreelist_max, 0,
	"maximum number of available thread entries before freeing some");


	/*
	* security.bsd.unprivileged_syspmcs -- allow non-root processes to
	* allocate system-wide PMCs.
	*
	* Allowing unprivileged processes to allocate system PMCs is convenient
	* if system-wide measurements need to be taken concurrently with other
	* per-process measurements. This feature is turned off by default.
	*/

	static int pmc_unprivileged_syspmcs = 0;
	SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN,
	&pmc_unprivileged_syspmcs, 0,
	"allow unprivileged process to allocate system PMCs");

	/*
	* Hash function. Discard the lower 2 bits of the pointer since
	* these are always zero for our uses. The hash multiplier is
	* round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
	*/

	#if LONG_BIT == 64
	#define _PMC_HM 11400714819323198486u
	#elif LONG_BIT == 32
	#define _PMC_HM 2654435769u
	#else
	#error Must know the size of 'long' to compile
	#endif

	#define PMC_HASH_PTR(P,M) ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))

	/*
	* Syscall structures
	*/

	/* The `sysent' for the new syscall */
	static struct sysent pmc_sysent = {
	.sy_narg = 2,
	.sy_call = pmc_syscall_handler,
	};

	static struct syscall_module_data pmc_syscall_mod = {
	.chainevh = load,
	.chainarg = NULL,
	.offset = &pmc_syscall_num,
	.new_sysent = &pmc_sysent,
	.old_sysent = { .sy_narg = 0, .sy_call = NULL },
	.flags = SY_THR_STATIC_KLD,
	};

	static moduledata_t pmc_mod = {
	.name = PMC_MODULE_NAME,
	.evhand = syscall_module_handler,
	.priv = &pmc_syscall_mod,
	};

	#ifdef EARLY_AP_STARTUP
	DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
	#else
	DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
	#endif
	MODULE_VERSION(pmc, PMC_VERSION);

	#ifdef HWPMC_DEBUG
	enum pmc_dbgparse_state {
	PMCDS_WS, /* in whitespace */
	PMCDS_MAJOR, /* seen a major keyword */
	PMCDS_MINOR
	};

	static int
	pmc_debugflags_parse(char newstr, char fence)
	{
	char c, p, q;
	struct pmc_debugflags *tmpflags;
	int error, found, *newbits, tmp;
	size_t kwlen;

	tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK\|M_ZERO);

	p = newstr;
	error = 0;

	for (; p < fence && (c = *p); p++) {

	/* skip white space */
	if (c == ' ' \|\| c == '\t')
	continue;

	/* look for a keyword followed by "=" */
	for (q = p; p < fence && (c = *p) && c != '='; p++)
	;
	if (c != '=') {
	error = EINVAL;
	goto done;
	}

	kwlen = p - q;
	newbits = NULL;

	/* lookup flag group name */
	#define DBG_SET_FLAG_MAJ(S,F) \
	if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \
	newbits = &tmpflags->pdb_ ## F;

	DBG_SET_FLAG_MAJ("cpu", CPU);
	DBG_SET_FLAG_MAJ("csw", CSW);
	DBG_SET_FLAG_MAJ("logging", LOG);
	DBG_SET_FLAG_MAJ("module", MOD);
	DBG_SET_FLAG_MAJ("md", MDP);
	DBG_SET_FLAG_MAJ("owner", OWN);
	DBG_SET_FLAG_MAJ("pmc", PMC);
	DBG_SET_FLAG_MAJ("process", PRC);
	DBG_SET_FLAG_MAJ("sampling", SAM);

	if (newbits == NULL) {
	error = EINVAL;
	goto done;
	}

	p++; /* skip the '=' */

	/* Now parse the individual flags */
	tmp = 0;
	newflag:
	for (q = p; p < fence && (c = *p); p++)
	if (c == ' ' \|\| c == '\t' \|\| c == ',')
	break;

	/* p == fence or c == ws or c == "," or c == 0 */

	if ((kwlen = p - q) == 0) {
	*newbits = tmp;
	continue;
	}

	found = 0;
	#define DBG_SET_FLAG_MIN(S,F) \
	if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \
	tmp \|= found = (1 << PMC_DEBUG_MIN_ ## F)

	/* a '' denotes all possible flags in the group /
	if (kwlen == 1 && q == '')
	tmp = found = ~0;
	/* look for individual flag names */
	DBG_SET_FLAG_MIN("allocaterow", ALR);
	DBG_SET_FLAG_MIN("allocate", ALL);
	DBG_SET_FLAG_MIN("attach", ATT);
	DBG_SET_FLAG_MIN("bind", BND);
	DBG_SET_FLAG_MIN("config", CFG);
	DBG_SET_FLAG_MIN("exec", EXC);
	DBG_SET_FLAG_MIN("exit", EXT);
	DBG_SET_FLAG_MIN("find", FND);
	DBG_SET_FLAG_MIN("flush", FLS);
	DBG_SET_FLAG_MIN("fork", FRK);
	DBG_SET_FLAG_MIN("getbuf", GTB);
	DBG_SET_FLAG_MIN("hook", PMH);
	DBG_SET_FLAG_MIN("init", INI);
	DBG_SET_FLAG_MIN("intr", INT);
	DBG_SET_FLAG_MIN("linktarget", TLK);
	DBG_SET_FLAG_MIN("mayberemove", OMR);
	DBG_SET_FLAG_MIN("ops", OPS);
	DBG_SET_FLAG_MIN("read", REA);
	DBG_SET_FLAG_MIN("register", REG);
	DBG_SET_FLAG_MIN("release", REL);
	DBG_SET_FLAG_MIN("remove", ORM);
	DBG_SET_FLAG_MIN("sample", SAM);
	DBG_SET_FLAG_MIN("scheduleio", SIO);
	DBG_SET_FLAG_MIN("select", SEL);
	DBG_SET_FLAG_MIN("signal", SIG);
	DBG_SET_FLAG_MIN("swi", SWI);
	DBG_SET_FLAG_MIN("swo", SWO);
	DBG_SET_FLAG_MIN("start", STA);
	DBG_SET_FLAG_MIN("stop", STO);
	DBG_SET_FLAG_MIN("syscall", PMS);
	DBG_SET_FLAG_MIN("unlinktarget", TUL);
	DBG_SET_FLAG_MIN("write", WRI);
	if (found == 0) {
	/* unrecognized flag name */
	error = EINVAL;
	goto done;
	}

	if (c == 0 \|\| c == ' ' \|\| c == '\t') { /* end of flag group */
	*newbits = tmp;
	continue;
	}

	p++;
	goto newflag;
	}

	/* save the new flag set */
	bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));

	done:
	free(tmpflags, M_PMC);
	return error;
	}

	static int
	pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
	{
	char fence, newstr;
	int error;
	unsigned int n;

	(void) arg1; (void) arg2; /* unused parameters */

	n = sizeof(pmc_debugstr);
	newstr = malloc(n, M_PMC, M_WAITOK\|M_ZERO);
	(void) strlcpy(newstr, pmc_debugstr, n);

	error = sysctl_handle_string(oidp, newstr, n, req);

	/* if there is a new string, parse and copy it */
	if (error == 0 && req->newptr != NULL) {
	fence = newstr + (n < req->newlen ? n : req->newlen + 1);
	if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
	(void) strlcpy(pmc_debugstr, newstr,
	sizeof(pmc_debugstr));
	}

	free(newstr, M_PMC);

	return error;
	}
	#endif

	/*
	* Map a row index to a classdep structure and return the adjusted row
	* index for the PMC class index.
	*/
	static struct pmc_classdep *
	pmc_ri_to_classdep(struct pmc_mdep md, int ri, int adjri)
	{
	struct pmc_classdep *pcd;

	(void) md;

	KASSERT(ri >= 0 && ri < md->pmd_npmc,
	("[pmc,%d] illegal row-index %d", __LINE__, ri));

	pcd = pmc_rowindex_to_classdep[ri];

	KASSERT(pcd != NULL,
	("[pmc,%d] ri %d null pcd", __LINE__, ri));

	*adjri = ri - pcd->pcd_ri;

	KASSERT(adjri >= 0 && adjri < pcd->pcd_num,
	("[pmc,%d] adjusted row-index %d", __LINE__, *adjri));

	return (pcd);
	}

	/*
	* Concurrency Control
	*
	* The driver manages the following data structures:
	*
	* - target process descriptors, one per target process
	* - owner process descriptors (and attached lists), one per owner process
	* - lookup hash tables for owner and target processes
	* - PMC descriptors (and attached lists)
	* - per-cpu hardware state
	* - the 'hook' variable through which the kernel calls into
	* this module
	* - the machine hardware state (managed by the MD layer)
	*
	* These data structures are accessed from:
	*
	* - thread context-switch code
	* - interrupt handlers (possibly on multiple cpus)
	* - kernel threads on multiple cpus running on behalf of user
	* processes doing system calls
	* - this driver's private kernel threads
	*
	* = Locks and Locking strategy =
	*
	* The driver uses four locking strategies for its operation:
	*
	* - The global SX lock "pmc_sx" is used to protect internal
	* data structures.
	*
	* Calls into the module by syscall() start with this lock being
	* held in exclusive mode. Depending on the requested operation,
	* the lock may be downgraded to 'shared' mode to allow more
	* concurrent readers into the module. Calls into the module from
	* other parts of the kernel acquire the lock in shared mode.
	*
	* This SX lock is held in exclusive mode for any operations that
	* modify the linkages between the driver's internal data structures.
	*
	* The 'pmc_hook' function pointer is also protected by this lock.
	* It is only examined with the sx lock held in exclusive mode. The
	* kernel module is allowed to be unloaded only with the sx lock held
	* in exclusive mode. In normal syscall handling, after acquiring the
	* pmc_sx lock we first check that 'pmc_hook' is non-null before
	* proceeding. This prevents races between the thread unloading the module
	* and other threads seeking to use the module.
	*
	* - Lookups of target process structures and owner process structures
	* cannot use the global "pmc_sx" SX lock because these lookups need
	* to happen during context switches and in other critical sections
	* where sleeping is not allowed. We protect these lookup tables
	* with their own private spin-mutexes, "pmc_processhash_mtx" and
	* "pmc_ownerhash_mtx".
	*
	* - Interrupt handlers work in a lock free manner. At interrupt
	* time, handlers look at the PMC pointer (phw->phw_pmc) configured
	* when the PMC was started. If this pointer is NULL, the interrupt
	* is ignored after updating driver statistics. We ensure that this
	* pointer is set (using an atomic operation if necessary) before the
	* PMC hardware is started. Conversely, this pointer is unset atomically
	* only after the PMC hardware is stopped.
	*
	* We ensure that everything needed for the operation of an
	* interrupt handler is available without it needing to acquire any
	* locks. We also ensure that a PMC's software state is destroyed only
	* after the PMC is taken off hardware (on all CPUs).
	*
	* - Context-switch handling with process-private PMCs needs more
	* care.
	*
	* A given process may be the target of multiple PMCs. For example,
	* PMCATTACH and PMCDETACH may be requested by a process on one CPU
	* while the target process is running on another. A PMC could also
	* be getting released because its owner is exiting. We tackle
	* these situations in the following manner:
	*
	* - each target process structure 'pmc_process' has an array
	* of 'struct pmc *' pointers, one for each hardware PMC.
	*
	* - At context switch IN time, each "target" PMC in RUNNING state
	* gets started on hardware and a pointer to each PMC is copied into
	* the per-cpu phw array. The 'runcount' for the PMC is
	* incremented.
	*
	* - At context switch OUT time, all process-virtual PMCs are stopped
	* on hardware. The saved value is added to the PMCs value field
	* only if the PMC is in a non-deleted state (the PMCs state could
	* have changed during the current time slice).
	*
	* Note that since in-between a switch IN on a processor and a switch
	* OUT, the PMC could have been released on another CPU. Therefore
	* context switch OUT always looks at the hardware state to turn
	* OFF PMCs and will update a PMC's saved value only if reachable
	* from the target process record.
	*
	* - OP PMCRELEASE could be called on a PMC at any time (the PMC could
	* be attached to many processes at the time of the call and could
	* be active on multiple CPUs).
	*
	* We prevent further scheduling of the PMC by marking it as in
	* state 'DELETED'. If the runcount of the PMC is non-zero then
	* this PMC is currently running on a CPU somewhere. The thread
	* doing the PMCRELEASE operation waits by repeatedly doing a
	* pause() till the runcount comes to zero.
	*
	* The contents of a PMC descriptor (struct pmc) are protected using
	* a spin-mutex. In order to save space, we use a mutex pool.
	*
	* In terms of lock types used by witness(4), we use:
	* - Type "pmc-sx", used by the global SX lock.
	* - Type "pmc-sleep", for sleep mutexes used by logger threads.
	* - Type "pmc-per-proc", for protecting PMC owner descriptors.
	* - Type "pmc-leaf", used for all other spin mutexes.
	*/

	/*
	* save the cpu binding of the current kthread
	*/

	static void
	pmc_save_cpu_binding(struct pmc_binding *pb)
	{
	PMCDBG0(CPU,BND,2, "save-cpu");
	thread_lock(curthread);
	pb->pb_bound = sched_is_bound(curthread);
	pb->pb_cpu = curthread->td_oncpu;
	thread_unlock(curthread);
	PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
	}

	/*
	* restore the cpu binding of the current thread
	*/

	static void
	pmc_restore_cpu_binding(struct pmc_binding *pb)
	{
	PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
	curthread->td_oncpu, pb->pb_cpu);
	thread_lock(curthread);
	if (pb->pb_bound)
	sched_bind(curthread, pb->pb_cpu);
	else
	sched_unbind(curthread);
	thread_unlock(curthread);
	PMCDBG0(CPU,BND,2, "restore-cpu done");
	}

	/*
	* move execution over the specified cpu and bind it there.
	*/

	static void
	pmc_select_cpu(int cpu)
	{
	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	("[pmc,%d] bad cpu number %d", __LINE__, cpu));

	/* Never move to an inactive CPU. */
	KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive "
	"CPU %d", __LINE__, cpu));

	PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu);
	thread_lock(curthread);
	sched_bind(curthread, cpu);
	thread_unlock(curthread);

	KASSERT(curthread->td_oncpu == cpu,
	("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
	cpu, curthread->td_oncpu));

	PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
	}

	/*
	* Force a context switch.
	*
	* We do this by pause'ing for 1 tick -- invoking mi_switch() is not
	* guaranteed to force a context switch.
	*/

	static void
	pmc_force_context_switch(void)
	{

	pause("pmcctx", 1);
	}

	uint64_t
	pmc_rdtsc(void)
	{
	#if defined(__i386__) \|\| defined(__amd64__)
	if (__predict_true(amd_feature & AMDID_RDTSCP))
	return rdtscp();
	else
	return rdtsc();
	#else
	return get_cyclecount();
	#endif
	}

	/*
	* Get the file name for an executable. This is a simple wrapper
	* around vn_fullpath(9).
	*/

	static void
	pmc_getfilename(struct vnode v, char fullpath, char *freepath)
	{

	*fullpath = "unknown";
	*freepath = NULL;
	vn_fullpath(curthread, v, fullpath, freepath);
	}

	/*
	* remove an process owning PMCs
	*/

	void
	pmc_remove_owner(struct pmc_owner *po)
	{
	struct pmc pm, tmp;

	sx_assert(&pmc_sx, SX_XLOCKED);

	PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po);

	/* Remove descriptor from the owner hash table */
	LIST_REMOVE(po, po_next);

	/* release all owned PMC descriptors */
	LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
	PMCDBG1(OWN,ORM,2, "pmc=%p", pm);
	KASSERT(pm->pm_owner == po,
	("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));

	pmc_release_pmc_descriptor(pm); /* will unlink from the list */
	pmc_destroy_pmc_descriptor(pm);
	}

	KASSERT(po->po_sscount == 0,
	("[pmc,%d] SS count not zero", __LINE__));
	KASSERT(LIST_EMPTY(&po->po_pmcs),
	("[pmc,%d] PMC list not empty", __LINE__));

	/* de-configure the log file if present */
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_deconfigure_log(po);
	}

	/*
	* remove an owner process record if all conditions are met.
	*/

	static void
	pmc_maybe_remove_owner(struct pmc_owner *po)
	{

	PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po);

	/*
	* Remove owner record if
	* - this process does not own any PMCs
	* - this process has not allocated a system-wide sampling buffer
	*/

	if (LIST_EMPTY(&po->po_pmcs) &&
	((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
	pmc_remove_owner(po);
	pmc_destroy_owner_descriptor(po);
	}
	}

	/*
	* Add an association between a target process and a PMC.
	*/

	static void
	pmc_link_target_process(struct pmc pm, struct pmc_process pp)
	{
	int ri;
	struct pmc_target *pt;
	#ifdef INVARIANTS
	struct pmc_thread *pt_td;
	#endif

	sx_assert(&pmc_sx, SX_XLOCKED);

	KASSERT(pm != NULL && pp != NULL,
	("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
	KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
	("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
	__LINE__, pm, pp->pp_proc->p_pid));
	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1),
	("[pmc,%d] Illegal reference count %d for process record %p",
	__LINE__, pp->pp_refcnt, (void *) pp));

	ri = PMC_TO_ROWINDEX(pm);

	PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
	pm, ri, pp);

	#ifdef HWPMC_DEBUG
	LIST_FOREACH(pt, &pm->pm_targets, pt_next)
	if (pt->pt_process == pp)
	KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
	__LINE__, pp, pm));
	#endif

	pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK\|M_ZERO);
	pt->pt_process = pp;

	LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);

	atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
	(uintptr_t)pm);

	if (pm->pm_owner->po_owner == pp->pp_proc)
	pm->pm_flags \|= PMC_F_ATTACHED_TO_OWNER;

	/*
	* Initialize the per-process values at this row index.
	*/
	pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
	pm->pm_sc.pm_reloadcount : 0;

	pp->pp_refcnt++;

	#ifdef INVARIANTS
	/* Confirm that the per-thread values at this row index are cleared. */
	if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
	mtx_lock_spin(pp->pp_tdslock);
	LIST_FOREACH(pt_td, &pp->pp_tds, pt_next) {
	KASSERT(pt_td->pt_pmcs[ri].pt_pmcval == (pmc_value_t) 0,
	("[pmc,%d] pt_pmcval not cleared for pid=%d at "
	"ri=%d", __LINE__, pp->pp_proc->p_pid, ri));
	}
	mtx_unlock_spin(pp->pp_tdslock);
	}
	#endif
	}

	/*
	* Removes the association between a target process and a PMC.
	*/

	static void
	pmc_unlink_target_process(struct pmc pm, struct pmc_process pp)
	{
	int ri;
	struct proc *p;
	struct pmc_target *ptgt;
	struct pmc_thread *pt;

	sx_assert(&pmc_sx, SX_XLOCKED);

	KASSERT(pm != NULL && pp != NULL,
	("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));

	KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc,
	("[pmc,%d] Illegal ref count %d on process record %p",
	__LINE__, pp->pp_refcnt, (void *) pp));

	ri = PMC_TO_ROWINDEX(pm);

	PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
	pm, ri, pp);

	KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
	("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
	ri, pm, pp->pp_pmcs[ri].pp_pmc));

	pp->pp_pmcs[ri].pp_pmc = NULL;
	pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;

	/* Clear the per-thread values at this row index. */
	if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
	mtx_lock_spin(pp->pp_tdslock);
	LIST_FOREACH(pt, &pp->pp_tds, pt_next)
	pt->pt_pmcs[ri].pt_pmcval = (pmc_value_t) 0;
	mtx_unlock_spin(pp->pp_tdslock);
	}

	/* Remove owner-specific flags */
	if (pm->pm_owner->po_owner == pp->pp_proc) {
	pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
	pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
	}

	pp->pp_refcnt--;

	/* Remove the target process from the PMC structure */
	LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
	if (ptgt->pt_process == pp)
	break;

	KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
	"in pmc %p", __LINE__, pp->pp_proc, pp, pm));

	LIST_REMOVE(ptgt, pt_next);
	free(ptgt, M_PMC);

	/* if the PMC now lacks targets, send the owner a SIGIO */
	if (LIST_EMPTY(&pm->pm_targets)) {
	p = pm->pm_owner->po_owner;
	PROC_LOCK(p);
	kern_psignal(p, SIGIO);
	PROC_UNLOCK(p);

	PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p,
	SIGIO);
	}
	}

	/*
	* Check if PMC 'pm' may be attached to target process 't'.
	*/

	static int
	pmc_can_attach(struct pmc pm, struct proc t)
	{
	struct proc o; / pmc owner */
	struct ucred oc, tc; /* owner, target credentials */
	int decline_attach, i;

	/*
	* A PMC's owner can always attach that PMC to itself.
	*/

	if ((o = pm->pm_owner->po_owner) == t)
	return 0;

	PROC_LOCK(o);
	oc = o->p_ucred;
	crhold(oc);
	PROC_UNLOCK(o);

	PROC_LOCK(t);
	tc = t->p_ucred;
	crhold(tc);
	PROC_UNLOCK(t);

	/*
	* The effective uid of the PMC owner should match at least one
	* of the {effective,real,saved} uids of the target process.
	*/

	decline_attach = oc->cr_uid != tc->cr_uid &&
	oc->cr_uid != tc->cr_svuid &&
	oc->cr_uid != tc->cr_ruid;

	/*
	* Every one of the target's group ids, must be in the owner's
	* group list.
	*/
	for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
	decline_attach = !groupmember(tc->cr_groups[i], oc);

	/* check the read and saved gids too */
	if (decline_attach == 0)
	decline_attach = !groupmember(tc->cr_rgid, oc) \|\|
	!groupmember(tc->cr_svgid, oc);

	crfree(tc);
	crfree(oc);

	return !decline_attach;
	}

	/*
	* Attach a process to a PMC.
	*/

	static int
	pmc_attach_one_process(struct proc p, struct pmc pm)
	{
	int ri, error;
	char fullpath, freepath;
	struct pmc_process *pp;

	sx_assert(&pmc_sx, SX_XLOCKED);

	PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
	PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);

	/*
	* Locate the process descriptor corresponding to process 'p',
	* allocating space as needed.
	*
	* Verify that rowindex 'pm_rowindex' is free in the process
	* descriptor.
	*
	* If not, allocate space for a descriptor and link the
	* process descriptor and PMC.
	*/
	ri = PMC_TO_ROWINDEX(pm);

	/* mark process as using HWPMCs */
	PROC_LOCK(p);
	p->p_flag \|= P_HWPMC;
	PROC_UNLOCK(p);

	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) {
	error = ENOMEM;
	goto fail;
	}

	if (pp->pp_pmcs[ri].pp_pmc == pm) {/* already present at slot [ri] */
	error = EEXIST;
	goto fail;
	}

	if (pp->pp_pmcs[ri].pp_pmc != NULL) {
	error = EBUSY;
	goto fail;
	}

	pmc_link_target_process(pm, pp);

	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
	(pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
	pm->pm_flags \|= PMC_F_NEEDS_LOGFILE;

	pm->pm_flags \|= PMC_F_ATTACH_DONE; /* mark as attached */

	/* issue an attach event to a configured log file */
	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
	if (p->p_flag & P_KPROC) {
	fullpath = kernelname;
	freepath = NULL;
	} else {
	pmc_getfilename(p->p_textvp, &fullpath, &freepath);
	pmclog_process_pmcattach(pm, p->p_pid, fullpath);
	}
	free(freepath, M_TEMP);
	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
	pmc_log_process_mappings(pm->pm_owner, p);
	}

	return (0);
	fail:
	PROC_LOCK(p);
	p->p_flag &= ~P_HWPMC;
	PROC_UNLOCK(p);
	return (error);
	}

	/*
	* Attach a process and optionally its children
	*/

	static int
	pmc_attach_process(struct proc p, struct pmc pm)
	{
	int error;
	struct proc *top;

	sx_assert(&pmc_sx, SX_XLOCKED);

	PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
	PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);


	/*
	* If this PMC successfully allowed a GETMSR operation
	* in the past, disallow further ATTACHes.
	*/

	if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
	return EPERM;

	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
	return pmc_attach_one_process(p, pm);

	/*
	* Traverse all child processes, attaching them to
	* this PMC.
	*/

	sx_slock(&proctree_lock);

	top = p;

	for (;;) {
	if ((error = pmc_attach_one_process(p, pm)) != 0)
	break;
	if (!LIST_EMPTY(&p->p_children))
	p = LIST_FIRST(&p->p_children);
	else for (;;) {
	if (p == top)
	goto done;
	if (LIST_NEXT(p, p_sibling)) {
	p = LIST_NEXT(p, p_sibling);
	break;
	}
	p = p->p_pptr;
	}
	}

	if (error)
	(void) pmc_detach_process(top, pm);

	done:
	sx_sunlock(&proctree_lock);
	return error;
	}

	/*
	* Detach a process from a PMC. If there are no other PMCs tracking
	* this process, remove the process structure from its hash table. If
	* 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
	*/

	static int
	pmc_detach_one_process(struct proc p, struct pmc pm, int flags)
	{
	int ri;
	struct pmc_process *pp;

	sx_assert(&pmc_sx, SX_XLOCKED);

	KASSERT(pm != NULL,
	("[pmc,%d] null pm pointer", __LINE__));

	ri = PMC_TO_ROWINDEX(pm);

	PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
	pm, ri, p, p->p_pid, p->p_comm, flags);

	if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
	return ESRCH;

	if (pp->pp_pmcs[ri].pp_pmc != pm)
	return EINVAL;

	pmc_unlink_target_process(pm, pp);

	/* Issue a detach entry if a log file is configured */
	if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_pmcdetach(pm, p->p_pid);

	/*
	* If there are no PMCs targeting this process, we remove its
	* descriptor from the target hash table and unset the P_HWPMC
	* flag in the struct proc.
	*/
	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
	("[pmc,%d] Illegal refcnt %d for process struct %p",
	__LINE__, pp->pp_refcnt, pp));

	if (pp->pp_refcnt != 0) /* still a target of some PMC */
	return 0;

	pmc_remove_process_descriptor(pp);

	if (flags & PMC_FLAG_REMOVE)
	pmc_destroy_process_descriptor(pp);

	PROC_LOCK(p);
	p->p_flag &= ~P_HWPMC;
	PROC_UNLOCK(p);

	return 0;
	}

	/*
	* Detach a process and optionally its descendants from a PMC.
	*/

	static int
	pmc_detach_process(struct proc p, struct pmc pm)
	{
	struct proc *top;

	sx_assert(&pmc_sx, SX_XLOCKED);

	PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
	PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);

	if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
	return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);

	/*
	* Traverse all children, detaching them from this PMC. We
	* ignore errors since we could be detaching a PMC from a
	* partially attached proc tree.
	*/

	sx_slock(&proctree_lock);

	top = p;

	for (;;) {
	(void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);

	if (!LIST_EMPTY(&p->p_children))
	p = LIST_FIRST(&p->p_children);
	else for (;;) {
	if (p == top)
	goto done;
	if (LIST_NEXT(p, p_sibling)) {
	p = LIST_NEXT(p, p_sibling);
	break;
	}
	p = p->p_pptr;
	}
	}

	done:
	sx_sunlock(&proctree_lock);

	if (LIST_EMPTY(&pm->pm_targets))
	pm->pm_flags &= ~PMC_F_ATTACH_DONE;

	return 0;
	}


	/*
	* Thread context switch IN
	*/

	static void
	pmc_process_csw_in(struct thread *td)
	{
	int cpu;
	unsigned int adjri, ri;
	struct pmc *pm;
	struct proc *p;
	struct pmc_cpu *pc;
	struct pmc_hw *phw;
	pmc_value_t newvalue;
	struct pmc_process *pp;
	struct pmc_thread *pt;
	struct pmc_classdep *pcd;

	p = td->td_proc;
	pt = NULL;
	if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
	return;

	KASSERT(pp->pp_proc == td->td_proc,
	("[pmc,%d] not my thread state", __LINE__));

	critical_enter(); /* no preemption from this point */

	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */

	PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
	p->p_pid, p->p_comm, pp);

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	("[pmc,%d] weird CPU id %d", __LINE__, cpu));

	pc = pmc_pcpu[cpu];

	for (ri = 0; ri < md->pmd_npmc; ri++) {

	if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
	continue;

	KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
	("[pmc,%d] Target PMC in non-virtual mode (%d)",
	__LINE__, PMC_TO_MODE(pm)));

	KASSERT(PMC_TO_ROWINDEX(pm) == ri,
	("[pmc,%d] Row index mismatch pmc %d != ri %d",
	__LINE__, PMC_TO_ROWINDEX(pm), ri));

	/*
	* Only PMCs that are marked as 'RUNNING' need
	* be placed on hardware.
	*/

	if (pm->pm_state != PMC_STATE_RUNNING)
	continue;

	KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
	("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	(unsigned long)counter_u64_fetch(pm->pm_runcount)));

	/* increment PMC runcount */
	counter_u64_add(pm->pm_runcount, 1);

	/* configure the HWPMC we are going to use. */
	pcd = pmc_ri_to_classdep(md, ri, &adjri);
	pcd->pcd_config_pmc(cpu, adjri, pm);

	phw = pc->pc_hwpmcs[ri];

	KASSERT(phw != NULL,
	("[pmc,%d] null hw pointer", __LINE__));

	KASSERT(phw->phw_pmc == pm,
	("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
	phw->phw_pmc, pm));

	/*
	* Write out saved value and start the PMC.
	*
	* Sampling PMCs use a per-thread value, while
	* counting mode PMCs use a per-pmc value that is
	* inherited across descendants.
	*/
	if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
	if (pt == NULL)
	pt = pmc_find_thread_descriptor(pp, td,
	PMC_FLAG_NONE);

	KASSERT(pt != NULL,
	("[pmc,%d] No thread found for td=%p", __LINE__,
	td));

	mtx_pool_lock_spin(pmc_mtxpool, pm);

	/*
	* If we have a thread descriptor, use the per-thread
	* counter in the descriptor. If not, we will use
	* a per-process counter.
	*
	* TODO: Remove the per-process "safety net" once
	* we have thoroughly tested that we don't hit the
	* above assert.
	*/
	if (pt != NULL) {
	if (pt->pt_pmcs[ri].pt_pmcval > 0)
	newvalue = pt->pt_pmcs[ri].pt_pmcval;
	else
	newvalue = pm->pm_sc.pm_reloadcount;
	} else {
	/*
	* Use the saved value calculated after the most
	* recent time a thread using the shared counter
	* switched out. Reset the saved count in case
	* another thread from this process switches in
	* before any threads switch out.
	*/

	newvalue = pp->pp_pmcs[ri].pp_pmcval;
	pp->pp_pmcs[ri].pp_pmcval =
	pm->pm_sc.pm_reloadcount;
	}
	mtx_pool_unlock_spin(pmc_mtxpool, pm);
	KASSERT(newvalue > 0 && newvalue <=
	pm->pm_sc.pm_reloadcount,
	("[pmc,%d] pmcval outside of expected range cpu=%d "
	"ri=%d pmcval=%jx pm_reloadcount=%jx", __LINE__,
	cpu, ri, newvalue, pm->pm_sc.pm_reloadcount));
	} else {
	KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
	("[pmc,%d] illegal mode=%d", __LINE__,
	PMC_TO_MODE(pm)));
	mtx_pool_lock_spin(pmc_mtxpool, pm);
	newvalue = PMC_PCPU_SAVED(cpu, ri) =
	pm->pm_gv.pm_savedvalue;
	mtx_pool_unlock_spin(pmc_mtxpool, pm);
	}

	PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);

	pcd->pcd_write_pmc(cpu, adjri, newvalue);

	/* If a sampling mode PMC, reset stalled state. */
	if (PMC_TO_MODE(pm) == PMC_MODE_TS)
	pm->pm_pcpu_state[cpu].pps_stalled = 0;

	/* Indicate that we desire this to run. */
	pm->pm_pcpu_state[cpu].pps_cpustate = 1;

	/* Start the PMC. */
	pcd->pcd_start_pmc(cpu, adjri);
	}

	/*
	* perform any other architecture/cpu dependent thread
	* switch-in actions.
	*/

	(void) (*md->pmd_switch_in)(pc, pp);

	critical_exit();

	}

	/*
	* Thread context switch OUT.
	*/

	static void
	pmc_process_csw_out(struct thread *td)
	{
	int cpu;
	int64_t tmp;
	struct pmc *pm;
	struct proc *p;
	enum pmc_mode mode;
	struct pmc_cpu *pc;
	pmc_value_t newvalue;
	unsigned int adjri, ri;
	struct pmc_process *pp;
	struct pmc_thread *pt = NULL;
	struct pmc_classdep *pcd;


	/*
	* Locate our process descriptor; this may be NULL if
	* this process is exiting and we have already removed
	* the process from the target process table.
	*
	* Note that due to kernel preemption, multiple
	* context switches may happen while the process is
	* exiting.
	*
	* Note also that if the target process cannot be
	* found we still need to deconfigure any PMCs that
	* are currently running on hardware.
	*/

	p = td->td_proc;
	pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);

	/*
	* save PMCs
	*/

	critical_enter();

	cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */

	PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
	p->p_pid, p->p_comm, pp);

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	("[pmc,%d weird CPU id %d", __LINE__, cpu));

	pc = pmc_pcpu[cpu];

	/*
	* When a PMC gets unlinked from a target PMC, it will
	* be removed from the target's pp_pmc[] array.
	*
	* However, on a MP system, the target could have been
	* executing on another CPU at the time of the unlink.
	* So, at context switch OUT time, we need to look at
	* the hardware to determine if a PMC is scheduled on
	* it.
	*/

	for (ri = 0; ri < md->pmd_npmc; ri++) {

	pcd = pmc_ri_to_classdep(md, ri, &adjri);
	pm = NULL;
	(void) (*pcd->pcd_get_config)(cpu, adjri, &pm);

	if (pm == NULL) /* nothing at this row index */
	continue;

	mode = PMC_TO_MODE(pm);
	if (!PMC_IS_VIRTUAL_MODE(mode))
	continue; /* not a process virtual PMC */

	KASSERT(PMC_TO_ROWINDEX(pm) == ri,
	("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
	__LINE__, PMC_TO_ROWINDEX(pm), ri));

	/*
	* Change desired state, and then stop if not stalled.
	* This two-step dance should avoid race conditions where
	* an interrupt re-enables the PMC after this code has
	* already checked the pm_stalled flag.
	*/
	pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	if (pm->pm_pcpu_state[cpu].pps_stalled == 0)
	pcd->pcd_stop_pmc(cpu, adjri);

	KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	(unsigned long)counter_u64_fetch(pm->pm_runcount)));

	/* reduce this PMC's runcount */
	counter_u64_add(pm->pm_runcount, -1);

	/*
	* If this PMC is associated with this process,
	* save the reading.
	*/

	if (pm->pm_state != PMC_STATE_DELETED && pp != NULL &&
	pp->pp_pmcs[ri].pp_pmc != NULL) {
	KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
	("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
	pm, ri, pp->pp_pmcs[ri].pp_pmc));

	KASSERT(pp->pp_refcnt > 0,
	("[pmc,%d] pp refcnt = %d", __LINE__,
	pp->pp_refcnt));

	pcd->pcd_read_pmc(cpu, adjri, &newvalue);

	if (mode == PMC_MODE_TS) {
	PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d val=%jd (samp)",
	cpu, ri, newvalue);

	if (pt == NULL)
	pt = pmc_find_thread_descriptor(pp, td,
	PMC_FLAG_NONE);

	KASSERT(pt != NULL,
	("[pmc,%d] No thread found for td=%p",
	__LINE__, td));

	mtx_pool_lock_spin(pmc_mtxpool, pm);

	/*
	* If we have a thread descriptor, save the
	* per-thread counter in the descriptor. If not,
	* we will update the per-process counter.
	*
	* TODO: Remove the per-process "safety net"
	* once we have thoroughly tested that we
	* don't hit the above assert.
	*/
	if (pt != NULL)
	pt->pt_pmcs[ri].pt_pmcval = newvalue;
	else {
	/*
	* For sampling process-virtual PMCs,
	* newvalue is the number of events to
	* be seen until the next sampling
	* interrupt. We can just add the events
	* left from this invocation to the
	* counter, then adjust in case we
	* overflow our range.
	*
	* (Recall that we reload the counter
	* every time we use it.)
	*/
	pp->pp_pmcs[ri].pp_pmcval += newvalue;
	if (pp->pp_pmcs[ri].pp_pmcval >
	pm->pm_sc.pm_reloadcount)
	pp->pp_pmcs[ri].pp_pmcval -=
	pm->pm_sc.pm_reloadcount;
	}
	mtx_pool_unlock_spin(pmc_mtxpool, pm);
	} else {
	tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);

	PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)",
	cpu, ri, tmp);

	/*
	* For counting process-virtual PMCs,
	* we expect the count to be
	* increasing monotonically, modulo a 64
	* bit wraparound.
	*/
	KASSERT(tmp >= 0,
	("[pmc,%d] negative increment cpu=%d "
	"ri=%d newvalue=%jx saved=%jx "
	"incr=%jx", __LINE__, cpu, ri,
	newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));

	mtx_pool_lock_spin(pmc_mtxpool, pm);
	pm->pm_gv.pm_savedvalue += tmp;
	pp->pp_pmcs[ri].pp_pmcval += tmp;
	mtx_pool_unlock_spin(pmc_mtxpool, pm);

	if (pm->pm_flags & PMC_F_LOG_PROCCSW)
	pmclog_process_proccsw(pm, pp, tmp, td);
	}
	}

	/* mark hardware as free */
	pcd->pcd_config_pmc(cpu, adjri, NULL);
	}

	/*
	* perform any other architecture/cpu dependent thread
	* switch out functions.
	*/

	(void) (*md->pmd_switch_out)(pc, pp);

	critical_exit();
	}

	/*
	* A new thread for a process.
	*/
	static void
	pmc_process_thread_add(struct thread *td)
	{
	struct pmc_process *pmc;

	pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
	if (pmc != NULL)
	pmc_find_thread_descriptor(pmc, td, PMC_FLAG_ALLOCATE);
	}

	/*
	* A thread delete for a process.
	*/
	static void
	pmc_process_thread_delete(struct thread *td)
	{
	struct pmc_process *pmc;

	pmc = pmc_find_process_descriptor(td->td_proc, PMC_FLAG_NONE);
	if (pmc != NULL)
	pmc_thread_descriptor_pool_free(pmc_find_thread_descriptor(pmc,
	td, PMC_FLAG_REMOVE));
	}

	/*
	* A userret() call for a thread.
	*/
	static void
	pmc_process_thread_userret(struct thread *td)
	{
	sched_pin();
	pmc_capture_user_callchain(curcpu, PMC_UR, td->td_frame);
	sched_unpin();
	}

	/*
	* A mapping change for a process.
	*/

	static void
	pmc_process_mmap(struct thread td, struct pmckern_map_in pkm)
	{
	int ri;
	pid_t pid;
	char fullpath, freepath;
	const struct pmc *pm;
	struct pmc_owner *po;
	const struct pmc_process *pp;

	freepath = fullpath = NULL;
	MPASS(!in_epoch(global_epoch_preempt));
	pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath);

	pid = td->td_proc->p_pid;

	PMC_EPOCH_ENTER();
	/* Inform owners of all system-wide sampling PMCs. */
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_map_in(po, pid, pkm->pm_address, fullpath);

	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
	goto done;

	/*
	* Inform sampling PMC owners tracking this process.
	*/
	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
	PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
	pmclog_process_map_in(pm->pm_owner,
	pid, pkm->pm_address, fullpath);

	done:
	if (freepath)
	free(freepath, M_TEMP);
	PMC_EPOCH_EXIT();
	}


	/*
	* Log an munmap request.
	*/

	static void
	pmc_process_munmap(struct thread td, struct pmckern_map_out pkm)
	{
	int ri;
	pid_t pid;
	struct pmc_owner *po;
	const struct pmc *pm;
	const struct pmc_process *pp;

	pid = td->td_proc->p_pid;

	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_map_out(po, pid, pkm->pm_address,
	pkm->pm_address + pkm->pm_size);
	PMC_EPOCH_EXIT();

	if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL)
	return;

	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
	PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
	pmclog_process_map_out(pm->pm_owner, pid,
	pkm->pm_address, pkm->pm_address + pkm->pm_size);
	}

	/*
	* Log mapping information about the kernel.
	*/

	static void
	pmc_log_kernel_mappings(struct pmc *pm)
	{
	struct pmc_owner *po;
	struct pmckern_map_in km, kmbase;

	MPASS(in_epoch(global_epoch_preempt) \|\| sx_xlocked(&pmc_sx));
	KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
	("[pmc,%d] non-sampling PMC (%p) desires mapping information",
	__LINE__, (void *) pm));

	po = pm->pm_owner;

	if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
	return;
	if (PMC_TO_MODE(pm) == PMC_MODE_SS)
	pmc_process_allproc(pm);
	/*
	* Log the current set of kernel modules.
	*/
	kmbase = linker_hwpmc_list_objects();
	for (km = kmbase; km->pm_file != NULL; km++) {
	PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file,
	(void *) km->pm_address);
	pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
	km->pm_file);
	}
	free(kmbase, M_LINKER);

	po->po_flags \|= PMC_PO_INITIAL_MAPPINGS_DONE;
	}

	/*
	* Log the mappings for a single process.
	*/

	static void
	pmc_log_process_mappings(struct pmc_owner po, struct proc p)
	{
	vm_map_t map;
	struct vnode *vp;
	struct vmspace *vm;
	vm_map_entry_t entry;
	vm_offset_t last_end;
	u_int last_timestamp;
	struct vnode *last_vp;
	vm_offset_t start_addr;
	vm_object_t obj, lobj, tobj;
	char fullpath, freepath;

	last_vp = NULL;
	last_end = (vm_offset_t) 0;
	fullpath = freepath = NULL;

	if ((vm = vmspace_acquire_ref(p)) == NULL)
	return;

	map = &vm->vm_map;
	vm_map_lock_read(map);

	VM_MAP_ENTRY_FOREACH(entry, map) {

	if (entry == NULL) {
	PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly "
	"NULL! pid=%d vm_map=%p\n", p->p_pid, map);
	break;
	}

	/*
	* We only care about executable map entries.
	*/
	if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) \|\|
	!(entry->protection & VM_PROT_EXECUTE) \|\|
	(entry->object.vm_object == NULL)) {
	continue;
	}

	obj = entry->object.vm_object;
	VM_OBJECT_RLOCK(obj);

	/*
	* Walk the backing_object list to find the base
	* (non-shadowed) vm_object.
	*/
	for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
	if (tobj != obj)
	VM_OBJECT_RLOCK(tobj);
	if (lobj != obj)
	VM_OBJECT_RUNLOCK(lobj);
	lobj = tobj;
	}

	/*
	* At this point lobj is the base vm_object and it is locked.
	*/
	if (lobj == NULL) {
	PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d "
	"vm_map=%p vm_obj=%p\n", p->p_pid, map, obj);
	VM_OBJECT_RUNLOCK(obj);
	continue;
	}

	vp = vm_object_vnode(lobj);
	if (vp == NULL) {
	if (lobj != obj)
	VM_OBJECT_RUNLOCK(lobj);
	VM_OBJECT_RUNLOCK(obj);
	continue;
	}

	/*
	* Skip contiguous regions that point to the same
	* vnode, so we don't emit redundant MAP-IN
	* directives.
	*/
	if (entry->start == last_end && vp == last_vp) {
	last_end = entry->end;
	if (lobj != obj)
	VM_OBJECT_RUNLOCK(lobj);
	VM_OBJECT_RUNLOCK(obj);
	continue;
	}

	/*
	* We don't want to keep the proc's vm_map or this
	* vm_object locked while we walk the pathname, since
	* vn_fullpath() can sleep. However, if we drop the
	* lock, it's possible for concurrent activity to
	* modify the vm_map list. To protect against this,
	* we save the vm_map timestamp before we release the
	* lock, and check it after we reacquire the lock
	* below.
	*/
	start_addr = entry->start;
	last_end = entry->end;
	last_timestamp = map->timestamp;
	vm_map_unlock_read(map);

	vref(vp);
	if (lobj != obj)
	VM_OBJECT_RUNLOCK(lobj);

	VM_OBJECT_RUNLOCK(obj);

	freepath = NULL;
	pmc_getfilename(vp, &fullpath, &freepath);
	last_vp = vp;

	vrele(vp);

	vp = NULL;
	pmclog_process_map_in(po, p->p_pid, start_addr, fullpath);
	if (freepath)
	free(freepath, M_TEMP);

	vm_map_lock_read(map);

	/*
	* If our saved timestamp doesn't match, this means
	* that the vm_map was modified out from under us and
	* we can't trust our current "entry" pointer. Do a
	* new lookup for this entry. If there is no entry
	* for this address range, vm_map_lookup_entry() will
	* return the previous one, so we always want to go to
	* the next entry on the next loop iteration.
	*
	* There is an edge condition here that can occur if
	* there is no entry at or before this address. In
	* this situation, vm_map_lookup_entry returns
	* &map->header, which would cause our loop to abort
	* without processing the rest of the map. However,
	* in practice this will never happen for process
	* vm_map. This is because the executable's text
	* segment is the first mapping in the proc's address
	* space, and this mapping is never removed until the
	* process exits, so there will always be a non-header
	* entry at or before the requested address for
	* vm_map_lookup_entry to return.
	*/
	if (map->timestamp != last_timestamp)
	vm_map_lookup_entry(map, last_end - 1, &entry);
	}

	vm_map_unlock_read(map);
	vmspace_free(vm);
	return;
	}

	/*
	* Log mappings for all processes in the system.
	*/

	static void
	pmc_log_all_process_mappings(struct pmc_owner *po)
	{
	struct proc p, top;

	sx_assert(&pmc_sx, SX_XLOCKED);

	if ((p = pfind(1)) == NULL)
	panic("[pmc,%d] Cannot find init", __LINE__);

	PROC_UNLOCK(p);

	sx_slock(&proctree_lock);

	top = p;

	for (;;) {
	pmc_log_process_mappings(po, p);
	if (!LIST_EMPTY(&p->p_children))
	p = LIST_FIRST(&p->p_children);
	else for (;;) {
	if (p == top)
	goto done;
	if (LIST_NEXT(p, p_sibling)) {
	p = LIST_NEXT(p, p_sibling);
	break;
	}
	p = p->p_pptr;
	}
	}
	done:
	sx_sunlock(&proctree_lock);
	}

	/*
	* The 'hook' invoked from the kernel proper
	*/


	#ifdef HWPMC_DEBUG
	const char *pmc_hooknames[] = {
	/* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
	"",
	"EXEC",
	"CSW-IN",
	"CSW-OUT",
	"SAMPLE",
	"UNUSED1",
	"UNUSED2",
	"MMAP",
	"MUNMAP",
	"CALLCHAIN-NMI",
	"CALLCHAIN-SOFT",
	"SOFTSAMPLING",
	"THR-CREATE",
	"THR-EXIT",
	"THR-USERRET",
	"THR-CREATE-LOG",
	"THR-EXIT-LOG",
	"PROC-CREATE-LOG"
	};
	#endif

	static int
	pmc_hook_handler(struct thread td, int function, void arg)
	{
	int cpu;

	PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
	pmc_hooknames[function], arg);

	switch (function)
	{

	/*
	* Process exec()
	*/

	case PMC_FN_PROCESS_EXEC:
	{
	char fullpath, freepath;
	unsigned int ri;
	int is_using_hwpmcs;
	struct pmc *pm;
	struct proc *p;
	struct pmc_owner *po;
	struct pmc_process *pp;
	struct pmckern_procexec *pk;

	sx_assert(&pmc_sx, SX_XLOCKED);

	p = td->td_proc;
	pmc_getfilename(p->p_textvp, &fullpath, &freepath);

	pk = (struct pmckern_procexec *) arg;

	PMC_EPOCH_ENTER();
	/* Inform owners of SS mode PMCs of the exec event. */
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_procexec(po, PMC_ID_INVALID,
	p->p_pid, pk->pm_entryaddr, fullpath);
	PMC_EPOCH_EXIT();

	PROC_LOCK(p);
	is_using_hwpmcs = p->p_flag & P_HWPMC;
	PROC_UNLOCK(p);

	if (!is_using_hwpmcs) {
	if (freepath)
	free(freepath, M_TEMP);
	break;
	}

	/*
	* PMCs are not inherited across an exec(): remove any
	* PMCs that this process is the owner of.
	*/

	if ((po = pmc_find_owner_descriptor(p)) != NULL) {
	pmc_remove_owner(po);
	pmc_destroy_owner_descriptor(po);
	}

	/*
	* If the process being exec'ed is not the target of any
	* PMC, we are done.
	*/
	if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) {
	if (freepath)
	free(freepath, M_TEMP);
	break;
	}

	/*
	* Log the exec event to all monitoring owners. Skip
	* owners who have already received the event because
	* they had system sampling PMCs active.
	*/
	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
	po = pm->pm_owner;
	if (po->po_sscount == 0 &&
	po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_procexec(po, pm->pm_id,
	p->p_pid, pk->pm_entryaddr,
	fullpath);
	}

	if (freepath)
	free(freepath, M_TEMP);


	PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
	p, p->p_pid, p->p_comm, pk->pm_credentialschanged);

	if (pk->pm_credentialschanged == 0) /* no change */
	break;

	/*
	* If the newly exec()'ed process has a different credential
	* than before, allow it to be the target of a PMC only if
	* the PMC's owner has sufficient privilege.
	*/

	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
	if (pmc_can_attach(pm, td->td_proc) != 0)
	pmc_detach_one_process(td->td_proc,
	pm, PMC_FLAG_NONE);

	KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc,
	("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
	pp->pp_refcnt, pp));

	/*
	* If this process is no longer the target of any
	* PMCs, we can remove the process entry and free
	* up space.
	*/

	if (pp->pp_refcnt == 0) {
	pmc_remove_process_descriptor(pp);
	pmc_destroy_process_descriptor(pp);
	break;
	}

	}
	break;

	case PMC_FN_CSW_IN:
	pmc_process_csw_in(td);
	break;

	case PMC_FN_CSW_OUT:
	pmc_process_csw_out(td);
	break;

	/*
	* Process accumulated PC samples.
	*
	* This function is expected to be called by hardclock() for
	* each CPU that has accumulated PC samples.
	*
	* This function is to be executed on the CPU whose samples
	* are being processed.
	*/
	case PMC_FN_DO_SAMPLES:

	/*
	* Clear the cpu specific bit in the CPU mask before
	* do the rest of the processing. If the NMI handler
	* gets invoked after the "atomic_clear_int()" call
	* below but before "pmc_process_samples()" gets
	* around to processing the interrupt, then we will
	* come back here at the next hardclock() tick (and
	* may find nothing to do if "pmc_process_samples()"
	* had already processed the interrupt). We don't
	* lose the interrupt sample.
	*/
	DPCPU_SET(pmc_sampled, 0);
	cpu = PCPU_GET(cpuid);
	pmc_process_samples(cpu, PMC_HR);
	pmc_process_samples(cpu, PMC_SR);
	pmc_process_samples(cpu, PMC_UR);
	break;

	case PMC_FN_MMAP:
	pmc_process_mmap(td, (struct pmckern_map_in *) arg);
	break;

	case PMC_FN_MUNMAP:
	MPASS(in_epoch(global_epoch_preempt) \|\| sx_xlocked(&pmc_sx));
	pmc_process_munmap(td, (struct pmckern_map_out *) arg);
	break;

	case PMC_FN_PROC_CREATE_LOG:
	pmc_process_proccreate((struct proc *)arg);
	break;

	case PMC_FN_USER_CALLCHAIN:
	/*
	* Record a call chain.
	*/
	KASSERT(td == curthread, ("[pmc,%d] td != curthread",
	__LINE__));

	pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
	(struct trapframe *) arg);

	KASSERT(td->td_pinned == 1,
	("[pmc,%d] invalid td_pinned value", __LINE__));
	sched_unpin(); /* Can migrate safely now. */

	td->td_pflags &= ~TDP_CALLCHAIN;
	break;

	case PMC_FN_USER_CALLCHAIN_SOFT:
	/*
	* Record a call chain.
	*/
	KASSERT(td == curthread, ("[pmc,%d] td != curthread",
	__LINE__));

	cpu = PCPU_GET(cpuid);
	pmc_capture_user_callchain(cpu, PMC_SR,
	(struct trapframe *) arg);

	KASSERT(td->td_pinned == 1,
	("[pmc,%d] invalid td_pinned value", __LINE__));

	sched_unpin(); /* Can migrate safely now. */

	td->td_pflags &= ~TDP_CALLCHAIN;
	break;

	case PMC_FN_SOFT_SAMPLING:
	/*
	* Call soft PMC sampling intr.
	*/
	pmc_soft_intr((struct pmckern_soft *) arg);
	break;

	case PMC_FN_THR_CREATE:
	pmc_process_thread_add(td);
	pmc_process_threadcreate(td);
	break;

	case PMC_FN_THR_CREATE_LOG:
	pmc_process_threadcreate(td);
	break;

	case PMC_FN_THR_EXIT:
	KASSERT(td == curthread, ("[pmc,%d] td != curthread",
	__LINE__));
	pmc_process_thread_delete(td);
	pmc_process_threadexit(td);
	break;
	case PMC_FN_THR_EXIT_LOG:
	pmc_process_threadexit(td);
	break;
	case PMC_FN_THR_USERRET:
	KASSERT(td == curthread, ("[pmc,%d] td != curthread",
	__LINE__));
	pmc_process_thread_userret(td);
	break;

	default:
	#ifdef HWPMC_DEBUG
	KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
	#endif
	break;

	}

	return 0;
	}

	/*
	* allocate a 'struct pmc_owner' descriptor in the owner hash table.
	*/

	static struct pmc_owner *
	pmc_allocate_owner_descriptor(struct proc *p)
	{
	uint32_t hindex;
	struct pmc_owner *po;
	struct pmc_ownerhash *poh;

	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
	poh = &pmc_ownerhash[hindex];

	/* allocate space for N pointers and one descriptor struct */
	po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK\|M_ZERO);
	po->po_owner = p;
	LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */

	TAILQ_INIT(&po->po_logbuffers);
	mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN);

	PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
	p, p->p_pid, p->p_comm, po);

	return po;
	}

	static void
	pmc_destroy_owner_descriptor(struct pmc_owner *po)
	{

	PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
	po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);

	mtx_destroy(&po->po_mtx);
	free(po, M_PMC);
	}

	/*
	* Allocate a thread descriptor from the free pool.
	*
	* NOTE: This can return NULL.
	*/
	static struct pmc_thread *
	pmc_thread_descriptor_pool_alloc(void)
	{
	struct pmc_thread *pt;

	mtx_lock_spin(&pmc_threadfreelist_mtx);
	if ((pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
	LIST_REMOVE(pt, pt_next);
	pmc_threadfreelist_entries--;
	}
	mtx_unlock_spin(&pmc_threadfreelist_mtx);

	return (pt);
	}

	/*
	* Add a thread descriptor to the free pool. We use this instead of free()
	* to maintain a cache of free entries. Additionally, we can safely call
	* this function when we cannot call free(), such as in a critical section.
	*
	*/
	static void
	pmc_thread_descriptor_pool_free(struct pmc_thread *pt)
	{

	if (pt == NULL)
	return;

	memset(pt, 0, THREADENTRY_SIZE);
	mtx_lock_spin(&pmc_threadfreelist_mtx);
	LIST_INSERT_HEAD(&pmc_threadfreelist, pt, pt_next);
	pmc_threadfreelist_entries++;
	if (pmc_threadfreelist_entries > pmc_threadfreelist_max)
	taskqueue_enqueue(taskqueue_fast, &free_task);
	mtx_unlock_spin(&pmc_threadfreelist_mtx);
	}

	/*
	* An asynchronous task to manage the free list.
	*/
	static void
	pmc_thread_descriptor_pool_free_task(void *arg __unused, int pending __unused)
	{
	struct pmc_thread *pt;
	LIST_HEAD(, pmc_thread) tmplist;
	int delta;

	LIST_INIT(&tmplist);

	/* Determine what changes, if any, we need to make. */
	mtx_lock_spin(&pmc_threadfreelist_mtx);
	delta = pmc_threadfreelist_entries - pmc_threadfreelist_max;
	while (delta > 0 && (pt = LIST_FIRST(&pmc_threadfreelist)) != NULL) {
	delta--;
	pmc_threadfreelist_entries--;
	LIST_REMOVE(pt, pt_next);
	LIST_INSERT_HEAD(&tmplist, pt, pt_next);
	}
	mtx_unlock_spin(&pmc_threadfreelist_mtx);

	/* If there are entries to free, free them. */
	while (!LIST_EMPTY(&tmplist)) {
	pt = LIST_FIRST(&tmplist);
	LIST_REMOVE(pt, pt_next);
	free(pt, M_PMC);
	}
	}

	/*
	* Drain the thread free pool, freeing all allocations.
	*/
	static void
	pmc_thread_descriptor_pool_drain()
	{
	struct pmc_thread pt, next;

	LIST_FOREACH_SAFE(pt, &pmc_threadfreelist, pt_next, next) {
	LIST_REMOVE(pt, pt_next);
	free(pt, M_PMC);
	}
	}

	/*
	* find the descriptor corresponding to thread 'td', adding or removing it
	* as specified by 'mode'.
	*
	* Note that this supports additional mode flags in addition to those
	* supported by pmc_find_process_descriptor():
	* PMC_FLAG_NOWAIT: Causes the function to not wait for mallocs.
	* This makes it safe to call while holding certain other locks.
	*/

	static struct pmc_thread *
	pmc_find_thread_descriptor(struct pmc_process pp, struct thread td,
	uint32_t mode)
	{
	struct pmc_thread pt = NULL, ptnew = NULL;
	int wait_flag;

	KASSERT(td != NULL, ("[pmc,%d] called to add NULL td", __LINE__));

	/*
	* Pre-allocate memory in the PMC_FLAG_ALLOCATE case prior to
	* acquiring the lock.
	*/
	if (mode & PMC_FLAG_ALLOCATE) {
	if ((ptnew = pmc_thread_descriptor_pool_alloc()) == NULL) {
	wait_flag = M_WAITOK;
	if ((mode & PMC_FLAG_NOWAIT) \|\| in_epoch(global_epoch_preempt))
	wait_flag = M_NOWAIT;

	ptnew = malloc(THREADENTRY_SIZE, M_PMC,
	wait_flag\|M_ZERO);
	}
	}

	mtx_lock_spin(pp->pp_tdslock);

	LIST_FOREACH(pt, &pp->pp_tds, pt_next)
	if (pt->pt_td == td)
	break;

	if ((mode & PMC_FLAG_REMOVE) && pt != NULL)
	LIST_REMOVE(pt, pt_next);

	if ((mode & PMC_FLAG_ALLOCATE) && pt == NULL && ptnew != NULL) {
	pt = ptnew;
	ptnew = NULL;
	pt->pt_td = td;
	LIST_INSERT_HEAD(&pp->pp_tds, pt, pt_next);
	}

	mtx_unlock_spin(pp->pp_tdslock);

	if (ptnew != NULL) {
	free(ptnew, M_PMC);
	}

	return pt;
	}

	/*
	* Try to add thread descriptors for each thread in a process.
	*/

	static void
	pmc_add_thread_descriptors_from_proc(struct proc p, struct pmc_process pp)
	{
	struct thread *curtd;
	struct pmc_thread **tdlist;
	int i, tdcnt, tdlistsz;

	KASSERT(!PROC_LOCKED(p), ("[pmc,%d] proc unexpectedly locked",
	__LINE__));
	tdcnt = 32;
	restart:
	tdlistsz = roundup2(tdcnt, 32);

	tdcnt = 0;
	tdlist = malloc(sizeof(struct pmc_thread) tdlistsz, M_TEMP, M_WAITOK);

	PROC_LOCK(p);
	FOREACH_THREAD_IN_PROC(p, curtd)
	tdcnt++;
	if (tdcnt >= tdlistsz) {
	PROC_UNLOCK(p);
	free(tdlist, M_TEMP);
	goto restart;
	}
	/*
	* Try to add each thread to the list without sleeping. If unable,
	* add to a queue to retry after dropping the process lock.
	*/
	tdcnt = 0;
	FOREACH_THREAD_IN_PROC(p, curtd) {
	tdlist[tdcnt] = pmc_find_thread_descriptor(pp, curtd,
	PMC_FLAG_ALLOCATE\|PMC_FLAG_NOWAIT);
	if (tdlist[tdcnt] == NULL) {
	PROC_UNLOCK(p);
	for (i = 0; i <= tdcnt; i++)
	pmc_thread_descriptor_pool_free(tdlist[i]);
	free(tdlist, M_TEMP);
	goto restart;
	}
	tdcnt++;
	}
	PROC_UNLOCK(p);
	free(tdlist, M_TEMP);
	}

	/*
	* find the descriptor corresponding to process 'p', adding or removing it
	* as specified by 'mode'.
	*/

	static struct pmc_process *
	pmc_find_process_descriptor(struct proc *p, uint32_t mode)
	{
	uint32_t hindex;
	struct pmc_process pp, ppnew;
	struct pmc_processhash *pph;

	hindex = PMC_HASH_PTR(p, pmc_processhashmask);
	pph = &pmc_processhash[hindex];

	ppnew = NULL;

	/*
	* Pre-allocate memory in the PMC_FLAG_ALLOCATE case since we
	* cannot call malloc(9) once we hold a spin lock.
	*/
	if (mode & PMC_FLAG_ALLOCATE)
	ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc *
	sizeof(struct pmc_targetstate), M_PMC, M_WAITOK\|M_ZERO);

	mtx_lock_spin(&pmc_processhash_mtx);
	LIST_FOREACH(pp, pph, pp_next)
	if (pp->pp_proc == p)
	break;

	if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
	LIST_REMOVE(pp, pp_next);

	if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
	ppnew != NULL) {
	ppnew->pp_proc = p;
	LIST_INIT(&ppnew->pp_tds);
	ppnew->pp_tdslock = mtx_pool_find(pmc_mtxpool, ppnew);
	LIST_INSERT_HEAD(pph, ppnew, pp_next);
	mtx_unlock_spin(&pmc_processhash_mtx);
	pp = ppnew;
	ppnew = NULL;

	/* Add thread descriptors for this process' current threads. */
	pmc_add_thread_descriptors_from_proc(p, pp);
	}
	else
	mtx_unlock_spin(&pmc_processhash_mtx);

	if (ppnew != NULL)
	free(ppnew, M_PMC);

	return pp;
	}

	/*
	* remove a process descriptor from the process hash table.
	*/

	static void
	pmc_remove_process_descriptor(struct pmc_process *pp)
	{
	KASSERT(pp->pp_refcnt == 0,
	("[pmc,%d] Removing process descriptor %p with count %d",
	__LINE__, pp, pp->pp_refcnt));

	mtx_lock_spin(&pmc_processhash_mtx);
	LIST_REMOVE(pp, pp_next);
	mtx_unlock_spin(&pmc_processhash_mtx);
	}

	/*
	* destroy a process descriptor.
	*/

	static void
	pmc_destroy_process_descriptor(struct pmc_process *pp)
	{
	struct pmc_thread *pmc_td;

	while ((pmc_td = LIST_FIRST(&pp->pp_tds)) != NULL) {
	LIST_REMOVE(pmc_td, pt_next);
	pmc_thread_descriptor_pool_free(pmc_td);
	}
	free(pp, M_PMC);
	}


	/*
	* find an owner descriptor corresponding to proc 'p'
	*/

	static struct pmc_owner *
	pmc_find_owner_descriptor(struct proc *p)
	{
	uint32_t hindex;
	struct pmc_owner *po;
	struct pmc_ownerhash *poh;

	hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
	poh = &pmc_ownerhash[hindex];

	po = NULL;
	LIST_FOREACH(po, poh, po_next)
	if (po->po_owner == p)
	break;

	PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
	"pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);

	return po;
	}

	/*
	* pmc_allocate_pmc_descriptor
	*
	* Allocate a pmc descriptor and initialize its
	* fields.
	*/

	static struct pmc *
	pmc_allocate_pmc_descriptor(void)
	{
	struct pmc *pmc;

	pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK\|M_ZERO);
	pmc->pm_runcount = counter_u64_alloc(M_WAITOK);
	pmc->pm_pcpu_state = malloc(sizeof(struct pmc_pcpu_state)*mp_ncpus, M_PMC, M_WAITOK\|M_ZERO);
	PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);

	return pmc;
	}

	/*
	* Destroy a pmc descriptor.
	*/

	static void
	pmc_destroy_pmc_descriptor(struct pmc *pm)
	{

	KASSERT(pm->pm_state == PMC_STATE_DELETED \|\|
	pm->pm_state == PMC_STATE_FREE,
	("[pmc,%d] destroying non-deleted PMC", __LINE__));
	KASSERT(LIST_EMPTY(&pm->pm_targets),
	("[pmc,%d] destroying pmc with targets", __LINE__));
	KASSERT(pm->pm_owner == NULL,
	("[pmc,%d] destroying pmc attached to an owner", __LINE__));
	KASSERT(counter_u64_fetch(pm->pm_runcount) == 0,
	("[pmc,%d] pmc has non-zero run count %ld", __LINE__,
	(unsigned long)counter_u64_fetch(pm->pm_runcount)));

	counter_u64_free(pm->pm_runcount);
	free(pm->pm_pcpu_state, M_PMC);
	free(pm, M_PMC);
	}

	static void
	pmc_wait_for_pmc_idle(struct pmc *pm)
	{
	#ifdef INVARIANTS
	volatile int maxloop;

	maxloop = 100 * pmc_cpu_max();
	#endif
	/*
	* Loop (with a forced context switch) till the PMC's runcount
	* comes down to zero.
	*/
	pmclog_flush(pm->pm_owner, 1);
	while (counter_u64_fetch(pm->pm_runcount) > 0) {
	pmclog_flush(pm->pm_owner, 1);
	#ifdef INVARIANTS
	maxloop--;
	KASSERT(maxloop > 0,
	("[pmc,%d] (ri%d, rc%ld) waiting too long for "
	"pmc to be free", __LINE__,
	PMC_TO_ROWINDEX(pm), (unsigned long)counter_u64_fetch(pm->pm_runcount)));
	#endif
	pmc_force_context_switch();
	}
	}

	/*
	* This function does the following things:
	*
	* - detaches the PMC from hardware
	* - unlinks all target threads that were attached to it
	* - removes the PMC from its owner's list
	* - destroys the PMC private mutex
	*
	* Once this function completes, the given pmc pointer can be freed by
	* calling pmc_destroy_pmc_descriptor().
	*/

	static void
	pmc_release_pmc_descriptor(struct pmc *pm)
	{
	enum pmc_mode mode;
	struct pmc_hw *phw;
	u_int adjri, ri, cpu;
	struct pmc_owner *po;
	struct pmc_binding pb;
	struct pmc_process *pp;
	struct pmc_classdep *pcd;
	struct pmc_target ptgt, tmp;

	sx_assert(&pmc_sx, SX_XLOCKED);

	KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));

	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);
	mode = PMC_TO_MODE(pm);

	PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
	mode);

	/*
	* First, we take the PMC off hardware.
	*/
	cpu = 0;
	if (PMC_IS_SYSTEM_MODE(mode)) {

	/*
	* A system mode PMC runs on a specific CPU. Switch
	* to this CPU and turn hardware off.
	*/
	pmc_save_cpu_binding(&pb);

	cpu = PMC_TO_CPU(pm);

	pmc_select_cpu(cpu);

	/* switch off non-stalled CPUs */
	pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	if (pm->pm_state == PMC_STATE_RUNNING &&
	pm->pm_pcpu_state[cpu].pps_stalled == 0) {

	phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];

	KASSERT(phw->phw_pmc == pm,
	("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
	__LINE__, ri, phw->phw_pmc, pm));
	PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);

	critical_enter();
	pcd->pcd_stop_pmc(cpu, adjri);
	critical_exit();
	}

	PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);

	critical_enter();
	pcd->pcd_config_pmc(cpu, adjri, NULL);
	critical_exit();

	/* adjust the global and process count of SS mode PMCs */
	if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
	po = pm->pm_owner;
	po->po_sscount--;
	if (po->po_sscount == 0) {
	atomic_subtract_rel_int(&pmc_ss_count, 1);
	CK_LIST_REMOVE(po, po_ssnext);
	epoch_wait_preempt(global_epoch_preempt);
	}
	}

	pm->pm_state = PMC_STATE_DELETED;

	pmc_restore_cpu_binding(&pb);

	/*
	* We could have references to this PMC structure in
	* the per-cpu sample queues. Wait for the queue to
	* drain.
	*/
	pmc_wait_for_pmc_idle(pm);

	} else if (PMC_IS_VIRTUAL_MODE(mode)) {

	/*
	* A virtual PMC could be running on multiple CPUs at
	* a given instant.
	*
	* By marking its state as DELETED, we ensure that
	* this PMC is never further scheduled on hardware.
	*
	* Then we wait till all CPUs are done with this PMC.
	*/
	pm->pm_state = PMC_STATE_DELETED;


	/* Wait for the PMCs runcount to come to zero. */
	pmc_wait_for_pmc_idle(pm);

	/*
	* At this point the PMC is off all CPUs and cannot be
	* freshly scheduled onto a CPU. It is now safe to
	* unlink all targets from this PMC. If a
	* process-record's refcount falls to zero, we remove
	* it from the hash table. The module-wide SX lock
	* protects us from races.
	*/
	LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
	pp = ptgt->pt_process;
	pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */

	PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);

	/*
	* If the target process record shows that no
	* PMCs are attached to it, reclaim its space.
	*/

	if (pp->pp_refcnt == 0) {
	pmc_remove_process_descriptor(pp);
	pmc_destroy_process_descriptor(pp);
	}
	}

	cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */

	}

	/*
	* Release any MD resources
	*/
	(void) pcd->pcd_release_pmc(cpu, adjri, pm);

	/*
	* Update row disposition
	*/

	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
	PMC_UNMARK_ROW_STANDALONE(ri);
	else
	PMC_UNMARK_ROW_THREAD(ri);

	/* unlink from the owner's list */
	if (pm->pm_owner) {
	LIST_REMOVE(pm, pm_next);
	pm->pm_owner = NULL;
	}
	}

	/*
	* Register an owner and a pmc.
	*/

	static int
	pmc_register_owner(struct proc p, struct pmc pmc)
	{
	struct pmc_owner *po;

	sx_assert(&pmc_sx, SX_XLOCKED);

	if ((po = pmc_find_owner_descriptor(p)) == NULL)
	if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
	return ENOMEM;

	KASSERT(pmc->pm_owner == NULL,
	("[pmc,%d] attempting to own an initialized PMC", __LINE__));
	pmc->pm_owner = po;

	LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);

	PROC_LOCK(p);
	p->p_flag \|= P_HWPMC;
	PROC_UNLOCK(p);

	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_pmcallocate(pmc);

	PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
	po, pmc);

	return 0;
	}

	/*
	* Return the current row disposition:
	* == 0 => FREE
	* > 0 => PROCESS MODE
	* < 0 => SYSTEM MODE
	*/

	int
	pmc_getrowdisp(int ri)
	{
	return pmc_pmcdisp[ri];
	}

	/*
	* Check if a PMC at row index 'ri' can be allocated to the current
	* process.
	*
	* Allocation can fail if:
	* - the current process is already being profiled by a PMC at index 'ri',
	* attached to it via OP_PMCATTACH.
	* - the current process has already allocated a PMC at index 'ri'
	* via OP_ALLOCATE.
	*/

	static int
	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
	{
	enum pmc_mode mode;
	struct pmc *pm;
	struct pmc_owner *po;
	struct pmc_process *pp;

	PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
	"cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);

	/*
	* We shouldn't have already allocated a process-mode PMC at
	* row index 'ri'.
	*
	* We shouldn't have allocated a system-wide PMC on the same
	* CPU and same RI.
	*/
	if ((po = pmc_find_owner_descriptor(p)) != NULL)
	LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
	if (PMC_TO_ROWINDEX(pm) == ri) {
	mode = PMC_TO_MODE(pm);
	if (PMC_IS_VIRTUAL_MODE(mode))
	return EEXIST;
	if (PMC_IS_SYSTEM_MODE(mode) &&
	(int) PMC_TO_CPU(pm) == cpu)
	return EEXIST;
	}
	}

	/*
	* We also shouldn't be the target of any PMC at this index
	* since otherwise a PMC_ATTACH to ourselves will fail.
	*/
	if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
	if (pp->pp_pmcs[ri].pp_pmc)
	return EEXIST;

	PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
	p, p->p_pid, p->p_comm, ri);

	return 0;
	}

	/*
	* Check if a given PMC at row index 'ri' can be currently used in
	* mode 'mode'.
	*/

	static int
	pmc_can_allocate_row(int ri, enum pmc_mode mode)
	{
	enum pmc_disp disp;

	sx_assert(&pmc_sx, SX_XLOCKED);

	PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);

	if (PMC_IS_SYSTEM_MODE(mode))
	disp = PMC_DISP_STANDALONE;
	else
	disp = PMC_DISP_THREAD;

	/*
	* check disposition for PMC row 'ri':
	*
	* Expected disposition Row-disposition Result
	*
	* STANDALONE STANDALONE or FREE proceed
	* STANDALONE THREAD fail
	* THREAD THREAD or FREE proceed
	* THREAD STANDALONE fail
	*/

	if (!PMC_ROW_DISP_IS_FREE(ri) &&
	!(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
	!(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
	return EBUSY;

	/*
	* All OK
	*/

	PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);

	return 0;

	}

	/*
	* Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
	*/

	static struct pmc *
	pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
	{
	struct pmc *pm;

	KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
	("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
	PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));

	LIST_FOREACH(pm, &po->po_pmcs, pm_next)
	if (pm->pm_id == pmcid)
	return pm;

	return NULL;
	}

	static int
	pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
	{

	struct pmc pm, opm;
	struct pmc_owner *po;
	struct pmc_process *pp;

	PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid);
	if (PMC_ID_TO_ROWINDEX(pmcid) >= md->pmd_npmc)
	return (EINVAL);

	if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) {
	/*
	* In case of PMC_F_DESCENDANTS child processes we will not find
	* the current process in the owners hash list. Find the owner
	* process first and from there lookup the po.
	*/
	if ((pp = pmc_find_process_descriptor(curthread->td_proc,
	PMC_FLAG_NONE)) == NULL) {
	return ESRCH;
	} else {
	opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc;
	if (opm == NULL)
	return ESRCH;
	if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER\|
	PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER\|
	PMC_F_DESCENDANTS))
	return ESRCH;
	po = opm->pm_owner;
	}
	}

	if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
	return EINVAL;

	PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);

	*pmc = pm;
	return 0;
	}

	/*
	* Start a PMC.
	*/

	static int
	pmc_start(struct pmc *pm)
	{
	enum pmc_mode mode;
	struct pmc_owner *po;
	struct pmc_binding pb;
	struct pmc_classdep *pcd;
	int adjri, error, cpu, ri;

	KASSERT(pm != NULL,
	("[pmc,%d] null pm", __LINE__));

	mode = PMC_TO_MODE(pm);
	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	error = 0;

	PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);

	po = pm->pm_owner;

	/*
	* Disallow PMCSTART if a logfile is required but has not been
	* configured yet.
	*/
	if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
	(po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
	return (EDOOFUS); /* programming error */

	/*
	* If this is a sampling mode PMC, log mapping information for
	* the kernel modules that are currently loaded.
	*/
	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
	pmc_log_kernel_mappings(pm);

	if (PMC_IS_VIRTUAL_MODE(mode)) {

	/*
	* If a PMCATTACH has never been done on this PMC,
	* attach it to its owner process.
	*/

	if (LIST_EMPTY(&pm->pm_targets))
	error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
	pmc_attach_process(po->po_owner, pm);

	/*
	* If the PMC is attached to its owner, then force a context
	* switch to ensure that the MD state gets set correctly.
	*/

	if (error == 0) {
	pm->pm_state = PMC_STATE_RUNNING;
	if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
	pmc_force_context_switch();
	}

	return (error);
	}


	/*
	* A system-wide PMC.
	*
	* Add the owner to the global list if this is a system-wide
	* sampling PMC.
	*/

	if (mode == PMC_MODE_SS) {
	/*
	* Log mapping information for all existing processes in the
	* system. Subsequent mappings are logged as they happen;
	* see pmc_process_mmap().
	*/
	if (po->po_logprocmaps == 0) {
	pmc_log_all_process_mappings(po);
	po->po_logprocmaps = 1;
	}
	po->po_sscount++;
	if (po->po_sscount == 1) {
	atomic_add_rel_int(&pmc_ss_count, 1);
	CK_LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
	PMCDBG1(PMC,OPS,1, "po=%p in global list", po);
	}
	}

	/*
	* Move to the CPU associated with this
	* PMC, and start the hardware.
	*/

	pmc_save_cpu_binding(&pb);

	cpu = PMC_TO_CPU(pm);

	if (!pmc_cpu_is_active(cpu))
	return (ENXIO);

	pmc_select_cpu(cpu);

	/*
	* global PMCs are configured at allocation time
	* so write out the initial value and start the PMC.
	*/

	pm->pm_state = PMC_STATE_RUNNING;

	critical_enter();
	if ((error = pcd->pcd_write_pmc(cpu, adjri,
	PMC_IS_SAMPLING_MODE(mode) ?
	pm->pm_sc.pm_reloadcount :
	pm->pm_sc.pm_initial)) == 0) {
	/* If a sampling mode PMC, reset stalled state. */
	if (PMC_IS_SAMPLING_MODE(mode))
	pm->pm_pcpu_state[cpu].pps_stalled = 0;

	/* Indicate that we desire this to run. Start it. */
	pm->pm_pcpu_state[cpu].pps_cpustate = 1;
	error = pcd->pcd_start_pmc(cpu, adjri);
	}
	critical_exit();

	pmc_restore_cpu_binding(&pb);

	return (error);
	}

	/*
	* Stop a PMC.
	*/

	static int
	pmc_stop(struct pmc *pm)
	{
	struct pmc_owner *po;
	struct pmc_binding pb;
	struct pmc_classdep *pcd;
	int adjri, cpu, error, ri;

	KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));

	PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
	PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));

	pm->pm_state = PMC_STATE_STOPPED;

	/*
	* If the PMC is a virtual mode one, changing the state to
	* non-RUNNING is enough to ensure that the PMC never gets
	* scheduled.
	*
	* If this PMC is current running on a CPU, then it will
	* handled correctly at the time its target process is context
	* switched out.
	*/

	if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
	return 0;

	/*
	* A system-mode PMC. Move to the CPU associated with
	* this PMC, and stop the hardware. We update the
	* 'initial count' so that a subsequent PMCSTART will
	* resume counting from the current hardware count.
	*/

	pmc_save_cpu_binding(&pb);

	cpu = PMC_TO_CPU(pm);

	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
	("[pmc,%d] illegal cpu=%d", __LINE__, cpu));

	if (!pmc_cpu_is_active(cpu))
	return ENXIO;

	pmc_select_cpu(cpu);

	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	critical_enter();
	if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0)
	error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial);
	critical_exit();

	pmc_restore_cpu_binding(&pb);

	po = pm->pm_owner;

	/* remove this owner from the global list of SS PMC owners */
	if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
	po->po_sscount--;
	if (po->po_sscount == 0) {
	atomic_subtract_rel_int(&pmc_ss_count, 1);
	CK_LIST_REMOVE(po, po_ssnext);
	epoch_wait_preempt(global_epoch_preempt);
	PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po);
	}
	}

	return (error);
	}

	static struct pmc_classdep *
	pmc_class_to_classdep(enum pmc_class class)
	{
	int n;

	for (n = 0; n < md->pmd_nclass; n++)
	if (md->pmd_classdep[n].pcd_class == class)
	return (&md->pmd_classdep[n]);
	return (NULL);
	}

	#if defined(HWPMC_DEBUG) && defined(KTR)
	static const char *pmc_op_to_name[] = {
	#undef __PMC_OP
	#define __PMC_OP(N, D) #N ,
	__PMC_OPS()
	NULL
	};
	#endif

	/*
	* The syscall interface
	*/

	#define PMC_GET_SX_XLOCK(...) do { \
	sx_xlock(&pmc_sx); \
	if (pmc_hook == NULL) { \
	sx_xunlock(&pmc_sx); \
	return __VA_ARGS__; \
	} \
	} while (0)

	#define PMC_DOWNGRADE_SX() do { \
	sx_downgrade(&pmc_sx); \
	is_sx_downgraded = 1; \
	} while (0)

	static int
	pmc_syscall_handler(struct thread td, void syscall_args)
	{
	int error, is_sx_downgraded, op;
	struct pmc_syscall_args *c;
	void *pmclog_proc_handle;
	void *arg;

	c = (struct pmc_syscall_args *)syscall_args;
	op = c->pmop_code;
	arg = c->pmop_data;
	/* PMC isn't set up yet */
	if (pmc_hook == NULL)
	return (EINVAL);
	if (op == PMC_OP_CONFIGURELOG) {
	/*
	* We cannot create the logging process inside
	* pmclog_configure_log() because there is a LOR
	* between pmc_sx and process structure locks.
	* Instead, pre-create the process and ignite the loop
	* if everything is fine, otherwise direct the process
	* to exit.
	*/
	error = pmclog_proc_create(td, &pmclog_proc_handle);
	if (error != 0)
	goto done_syscall;
	}

	PMC_GET_SX_XLOCK(ENOSYS);
	is_sx_downgraded = 0;
	PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
	pmc_op_to_name[op], arg);

	error = 0;
	counter_u64_add(pmc_stats.pm_syscalls, 1);

	switch (op) {


	/*
	* Configure a log file.
	*
	* XXX This OP will be reworked.
	*/

	case PMC_OP_CONFIGURELOG:
	{
	struct proc *p;
	struct pmc *pm;
	struct pmc_owner *po;
	struct pmc_op_configurelog cl;

	if ((error = copyin(arg, &cl, sizeof(cl))) != 0) {
	pmclog_proc_ignite(pmclog_proc_handle, NULL);
	break;
	}

	/* mark this process as owning a log file */
	p = td->td_proc;
	if ((po = pmc_find_owner_descriptor(p)) == NULL)
	if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
	pmclog_proc_ignite(pmclog_proc_handle, NULL);
	error = ENOMEM;
	break;
	}

	/*
	* If a valid fd was passed in, try to configure that,
	* otherwise if 'fd' was less than zero and there was
	* a log file configured, flush its buffers and
	* de-configure it.
	*/
	if (cl.pm_logfd >= 0) {
	error = pmclog_configure_log(md, po, cl.pm_logfd);
	pmclog_proc_ignite(pmclog_proc_handle, error == 0 ?
	po : NULL);
	} else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
	pmclog_proc_ignite(pmclog_proc_handle, NULL);
	error = pmclog_close(po);
	if (error == 0) {
	LIST_FOREACH(pm, &po->po_pmcs, pm_next)
	if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
	pm->pm_state == PMC_STATE_RUNNING)
	pmc_stop(pm);
	error = pmclog_deconfigure_log(po);
	}
	} else {
	pmclog_proc_ignite(pmclog_proc_handle, NULL);
	error = EINVAL;
	}
	}
	break;

	/*
	* Flush a log file.
	*/

	case PMC_OP_FLUSHLOG:
	{
	struct pmc_owner *po;

	sx_assert(&pmc_sx, SX_XLOCKED);

	if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
	error = EINVAL;
	break;
	}

	error = pmclog_flush(po, 0);
	}
	break;

	/*
	* Close a log file.
	*/

	case PMC_OP_CLOSELOG:
	{
	struct pmc_owner *po;

	sx_assert(&pmc_sx, SX_XLOCKED);

	if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
	error = EINVAL;
	break;
	}

	error = pmclog_close(po);
	}
	break;

	/*
	* Retrieve hardware configuration.
	*/

	case PMC_OP_GETCPUINFO: /* CPU information */
	{
	struct pmc_op_getcpuinfo gci;
	struct pmc_classinfo *pci;
	struct pmc_classdep *pcd;
	int cl;

	memset(&gci, 0, sizeof(gci));
	gci.pm_cputype = md->pmd_cputype;
	gci.pm_ncpu = pmc_cpu_max();
	gci.pm_npmc = md->pmd_npmc;
	gci.pm_nclass = md->pmd_nclass;
	pci = gci.pm_classes;
	pcd = md->pmd_classdep;
	for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) {
	pci->pm_caps = pcd->pcd_caps;
	pci->pm_class = pcd->pcd_class;
	pci->pm_width = pcd->pcd_width;
	pci->pm_num = pcd->pcd_num;
	}
	error = copyout(&gci, arg, sizeof(gci));
	}
	break;

	/*
	* Retrieve soft events list.
	*/
	case PMC_OP_GETDYNEVENTINFO:
	{
	enum pmc_class cl;
	enum pmc_event ev;
	struct pmc_op_getdyneventinfo *gei;
	struct pmc_dyn_event_descr dev;
	struct pmc_soft *ps;
	uint32_t nevent;

	sx_assert(&pmc_sx, SX_LOCKED);

	gei = (struct pmc_op_getdyneventinfo *) arg;

	if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
	break;

	/* Only SOFT class is dynamic. */
	if (cl != PMC_CLASS_SOFT) {
	error = EINVAL;
	break;
	}

	nevent = 0;
	for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
	ps = pmc_soft_ev_acquire(ev);
	if (ps == NULL)
	continue;
	bcopy(&ps->ps_ev, &dev, sizeof(dev));
	pmc_soft_ev_release(ps);

	error = copyout(&dev,
	&gei->pm_events[nevent],
	sizeof(struct pmc_dyn_event_descr));
	if (error != 0)
	break;
	nevent++;
	}
	if (error != 0)
	break;

	error = copyout(&nevent, &gei->pm_nevent,
	sizeof(nevent));
	}
	break;

	/*
	* Get module statistics
	*/

	case PMC_OP_GETDRIVERSTATS:
	{
	struct pmc_op_getdriverstats gms;
	#define CFETCH(a, b, field) a.field = counter_u64_fetch(b.field)
	CFETCH(gms, pmc_stats, pm_intr_ignored);
	CFETCH(gms, pmc_stats, pm_intr_processed);
	CFETCH(gms, pmc_stats, pm_intr_bufferfull);
	CFETCH(gms, pmc_stats, pm_syscalls);
	CFETCH(gms, pmc_stats, pm_syscall_errors);
	CFETCH(gms, pmc_stats, pm_buffer_requests);
	CFETCH(gms, pmc_stats, pm_buffer_requests_failed);
	CFETCH(gms, pmc_stats, pm_log_sweeps);
	#undef CFETCH
	error = copyout(&gms, arg, sizeof(gms));
	}
	break;


	/*
	* Retrieve module version number
	*/

	case PMC_OP_GETMODULEVERSION:
	{
	uint32_t cv, modv;

	/* retrieve the client's idea of the ABI version */
	if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
	break;
	/* don't service clients newer than our driver */
	modv = PMC_VERSION;
	if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
	error = EPROGMISMATCH;
	break;
	}
	error = copyout(&modv, arg, sizeof(int));
	}
	break;


	/*
	* Retrieve the state of all the PMCs on a given
	* CPU.
	*/

	case PMC_OP_GETPMCINFO:
	{
	int ari;
	struct pmc *pm;
	size_t pmcinfo_size;
	uint32_t cpu, n, npmc;
	struct pmc_owner *po;
	struct pmc_binding pb;
	struct pmc_classdep *pcd;
	struct pmc_info p, pmcinfo;
	struct pmc_op_getpmcinfo *gpi;

	PMC_DOWNGRADE_SX();

	gpi = (struct pmc_op_getpmcinfo *) arg;

	if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
	break;

	if (cpu >= pmc_cpu_max()) {
	error = EINVAL;
	break;
	}

	if (!pmc_cpu_is_active(cpu)) {
	error = ENXIO;
	break;
	}

	/* switch to CPU 'cpu' */
	pmc_save_cpu_binding(&pb);
	pmc_select_cpu(cpu);

	npmc = md->pmd_npmc;

	pmcinfo_size = npmc * sizeof(struct pmc_info);
	pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK \| M_ZERO);

	p = pmcinfo;

	for (n = 0; n < md->pmd_npmc; n++, p++) {

	pcd = pmc_ri_to_classdep(md, n, &ari);

	KASSERT(pcd != NULL,
	("[pmc,%d] null pcd ri=%d", __LINE__, n));

	if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0)
	break;

	if (PMC_ROW_DISP_IS_STANDALONE(n))
	p->pm_rowdisp = PMC_DISP_STANDALONE;
	else if (PMC_ROW_DISP_IS_THREAD(n))
	p->pm_rowdisp = PMC_DISP_THREAD;
	else
	p->pm_rowdisp = PMC_DISP_FREE;

	p->pm_ownerpid = -1;

	if (pm == NULL) /* no PMC associated */
	continue;

	po = pm->pm_owner;

	KASSERT(po->po_owner != NULL,
	("[pmc,%d] pmc_owner had a null proc pointer",
	__LINE__));

	p->pm_ownerpid = po->po_owner->p_pid;
	p->pm_mode = PMC_TO_MODE(pm);
	p->pm_event = pm->pm_event;
	p->pm_flags = pm->pm_flags;

	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
	p->pm_reloadcount =
	pm->pm_sc.pm_reloadcount;
	}

	pmc_restore_cpu_binding(&pb);

	/* now copy out the PMC info collected */
	if (error == 0)
	error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);

	free(pmcinfo, M_PMC);
	}
	break;


	/*
	* Set the administrative state of a PMC. I.e. whether
	* the PMC is to be used or not.
	*/

	case PMC_OP_PMCADMIN:
	{
	int cpu, ri;
	enum pmc_state request;
	struct pmc_cpu *pc;
	struct pmc_hw *phw;
	struct pmc_op_pmcadmin pma;
	struct pmc_binding pb;

	sx_assert(&pmc_sx, SX_XLOCKED);

	KASSERT(td == curthread,
	("[pmc,%d] td != curthread", __LINE__));

	error = priv_check(td, PRIV_PMC_MANAGE);
	if (error)
	break;

	if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
	break;

	cpu = pma.pm_cpu;

	if (cpu < 0 \|\| cpu >= (int) pmc_cpu_max()) {
	error = EINVAL;
	break;
	}

	if (!pmc_cpu_is_active(cpu)) {
	error = ENXIO;
	break;
	}

	request = pma.pm_state;

	if (request != PMC_STATE_DISABLED &&
	request != PMC_STATE_FREE) {
	error = EINVAL;
	break;
	}

	ri = pma.pm_pmc; /* pmc id == row index */
	if (ri < 0 \|\| ri >= (int) md->pmd_npmc) {
	error = EINVAL;
	break;
	}

	/*
	* We can't disable a PMC with a row-index allocated
	* for process virtual PMCs.
	*/

	if (PMC_ROW_DISP_IS_THREAD(ri) &&
	request == PMC_STATE_DISABLED) {
	error = EBUSY;
	break;
	}

	/*
	* otherwise, this PMC on this CPU is either free or
	* in system-wide mode.
	*/

	pmc_save_cpu_binding(&pb);
	pmc_select_cpu(cpu);

	pc = pmc_pcpu[cpu];
	phw = pc->pc_hwpmcs[ri];

	/*
	* XXX do we need some kind of 'forced' disable?
	*/

	if (phw->phw_pmc == NULL) {
	if (request == PMC_STATE_DISABLED &&
	(phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
	phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
	PMC_MARK_ROW_STANDALONE(ri);
	} else if (request == PMC_STATE_FREE &&
	(phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
	phw->phw_state \|= PMC_PHW_FLAG_IS_ENABLED;
	PMC_UNMARK_ROW_STANDALONE(ri);
	}
	/* other cases are a no-op */
	} else
	error = EBUSY;

	pmc_restore_cpu_binding(&pb);
	}
	break;


	/*
	* Allocate a PMC.
	*/

	case PMC_OP_PMCALLOCATE:
	{
	int adjri, n;
	u_int cpu;
	uint32_t caps;
	struct pmc *pmc;
	enum pmc_mode mode;
	struct pmc_hw *phw;
	struct pmc_binding pb;
	struct pmc_classdep *pcd;
	struct pmc_op_pmcallocate pa;

	if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
	break;

	caps = pa.pm_caps;
	mode = pa.pm_mode;
	cpu = pa.pm_cpu;

	if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC &&
	mode != PMC_MODE_TS && mode != PMC_MODE_TC) \|\|
	(cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) {
	error = EINVAL;
	break;
	}

	/*
	* Virtual PMCs should only ask for a default CPU.
	* System mode PMCs need to specify a non-default CPU.
	*/

	if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) \|\|
	(PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
	error = EINVAL;
	break;
	}

	/*
	* Check that an inactive CPU is not being asked for.
	*/

	if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) {
	error = ENXIO;
	break;
	}

	/*
	* Refuse an allocation for a system-wide PMC if this
	* process has been jailed, or if this process lacks
	* super-user credentials and the sysctl tunable
	* 'security.bsd.unprivileged_syspmcs' is zero.
	*/

	if (PMC_IS_SYSTEM_MODE(mode)) {
	if (jailed(curthread->td_ucred)) {
	error = EPERM;
	break;
	}
	if (!pmc_unprivileged_syspmcs) {
	error = priv_check(curthread,
	PRIV_PMC_SYSTEM);
	if (error)
	break;
	}
	}

	/*
	* Look for valid values for 'pm_flags'
	*/

	if ((pa.pm_flags & ~(PMC_F_DESCENDANTS \| PMC_F_LOG_PROCCSW \|
	PMC_F_LOG_PROCEXIT \| PMC_F_CALLCHAIN \|
	PMC_F_USERCALLCHAIN)) != 0) {
	error = EINVAL;
	break;
	}

	/* PMC_F_USERCALLCHAIN is only valid with PMC_F_CALLCHAIN */
	if ((pa.pm_flags & (PMC_F_CALLCHAIN \| PMC_F_USERCALLCHAIN)) ==
	PMC_F_USERCALLCHAIN) {
	error = EINVAL;
	break;
	}

	/* PMC_F_USERCALLCHAIN is only valid for sampling mode */
	if (pa.pm_flags & PMC_F_USERCALLCHAIN &&
	mode != PMC_MODE_TS && mode != PMC_MODE_SS) {
	error = EINVAL;
	break;
	}

	/* process logging options are not allowed for system PMCs */
	if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
	(PMC_F_LOG_PROCCSW \| PMC_F_LOG_PROCEXIT))) {
	error = EINVAL;
	break;
	}

	/*
	* All sampling mode PMCs need to be able to interrupt the
	* CPU.
	*/
	if (PMC_IS_SAMPLING_MODE(mode))
	caps \|= PMC_CAP_INTERRUPT;

	/* A valid class specifier should have been passed in. */
	pcd = pmc_class_to_classdep(pa.pm_class);
	if (pcd == NULL) {
	error = EINVAL;
	break;
	}

	/* The requested PMC capabilities should be feasible. */
	if ((pcd->pcd_caps & caps) != caps) {
	error = EOPNOTSUPP;
	break;
	}

	PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
	pa.pm_ev, caps, mode, cpu);

	pmc = pmc_allocate_pmc_descriptor();
	pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
	PMC_ID_INVALID);
	pmc->pm_event = pa.pm_ev;
	pmc->pm_state = PMC_STATE_FREE;
	pmc->pm_caps = caps;
	pmc->pm_flags = pa.pm_flags;

	/* XXX set lower bound on sampling for process counters */
	if (PMC_IS_SAMPLING_MODE(mode)) {
	/*
	* Don't permit requested sample rate to be less than 1000
	*/
	if (pa.pm_count < 1000)
	log(LOG_WARNING,
	"pmcallocate: passed sample rate %ju - setting to 1000\n",
	(uintmax_t)pa.pm_count);
	pmc->pm_sc.pm_reloadcount = MAX(1000, pa.pm_count);
	} else
	pmc->pm_sc.pm_initial = pa.pm_count;

	/* switch thread to CPU 'cpu' */
	pmc_save_cpu_binding(&pb);

	#define PMC_IS_SHAREABLE_PMC(cpu, n) \
	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
	PMC_PHW_FLAG_IS_SHAREABLE)
	#define PMC_IS_UNALLOCATED(cpu, n) \
	(pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)

	if (PMC_IS_SYSTEM_MODE(mode)) {
	pmc_select_cpu(cpu);
	for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
	pcd = pmc_ri_to_classdep(md, n, &adjri);
	if (pmc_can_allocate_row(n, mode) == 0 &&
	pmc_can_allocate_rowindex(
	curthread->td_proc, n, cpu) == 0 &&
	(PMC_IS_UNALLOCATED(cpu, n) \|\|
	PMC_IS_SHAREABLE_PMC(cpu, n)) &&
	pcd->pcd_allocate_pmc(cpu, adjri, pmc,
	&pa) == 0)
	break;
	}
	} else {
	/* Process virtual mode */
	for (n = pcd->pcd_ri; n < (int) md->pmd_npmc; n++) {
	pcd = pmc_ri_to_classdep(md, n, &adjri);
	if (pmc_can_allocate_row(n, mode) == 0 &&
	pmc_can_allocate_rowindex(
	curthread->td_proc, n,
	PMC_CPU_ANY) == 0 &&
	pcd->pcd_allocate_pmc(curthread->td_oncpu,
	adjri, pmc, &pa) == 0)
	break;
	}
	}

	#undef PMC_IS_UNALLOCATED
	#undef PMC_IS_SHAREABLE_PMC

	pmc_restore_cpu_binding(&pb);

	if (n == (int) md->pmd_npmc) {
	pmc_destroy_pmc_descriptor(pmc);
	pmc = NULL;
	error = EINVAL;
	break;
	}

	/* Fill in the correct value in the ID field */
	pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);

	PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
	pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);

	/* Process mode PMCs with logging enabled need log files */
	if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT \| PMC_F_LOG_PROCCSW))
	pmc->pm_flags \|= PMC_F_NEEDS_LOGFILE;

	/* All system mode sampling PMCs require a log file */
	if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
	pmc->pm_flags \|= PMC_F_NEEDS_LOGFILE;

	/*
	* Configure global pmc's immediately
	*/

	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {

	pmc_save_cpu_binding(&pb);
	pmc_select_cpu(cpu);

	phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
	pcd = pmc_ri_to_classdep(md, n, &adjri);

	if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 \|\|
	(error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) {
	(void) pcd->pcd_release_pmc(cpu, adjri, pmc);
	pmc_destroy_pmc_descriptor(pmc);
	pmc = NULL;
	pmc_restore_cpu_binding(&pb);
	error = EPERM;
	break;
	}

	pmc_restore_cpu_binding(&pb);
	}

	pmc->pm_state = PMC_STATE_ALLOCATED;
	pmc->pm_class = pa.pm_class;

	/*
	* mark row disposition
	*/

	if (PMC_IS_SYSTEM_MODE(mode))
	PMC_MARK_ROW_STANDALONE(n);
	else
	PMC_MARK_ROW_THREAD(n);

	/*
	* Register this PMC with the current thread as its owner.
	*/

	if ((error =
	pmc_register_owner(curthread->td_proc, pmc)) != 0) {
	pmc_release_pmc_descriptor(pmc);
	pmc_destroy_pmc_descriptor(pmc);
	pmc = NULL;
	break;
	}


	/*
	* Return the allocated index.
	*/

	pa.pm_pmcid = pmc->pm_id;

	error = copyout(&pa, arg, sizeof(pa));
	}
	break;


	/*
	* Attach a PMC to a process.
	*/

	case PMC_OP_PMCATTACH:
	{
	struct pmc *pm;
	struct proc *p;
	struct pmc_op_pmcattach a;

	sx_assert(&pmc_sx, SX_XLOCKED);

	if ((error = copyin(arg, &a, sizeof(a))) != 0)
	break;

	if (a.pm_pid < 0) {
	error = EINVAL;
	break;
	} else if (a.pm_pid == 0)
	a.pm_pid = td->td_proc->p_pid;

	if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
	break;

	if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
	error = EINVAL;
	break;
	}

	/* PMCs may be (re)attached only when allocated or stopped */
	if (pm->pm_state == PMC_STATE_RUNNING) {
	error = EBUSY;
	break;
	} else if (pm->pm_state != PMC_STATE_ALLOCATED &&
	pm->pm_state != PMC_STATE_STOPPED) {
	error = EINVAL;
	break;
	}

	/* lookup pid */
	if ((p = pfind(a.pm_pid)) == NULL) {
	error = ESRCH;
	break;
	}

	/*
	* Ignore processes that are working on exiting.
	*/
	if (p->p_flag & P_WEXIT) {
	error = ESRCH;
	PROC_UNLOCK(p); /* pfind() returns a locked process */
	break;
	}

	/*
	* we are allowed to attach a PMC to a process if
	* we can debug it.
	*/
	error = p_candebug(curthread, p);

	PROC_UNLOCK(p);

	if (error == 0)
	error = pmc_attach_process(p, pm);
	}
	break;


	/*
	* Detach an attached PMC from a process.
	*/

	case PMC_OP_PMCDETACH:
	{
	struct pmc *pm;
	struct proc *p;
	struct pmc_op_pmcattach a;

	if ((error = copyin(arg, &a, sizeof(a))) != 0)
	break;

	if (a.pm_pid < 0) {
	error = EINVAL;
	break;
	} else if (a.pm_pid == 0)
	a.pm_pid = td->td_proc->p_pid;

	if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
	break;

	if ((p = pfind(a.pm_pid)) == NULL) {
	error = ESRCH;
	break;
	}

	/*
	* Treat processes that are in the process of exiting
	* as if they were not present.
	*/

	if (p->p_flag & P_WEXIT)
	error = ESRCH;

	PROC_UNLOCK(p); /* pfind() returns a locked process */

	if (error == 0)
	error = pmc_detach_process(p, pm);
	}
	break;


	/*
	* Retrieve the MSR number associated with the counter
	* 'pmc_id'. This allows processes to directly use RDPMC
	* instructions to read their PMCs, without the overhead of a
	* system call.
	*/

	case PMC_OP_PMCGETMSR:
	{
	int adjri, ri;
	struct pmc *pm;
	struct pmc_target *pt;
	struct pmc_op_getmsr gm;
	struct pmc_classdep *pcd;

	PMC_DOWNGRADE_SX();

	if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
	break;

	if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
	break;

	/*
	* The allocated PMC has to be a process virtual PMC,
	* i.e., of type MODE_T[CS]. Global PMCs can only be
	* read using the PMCREAD operation since they may be
	* allocated on a different CPU than the one we could
	* be running on at the time of the RDPMC instruction.
	*
	* The GETMSR operation is not allowed for PMCs that
	* are inherited across processes.
	*/

	if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) \|\|
	(pm->pm_flags & PMC_F_DESCENDANTS)) {
	error = EINVAL;
	break;
	}

	/*
	* It only makes sense to use a RDPMC (or its
	* equivalent instruction on non-x86 architectures) on
	* a process that has allocated and attached a PMC to
	* itself. Conversely the PMC is only allowed to have
	* one process attached to it -- its owner.
	*/

	if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL \|\|
	LIST_NEXT(pt, pt_next) != NULL \|\|
	pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
	error = EINVAL;
	break;
	}

	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	/* PMC class has no 'GETMSR' support */
	if (pcd->pcd_get_msr == NULL) {
	error = ENOSYS;
	break;
	}

	if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0)
	break;

	if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
	break;

	/*
	* Mark our process as using MSRs. Update machine
	* state using a forced context switch.
	*/

	pt->pt_process->pp_flags \|= PMC_PP_ENABLE_MSR_ACCESS;
	pmc_force_context_switch();

	}
	break;

	/*
	* Release an allocated PMC
	*/

	case PMC_OP_PMCRELEASE:
	{
	pmc_id_t pmcid;
	struct pmc *pm;
	struct pmc_owner *po;
	struct pmc_op_simple sp;

	/*
	* Find PMC pointer for the named PMC.
	*
	* Use pmc_release_pmc_descriptor() to switch off the
	* PMC, remove all its target threads, and remove the
	* PMC from its owner's list.
	*
	* Remove the owner record if this is the last PMC
	* owned.
	*
	* Free up space.
	*/

	if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
	break;

	pmcid = sp.pm_pmcid;

	if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
	break;

	po = pm->pm_owner;
	pmc_release_pmc_descriptor(pm);
	pmc_maybe_remove_owner(po);
	pmc_destroy_pmc_descriptor(pm);
	}
	break;


	/*
	* Read and/or write a PMC.
	*/

	case PMC_OP_PMCRW:
	{
	int adjri;
	struct pmc *pm;
	uint32_t cpu, ri;
	pmc_value_t oldvalue;
	struct pmc_binding pb;
	struct pmc_op_pmcrw prw;
	struct pmc_classdep *pcd;
	struct pmc_op_pmcrw *pprw;

	PMC_DOWNGRADE_SX();

	if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
	break;

	ri = 0;
	PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
	prw.pm_flags);

	/* must have at least one flag set */
	if ((prw.pm_flags & (PMC_F_OLDVALUE\|PMC_F_NEWVALUE)) == 0) {
	error = EINVAL;
	break;
	}

	/* locate pmc descriptor */
	if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
	break;

	/* Can't read a PMC that hasn't been started. */
	if (pm->pm_state != PMC_STATE_ALLOCATED &&
	pm->pm_state != PMC_STATE_STOPPED &&
	pm->pm_state != PMC_STATE_RUNNING) {
	error = EINVAL;
	break;
	}

	/* writing a new value is allowed only for 'STOPPED' pmcs */
	if (pm->pm_state == PMC_STATE_RUNNING &&
	(prw.pm_flags & PMC_F_NEWVALUE)) {
	error = EBUSY;
	break;
	}

	if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {

	/*
	* If this PMC is attached to its owner (i.e.,
	* the process requesting this operation) and
	* is running, then attempt to get an
	* upto-date reading from hardware for a READ.
	* Writes are only allowed when the PMC is
	* stopped, so only update the saved value
	* field.
	*
	* If the PMC is not running, or is not
	* attached to its owner, read/write to the
	* savedvalue field.
	*/

	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	mtx_pool_lock_spin(pmc_mtxpool, pm);
	cpu = curthread->td_oncpu;

	if (prw.pm_flags & PMC_F_OLDVALUE) {
	if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
	(pm->pm_state == PMC_STATE_RUNNING))
	error = (*pcd->pcd_read_pmc)(cpu, adjri,
	&oldvalue);
	else
	oldvalue = pm->pm_gv.pm_savedvalue;
	}
	if (prw.pm_flags & PMC_F_NEWVALUE)
	pm->pm_gv.pm_savedvalue = prw.pm_value;

	mtx_pool_unlock_spin(pmc_mtxpool, pm);

	} else { /* System mode PMCs */
	cpu = PMC_TO_CPU(pm);
	ri = PMC_TO_ROWINDEX(pm);
	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	if (!pmc_cpu_is_active(cpu)) {
	error = ENXIO;
	break;
	}

	/* move this thread to CPU 'cpu' */
	pmc_save_cpu_binding(&pb);
	pmc_select_cpu(cpu);

	critical_enter();
	/* save old value */
	if (prw.pm_flags & PMC_F_OLDVALUE)
	if ((error = (*pcd->pcd_read_pmc)(cpu, adjri,
	&oldvalue)))
	goto error;
	/* write out new value */
	if (prw.pm_flags & PMC_F_NEWVALUE)
	error = (*pcd->pcd_write_pmc)(cpu, adjri,
	prw.pm_value);
	error:
	critical_exit();
	pmc_restore_cpu_binding(&pb);
	if (error)
	break;
	}

	pprw = (struct pmc_op_pmcrw *) arg;

	#ifdef HWPMC_DEBUG
	if (prw.pm_flags & PMC_F_NEWVALUE)
	PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
	ri, prw.pm_value, oldvalue);
	else if (prw.pm_flags & PMC_F_OLDVALUE)
	PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
	#endif

	/* return old value if requested */
	if (prw.pm_flags & PMC_F_OLDVALUE)
	if ((error = copyout(&oldvalue, &pprw->pm_value,
	sizeof(prw.pm_value))))
	break;

	}
	break;


	/*
	* Set the sampling rate for a sampling mode PMC and the
	* initial count for a counting mode PMC.
	*/

	case PMC_OP_PMCSETCOUNT:
	{
	struct pmc *pm;
	struct pmc_op_pmcsetcount sc;

	PMC_DOWNGRADE_SX();

	if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
	break;

	if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
	break;

	if (pm->pm_state == PMC_STATE_RUNNING) {
	error = EBUSY;
	break;
	}

	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
	/*
	* Don't permit requested sample rate to be less than 1000
	*/
	if (sc.pm_count < 1000)
	log(LOG_WARNING,
	"pmcsetcount: passed sample rate %ju - setting to 1000\n",
	(uintmax_t)sc.pm_count);
	pm->pm_sc.pm_reloadcount = MAX(1000, sc.pm_count);
	} else
	pm->pm_sc.pm_initial = sc.pm_count;
	}
	break;


	/*
	* Start a PMC.
	*/

	case PMC_OP_PMCSTART:
	{
	pmc_id_t pmcid;
	struct pmc *pm;
	struct pmc_op_simple sp;

	sx_assert(&pmc_sx, SX_XLOCKED);

	if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
	break;

	pmcid = sp.pm_pmcid;

	if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
	break;

	KASSERT(pmcid == pm->pm_id,
	("[pmc,%d] pmcid %x != id %x", __LINE__,
	pm->pm_id, pmcid));

	if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
	break;
	else if (pm->pm_state != PMC_STATE_STOPPED &&
	pm->pm_state != PMC_STATE_ALLOCATED) {
	error = EINVAL;
	break;
	}

	error = pmc_start(pm);
	}
	break;


	/*
	* Stop a PMC.
	*/

	case PMC_OP_PMCSTOP:
	{
	pmc_id_t pmcid;
	struct pmc *pm;
	struct pmc_op_simple sp;

	PMC_DOWNGRADE_SX();

	if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
	break;

	pmcid = sp.pm_pmcid;

	/*
	* Mark the PMC as inactive and invoke the MD stop
	* routines if needed.
	*/

	if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
	break;

	KASSERT(pmcid == pm->pm_id,
	("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
	pm->pm_id, pmcid));

	if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
	break;
	else if (pm->pm_state != PMC_STATE_RUNNING) {
	error = EINVAL;
	break;
	}

	error = pmc_stop(pm);
	}
	break;


	/*
	* Write a user supplied value to the log file.
	*/

	case PMC_OP_WRITELOG:
	{
	struct pmc_op_writelog wl;
	struct pmc_owner *po;

	PMC_DOWNGRADE_SX();

	if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
	break;

	if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
	error = EINVAL;
	break;
	}

	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
	error = EINVAL;
	break;
	}

	error = pmclog_process_userlog(po, &wl);
	}
	break;


	default:
	error = EINVAL;
	break;
	}

	if (is_sx_downgraded)
	sx_sunlock(&pmc_sx);
	else
	sx_xunlock(&pmc_sx);
	done_syscall:
	if (error)
	counter_u64_add(pmc_stats.pm_syscall_errors, 1);

	return (error);
	}

	/*
	* Helper functions
	*/


	/*
	* Mark the thread as needing callchain capture and post an AST. The
	* actual callchain capture will be done in a context where it is safe
	* to take page faults.
	*/

	static void
	pmc_post_callchain_callback(void)
	{
	struct thread *td;

	td = curthread;

	/*
	* If there is multiple PMCs for the same interrupt ignore new post
	*/
	if (td->td_pflags & TDP_CALLCHAIN)
	return;

	/*
	* Mark this thread as needing callchain capture.
	* `td->td_pflags' will be safe to touch because this thread
	* was in user space when it was interrupted.
	*/
	td->td_pflags \|= TDP_CALLCHAIN;

	/*
	* Don't let this thread migrate between CPUs until callchain
	* capture completes.
	*/
	sched_pin();

	return;
	}

	/*
	* Find a free slot in the per-cpu array of samples and capture the
	* current callchain there. If a sample was successfully added, a bit
	* is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
	* needs to be invoked from the clock handler.
	*
	* This function is meant to be called from an NMI handler. It cannot
	* use any of the locking primitives supplied by the OS.
	*/

	static int
	pmc_add_sample(ring_type_t ring, struct pmc pm, struct trapframe tf)
	{
	int error, cpu, callchaindepth, inuserspace;
	struct thread *td;
	struct pmc_sample *ps;
	struct pmc_samplebuffer *psb;

	error = 0;

	/*
	* Allocate space for a sample buffer.
	*/
	cpu = curcpu;
	psb = pmc_pcpu[cpu]->pc_sb[ring];
	inuserspace = TRAPF_USERMODE(tf);
	ps = PMC_PROD_SAMPLE(psb);
	if (psb->ps_considx != psb->ps_prodidx &&
	ps->ps_nsamples) { /* in use, reader hasn't caught up */
	pm->pm_pcpu_state[cpu].pps_stalled = 1;
	counter_u64_add(pmc_stats.pm_intr_bufferfull, 1);
	PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
	cpu, pm, (void *) tf, inuserspace,
	(int) (psb->ps_prodidx & pmc_sample_mask),
	(int) (psb->ps_considx & pmc_sample_mask));
	callchaindepth = 1;
	error = ENOMEM;
	goto done;
	}

	/* Fill in entry. */
	PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
	(void *) tf, inuserspace,
	(int) (psb->ps_prodidx & pmc_sample_mask),
	(int) (psb->ps_considx & pmc_sample_mask));

	td = curthread;
	ps->ps_pmc = pm;
	ps->ps_td = td;
	ps->ps_pid = td->td_proc->p_pid;
	ps->ps_tid = td->td_tid;
	ps->ps_tsc = pmc_rdtsc();
	ps->ps_ticks = ticks;
	ps->ps_cpu = cpu;
	ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;

	callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
	pmc_callchaindepth : 1;

	MPASS(ps->ps_pc != NULL);
	if (callchaindepth == 1)
	ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
	else {
	/*
	* Kernel stack traversals can be done immediately,
	* while we defer to an AST for user space traversals.
	*/
	if (!inuserspace) {
	callchaindepth =
	pmc_save_kernel_callchain(ps->ps_pc,
	callchaindepth, tf);
	} else {
	pmc_post_callchain_callback();
	callchaindepth = PMC_USER_CALLCHAIN_PENDING;
	}
	}

	ps->ps_nsamples = callchaindepth; /* mark entry as in use */
	if (ring == PMC_UR) {
	ps->ps_nsamples_actual = callchaindepth; /* mark entry as in use */
	ps->ps_nsamples = PMC_USER_CALLCHAIN_PENDING;
	} else
	ps->ps_nsamples = callchaindepth; /* mark entry as in use */

	KASSERT(counter_u64_fetch(pm->pm_runcount) >= 0,
	("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	(unsigned long)counter_u64_fetch(pm->pm_runcount)));

	counter_u64_add(pm->pm_runcount, 1); /* hold onto PMC */
	/* increment write pointer */
	psb->ps_prodidx++;
	done:
	/* mark CPU as needing processing */
	if (callchaindepth != PMC_USER_CALLCHAIN_PENDING)
	DPCPU_SET(pmc_sampled, 1);

	return (error);
	}

	/*
	* Interrupt processing.
	*
	* This function is meant to be called from an NMI handler. It cannot
	* use any of the locking primitives supplied by the OS.
	*/

	int
	pmc_process_interrupt(int ring, struct pmc pm, struct trapframe tf)
	{
	struct thread *td;

	td = curthread;
	if ((pm->pm_flags & PMC_F_USERCALLCHAIN) &&
	(td->td_proc->p_flag & P_KPROC) == 0 &&
	!TRAPF_USERMODE(tf)) {
	atomic_add_int(&td->td_pmcpend, 1);
	return (pmc_add_sample(PMC_UR, pm, tf));
	}
	return (pmc_add_sample(ring, pm, tf));
	}

	/*
	* Capture a user call chain. This function will be called from ast()
	* before control returns to userland and before the process gets
	* rescheduled.
	*/

	static void
	pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
	{
	struct pmc *pm;
	struct thread *td;
	struct pmc_sample *ps;
	struct pmc_samplebuffer *psb;
	uint64_t considx, prodidx;
	int nsamples, nrecords, pass, iter;
	#ifdef INVARIANTS
	int ncallchains;
	int nfree;
	int start_ticks = ticks;
	#endif
	psb = pmc_pcpu[cpu]->pc_sb[ring];
	td = curthread;

	KASSERT(td->td_pflags & TDP_CALLCHAIN,
	("[pmc,%d] Retrieving callchain for thread that doesn't want it",
	__LINE__));

	#ifdef INVARIANTS
	ncallchains = 0;
	nfree = 0;
	#endif
	nrecords = INT_MAX;
	pass = 0;
	restart:
	if (ring == PMC_UR)
	nrecords = atomic_readandclear_32(&td->td_pmcpend);

	for (iter = 0, considx = psb->ps_considx, prodidx = psb->ps_prodidx;
	considx < prodidx && iter < pmc_nsamples; considx++, iter++) {
	ps = PMC_CONS_SAMPLE_OFF(psb, considx);

	/*
	* Iterate through all deferred callchain requests.
	* Walk from the current read pointer to the current
	* write pointer.
	*/

	#ifdef INVARIANTS
	if (ps->ps_nsamples == PMC_SAMPLE_FREE) {
	nfree++;
	continue;
	}

	if ((ps->ps_pmc == NULL) \|\|
	(ps->ps_pmc->pm_state != PMC_STATE_RUNNING))
	nfree++;
	#endif
	if (ps->ps_td != td \|\|
	ps->ps_nsamples != PMC_USER_CALLCHAIN_PENDING \|\|
	ps->ps_pmc->pm_state != PMC_STATE_RUNNING)
	continue;

	KASSERT(ps->ps_cpu == cpu,
	("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
	ps->ps_cpu, PCPU_GET(cpuid)));

	pm = ps->ps_pmc;

	KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
	("[pmc,%d] Retrieving callchain for PMC that doesn't "
	"want it", __LINE__));

	KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	("[pmc,%d] runcount %ld", __LINE__, (unsigned long)counter_u64_fetch(pm->pm_runcount)));

	if (ring == PMC_UR) {
	nsamples = ps->ps_nsamples_actual;
	counter_u64_add(pmc_stats.pm_merges, 1);
	} else
	nsamples = 0;

	/*
	* Retrieve the callchain and mark the sample buffer
	* as 'processable' by the timer tick sweep code.
	*/

	#ifdef INVARIANTS
	ncallchains++;
	#endif

	if (__predict_true(nsamples < pmc_callchaindepth - 1))
	nsamples += pmc_save_user_callchain(ps->ps_pc + nsamples,
	pmc_callchaindepth - nsamples - 1, tf);

	/*
	* We have to prevent hardclock from potentially overwriting
	* this sample between when we read the value and when we set
	* it
	*/
	spinlock_enter();
	/*
	* Verify that the sample hasn't been dropped in the meantime
	*/
	if (ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING) {
	ps->ps_nsamples = nsamples;
	/*
	* If we couldn't get a sample, simply drop the reference
	*/
	if (nsamples == 0)
	counter_u64_add(pm->pm_runcount, -1);
	}
	spinlock_exit();
	if (nrecords-- == 1)
	break;
	}
	if (__predict_false(ring == PMC_UR && td->td_pmcpend)) {
	if (pass == 0) {
	pass = 1;
	goto restart;
	}
	/* only collect samples for this part once */
	td->td_pmcpend = 0;
	}

	#ifdef INVARIANTS
	if ((ticks - start_ticks) > hz)
	log(LOG_ERR, "%s took %d ticks\n", __func__, (ticks - start_ticks));
	#endif

	/* mark CPU as needing processing */
	DPCPU_SET(pmc_sampled, 1);
	}

	/*
	* Process saved PC samples.
	*/

	static void
	pmc_process_samples(int cpu, ring_type_t ring)
	{
	struct pmc *pm;
	int adjri, n;
	struct thread *td;
	struct pmc_owner *po;
	struct pmc_sample *ps;
	struct pmc_classdep *pcd;
	struct pmc_samplebuffer *psb;
	uint64_t delta;

	KASSERT(PCPU_GET(cpuid) == cpu,
	("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
	PCPU_GET(cpuid), cpu));

	psb = pmc_pcpu[cpu]->pc_sb[ring];
	delta = psb->ps_prodidx - psb->ps_considx;
	MPASS(delta <= pmc_nsamples);
	MPASS(psb->ps_considx <= psb->ps_prodidx);
	for (n = 0; psb->ps_considx < psb->ps_prodidx; psb->ps_considx++, n++) {
	ps = PMC_CONS_SAMPLE(psb);

	if (__predict_false(ps->ps_nsamples == PMC_SAMPLE_FREE))
	continue;
	pm = ps->ps_pmc;
	/* skip non-running samples */
	if (pm->pm_state != PMC_STATE_RUNNING)
	goto entrydone;

	KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	(unsigned long)counter_u64_fetch(pm->pm_runcount)));

	po = pm->pm_owner;

	KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
	("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
	pm, PMC_TO_MODE(pm)));


	/* If there is a pending AST wait for completion */
	if (ps->ps_nsamples == PMC_USER_CALLCHAIN_PENDING) {
	/* if we've been waiting more than 1 tick to
	* collect a callchain for this record then
	* drop it and move on.
	*/
	if (ticks - ps->ps_ticks > 1) {
	/*
	* track how often we hit this as it will
	* preferentially lose user samples
	* for long running system calls
	*/
	counter_u64_add(pmc_stats.pm_overwrites, 1);
	goto entrydone;
	}
	/* Need a rescan at a later time. */
	DPCPU_SET(pmc_sampled, 1);
	break;
	}

	PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
	pm, ps->ps_nsamples, ps->ps_flags,
	(int) (psb->ps_prodidx & pmc_sample_mask),
	(int) (psb->ps_considx & pmc_sample_mask));

	/*
	* If this is a process-mode PMC that is attached to
	* its owner, and if the PC is in user mode, update
	* profiling statistics like timer-based profiling
	* would have done.
	*
	* Otherwise, this is either a sampling-mode PMC that
	* is attached to a different process than its owner,
	* or a system-wide sampling PMC. Dispatch a log
	* entry to the PMC's owner process.
	*/
	if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
	if (ps->ps_flags & PMC_CC_F_USERSPACE) {
	td = FIRST_THREAD_IN_PROC(po->po_owner);
	addupc_intr(td, ps->ps_pc[0], 1);
	}
	} else
	pmclog_process_callchain(pm, ps);

	entrydone:
	ps->ps_nsamples = 0; /* mark entry as free */
	KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	("[pmc,%d] pm=%p runcount %ld", __LINE__, (void *) pm,
	(unsigned long)counter_u64_fetch(pm->pm_runcount)));

	counter_u64_add(pm->pm_runcount, -1);
	}

	counter_u64_add(pmc_stats.pm_log_sweeps, 1);

	/* Do not re-enable stalled PMCs if we failed to process any samples */
	if (n == 0)
	return;

	/*
	* Restart any stalled sampling PMCs on this CPU.
	*
	* If the NMI handler sets the pm_stalled field of a PMC after
	* the check below, we'll end up processing the stalled PMC at
	* the next hardclock tick.
	*/
	for (n = 0; n < md->pmd_npmc; n++) {
	pcd = pmc_ri_to_classdep(md, n, &adjri);
	KASSERT(pcd != NULL,
	("[pmc,%d] null pcd ri=%d", __LINE__, n));
	(void) (*pcd->pcd_get_config)(cpu,adjri,&pm);

	if (pm == NULL \|\| /* !cfg'ed */
	pm->pm_state != PMC_STATE_RUNNING \|\| /* !active */
	!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) \|\| /* !sampling */
	!pm->pm_pcpu_state[cpu].pps_cpustate \|\| /* !desired */
	!pm->pm_pcpu_state[cpu].pps_stalled) /* !stalled */
	continue;

	pm->pm_pcpu_state[cpu].pps_stalled = 0;
	(*pcd->pcd_start_pmc)(cpu, adjri);
	}
	}

	/*
	* Event handlers.
	*/

	/*
	* Handle a process exit.
	*
	* Remove this process from all hash tables. If this process
	* owned any PMCs, turn off those PMCs and deallocate them,
	* removing any associations with target processes.
	*
	* This function will be called by the last 'thread' of a
	* process.
	*
	* XXX This eventhandler gets called early in the exit process.
	* Consider using a 'hook' invocation from thread_exit() or equivalent
	* spot. Another negative is that kse_exit doesn't seem to call
	* exit1() [??].
	*
	*/

	static void
	pmc_process_exit(void arg __unused, struct proc p)
	{
	struct pmc *pm;
	int adjri, cpu;
	unsigned int ri;
	int is_using_hwpmcs;
	struct pmc_owner *po;
	struct pmc_process *pp;
	struct pmc_classdep *pcd;
	pmc_value_t newvalue, tmp;

	PROC_LOCK(p);
	is_using_hwpmcs = p->p_flag & P_HWPMC;
	PROC_UNLOCK(p);

	/*
	* Log a sysexit event to all SS PMC owners.
	*/
	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_sysexit(po, p->p_pid);
	PMC_EPOCH_EXIT();

	if (!is_using_hwpmcs)
	return;

	PMC_GET_SX_XLOCK();
	PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
	p->p_comm);

	/*
	* Since this code is invoked by the last thread in an exiting
	* process, we would have context switched IN at some prior
	* point. However, with PREEMPTION, kernel mode context
	* switches may happen any time, so we want to disable a
	* context switch OUT till we get any PMCs targeting this
	* process off the hardware.
	*
	* We also need to atomically remove this process'
	* entry from our target process hash table, using
	* PMC_FLAG_REMOVE.
	*/
	PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
	p->p_comm);

	critical_enter(); /* no preemption */

	cpu = curthread->td_oncpu;

	if ((pp = pmc_find_process_descriptor(p,
	PMC_FLAG_REMOVE)) != NULL) {

	PMCDBG2(PRC,EXT,2,
	"process-exit proc=%p pmc-process=%p", p, pp);

	/*
	* The exiting process could the target of
	* some PMCs which will be running on
	* currently executing CPU.
	*
	* We need to turn these PMCs off like we
	* would do at context switch OUT time.
	*/
	for (ri = 0; ri < md->pmd_npmc; ri++) {

	/*
	* Pick up the pmc pointer from hardware
	* state similar to the CSW_OUT code.
	*/
	pm = NULL;

	pcd = pmc_ri_to_classdep(md, ri, &adjri);

	(void) (*pcd->pcd_get_config)(cpu, adjri, &pm);

	PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm);

	if (pm == NULL \|\|
	!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
	continue;

	PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
	"state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
	pm, pm->pm_state);

	KASSERT(PMC_TO_ROWINDEX(pm) == ri,
	("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
	__LINE__, PMC_TO_ROWINDEX(pm), ri));

	KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
	("[pmc,%d] pm %p != pp_pmcs[%d] %p",
	__LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));

	KASSERT(counter_u64_fetch(pm->pm_runcount) > 0,
	("[pmc,%d] bad runcount ri %d rc %ld",
	__LINE__, ri, (unsigned long)counter_u64_fetch(pm->pm_runcount)));

	/*
	* Change desired state, and then stop if not
	* stalled. This two-step dance should avoid
	* race conditions where an interrupt re-enables
	* the PMC after this code has already checked
	* the pm_stalled flag.
	*/
	if (pm->pm_pcpu_state[cpu].pps_cpustate) {
	pm->pm_pcpu_state[cpu].pps_cpustate = 0;
	if (!pm->pm_pcpu_state[cpu].pps_stalled) {
	(void) pcd->pcd_stop_pmc(cpu, adjri);

	if (PMC_TO_MODE(pm) == PMC_MODE_TC) {
	pcd->pcd_read_pmc(cpu, adjri,
	&newvalue);
	tmp = newvalue -
	PMC_PCPU_SAVED(cpu,ri);

	mtx_pool_lock_spin(pmc_mtxpool,
	pm);
	pm->pm_gv.pm_savedvalue += tmp;
	pp->pp_pmcs[ri].pp_pmcval +=
	tmp;
	mtx_pool_unlock_spin(
	pmc_mtxpool, pm);
	}
	}
	}

	KASSERT((int64_t) counter_u64_fetch(pm->pm_runcount) > 0,
	("[pmc,%d] runcount is %d", __LINE__, ri));

	counter_u64_add(pm->pm_runcount, -1);

	(void) pcd->pcd_config_pmc(cpu, adjri, NULL);
	}

	/*
	* Inform the MD layer of this pseudo "context switch
	* out"
	*/
	(void) md->pmd_switch_out(pmc_pcpu[cpu], pp);

	critical_exit(); /* ok to be pre-empted now */

	/*
	* Unlink this process from the PMCs that are
	* targeting it. This will send a signal to
	* all PMC owner's whose PMCs are orphaned.
	*
	* Log PMC value at exit time if requested.
	*/
	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
	if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
	PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
	pmclog_process_procexit(pm, pp);
	pmc_unlink_target_process(pm, pp);
	}
	free(pp, M_PMC);

	} else
	critical_exit(); /* pp == NULL */


	/*
	* If the process owned PMCs, free them up and free up
	* memory.
	*/
	if ((po = pmc_find_owner_descriptor(p)) != NULL) {
	pmc_remove_owner(po);
	pmc_destroy_owner_descriptor(po);
	}

	sx_xunlock(&pmc_sx);
	}

	/*
	* Handle a process fork.
	*
	* If the parent process 'p1' is under HWPMC monitoring, then copy
	* over any attached PMCs that have 'do_descendants' semantics.
	*/

	static void
	pmc_process_fork(void arg __unused, struct proc p1, struct proc *newproc,
	int flags)
	{
	int is_using_hwpmcs;
	unsigned int ri;
	uint32_t do_descendants;
	struct pmc *pm;
	struct pmc_owner *po;
	struct pmc_process ppnew, ppold;

	(void) flags; /* unused parameter */

	PROC_LOCK(p1);
	is_using_hwpmcs = p1->p_flag & P_HWPMC;
	PROC_UNLOCK(p1);

	/*
	* If there are system-wide sampling PMCs active, we need to
	* log all fork events to their owner's logs.
	*/
	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
	pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
	pmclog_process_proccreate(po, newproc, 1);
	}
	PMC_EPOCH_EXIT();

	if (!is_using_hwpmcs)
	return;

	PMC_GET_SX_XLOCK();
	PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
	p1->p_pid, p1->p_comm, newproc);

	/*
	* If the parent process (curthread->td_proc) is a
	* target of any PMCs, look for PMCs that are to be
	* inherited, and link these into the new process
	* descriptor.
	*/
	if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
	PMC_FLAG_NONE)) == NULL)
	goto done; /* nothing to do */

	do_descendants = 0;
	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
	do_descendants \|= pm->pm_flags & PMC_F_DESCENDANTS;
	if (do_descendants == 0) /* nothing to do */
	goto done;

	/*
	* Now mark the new process as being tracked by this driver.
	*/
	PROC_LOCK(newproc);
	newproc->p_flag \|= P_HWPMC;
	PROC_UNLOCK(newproc);

	/* allocate a descriptor for the new process */
	if ((ppnew = pmc_find_process_descriptor(newproc,
	PMC_FLAG_ALLOCATE)) == NULL)
	goto done;

	/*
	* Run through all PMCs that were targeting the old process
	* and which specified F_DESCENDANTS and attach them to the
	* new process.
	*
	* Log the fork event to all owners of PMCs attached to this
	* process, if not already logged.
	*/
	for (ri = 0; ri < md->pmd_npmc; ri++)
	if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
	(pm->pm_flags & PMC_F_DESCENDANTS)) {
	pmc_link_target_process(pm, ppnew);
	po = pm->pm_owner;
	if (po->po_sscount == 0 &&
	po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_procfork(po, p1->p_pid,
	newproc->p_pid);
	}

	done:
	sx_xunlock(&pmc_sx);
	}

	static void
	pmc_process_threadcreate(struct thread *td)
	{
	struct pmc_owner *po;

	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_threadcreate(po, td, 1);
	PMC_EPOCH_EXIT();
	}

	static void
	pmc_process_threadexit(struct thread *td)
	{
	struct pmc_owner *po;

	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_threadexit(po, td);
	PMC_EPOCH_EXIT();
	}

	static void
	pmc_process_proccreate(struct proc *p)
	{
	struct pmc_owner *po;

	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_proccreate(po, p, 1 /* sync */);
	PMC_EPOCH_EXIT();
	}

	static void
	pmc_process_allproc(struct pmc *pm)
	{
	struct pmc_owner *po;
	struct thread *td;
	struct proc *p;

	po = pm->pm_owner;
	if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
	return;
	sx_slock(&allproc_lock);
	FOREACH_PROC_IN_SYSTEM(p) {
	pmclog_process_proccreate(po, p, 0 /* sync */);
	PROC_LOCK(p);
	FOREACH_THREAD_IN_PROC(p, td)
	pmclog_process_threadcreate(po, td, 0 /* sync */);
	PROC_UNLOCK(p);
	}
	sx_sunlock(&allproc_lock);
	pmclog_flush(po, 0);
	}

	static void
	pmc_kld_load(void *arg __unused, linker_file_t lf)
	{
	struct pmc_owner *po;

	/*
	* Notify owners of system sampling PMCs about KLD operations.
	*/
	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_map_in(po, (pid_t) -1,
	(uintfptr_t) lf->address, lf->filename);
	PMC_EPOCH_EXIT();

	/*
	* TODO: Notify owners of (all) process-sampling PMCs too.
	*/
	}

	static void
	pmc_kld_unload(void arg __unused, const char filename __unused,
	caddr_t address, size_t size)
	{
	struct pmc_owner *po;

	PMC_EPOCH_ENTER();
	CK_LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
	if (po->po_flags & PMC_PO_OWNS_LOGFILE)
	pmclog_process_map_out(po, (pid_t) -1,
	(uintfptr_t) address, (uintfptr_t) address + size);
	PMC_EPOCH_EXIT();

	/*
	* TODO: Notify owners of process-sampling PMCs.
	*/
	}

	/*
	* initialization
	*/
	static const char *
	pmc_name_of_pmcclass(enum pmc_class class)
	{

	switch (class) {
	#undef __PMC_CLASS
	#define __PMC_CLASS(S,V,D) \
	case PMC_CLASS_##S: \
	return #S;
	__PMC_CLASSES();
	default:
	return ("<unknown>");
	}
	}

	/*
	* Base class initializer: allocate structure and set default classes.
	*/
	struct pmc_mdep *
	pmc_mdep_alloc(int nclasses)
	{
	struct pmc_mdep *md;
	int n;

	/* SOFT + md classes */
	n = 1 + nclasses;
	md = malloc(sizeof(struct pmc_mdep) + n *
	sizeof(struct pmc_classdep), M_PMC, M_WAITOK\|M_ZERO);
	md->pmd_nclass = n;

	/* Add base class. */
	pmc_soft_initialize(md);
	return md;
	}

	void
	pmc_mdep_free(struct pmc_mdep *md)
	{
	pmc_soft_finalize(md);
	free(md, M_PMC);
	}

	static int
	generic_switch_in(struct pmc_cpu pc, struct pmc_process pp)
	{
	(void) pc; (void) pp;

	return (0);
	}

	static int
	generic_switch_out(struct pmc_cpu pc, struct pmc_process pp)
	{
	(void) pc; (void) pp;

	return (0);
	}

	static struct pmc_mdep *
	pmc_generic_cpu_initialize(void)
	{
	struct pmc_mdep *md;

	md = pmc_mdep_alloc(0);

	md->pmd_cputype = PMC_CPU_GENERIC;

	md->pmd_pcpu_init = NULL;
	md->pmd_pcpu_fini = NULL;
	md->pmd_switch_in = generic_switch_in;
	md->pmd_switch_out = generic_switch_out;

	return (md);
	}

	static void
	pmc_generic_cpu_finalize(struct pmc_mdep *md)
	{
	(void) md;
	}


	static int
	pmc_initialize(void)
	{
	int c, cpu, error, n, ri;
	unsigned int maxcpu, domain;
	struct pcpu *pc;
	struct pmc_binding pb;
	struct pmc_sample *ps;
	struct pmc_classdep *pcd;
	struct pmc_samplebuffer *sb;

	md = NULL;
	error = 0;

	pmc_stats.pm_intr_ignored = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_intr_processed = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_intr_bufferfull = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_syscalls = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_syscall_errors = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_buffer_requests = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_buffer_requests_failed = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_log_sweeps = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_merges = counter_u64_alloc(M_WAITOK);
	pmc_stats.pm_overwrites = counter_u64_alloc(M_WAITOK);

	#ifdef HWPMC_DEBUG
	/* parse debug flags first */
	if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
	pmc_debugstr, sizeof(pmc_debugstr)))
	pmc_debugflags_parse(pmc_debugstr,
	pmc_debugstr+strlen(pmc_debugstr));
	#endif

	PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);

	/* check kernel version */
	if (pmc_kernel_version != PMC_VERSION) {
	if (pmc_kernel_version == 0)
	printf("hwpmc: this kernel has not been compiled with "
	"'options HWPMC_HOOKS'.\n");
	else
	printf("hwpmc: kernel version (0x%x) does not match "
	"module version (0x%x).\n", pmc_kernel_version,
	PMC_VERSION);
	return EPROGMISMATCH;
	}

	/*
	* check sysctl parameters
	*/

	if (pmc_hashsize <= 0) {
	(void) printf("hwpmc: tunable \"hashsize\"=%d must be "
	"greater than zero.\n", pmc_hashsize);
	pmc_hashsize = PMC_HASH_SIZE;
	}

	if (pmc_nsamples <= 0 \|\| pmc_nsamples > 65535) {
	(void) printf("hwpmc: tunable \"nsamples\"=%d out of "
	"range.\n", pmc_nsamples);
	pmc_nsamples = PMC_NSAMPLES;
	}
	pmc_sample_mask = pmc_nsamples-1;

	if (pmc_callchaindepth <= 0 \|\|
	pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
	(void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
	"range - using %d.\n", pmc_callchaindepth,
	PMC_CALLCHAIN_DEPTH_MAX);
	pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX;
	}

	md = pmc_md_initialize();
	if (md == NULL) {
	/* Default to generic CPU. */
	md = pmc_generic_cpu_initialize();
	if (md == NULL)
	return (ENOSYS);
	}

	KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
	("[pmc,%d] no classes or pmcs", __LINE__));

	/* Compute the map from row-indices to classdep pointers. */
	pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep )
	md->pmd_npmc, M_PMC, M_WAITOK\|M_ZERO);

	for (n = 0; n < md->pmd_npmc; n++)
	pmc_rowindex_to_classdep[n] = NULL;
	for (ri = c = 0; c < md->pmd_nclass; c++) {
	pcd = &md->pmd_classdep[c];
	for (n = 0; n < pcd->pcd_num; n++, ri++)
	pmc_rowindex_to_classdep[ri] = pcd;
	}

	KASSERT(ri == md->pmd_npmc,
	("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__,
	ri, md->pmd_npmc));

	maxcpu = pmc_cpu_max();

	/* allocate space for the per-cpu array */
	pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC,
	M_WAITOK\|M_ZERO);

	/* per-cpu 'saved values' for managing process-mode PMCs */
	pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc,
	M_PMC, M_WAITOK);

	/* Perform CPU-dependent initialization. */
	pmc_save_cpu_binding(&pb);
	error = 0;
	for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) {
	if (!pmc_cpu_is_active(cpu))
	continue;
	pmc_select_cpu(cpu);
	pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) +
	md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC,
	M_WAITOK\|M_ZERO);
	if (md->pmd_pcpu_init)
	error = md->pmd_pcpu_init(md, cpu);
	for (n = 0; error == 0 && n < md->pmd_nclass; n++)
	error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu);
	}
	pmc_restore_cpu_binding(&pb);

	if (error)
	return (error);

	/* allocate space for the sample array */
	for (cpu = 0; cpu < maxcpu; cpu++) {
	if (!pmc_cpu_is_active(cpu))
	continue;
	pc = pcpu_find(cpu);
	domain = pc->pc_domain;
	sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
	pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
	DOMAINSET_PREF(domain), M_WAITOK \| M_ZERO);

	KASSERT(pmc_pcpu[cpu] != NULL,
	("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));

	sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
	pmc_nsamples * sizeof(uintptr_t), M_PMC,
	DOMAINSET_PREF(domain), M_WAITOK \| M_ZERO);

	for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
	ps->ps_pc = sb->ps_callchains +
	(n * pmc_callchaindepth);

	pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;

	sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
	pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
	DOMAINSET_PREF(domain), M_WAITOK \| M_ZERO);

	sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
	pmc_nsamples * sizeof(uintptr_t), M_PMC,
	DOMAINSET_PREF(domain), M_WAITOK \| M_ZERO);
	for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
	ps->ps_pc = sb->ps_callchains +
	(n * pmc_callchaindepth);

	pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;

	sb = malloc_domainset(sizeof(struct pmc_samplebuffer) +
	pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
	DOMAINSET_PREF(domain), M_WAITOK \| M_ZERO);
	sb->ps_callchains = malloc_domainset(pmc_callchaindepth *
	pmc_nsamples * sizeof(uintptr_t), M_PMC,
	DOMAINSET_PREF(domain), M_WAITOK \| M_ZERO);
	for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
	ps->ps_pc = sb->ps_callchains + n * pmc_callchaindepth;

	pmc_pcpu[cpu]->pc_sb[PMC_UR] = sb;
	}

	/* allocate space for the row disposition array */
	pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
	M_PMC, M_WAITOK\|M_ZERO);

	/* mark all PMCs as available */
	for (n = 0; n < (int) md->pmd_npmc; n++)
	PMC_MARK_ROW_FREE(n);

	/* allocate thread hash tables */
	pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
	&pmc_ownerhashmask);

	pmc_processhash = hashinit(pmc_hashsize, M_PMC,
	&pmc_processhashmask);
	mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf",
	MTX_SPIN);

	CK_LIST_INIT(&pmc_ss_owners);
	pmc_ss_count = 0;

	/* allocate a pool of spin mutexes */
	pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size,
	MTX_SPIN);

	PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
	"targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
	pmc_processhash, pmc_processhashmask);

	/* Initialize a spin mutex for the thread free list. */
	mtx_init(&pmc_threadfreelist_mtx, "pmc-threadfreelist", "pmc-leaf",
	MTX_SPIN);

	/* Initialize the task to prune the thread free list. */
	TASK_INIT(&free_task, 0, pmc_thread_descriptor_pool_free_task, NULL);

	/* register process {exit,fork,exec} handlers */
	pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
	pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
	pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
	pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);

	/* register kld event handlers */
	pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load,
	NULL, EVENTHANDLER_PRI_ANY);
	pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload,
	NULL, EVENTHANDLER_PRI_ANY);

	/* initialize logging */
	pmclog_initialize();

	/* set hook functions */
	pmc_intr = md->pmd_intr;
	wmb();
	pmc_hook = pmc_hook_handler;

	if (error == 0) {
	printf(PMC_MODULE_NAME ":");
	for (n = 0; n < (int) md->pmd_nclass; n++) {
	pcd = &md->pmd_classdep[n];
	printf(" %s/%d/%d/0x%b",
	pmc_name_of_pmcclass(pcd->pcd_class),
	pcd->pcd_num,
	pcd->pcd_width,
	pcd->pcd_caps,
	"\20"
	"\1INT\2USR\3SYS\4EDG\5THR"
	"\6REA\7WRI\10INV\11QUA\12PRC"
	"\13TAG\14CSC");
	}
	printf("\n");
	}

	return (error);
	}

	/* prepare to be unloaded */
	static void
	pmc_cleanup(void)
	{
	int c, cpu;
	unsigned int maxcpu;
	struct pmc_ownerhash *ph;
	struct pmc_owner po, tmp;
	struct pmc_binding pb;
	#ifdef HWPMC_DEBUG
	struct pmc_processhash *prh;
	#endif

	PMCDBG0(MOD,INI,0, "cleanup");

	/* switch off sampling */
	CPU_FOREACH(cpu)
	DPCPU_ID_SET(cpu, pmc_sampled, 0);
	pmc_intr = NULL;

	sx_xlock(&pmc_sx);
	if (pmc_hook == NULL) { /* being unloaded already */
	sx_xunlock(&pmc_sx);
	return;
	}

	pmc_hook = NULL; /* prevent new threads from entering module */

	/* deregister event handlers */
	EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
	EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
	EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag);
	EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag);

	/* send SIGBUS to all owner threads, free up allocations */
	if (pmc_ownerhash)
	for (ph = pmc_ownerhash;
	ph <= &pmc_ownerhash[pmc_ownerhashmask];
	ph++) {
	LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
	pmc_remove_owner(po);

	/* send SIGBUS to owner processes */
	PMCDBG3(MOD,INI,2, "cleanup signal proc=%p "
	"(%d, %s)", po->po_owner,
	po->po_owner->p_pid,
	po->po_owner->p_comm);

	PROC_LOCK(po->po_owner);
	kern_psignal(po->po_owner, SIGBUS);
	PROC_UNLOCK(po->po_owner);

	pmc_destroy_owner_descriptor(po);
	}
	}

	/* reclaim allocated data structures */
	taskqueue_drain(taskqueue_fast, &free_task);
	mtx_destroy(&pmc_threadfreelist_mtx);
	pmc_thread_descriptor_pool_drain();

	if (pmc_mtxpool)
	mtx_pool_destroy(&pmc_mtxpool);

	mtx_destroy(&pmc_processhash_mtx);
	if (pmc_processhash) {
	#ifdef HWPMC_DEBUG
	struct pmc_process *pp;

	PMCDBG0(MOD,INI,3, "destroy process hash");
	for (prh = pmc_processhash;
	prh <= &pmc_processhash[pmc_processhashmask];
	prh++)
	LIST_FOREACH(pp, prh, pp_next)
	PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
	#endif

	hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
	pmc_processhash = NULL;
	}

	if (pmc_ownerhash) {
	PMCDBG0(MOD,INI,3, "destroy owner hash");
	hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
	pmc_ownerhash = NULL;
	}

	KASSERT(CK_LIST_EMPTY(&pmc_ss_owners),
	("[pmc,%d] Global SS owner list not empty", __LINE__));
	KASSERT(pmc_ss_count == 0,
	("[pmc,%d] Global SS count not empty", __LINE__));

	/* do processor and pmc-class dependent cleanup */
	maxcpu = pmc_cpu_max();

	PMCDBG0(MOD,INI,3, "md cleanup");
	if (md) {
	pmc_save_cpu_binding(&pb);
	for (cpu = 0; cpu < maxcpu; cpu++) {
	PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
	cpu, pmc_pcpu[cpu]);
	if (!pmc_cpu_is_active(cpu) \|\| pmc_pcpu[cpu] == NULL)
	continue;
	pmc_select_cpu(cpu);
	for (c = 0; c < md->pmd_nclass; c++)
	md->pmd_classdep[c].pcd_pcpu_fini(md, cpu);
	if (md->pmd_pcpu_fini)
	md->pmd_pcpu_fini(md, cpu);
	}

	if (md->pmd_cputype == PMC_CPU_GENERIC)
	pmc_generic_cpu_finalize(md);
	else
	pmc_md_finalize(md);

	pmc_mdep_free(md);
	md = NULL;
	pmc_restore_cpu_binding(&pb);
	}

	/* Free per-cpu descriptors. */
	for (cpu = 0; cpu < maxcpu; cpu++) {
	if (!pmc_cpu_is_active(cpu))
	continue;
	KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
	("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
	cpu));
	KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
	("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
	cpu));
	KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_UR] != NULL,
	("[pmc,%d] Null userret cpu sample buffer cpu=%d", __LINE__,
	cpu));
	free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
	free_domain(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
	free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
	free_domain(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
	free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR]->ps_callchains, M_PMC);
	free_domain(pmc_pcpu[cpu]->pc_sb[PMC_UR], M_PMC);
	free_domain(pmc_pcpu[cpu], M_PMC);
	}

	free(pmc_pcpu, M_PMC);
	pmc_pcpu = NULL;

	free(pmc_pcpu_saved, M_PMC);
	pmc_pcpu_saved = NULL;

	if (pmc_pmcdisp) {
	free(pmc_pmcdisp, M_PMC);
	pmc_pmcdisp = NULL;
	}

	if (pmc_rowindex_to_classdep) {
	free(pmc_rowindex_to_classdep, M_PMC);
	pmc_rowindex_to_classdep = NULL;
	}

	pmclog_shutdown();
	counter_u64_free(pmc_stats.pm_intr_ignored);
	counter_u64_free(pmc_stats.pm_intr_processed);
	counter_u64_free(pmc_stats.pm_intr_bufferfull);
	counter_u64_free(pmc_stats.pm_syscalls);
	counter_u64_free(pmc_stats.pm_syscall_errors);
	counter_u64_free(pmc_stats.pm_buffer_requests);
	counter_u64_free(pmc_stats.pm_buffer_requests_failed);
	counter_u64_free(pmc_stats.pm_log_sweeps);
	counter_u64_free(pmc_stats.pm_merges);
	counter_u64_free(pmc_stats.pm_overwrites);
	sx_xunlock(&pmc_sx); /* we are done */
	}

	/*
	* The function called at load/unload.
	*/

	static int
	load (struct module module __unused, int cmd, void arg __unused)
	{
	int error;

	error = 0;

	switch (cmd) {
	case MOD_LOAD :
	/* initialize the subsystem */
	error = pmc_initialize();
	if (error != 0)
	break;
	PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d",
	pmc_syscall_num, pmc_cpu_max());
	break;


	case MOD_UNLOAD :
	case MOD_SHUTDOWN:
	pmc_cleanup();
	PMCDBG0(MOD,INI,1, "unloaded");
	break;

	default :
	error = EINVAL; /* XXX should panic(9) */
	break;
	}

	return error;
	}
	diff --git a/sys/dev/hyperv/vmbus/vmbus.c b/sys/dev/hyperv/vmbus/vmbus.c
	index 9d4f5e77fa97..38e9cd7b4c26 100644
	--- a/sys/dev/hyperv/vmbus/vmbus.c
	+++ b/sys/dev/hyperv/vmbus/vmbus.c
	@@ -1,1593 +1,1650 @@
	/*-
	* Copyright (c) 2009-2012,2016-2017 Microsoft Corp.
	* Copyright (c) 2012 NetApp Inc.
	* Copyright (c) 2012 Citrix Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice unmodified, this list of conditions, and the following
	* disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	/*
	* VM Bus Driver Implementation
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	+#include <sys/linker.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/smp.h>
	#include <sys/sysctl.h>
	#include <sys/systm.h>
	#include <sys/taskqueue.h>

	#include <machine/bus.h>
	#include <machine/intr_machdep.h>
	+#include <machine/metadata.h>
	#include <machine/md_var.h>
	#include <machine/resource.h>
	#include <x86/include/apicvar.h>

	#include <contrib/dev/acpica/include/acpi.h>
	#include <dev/acpica/acpivar.h>

	#include <dev/hyperv/include/hyperv.h>
	#include <dev/hyperv/include/vmbus_xact.h>
	#include <dev/hyperv/vmbus/hyperv_reg.h>
	#include <dev/hyperv/vmbus/hyperv_var.h>
	#include <dev/hyperv/vmbus/vmbus_reg.h>
	#include <dev/hyperv/vmbus/vmbus_var.h>
	#include <dev/hyperv/vmbus/vmbus_chanvar.h>

	#include "acpi_if.h"
	#include "pcib_if.h"
	#include "vmbus_if.h"

	#define VMBUS_GPADL_START 0xe1e10

	struct vmbus_msghc {
	struct vmbus_xact *mh_xact;
	struct hypercall_postmsg_in mh_inprm_save;
	};

	static void vmbus_identify(driver_t *, device_t);
	static int vmbus_probe(device_t);
	static int vmbus_attach(device_t);
	static int vmbus_detach(device_t);
	static int vmbus_read_ivar(device_t, device_t, int,
	uintptr_t *);
	static int vmbus_child_pnpinfo_str(device_t, device_t,
	char *, size_t);
	static struct resource *vmbus_alloc_resource(device_t dev,
	device_t child, int type, int *rid,
	rman_res_t start, rman_res_t end,
	rman_res_t count, u_int flags);
	static int vmbus_alloc_msi(device_t bus, device_t dev,
	int count, int maxcount, int *irqs);
	static int vmbus_release_msi(device_t bus, device_t dev,
	int count, int *irqs);
	static int vmbus_alloc_msix(device_t bus, device_t dev,
	int *irq);
	static int vmbus_release_msix(device_t bus, device_t dev,
	int irq);
	static int vmbus_map_msi(device_t bus, device_t dev,
	int irq, uint64_t addr, uint32_t data);
	static uint32_t vmbus_get_version_method(device_t, device_t);
	static int vmbus_probe_guid_method(device_t, device_t,
	const struct hyperv_guid *);
	static uint32_t vmbus_get_vcpu_id_method(device_t bus,
	device_t dev, int cpu);
	static struct taskqueue *vmbus_get_eventtq_method(device_t, device_t,
	int);
	#ifdef EARLY_AP_STARTUP
	static void vmbus_intrhook(void *);
	#endif

	static int vmbus_init(struct vmbus_softc *);
	static int vmbus_connect(struct vmbus_softc *, uint32_t);
	static int vmbus_req_channels(struct vmbus_softc *sc);
	static void vmbus_disconnect(struct vmbus_softc *);
	static int vmbus_scan(struct vmbus_softc *);
	static void vmbus_scan_teardown(struct vmbus_softc *);
	static void vmbus_scan_done(struct vmbus_softc *,
	const struct vmbus_message *);
	static void vmbus_chanmsg_handle(struct vmbus_softc *,
	const struct vmbus_message *);
	static void vmbus_msg_task(void *, int);
	static void vmbus_synic_setup(void *);
	static void vmbus_synic_teardown(void *);
	static int vmbus_sysctl_version(SYSCTL_HANDLER_ARGS);
	static int vmbus_dma_alloc(struct vmbus_softc *);
	static void vmbus_dma_free(struct vmbus_softc *);
	static int vmbus_intr_setup(struct vmbus_softc *);
	static void vmbus_intr_teardown(struct vmbus_softc *);
	static int vmbus_doattach(struct vmbus_softc *);
	static void vmbus_event_proc_dummy(struct vmbus_softc *,
	int);

	static struct vmbus_softc *vmbus_sc;

	SYSCTL_NODE(_hw, OID_AUTO, vmbus, CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"Hyper-V vmbus");

	static int vmbus_pin_evttask = 1;
	SYSCTL_INT(_hw_vmbus, OID_AUTO, pin_evttask, CTLFLAG_RDTUN,
	&vmbus_pin_evttask, 0, "Pin event tasks to their respective CPU");

	extern inthand_t IDTVEC(vmbus_isr), IDTVEC(vmbus_isr_pti);

	uint32_t vmbus_current_version;

	static const uint32_t vmbus_version[] = {
	VMBUS_VERSION_WIN10,
	VMBUS_VERSION_WIN8_1,
	VMBUS_VERSION_WIN8,
	VMBUS_VERSION_WIN7,
	VMBUS_VERSION_WS2008
	};

	static const vmbus_chanmsg_proc_t
	vmbus_chanmsg_handlers[VMBUS_CHANMSG_TYPE_MAX] = {
	VMBUS_CHANMSG_PROC(CHOFFER_DONE, vmbus_scan_done),
	VMBUS_CHANMSG_PROC_WAKEUP(CONNECT_RESP)
	};

	static device_method_t vmbus_methods[] = {
	/* Device interface */
	DEVMETHOD(device_identify, vmbus_identify),
	DEVMETHOD(device_probe, vmbus_probe),
	DEVMETHOD(device_attach, vmbus_attach),
	DEVMETHOD(device_detach, vmbus_detach),
	DEVMETHOD(device_shutdown, bus_generic_shutdown),
	DEVMETHOD(device_suspend, bus_generic_suspend),
	DEVMETHOD(device_resume, bus_generic_resume),

	/* Bus interface */
	DEVMETHOD(bus_add_child, bus_generic_add_child),
	DEVMETHOD(bus_print_child, bus_generic_print_child),
	DEVMETHOD(bus_read_ivar, vmbus_read_ivar),
	DEVMETHOD(bus_child_pnpinfo_str, vmbus_child_pnpinfo_str),
	DEVMETHOD(bus_alloc_resource, vmbus_alloc_resource),
	DEVMETHOD(bus_release_resource, bus_generic_release_resource),
	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
	DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
	DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
	#if __FreeBSD_version >= 1100000
	DEVMETHOD(bus_get_cpus, bus_generic_get_cpus),
	#endif

	/* pcib interface */
	DEVMETHOD(pcib_alloc_msi, vmbus_alloc_msi),
	DEVMETHOD(pcib_release_msi, vmbus_release_msi),
	DEVMETHOD(pcib_alloc_msix, vmbus_alloc_msix),
	DEVMETHOD(pcib_release_msix, vmbus_release_msix),
	DEVMETHOD(pcib_map_msi, vmbus_map_msi),

	/* Vmbus interface */
	DEVMETHOD(vmbus_get_version, vmbus_get_version_method),
	DEVMETHOD(vmbus_probe_guid, vmbus_probe_guid_method),
	DEVMETHOD(vmbus_get_vcpu_id, vmbus_get_vcpu_id_method),
	DEVMETHOD(vmbus_get_event_taskq, vmbus_get_eventtq_method),

	DEVMETHOD_END
	};

	static driver_t vmbus_driver = {
	"vmbus",
	vmbus_methods,
	sizeof(struct vmbus_softc)
	};

	static devclass_t vmbus_devclass;

	DRIVER_MODULE(vmbus, pcib, vmbus_driver, vmbus_devclass, NULL, NULL);
	DRIVER_MODULE(vmbus, acpi_syscontainer, vmbus_driver, vmbus_devclass,
	NULL, NULL);

	MODULE_DEPEND(vmbus, acpi, 1, 1, 1);
	MODULE_DEPEND(vmbus, pci, 1, 1, 1);
	MODULE_VERSION(vmbus, 1);

	static __inline struct vmbus_softc *
	vmbus_get_softc(void)
	{
	return vmbus_sc;
	}

	void
	vmbus_msghc_reset(struct vmbus_msghc *mh, size_t dsize)
	{
	struct hypercall_postmsg_in *inprm;

	if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX)
	panic("invalid data size %zu", dsize);

	inprm = vmbus_xact_req_data(mh->mh_xact);
	memset(inprm, 0, HYPERCALL_POSTMSGIN_SIZE);
	inprm->hc_connid = VMBUS_CONNID_MESSAGE;
	inprm->hc_msgtype = HYPERV_MSGTYPE_CHANNEL;
	inprm->hc_dsize = dsize;
	}

	struct vmbus_msghc *
	vmbus_msghc_get(struct vmbus_softc *sc, size_t dsize)
	{
	struct vmbus_msghc *mh;
	struct vmbus_xact *xact;

	if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX)
	panic("invalid data size %zu", dsize);

	xact = vmbus_xact_get(sc->vmbus_xc,
	dsize + __offsetof(struct hypercall_postmsg_in, hc_data[0]));
	if (xact == NULL)
	return (NULL);

	mh = vmbus_xact_priv(xact, sizeof(*mh));
	mh->mh_xact = xact;

	vmbus_msghc_reset(mh, dsize);
	return (mh);
	}

	void
	vmbus_msghc_put(struct vmbus_softc sc __unused, struct vmbus_msghc mh)
	{

	vmbus_xact_put(mh->mh_xact);
	}

	void *
	vmbus_msghc_dataptr(struct vmbus_msghc *mh)
	{
	struct hypercall_postmsg_in *inprm;

	inprm = vmbus_xact_req_data(mh->mh_xact);
	return (inprm->hc_data);
	}

	int
	vmbus_msghc_exec_noresult(struct vmbus_msghc *mh)
	{
	sbintime_t time = SBT_1MS;
	struct hypercall_postmsg_in *inprm;
	bus_addr_t inprm_paddr;
	int i;

	inprm = vmbus_xact_req_data(mh->mh_xact);
	inprm_paddr = vmbus_xact_req_paddr(mh->mh_xact);

	/*
	* Save the input parameter so that we could restore the input
	* parameter if the Hypercall failed.
	*
	* XXX
	* Is this really necessary?! i.e. Will the Hypercall ever
	* overwrite the input parameter?
	*/
	memcpy(&mh->mh_inprm_save, inprm, HYPERCALL_POSTMSGIN_SIZE);

	/*
	* In order to cope with transient failures, e.g. insufficient
	* resources on host side, we retry the post message Hypercall
	* several times. 20 retries seem sufficient.
	*/
	#define HC_RETRY_MAX 20

	for (i = 0; i < HC_RETRY_MAX; ++i) {
	uint64_t status;

	status = hypercall_post_message(inprm_paddr);
	if (status == HYPERCALL_STATUS_SUCCESS)
	return 0;

	pause_sbt("hcpmsg", time, 0, C_HARDCLOCK);
	if (time < SBT_1S * 2)
	time *= 2;

	/* Restore input parameter and try again */
	memcpy(inprm, &mh->mh_inprm_save, HYPERCALL_POSTMSGIN_SIZE);
	}

	#undef HC_RETRY_MAX

	return EIO;
	}

	int
	vmbus_msghc_exec(struct vmbus_softc sc __unused, struct vmbus_msghc mh)
	{
	int error;

	vmbus_xact_activate(mh->mh_xact);
	error = vmbus_msghc_exec_noresult(mh);
	if (error)
	vmbus_xact_deactivate(mh->mh_xact);
	return error;
	}

	void
	vmbus_msghc_exec_cancel(struct vmbus_softc sc __unused, struct vmbus_msghc mh)
	{

	vmbus_xact_deactivate(mh->mh_xact);
	}

	const struct vmbus_message *
	vmbus_msghc_wait_result(struct vmbus_softc sc __unused, struct vmbus_msghc mh)
	{
	size_t resp_len;

	return (vmbus_xact_wait(mh->mh_xact, &resp_len));
	}

	const struct vmbus_message *
	vmbus_msghc_poll_result(struct vmbus_softc sc __unused, struct vmbus_msghc mh)
	{
	size_t resp_len;

	return (vmbus_xact_poll(mh->mh_xact, &resp_len));
	}

	void
	vmbus_msghc_wakeup(struct vmbus_softc sc, const struct vmbus_message msg)
	{

	vmbus_xact_ctx_wakeup(sc->vmbus_xc, msg, sizeof(*msg));
	}

	uint32_t
	vmbus_gpadl_alloc(struct vmbus_softc *sc)
	{
	uint32_t gpadl;

	again:
	gpadl = atomic_fetchadd_int(&sc->vmbus_gpadl, 1);
	if (gpadl == 0)
	goto again;
	return (gpadl);
	}

	/* Used for Hyper-V socket when guest client connects to host */
	int
	vmbus_req_tl_connect(struct hyperv_guid *guest_srv_id,
	struct hyperv_guid *host_srv_id)
	{
	struct vmbus_softc *sc = vmbus_get_softc();
	struct vmbus_chanmsg_tl_connect *req;
	struct vmbus_msghc *mh;
	int error;

	if (!sc)
	return ENXIO;

	mh = vmbus_msghc_get(sc, sizeof(*req));
	if (mh == NULL) {
	device_printf(sc->vmbus_dev,
	"can not get msg hypercall for tl connect\n");
	return ENXIO;
	}

	req = vmbus_msghc_dataptr(mh);
	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_TL_CONN;
	req->guest_endpoint_id = *guest_srv_id;
	req->host_service_id = *host_srv_id;

	error = vmbus_msghc_exec_noresult(mh);
	vmbus_msghc_put(sc, mh);

	if (error) {
	device_printf(sc->vmbus_dev,
	"tl connect msg hypercall failed\n");
	}

	return error;
	}

	static int
	vmbus_connect(struct vmbus_softc *sc, uint32_t version)
	{
	struct vmbus_chanmsg_connect *req;
	const struct vmbus_message *msg;
	struct vmbus_msghc *mh;
	int error, done = 0;

	mh = vmbus_msghc_get(sc, sizeof(*req));
	if (mh == NULL)
	return ENXIO;

	req = vmbus_msghc_dataptr(mh);
	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CONNECT;
	req->chm_ver = version;
	req->chm_evtflags = sc->vmbus_evtflags_dma.hv_paddr;
	req->chm_mnf1 = sc->vmbus_mnf1_dma.hv_paddr;
	req->chm_mnf2 = sc->vmbus_mnf2_dma.hv_paddr;

	error = vmbus_msghc_exec(sc, mh);
	if (error) {
	vmbus_msghc_put(sc, mh);
	return error;
	}

	msg = vmbus_msghc_wait_result(sc, mh);
	done = ((const struct vmbus_chanmsg_connect_resp *)
	msg->msg_data)->chm_done;

	vmbus_msghc_put(sc, mh);

	return (done ? 0 : EOPNOTSUPP);
	}

	static int
	vmbus_init(struct vmbus_softc *sc)
	{
	int i;

	for (i = 0; i < nitems(vmbus_version); ++i) {
	int error;

	error = vmbus_connect(sc, vmbus_version[i]);
	if (!error) {
	vmbus_current_version = vmbus_version[i];
	sc->vmbus_version = vmbus_version[i];
	device_printf(sc->vmbus_dev, "version %u.%u\n",
	VMBUS_VERSION_MAJOR(sc->vmbus_version),
	VMBUS_VERSION_MINOR(sc->vmbus_version));
	return 0;
	}
	}
	return ENXIO;
	}

	static void
	vmbus_disconnect(struct vmbus_softc *sc)
	{
	struct vmbus_chanmsg_disconnect *req;
	struct vmbus_msghc *mh;
	int error;

	mh = vmbus_msghc_get(sc, sizeof(*req));
	if (mh == NULL) {
	device_printf(sc->vmbus_dev,
	"can not get msg hypercall for disconnect\n");
	return;
	}

	req = vmbus_msghc_dataptr(mh);
	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_DISCONNECT;

	error = vmbus_msghc_exec_noresult(mh);
	vmbus_msghc_put(sc, mh);

	if (error) {
	device_printf(sc->vmbus_dev,
	"disconnect msg hypercall failed\n");
	}
	}

	static int
	vmbus_req_channels(struct vmbus_softc *sc)
	{
	struct vmbus_chanmsg_chrequest *req;
	struct vmbus_msghc *mh;
	int error;

	mh = vmbus_msghc_get(sc, sizeof(*req));
	if (mh == NULL)
	return ENXIO;

	req = vmbus_msghc_dataptr(mh);
	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHREQUEST;

	error = vmbus_msghc_exec_noresult(mh);
	vmbus_msghc_put(sc, mh);

	return error;
	}

	static void
	vmbus_scan_done_task(void *xsc, int pending __unused)
	{
	struct vmbus_softc *sc = xsc;

	mtx_lock(&Giant);
	sc->vmbus_scandone = true;
	mtx_unlock(&Giant);
	wakeup(&sc->vmbus_scandone);
	}

	static void
	vmbus_scan_done(struct vmbus_softc *sc,
	const struct vmbus_message *msg __unused)
	{

	taskqueue_enqueue(sc->vmbus_devtq, &sc->vmbus_scandone_task);
	}

	static int
	vmbus_scan(struct vmbus_softc *sc)
	{
	int error;

	/*
	* Identify, probe and attach for non-channel devices.
	*/
	bus_generic_probe(sc->vmbus_dev);
	bus_generic_attach(sc->vmbus_dev);

	/*
	* This taskqueue serializes vmbus devices' attach and detach
	* for channel offer and rescind messages.
	*/
	sc->vmbus_devtq = taskqueue_create("vmbus dev", M_WAITOK,
	taskqueue_thread_enqueue, &sc->vmbus_devtq);
	taskqueue_start_threads(&sc->vmbus_devtq, 1, PI_NET, "vmbusdev");
	TASK_INIT(&sc->vmbus_scandone_task, 0, vmbus_scan_done_task, sc);

	/*
	* This taskqueue handles sub-channel detach, so that vmbus
	* device's detach running in vmbus_devtq can drain its sub-
	* channels.
	*/
	sc->vmbus_subchtq = taskqueue_create("vmbus subch", M_WAITOK,
	taskqueue_thread_enqueue, &sc->vmbus_subchtq);
	taskqueue_start_threads(&sc->vmbus_subchtq, 1, PI_NET, "vmbussch");

	/*
	* Start vmbus scanning.
	*/
	error = vmbus_req_channels(sc);
	if (error) {
	device_printf(sc->vmbus_dev, "channel request failed: %d\n",
	error);
	return (error);
	}

	/*
	* Wait for all vmbus devices from the initial channel offers to be
	* attached.
	*/
	GIANT_REQUIRED;
	while (!sc->vmbus_scandone)
	mtx_sleep(&sc->vmbus_scandone, &Giant, 0, "vmbusdev", 0);

	if (bootverbose) {
	device_printf(sc->vmbus_dev, "device scan, probe and attach "
	"done\n");
	}
	return (0);
	}

	static void
	vmbus_scan_teardown(struct vmbus_softc *sc)
	{

	GIANT_REQUIRED;
	if (sc->vmbus_devtq != NULL) {
	mtx_unlock(&Giant);
	taskqueue_free(sc->vmbus_devtq);
	mtx_lock(&Giant);
	sc->vmbus_devtq = NULL;
	}
	if (sc->vmbus_subchtq != NULL) {
	mtx_unlock(&Giant);
	taskqueue_free(sc->vmbus_subchtq);
	mtx_lock(&Giant);
	sc->vmbus_subchtq = NULL;
	}
	}

	static void
	vmbus_chanmsg_handle(struct vmbus_softc sc, const struct vmbus_message msg)
	{
	vmbus_chanmsg_proc_t msg_proc;
	uint32_t msg_type;

	msg_type = ((const struct vmbus_chanmsg_hdr *)msg->msg_data)->chm_type;
	if (msg_type >= VMBUS_CHANMSG_TYPE_MAX) {
	device_printf(sc->vmbus_dev, "unknown message type 0x%x\n",
	msg_type);
	return;
	}

	msg_proc = vmbus_chanmsg_handlers[msg_type];
	if (msg_proc != NULL)
	msg_proc(sc, msg);

	/* Channel specific processing */
	vmbus_chan_msgproc(sc, msg);
	}

	static void
	vmbus_msg_task(void *xsc, int pending __unused)
	{
	struct vmbus_softc *sc = xsc;
	volatile struct vmbus_message *msg;

	msg = VMBUS_PCPU_GET(sc, message, curcpu) + VMBUS_SINT_MESSAGE;
	for (;;) {
	if (msg->msg_type == HYPERV_MSGTYPE_NONE) {
	/* No message */
	break;
	} else if (msg->msg_type == HYPERV_MSGTYPE_CHANNEL) {
	/* Channel message */
	vmbus_chanmsg_handle(sc,
	__DEVOLATILE(const struct vmbus_message *, msg));
	}

	msg->msg_type = HYPERV_MSGTYPE_NONE;
	/*
	* Make sure the write to msg_type (i.e. set to
	* HYPERV_MSGTYPE_NONE) happens before we read the
	* msg_flags and EOMing. Otherwise, the EOMing will
	* not deliver any more messages since there is no
	* empty slot
	*
	* NOTE:
	* mb() is used here, since atomic_thread_fence_seq_cst()
	* will become compiler fence on UP kernel.
	*/
	mb();
	if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) {
	/*
	* This will cause message queue rescan to possibly
	* deliver another msg from the hypervisor
	*/
	wrmsr(MSR_HV_EOM, 0);
	}
	}
	}

	static __inline int
	vmbus_handle_intr1(struct vmbus_softc sc, struct trapframe frame, int cpu)
	{
	volatile struct vmbus_message *msg;
	struct vmbus_message *msg_base;

	msg_base = VMBUS_PCPU_GET(sc, message, cpu);

	/*
	* Check event timer.
	*
	* TODO: move this to independent IDT vector.
	*/
	msg = msg_base + VMBUS_SINT_TIMER;
	if (msg->msg_type == HYPERV_MSGTYPE_TIMER_EXPIRED) {
	msg->msg_type = HYPERV_MSGTYPE_NONE;

	vmbus_et_intr(frame);

	/*
	* Make sure the write to msg_type (i.e. set to
	* HYPERV_MSGTYPE_NONE) happens before we read the
	* msg_flags and EOMing. Otherwise, the EOMing will
	* not deliver any more messages since there is no
	* empty slot
	*
	* NOTE:
	* mb() is used here, since atomic_thread_fence_seq_cst()
	* will become compiler fence on UP kernel.
	*/
	mb();
	if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) {
	/*
	* This will cause message queue rescan to possibly
	* deliver another msg from the hypervisor
	*/
	wrmsr(MSR_HV_EOM, 0);
	}
	}

	/*
	* Check events. Hot path for network and storage I/O data; high rate.
	*
	* NOTE:
	* As recommended by the Windows guest fellows, we check events before
	* checking messages.
	*/
	sc->vmbus_event_proc(sc, cpu);

	/*
	* Check messages. Mainly management stuffs; ultra low rate.
	*/
	msg = msg_base + VMBUS_SINT_MESSAGE;
	if (__predict_false(msg->msg_type != HYPERV_MSGTYPE_NONE)) {
	taskqueue_enqueue(VMBUS_PCPU_GET(sc, message_tq, cpu),
	VMBUS_PCPU_PTR(sc, message_task, cpu));
	}

	return (FILTER_HANDLED);
	}

	void
	vmbus_handle_intr(struct trapframe *trap_frame)
	{
	struct vmbus_softc *sc = vmbus_get_softc();
	int cpu = curcpu;

	/*
	* Disable preemption.
	*/
	critical_enter();

	/*
	* Do a little interrupt counting.
	*/
	(*VMBUS_PCPU_GET(sc, intr_cnt, cpu))++;

	vmbus_handle_intr1(sc, trap_frame, cpu);

	/*
	* Enable preemption.
	*/
	critical_exit();
	}

	static void
	vmbus_synic_setup(void *xsc)
	{
	struct vmbus_softc *sc = xsc;
	int cpu = curcpu;
	uint64_t val, orig;
	uint32_t sint;

	if (hyperv_features & CPUID_HV_MSR_VP_INDEX) {
	/* Save virtual processor id. */
	VMBUS_PCPU_GET(sc, vcpuid, cpu) = rdmsr(MSR_HV_VP_INDEX);
	} else {
	/* Set virtual processor id to 0 for compatibility. */
	VMBUS_PCPU_GET(sc, vcpuid, cpu) = 0;
	}

	/*
	* Setup the SynIC message.
	*/
	orig = rdmsr(MSR_HV_SIMP);
	val = MSR_HV_SIMP_ENABLE \| (orig & MSR_HV_SIMP_RSVD_MASK) \|
	((VMBUS_PCPU_GET(sc, message_dma.hv_paddr, cpu) >> PAGE_SHIFT) <<
	MSR_HV_SIMP_PGSHIFT);
	wrmsr(MSR_HV_SIMP, val);

	/*
	* Setup the SynIC event flags.
	*/
	orig = rdmsr(MSR_HV_SIEFP);
	val = MSR_HV_SIEFP_ENABLE \| (orig & MSR_HV_SIEFP_RSVD_MASK) \|
	((VMBUS_PCPU_GET(sc, event_flags_dma.hv_paddr, cpu)
	>> PAGE_SHIFT) << MSR_HV_SIEFP_PGSHIFT);
	wrmsr(MSR_HV_SIEFP, val);


	/*
	* Configure and unmask SINT for message and event flags.
	*/
	sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE;
	orig = rdmsr(sint);
	val = sc->vmbus_idtvec \| MSR_HV_SINT_AUTOEOI \|
	(orig & MSR_HV_SINT_RSVD_MASK);
	wrmsr(sint, val);

	/*
	* Configure and unmask SINT for timer.
	*/
	sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER;
	orig = rdmsr(sint);
	val = sc->vmbus_idtvec \| MSR_HV_SINT_AUTOEOI \|
	(orig & MSR_HV_SINT_RSVD_MASK);
	wrmsr(sint, val);

	/*
	* All done; enable SynIC.
	*/
	orig = rdmsr(MSR_HV_SCONTROL);
	val = MSR_HV_SCTRL_ENABLE \| (orig & MSR_HV_SCTRL_RSVD_MASK);
	wrmsr(MSR_HV_SCONTROL, val);
	}

	static void
	vmbus_synic_teardown(void *arg)
	{
	uint64_t orig;
	uint32_t sint;

	/*
	* Disable SynIC.
	*/
	orig = rdmsr(MSR_HV_SCONTROL);
	wrmsr(MSR_HV_SCONTROL, (orig & MSR_HV_SCTRL_RSVD_MASK));

	/*
	* Mask message and event flags SINT.
	*/
	sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE;
	orig = rdmsr(sint);
	wrmsr(sint, orig \| MSR_HV_SINT_MASKED);

	/*
	* Mask timer SINT.
	*/
	sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER;
	orig = rdmsr(sint);
	wrmsr(sint, orig \| MSR_HV_SINT_MASKED);

	/*
	* Teardown SynIC message.
	*/
	orig = rdmsr(MSR_HV_SIMP);
	wrmsr(MSR_HV_SIMP, (orig & MSR_HV_SIMP_RSVD_MASK));

	/*
	* Teardown SynIC event flags.
	*/
	orig = rdmsr(MSR_HV_SIEFP);
	wrmsr(MSR_HV_SIEFP, (orig & MSR_HV_SIEFP_RSVD_MASK));
	}

	static int
	vmbus_dma_alloc(struct vmbus_softc *sc)
	{
	bus_dma_tag_t parent_dtag;
	uint8_t *evtflags;
	int cpu;

	parent_dtag = bus_get_dma_tag(sc->vmbus_dev);
	CPU_FOREACH(cpu) {
	void *ptr;

	/*
	* Per-cpu messages and event flags.
	*/
	ptr = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	PAGE_SIZE, VMBUS_PCPU_PTR(sc, message_dma, cpu),
	BUS_DMA_WAITOK \| BUS_DMA_ZERO);
	if (ptr == NULL)
	return ENOMEM;
	VMBUS_PCPU_GET(sc, message, cpu) = ptr;

	ptr = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	PAGE_SIZE, VMBUS_PCPU_PTR(sc, event_flags_dma, cpu),
	BUS_DMA_WAITOK \| BUS_DMA_ZERO);
	if (ptr == NULL)
	return ENOMEM;
	VMBUS_PCPU_GET(sc, event_flags, cpu) = ptr;
	}

	evtflags = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	PAGE_SIZE, &sc->vmbus_evtflags_dma, BUS_DMA_WAITOK \| BUS_DMA_ZERO);
	if (evtflags == NULL)
	return ENOMEM;
	sc->vmbus_rx_evtflags = (u_long *)evtflags;
	sc->vmbus_tx_evtflags = (u_long *)(evtflags + (PAGE_SIZE / 2));
	sc->vmbus_evtflags = evtflags;

	sc->vmbus_mnf1 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	PAGE_SIZE, &sc->vmbus_mnf1_dma, BUS_DMA_WAITOK \| BUS_DMA_ZERO);
	if (sc->vmbus_mnf1 == NULL)
	return ENOMEM;

	sc->vmbus_mnf2 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	sizeof(struct vmbus_mnf), &sc->vmbus_mnf2_dma,
	BUS_DMA_WAITOK \| BUS_DMA_ZERO);
	if (sc->vmbus_mnf2 == NULL)
	return ENOMEM;

	return 0;
	}

	static void
	vmbus_dma_free(struct vmbus_softc *sc)
	{
	int cpu;

	if (sc->vmbus_evtflags != NULL) {
	hyperv_dmamem_free(&sc->vmbus_evtflags_dma, sc->vmbus_evtflags);
	sc->vmbus_evtflags = NULL;
	sc->vmbus_rx_evtflags = NULL;
	sc->vmbus_tx_evtflags = NULL;
	}
	if (sc->vmbus_mnf1 != NULL) {
	hyperv_dmamem_free(&sc->vmbus_mnf1_dma, sc->vmbus_mnf1);
	sc->vmbus_mnf1 = NULL;
	}
	if (sc->vmbus_mnf2 != NULL) {
	hyperv_dmamem_free(&sc->vmbus_mnf2_dma, sc->vmbus_mnf2);
	sc->vmbus_mnf2 = NULL;
	}

	CPU_FOREACH(cpu) {
	if (VMBUS_PCPU_GET(sc, message, cpu) != NULL) {
	hyperv_dmamem_free(
	VMBUS_PCPU_PTR(sc, message_dma, cpu),
	VMBUS_PCPU_GET(sc, message, cpu));
	VMBUS_PCPU_GET(sc, message, cpu) = NULL;
	}
	if (VMBUS_PCPU_GET(sc, event_flags, cpu) != NULL) {
	hyperv_dmamem_free(
	VMBUS_PCPU_PTR(sc, event_flags_dma, cpu),
	VMBUS_PCPU_GET(sc, event_flags, cpu));
	VMBUS_PCPU_GET(sc, event_flags, cpu) = NULL;
	}
	}
	}

	static int
	vmbus_intr_setup(struct vmbus_softc *sc)
	{
	int cpu;

	CPU_FOREACH(cpu) {
	char buf[MAXCOMLEN + 1];
	cpuset_t cpu_mask;

	/* Allocate an interrupt counter for Hyper-V interrupt */
	snprintf(buf, sizeof(buf), "cpu%d:hyperv", cpu);
	intrcnt_add(buf, VMBUS_PCPU_PTR(sc, intr_cnt, cpu));

	/*
	* Setup taskqueue to handle events. Task will be per-
	* channel.
	*/
	VMBUS_PCPU_GET(sc, event_tq, cpu) = taskqueue_create_fast(
	"hyperv event", M_WAITOK, taskqueue_thread_enqueue,
	VMBUS_PCPU_PTR(sc, event_tq, cpu));
	if (vmbus_pin_evttask) {
	CPU_SETOF(cpu, &cpu_mask);
	taskqueue_start_threads_cpuset(
	VMBUS_PCPU_PTR(sc, event_tq, cpu), 1, PI_NET,
	&cpu_mask, "hvevent%d", cpu);
	} else {
	taskqueue_start_threads(
	VMBUS_PCPU_PTR(sc, event_tq, cpu), 1, PI_NET,
	"hvevent%d", cpu);
	}

	/*
	* Setup tasks and taskqueues to handle messages.
	*/
	VMBUS_PCPU_GET(sc, message_tq, cpu) = taskqueue_create_fast(
	"hyperv msg", M_WAITOK, taskqueue_thread_enqueue,
	VMBUS_PCPU_PTR(sc, message_tq, cpu));
	CPU_SETOF(cpu, &cpu_mask);
	taskqueue_start_threads_cpuset(
	VMBUS_PCPU_PTR(sc, message_tq, cpu), 1, PI_NET, &cpu_mask,
	"hvmsg%d", cpu);
	TASK_INIT(VMBUS_PCPU_PTR(sc, message_task, cpu), 0,
	vmbus_msg_task, sc);
	}

	/*
	* All Hyper-V ISR required resources are setup, now let's find a
	* free IDT vector for Hyper-V ISR and set it up.
	*/
	sc->vmbus_idtvec = lapic_ipi_alloc(pti ? IDTVEC(vmbus_isr_pti) :
	IDTVEC(vmbus_isr));
	if (sc->vmbus_idtvec < 0) {
	device_printf(sc->vmbus_dev, "cannot find free IDT vector\n");
	return ENXIO;
	}
	if (bootverbose) {
	device_printf(sc->vmbus_dev, "vmbus IDT vector %d\n",
	sc->vmbus_idtvec);
	}
	return 0;
	}

	static void
	vmbus_intr_teardown(struct vmbus_softc *sc)
	{
	int cpu;

	if (sc->vmbus_idtvec >= 0) {
	lapic_ipi_free(sc->vmbus_idtvec);
	sc->vmbus_idtvec = -1;
	}

	CPU_FOREACH(cpu) {
	if (VMBUS_PCPU_GET(sc, event_tq, cpu) != NULL) {
	taskqueue_free(VMBUS_PCPU_GET(sc, event_tq, cpu));
	VMBUS_PCPU_GET(sc, event_tq, cpu) = NULL;
	}
	if (VMBUS_PCPU_GET(sc, message_tq, cpu) != NULL) {
	taskqueue_drain(VMBUS_PCPU_GET(sc, message_tq, cpu),
	VMBUS_PCPU_PTR(sc, message_task, cpu));
	taskqueue_free(VMBUS_PCPU_GET(sc, message_tq, cpu));
	VMBUS_PCPU_GET(sc, message_tq, cpu) = NULL;
	}
	}
	}

	static int
	vmbus_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
	{
	return (ENOENT);
	}

	static int
	vmbus_child_pnpinfo_str(device_t dev, device_t child, char *buf, size_t buflen)
	{
	const struct vmbus_channel *chan;
	char guidbuf[HYPERV_GUID_STRLEN];

	chan = vmbus_get_channel(child);
	if (chan == NULL) {
	/* Event timer device, which does not belong to a channel */
	return (0);
	}

	strlcat(buf, "classid=", buflen);
	hyperv_guid2str(&chan->ch_guid_type, guidbuf, sizeof(guidbuf));
	strlcat(buf, guidbuf, buflen);

	strlcat(buf, " deviceid=", buflen);
	hyperv_guid2str(&chan->ch_guid_inst, guidbuf, sizeof(guidbuf));
	strlcat(buf, guidbuf, buflen);

	return (0);
	}

	int
	vmbus_add_child(struct vmbus_channel *chan)
	{
	struct vmbus_softc *sc = chan->ch_vmbus;
	device_t parent = sc->vmbus_dev;

	mtx_lock(&Giant);

	chan->ch_dev = device_add_child(parent, NULL, -1);
	if (chan->ch_dev == NULL) {
	mtx_unlock(&Giant);
	device_printf(parent, "device_add_child for chan%u failed\n",
	chan->ch_id);
	return (ENXIO);
	}
	device_set_ivars(chan->ch_dev, chan);
	device_probe_and_attach(chan->ch_dev);

	mtx_unlock(&Giant);
	return (0);
	}

	int
	vmbus_delete_child(struct vmbus_channel *chan)
	{
	int error = 0;

	mtx_lock(&Giant);
	if (chan->ch_dev != NULL) {
	error = device_delete_child(chan->ch_vmbus->vmbus_dev,
	chan->ch_dev);
	chan->ch_dev = NULL;
	}
	mtx_unlock(&Giant);
	return (error);
	}

	static int
	vmbus_sysctl_version(SYSCTL_HANDLER_ARGS)
	{
	struct vmbus_softc *sc = arg1;
	char verstr[16];

	snprintf(verstr, sizeof(verstr), "%u.%u",
	VMBUS_VERSION_MAJOR(sc->vmbus_version),
	VMBUS_VERSION_MINOR(sc->vmbus_version));
	return sysctl_handle_string(oidp, verstr, sizeof(verstr), req);
	}

	/*
	* We need the function to make sure the MMIO resource is allocated from the
	* ranges found in _CRS.
	*
	* For the release function, we can use bus_generic_release_resource().
	*/
	static struct resource *
	vmbus_alloc_resource(device_t dev, device_t child, int type, int *rid,
	rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
	{
	device_t parent = device_get_parent(dev);
	struct resource *res;

	#ifdef NEW_PCIB
	if (type == SYS_RES_MEMORY) {
	struct vmbus_softc *sc = device_get_softc(dev);

	res = pcib_host_res_alloc(&sc->vmbus_mmio_res, child, type,
	rid, start, end, count, flags);
	} else
	#endif
	{
	res = BUS_ALLOC_RESOURCE(parent, child, type, rid, start,
	end, count, flags);
	}

	return (res);
	}

	static int
	vmbus_alloc_msi(device_t bus, device_t dev, int count, int maxcount, int *irqs)
	{

	return (PCIB_ALLOC_MSI(device_get_parent(bus), dev, count, maxcount,
	irqs));
	}

	static int
	vmbus_release_msi(device_t bus, device_t dev, int count, int *irqs)
	{

	return (PCIB_RELEASE_MSI(device_get_parent(bus), dev, count, irqs));
	}

	static int
	vmbus_alloc_msix(device_t bus, device_t dev, int *irq)
	{

	return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
	}

	static int
	vmbus_release_msix(device_t bus, device_t dev, int irq)
	{

	return (PCIB_RELEASE_MSIX(device_get_parent(bus), dev, irq));
	}

	static int
	vmbus_map_msi(device_t bus, device_t dev, int irq, uint64_t *addr,
	uint32_t *data)
	{

	return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
	}

	static uint32_t
	vmbus_get_version_method(device_t bus, device_t dev)
	{
	struct vmbus_softc *sc = device_get_softc(bus);

	return sc->vmbus_version;
	}

	static int
	vmbus_probe_guid_method(device_t bus, device_t dev,
	const struct hyperv_guid *guid)
	{
	const struct vmbus_channel *chan = vmbus_get_channel(dev);

	if (memcmp(&chan->ch_guid_type, guid, sizeof(struct hyperv_guid)) == 0)
	return 0;
	return ENXIO;
	}

	static uint32_t
	vmbus_get_vcpu_id_method(device_t bus, device_t dev, int cpu)
	{
	const struct vmbus_softc *sc = device_get_softc(bus);

	return (VMBUS_PCPU_GET(sc, vcpuid, cpu));
	}

	static struct taskqueue *
	vmbus_get_eventtq_method(device_t bus, device_t dev __unused, int cpu)
	{
	const struct vmbus_softc *sc = device_get_softc(bus);

	KASSERT(cpu >= 0 && cpu < mp_ncpus, ("invalid cpu%d", cpu));
	return (VMBUS_PCPU_GET(sc, event_tq, cpu));
	}

	#ifdef NEW_PCIB
	#define VTPM_BASE_ADDR 0xfed40000
	#define FOUR_GB (1ULL << 32)

	enum parse_pass { parse_64, parse_32 };

	struct parse_context {
	device_t vmbus_dev;
	enum parse_pass pass;
	};

	static ACPI_STATUS
	parse_crs(ACPI_RESOURCE res, void ctx)
	{
	const struct parse_context *pc = ctx;
	device_t vmbus_dev = pc->vmbus_dev;

	struct vmbus_softc *sc = device_get_softc(vmbus_dev);
	UINT64 start, end;

	switch (res->Type) {
	case ACPI_RESOURCE_TYPE_ADDRESS32:
	start = res->Data.Address32.Address.Minimum;
	end = res->Data.Address32.Address.Maximum;
	break;

	case ACPI_RESOURCE_TYPE_ADDRESS64:
	start = res->Data.Address64.Address.Minimum;
	end = res->Data.Address64.Address.Maximum;
	break;

	default:
	/* Unused types. */
	return (AE_OK);
	}

	/*
	* We don't use <1MB addresses.
	*/
	if (end < 0x100000)
	return (AE_OK);

	/* Don't conflict with vTPM. */
	if (end >= VTPM_BASE_ADDR && start < VTPM_BASE_ADDR)
	end = VTPM_BASE_ADDR - 1;

	if ((pc->pass == parse_32 && start < FOUR_GB) \|\|
	(pc->pass == parse_64 && start >= FOUR_GB))
	pcib_host_res_decodes(&sc->vmbus_mmio_res, SYS_RES_MEMORY,
	start, end, 0);

	return (AE_OK);
	}

	static void
	vmbus_get_crs(device_t dev, device_t vmbus_dev, enum parse_pass pass)
	{
	struct parse_context pc;
	ACPI_STATUS status;

	if (bootverbose)
	device_printf(dev, "walking _CRS, pass=%d\n", pass);

	pc.vmbus_dev = vmbus_dev;
	pc.pass = pass;
	status = AcpiWalkResources(acpi_get_handle(dev), "_CRS",
	parse_crs, &pc);

	if (bootverbose && ACPI_FAILURE(status))
	device_printf(dev, "_CRS: not found, pass=%d\n", pass);
	}

	static void
	vmbus_get_mmio_res_pass(device_t dev, enum parse_pass pass)
	{
	device_t acpi0, parent;

	parent = device_get_parent(dev);

	acpi0 = device_get_parent(parent);
	if (strcmp("acpi0", device_get_nameunit(acpi0)) == 0) {
	device_t *children;
	int count;

	/*
	* Try to locate VMBUS resources and find _CRS on them.
	*/
	if (device_get_children(acpi0, &children, &count) == 0) {
	int i;

	for (i = 0; i < count; ++i) {
	if (!device_is_attached(children[i]))
	continue;

	if (strcmp("vmbus_res",
	device_get_name(children[i])) == 0)
	vmbus_get_crs(children[i], dev, pass);
	}
	free(children, M_TEMP);
	}

	/*
	* Try to find _CRS on acpi.
	*/
	vmbus_get_crs(acpi0, dev, pass);
	} else {
	device_printf(dev, "not grandchild of acpi\n");
	}

	/*
	* Try to find _CRS on parent.
	*/
	vmbus_get_crs(parent, dev, pass);
	}

	static void
	vmbus_get_mmio_res(device_t dev)
	{
	struct vmbus_softc *sc = device_get_softc(dev);
	/*
	* We walk the resources twice to make sure that: in the resource
	* list, the 32-bit resources appear behind the 64-bit resources.
	* NB: resource_list_add() uses INSERT_TAIL. This way, when we
	* iterate through the list to find a range for a 64-bit BAR in
	* vmbus_alloc_resource(), we can make sure we try to use >4GB
	* ranges first.
	*/
	pcib_host_res_init(dev, &sc->vmbus_mmio_res);

	vmbus_get_mmio_res_pass(dev, parse_64);
	vmbus_get_mmio_res_pass(dev, parse_32);
	}

	+/*
	+ * On Gen2 VMs, Hyper-V provides mmio space for framebuffer.
	+ * This mmio address range is not useable for other PCI devices.
	+ * Currently only efifb driver is using this range without reserving
	+ * it from system.
	+ * Therefore, vmbus driver reserves it before any other PCI device
	+ * drivers start to request mmio addresses.
	+ */
	+static struct resource *hv_fb_res;
	+
	+static void
	+vmbus_fb_mmio_res(device_t dev)
	+{
	+ struct efi_fb *efifb;
	+ caddr_t kmdp;
	+
	+ struct vmbus_softc *sc = device_get_softc(dev);
	+ int rid = 0;
	+
	+ kmdp = preload_search_by_type("elf kernel");
	+ if (kmdp == NULL)
	+ kmdp = preload_search_by_type("elf64 kernel");
	+ efifb = (struct efi_fb *)preload_search_info(kmdp,
	+ MODINFO_METADATA \| MODINFOMD_EFI_FB);
	+ if (efifb == NULL) {
	+ if (bootverbose)
	+ device_printf(dev,
	+ "fb has no preloaded kernel efi information\n");
	+ /* We are on Gen1 VM, just return. */
	+ return;
	+ } else {
	+ if (bootverbose)
	+ device_printf(dev,
	+ "efifb: fb_addr: %#jx, size: %#jx, "
	+ "actual size needed: 0x%x\n",
	+ efifb->fb_addr, efifb->fb_size,
	+ (int) efifb->fb_height * efifb->fb_width);
	+ }
	+
	+ hv_fb_res = pcib_host_res_alloc(&sc->vmbus_mmio_res, dev,
	+ SYS_RES_MEMORY, &rid,
	+ efifb->fb_addr, efifb->fb_addr + efifb->fb_size, efifb->fb_size,
	+ RF_ACTIVE \| rman_make_alignment_flags(PAGE_SIZE));
	+
	+ if (hv_fb_res && bootverbose)
	+ device_printf(dev,
	+ "successfully reserved memory for framebuffer "
	+ "starting at %#jx, size %#jx\n",
	+ efifb->fb_addr, efifb->fb_size);
	+}
	+
	static void
	vmbus_free_mmio_res(device_t dev)
	{
	struct vmbus_softc *sc = device_get_softc(dev);

	pcib_host_res_free(dev, &sc->vmbus_mmio_res);
	+
	+ if (hv_fb_res)
	+ hv_fb_res = NULL;
	}
	#endif /* NEW_PCIB */

	static void
	vmbus_identify(driver_t *driver, device_t parent)
	{

	if (device_get_unit(parent) != 0 \|\| vm_guest != VM_GUEST_HV \|\|
	(hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
	return;
	device_add_child(parent, "vmbus", -1);
	}

	static int
	vmbus_probe(device_t dev)
	{

	if (device_get_unit(dev) != 0 \|\| vm_guest != VM_GUEST_HV \|\|
	(hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
	return (ENXIO);

	device_set_desc(dev, "Hyper-V Vmbus");
	return (BUS_PROBE_DEFAULT);
	}

	/**
	* @brief Main vmbus driver initialization routine.
	*
	* Here, we
	* - initialize the vmbus driver context
	* - setup various driver entry points
	* - invoke the vmbus hv main init routine
	* - get the irq resource
	* - invoke the vmbus to add the vmbus root device
	* - setup the vmbus root device
	* - retrieve the channel offers
	*/
	static int
	vmbus_doattach(struct vmbus_softc *sc)
	{
	struct sysctl_oid_list *child;
	struct sysctl_ctx_list *ctx;
	int ret;

	if (sc->vmbus_flags & VMBUS_FLAG_ATTACHED)
	return (0);

	#ifdef NEW_PCIB
	vmbus_get_mmio_res(sc->vmbus_dev);
	+ vmbus_fb_mmio_res(sc->vmbus_dev);
	#endif

	sc->vmbus_flags \|= VMBUS_FLAG_ATTACHED;

	sc->vmbus_gpadl = VMBUS_GPADL_START;
	mtx_init(&sc->vmbus_prichan_lock, "vmbus prichan", NULL, MTX_DEF);
	TAILQ_INIT(&sc->vmbus_prichans);
	mtx_init(&sc->vmbus_chan_lock, "vmbus channel", NULL, MTX_DEF);
	TAILQ_INIT(&sc->vmbus_chans);
	sc->vmbus_chmap = malloc(
	sizeof(struct vmbus_channel ) VMBUS_CHAN_MAX, M_DEVBUF,
	M_WAITOK \| M_ZERO);

	/*
	* Create context for "post message" Hypercalls
	*/
	sc->vmbus_xc = vmbus_xact_ctx_create(bus_get_dma_tag(sc->vmbus_dev),
	HYPERCALL_POSTMSGIN_SIZE, VMBUS_MSG_SIZE,
	sizeof(struct vmbus_msghc));
	if (sc->vmbus_xc == NULL) {
	ret = ENXIO;
	goto cleanup;
	}

	/*
	* Allocate DMA stuffs.
	*/
	ret = vmbus_dma_alloc(sc);
	if (ret != 0)
	goto cleanup;

	/*
	* Setup interrupt.
	*/
	ret = vmbus_intr_setup(sc);
	if (ret != 0)
	goto cleanup;

	/*
	* Setup SynIC.
	*/
	if (bootverbose)
	device_printf(sc->vmbus_dev, "smp_started = %d\n", smp_started);
	smp_rendezvous(NULL, vmbus_synic_setup, NULL, sc);
	sc->vmbus_flags \|= VMBUS_FLAG_SYNIC;

	/*
	* Initialize vmbus, e.g. connect to Hypervisor.
	*/
	ret = vmbus_init(sc);
	if (ret != 0)
	goto cleanup;

	if (sc->vmbus_version == VMBUS_VERSION_WS2008 \|\|
	sc->vmbus_version == VMBUS_VERSION_WIN7)
	sc->vmbus_event_proc = vmbus_event_proc_compat;
	else
	sc->vmbus_event_proc = vmbus_event_proc;

	ret = vmbus_scan(sc);
	if (ret != 0)
	goto cleanup;

	ctx = device_get_sysctl_ctx(sc->vmbus_dev);
	child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->vmbus_dev));
	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "version",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, sc, 0,
	vmbus_sysctl_version, "A", "vmbus version");

	return (ret);

	cleanup:
	vmbus_scan_teardown(sc);
	vmbus_intr_teardown(sc);
	vmbus_dma_free(sc);
	if (sc->vmbus_xc != NULL) {
	vmbus_xact_ctx_destroy(sc->vmbus_xc);
	sc->vmbus_xc = NULL;
	}
	free(__DEVOLATILE(void *, sc->vmbus_chmap), M_DEVBUF);
	mtx_destroy(&sc->vmbus_prichan_lock);
	mtx_destroy(&sc->vmbus_chan_lock);

	return (ret);
	}

	static void
	vmbus_event_proc_dummy(struct vmbus_softc *sc __unused, int cpu __unused)
	{
	}

	#ifdef EARLY_AP_STARTUP

	static void
	vmbus_intrhook(void *xsc)
	{
	struct vmbus_softc *sc = xsc;

	if (bootverbose)
	device_printf(sc->vmbus_dev, "intrhook\n");
	vmbus_doattach(sc);
	config_intrhook_disestablish(&sc->vmbus_intrhook);
	}

	#endif /* EARLY_AP_STARTUP */

	static int
	vmbus_attach(device_t dev)
	{
	vmbus_sc = device_get_softc(dev);
	vmbus_sc->vmbus_dev = dev;
	vmbus_sc->vmbus_idtvec = -1;

	/*
	* Event processing logic will be configured:
	* - After the vmbus protocol version negotiation.
	* - Before we request channel offers.
	*/
	vmbus_sc->vmbus_event_proc = vmbus_event_proc_dummy;

	#ifdef EARLY_AP_STARTUP
	/*
	* Defer the real attach until the pause(9) works as expected.
	*/
	vmbus_sc->vmbus_intrhook.ich_func = vmbus_intrhook;
	vmbus_sc->vmbus_intrhook.ich_arg = vmbus_sc;
	config_intrhook_establish(&vmbus_sc->vmbus_intrhook);
	#else /* !EARLY_AP_STARTUP */
	/*
	* If the system has already booted and thread
	* scheduling is possible indicated by the global
	* cold set to zero, we just call the driver
	* initialization directly.
	*/
	if (!cold)
	vmbus_doattach(vmbus_sc);
	#endif /* EARLY_AP_STARTUP */

	return (0);
	}

	static int
	vmbus_detach(device_t dev)
	{
	struct vmbus_softc *sc = device_get_softc(dev);

	bus_generic_detach(dev);
	vmbus_chan_destroy_all(sc);

	vmbus_scan_teardown(sc);

	vmbus_disconnect(sc);

	if (sc->vmbus_flags & VMBUS_FLAG_SYNIC) {
	sc->vmbus_flags &= ~VMBUS_FLAG_SYNIC;
	smp_rendezvous(NULL, vmbus_synic_teardown, NULL, NULL);
	}

	vmbus_intr_teardown(sc);
	vmbus_dma_free(sc);

	if (sc->vmbus_xc != NULL) {
	vmbus_xact_ctx_destroy(sc->vmbus_xc);
	sc->vmbus_xc = NULL;
	}

	free(__DEVOLATILE(void *, sc->vmbus_chmap), M_DEVBUF);
	mtx_destroy(&sc->vmbus_prichan_lock);
	mtx_destroy(&sc->vmbus_chan_lock);

	#ifdef NEW_PCIB
	vmbus_free_mmio_res(dev);
	#endif

	return (0);
	}

	#ifndef EARLY_AP_STARTUP

	static void
	vmbus_sysinit(void *arg __unused)
	{
	struct vmbus_softc *sc = vmbus_get_softc();

	if (vm_guest != VM_GUEST_HV \|\| sc == NULL)
	return;

	/*
	* If the system has already booted and thread
	* scheduling is possible, as indicated by the
	* global cold set to zero, we just call the driver
	* initialization directly.
	*/
	if (!cold)
	vmbus_doattach(sc);
	}
	/*
	* NOTE:
	* We have to start as the last step of SI_SUB_SMP, i.e. after SMP is
	* initialized.
	*/
	SYSINIT(vmbus_initialize, SI_SUB_SMP, SI_ORDER_ANY, vmbus_sysinit, NULL);

	#endif /* !EARLY_AP_STARTUP */
	diff --git a/sys/dev/iommu/busdma_iommu.c b/sys/dev/iommu/busdma_iommu.c
	index a4fe8ee9d063..dd1309fd7dc6 100644
	--- a/sys/dev/iommu/busdma_iommu.c
	+++ b/sys/dev/iommu/busdma_iommu.c
	@@ -1,1050 +1,1070 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/domainset.h>
	#include <sys/malloc.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/interrupt.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/proc.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/sysctl.h>
	#include <sys/rman.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	#include <sys/vmem.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_map.h>
	#include <machine/atomic.h>
	#include <machine/bus.h>
	#include <machine/md_var.h>
	#if defined(__amd64__) \|\| defined(__i386__)
	#include <machine/specialreg.h>
	#include <x86/include/busdma_impl.h>
	#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	#include <dev/iommu/iommu.h>
	#include <x86/iommu/intel_dmar.h>
	#endif

	/*
	* busdma_iommu.c, the implementation of the busdma(9) interface using
	* IOMMU units from Intel VT-d.
	*/

	static bool
	iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
	{
	char str[128], *env;
	int default_bounce;
	bool ret;
	static const char bounce_str[] = "bounce";
	static const char iommu_str[] = "iommu";
	static const char dmar_str[] = "dmar"; /* compatibility */

	default_bounce = 0;
	env = kern_getenv("hw.busdma.default");
	if (env != NULL) {
	if (strcmp(env, bounce_str) == 0)
	default_bounce = 1;
	else if (strcmp(env, iommu_str) == 0 \|\|
	strcmp(env, dmar_str) == 0)
	default_bounce = 0;
	freeenv(env);
	}

	snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
	domain, bus, slot, func);
	env = kern_getenv(str);
	if (env == NULL)
	return (default_bounce != 0);
	if (strcmp(env, bounce_str) == 0)
	ret = true;
	else if (strcmp(env, iommu_str) == 0 \|\|
	strcmp(env, dmar_str) == 0)
	ret = false;
	else
	ret = default_bounce != 0;
	freeenv(env);
	return (ret);
	}

	/*
	* Given original device, find the requester ID that will be seen by
	* the IOMMU unit and used for page table lookup. PCI bridges may take
	* ownership of transactions from downstream devices, so it may not be
	* the same as the BSF of the target device. In those cases, all
	* devices downstream of the bridge must share a single mapping
	* domain, and must collectively be assigned to use either IOMMU or
	* bounce mapping.
	*/
	device_t
	iommu_get_requester(device_t dev, uint16_t *rid)
	{
	devclass_t pci_class;
	device_t l, pci, pcib, pcip, pcibp, requester;
	int cap_offset;
	uint16_t pcie_flags;
	bool bridge_is_pcie;

	pci_class = devclass_find("pci");
	l = requester = dev;

	*rid = pci_get_rid(dev);

	/*
	* Walk the bridge hierarchy from the target device to the
	* host port to find the translating bridge nearest the IOMMU
	* unit.
	*/
	for (;;) {
	pci = device_get_parent(l);
	KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent "
	"for %s", device_get_name(dev), device_get_name(l)));
	KASSERT(device_get_devclass(pci) == pci_class,
	("iommu_get_requester(%s): non-pci parent %s for %s",
	device_get_name(dev), device_get_name(pci),
	device_get_name(l)));

	pcib = device_get_parent(pci);
	KASSERT(pcib != NULL, ("iommu_get_requester(%s): NULL bridge "
	"for %s", device_get_name(dev), device_get_name(pci)));

	/*
	* The parent of our "bridge" isn't another PCI bus,
	* so pcib isn't a PCI->PCI bridge but rather a host
	* port, and the requester ID won't be translated
	* further.
	*/
	pcip = device_get_parent(pcib);
	if (device_get_devclass(pcip) != pci_class)
	break;
	pcibp = device_get_parent(pcip);

	if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
	/*
	* Do not stop the loop even if the target
	* device is PCIe, because it is possible (but
	* unlikely) to have a PCI->PCIe bridge
	* somewhere in the hierarchy.
	*/
	l = pcib;
	} else {
	/*
	* Device is not PCIe, it cannot be seen as a
	* requester by IOMMU unit. Check whether the
	* bridge is PCIe.
	*/
	bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
	&cap_offset) == 0;
	requester = pcib;

	/*
	* Check for a buggy PCIe/PCI bridge that
	* doesn't report the express capability. If
	* the bridge above it is express but isn't a
	* PCI bridge, then we know pcib is actually a
	* PCIe/PCI bridge.
	*/
	if (!bridge_is_pcie && pci_find_cap(pcibp,
	PCIY_EXPRESS, &cap_offset) == 0) {
	pcie_flags = pci_read_config(pcibp,
	cap_offset + PCIER_FLAGS, 2);
	if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
	PCIEM_TYPE_PCI_BRIDGE)
	bridge_is_pcie = true;
	}

	if (bridge_is_pcie) {
	/*
	* The current device is not PCIe, but
	* the bridge above it is. This is a
	* PCIe->PCI bridge. Assume that the
	* requester ID will be the secondary
	* bus number with slot and function
	* set to zero.
	*
	* XXX: Doesn't handle the case where
	* the bridge is PCIe->PCI-X, and the
	* bridge will only take ownership of
	* requests in some cases. We should
	* provide context entries with the
	* same page tables for taken and
	* non-taken transactions.
	*/
	*rid = PCI_RID(pci_get_bus(l), 0, 0);
	l = pcibp;
	} else {
	/*
	* Neither the device nor the bridge
	* above it are PCIe. This is a
	* conventional PCI->PCI bridge, which
	* will use the bridge's BSF as the
	* requester ID.
	*/
	*rid = pci_get_rid(pcib);
	l = pcib;
	}
	}
	}
	return (requester);
	}

	struct iommu_ctx *
	iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
	{
	device_t requester;
	struct iommu_ctx *ctx;
	bool disabled;
	uint16_t rid;

	requester = iommu_get_requester(dev, &rid);

	/*
	* If the user requested the IOMMU disabled for the device, we
	* cannot disable the IOMMU unit, due to possibility of other
	* devices on the same IOMMU unit still requiring translation.
	* Instead provide the identity mapping for the device
	* context.
	*/
	disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester),
	pci_get_bus(requester), pci_get_slot(requester),
	pci_get_function(requester));
	ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr);
	if (ctx == NULL)
	return (NULL);
	if (disabled) {
	/*
	* Keep the first reference on context, release the
	* later refs.
	*/
	IOMMU_LOCK(unit);
	if ((ctx->flags & IOMMU_CTX_DISABLED) == 0) {
	ctx->flags \|= IOMMU_CTX_DISABLED;
	IOMMU_UNLOCK(unit);
	} else {
	iommu_free_ctx_locked(unit, ctx);
	}
	ctx = NULL;
	}
	return (ctx);
	}

	bus_dma_tag_t
	acpi_iommu_get_dma_tag(device_t dev, device_t child)
	{
	struct iommu_unit *unit;
	struct iommu_ctx *ctx;
	bus_dma_tag_t res;

	unit = iommu_find(child, bootverbose);
	/* Not in scope of any IOMMU ? */
	if (unit == NULL)
	return (NULL);
	if (!unit->dma_enabled)
	return (NULL);

	#if defined(__amd64__) \|\| defined(__i386__)
	dmar_quirks_pre_use(unit);
	dmar_instantiate_rmrr_ctxs(unit);
	#endif

	ctx = iommu_instantiate_ctx(unit, child, false);
	res = ctx == NULL ? NULL : (bus_dma_tag_t)ctx->tag;
	return (res);
	}

	bool
	-bus_dma_dmar_set_buswide(device_t dev)
	+bus_dma_iommu_set_buswide(device_t dev)
	{
	struct iommu_unit *unit;
	device_t parent;
	u_int busno, slot, func;

	parent = device_get_parent(dev);
	if (device_get_devclass(parent) != devclass_find("pci"))
	return (false);
	unit = iommu_find(dev, bootverbose);
	if (unit == NULL)
	return (false);
	busno = pci_get_bus(dev);
	slot = pci_get_slot(dev);
	func = pci_get_function(dev);
	if (slot != 0 \|\| func != 0) {
	if (bootverbose) {
	device_printf(dev,
	- "dmar%d pci%d:%d:%d requested buswide busdma\n",
	+ "iommu%d pci%d:%d:%d requested buswide busdma\n",
	unit->unit, busno, slot, func);
	}
	return (false);
	}
	- dmar_set_buswide_ctx(unit, busno);
	+ iommu_set_buswide_ctx(unit, busno);
	return (true);
	}

	+void
	+iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno)
	+{
	+
	+ MPASS(busno <= PCI_BUSMAX);
	+ IOMMU_LOCK(unit);
	+ unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] \|=
	+ 1 << (busno % (NBBY * sizeof(uint32_t)));
	+ IOMMU_UNLOCK(unit);
	+}
	+
	+bool
	+iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno)
	+{
	+
	+ MPASS(busno <= PCI_BUSMAX);
	+ return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] &
	+ (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0);
	+}
	+
	static MALLOC_DEFINE(M_IOMMU_DMAMAP, "iommu_dmamap", "IOMMU DMA Map");

	static void iommu_bus_schedule_dmamap(struct iommu_unit *unit,
	struct bus_dmamap_iommu *map);

	static int
	iommu_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
	bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
	bus_dma_filter_t filter, void filterarg, bus_size_t maxsize,
	int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
	void lockfuncarg, bus_dma_tag_t dmat)
	{
	struct bus_dma_tag_iommu newtag, oldtag;
	int error;

	*dmat = NULL;
	error = common_bus_dma_tag_create(parent != NULL ?
	&((struct bus_dma_tag_iommu *)parent)->common : NULL, alignment,
	boundary, lowaddr, highaddr, filter, filterarg, maxsize,
	nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
	sizeof(struct bus_dma_tag_iommu), (void **)&newtag);
	if (error != 0)
	goto out;

	oldtag = (struct bus_dma_tag_iommu *)parent;
	newtag->common.impl = &bus_dma_iommu_impl;
	newtag->ctx = oldtag->ctx;
	newtag->owner = oldtag->owner;

	*dmat = (bus_dma_tag_t)newtag;
	out:
	CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
	__func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
	error);
	return (error);
	}

	static int
	iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat)
	{

	return (0);
	}

	static int
	iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
	{
	struct bus_dma_tag_iommu dmat, dmat_copy, *parent;
	int error;

	error = 0;
	dmat_copy = dmat = (struct bus_dma_tag_iommu *)dmat1;

	if (dmat != NULL) {
	if (dmat->map_count != 0) {
	error = EBUSY;
	goto out;
	}
	while (dmat != NULL) {
	parent = (struct bus_dma_tag_iommu *)dmat->common.parent;
	if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
	1) {
	if (dmat == dmat->ctx->tag)
	iommu_free_ctx(dmat->ctx);
	free_domain(dmat->segments, M_IOMMU_DMAMAP);
	free(dmat, M_DEVBUF);
	dmat = parent;
	} else
	dmat = NULL;
	}
	}
	out:
	CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
	return (error);
	}

	static bool
	iommu_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
	{

	return (false);
	}

	static int
	iommu_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = malloc_domainset(sizeof(*map), M_IOMMU_DMAMAP,
	DOMAINSET_PREF(tag->common.domain), M_NOWAIT \| M_ZERO);
	if (map == NULL) {
	*mapp = NULL;
	return (ENOMEM);
	}
	if (tag->segments == NULL) {
	tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
	tag->common.nsegments, M_IOMMU_DMAMAP,
	DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
	if (tag->segments == NULL) {
	free_domain(map, M_IOMMU_DMAMAP);
	*mapp = NULL;
	return (ENOMEM);
	}
	}
	TAILQ_INIT(&map->map_entries);
	map->tag = tag;
	map->locked = true;
	map->cansleep = false;
	tag->map_count++;
	*mapp = (bus_dmamap_t)map;

	return (0);
	}

	static int
	iommu_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	struct iommu_domain *domain;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;
	if (map != NULL) {
	domain = tag->ctx->domain;
	IOMMU_DOMAIN_LOCK(domain);
	if (!TAILQ_EMPTY(&map->map_entries)) {
	IOMMU_DOMAIN_UNLOCK(domain);
	return (EBUSY);
	}
	IOMMU_DOMAIN_UNLOCK(domain);
	free_domain(map, M_IOMMU_DMAMAP);
	}
	tag->map_count--;
	return (0);
	}


	static int
	iommu_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
	bus_dmamap_t *mapp)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	int error, mflags;
	vm_memattr_t attr;

	error = iommu_bus_dmamap_create(dmat, flags, mapp);
	if (error != 0)
	return (error);

	mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
	mflags \|= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
	attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
	VM_MEMATTR_DEFAULT;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu )mapp;

	if (tag->common.maxsize < PAGE_SIZE &&
	tag->common.alignment <= tag->common.maxsize &&
	attr == VM_MEMATTR_DEFAULT) {
	*vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
	DOMAINSET_PREF(tag->common.domain), mflags);
	map->flags \|= BUS_DMAMAP_IOMMU_MALLOC;
	} else {
	vaddr = (void )kmem_alloc_attr_domainset(
	DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
	mflags, 0ul, BUS_SPACE_MAXADDR, attr);
	map->flags \|= BUS_DMAMAP_IOMMU_KMEM_ALLOC;
	}
	if (*vaddr == NULL) {
	iommu_bus_dmamap_destroy(dmat, *mapp);
	*mapp = NULL;
	return (ENOMEM);
	}
	return (0);
	}

	static void
	iommu_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;

	if ((map->flags & BUS_DMAMAP_IOMMU_MALLOC) != 0) {
	free_domain(vaddr, M_DEVBUF);
	map->flags &= ~BUS_DMAMAP_IOMMU_MALLOC;
	} else {
	KASSERT((map->flags & BUS_DMAMAP_IOMMU_KMEM_ALLOC) != 0,
	("iommu_bus_dmamem_free for non alloced map %p", map));
	kmem_free((vm_offset_t)vaddr, tag->common.maxsize);
	map->flags &= ~BUS_DMAMAP_IOMMU_KMEM_ALLOC;
	}

	iommu_bus_dmamap_destroy(dmat, map1);
	}

	static int
	iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu *tag,
	struct bus_dmamap_iommu map, vm_page_t ma, int offset, bus_size_t buflen,
	int flags, bus_dma_segment_t segs, int segp,
	struct iommu_map_entries_tailq *unroll_list)
	{
	struct iommu_ctx *ctx;
	struct iommu_domain *domain;
	struct iommu_map_entry *entry;
	iommu_gaddr_t size;
	bus_size_t buflen1;
	int error, idx, gas_flags, seg;

	KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset));
	if (segs == NULL)
	segs = tag->segments;
	ctx = tag->ctx;
	domain = ctx->domain;
	seg = *segp;
	error = 0;
	idx = 0;
	while (buflen > 0) {
	seg++;
	if (seg >= tag->common.nsegments) {
	error = EFBIG;
	break;
	}
	buflen1 = buflen > tag->common.maxsegsz ?
	tag->common.maxsegsz : buflen;
	size = round_page(offset + buflen1);

	/*
	* (Too) optimistically allow split if there are more
	* then one segments left.
	*/
	gas_flags = map->cansleep ? IOMMU_MF_CANWAIT : 0;
	if (seg + 1 < tag->common.nsegments)
	gas_flags \|= IOMMU_MF_CANSPLIT;

	error = iommu_map(domain, &tag->common, size, offset,
	IOMMU_MAP_ENTRY_READ \|
	((flags & BUS_DMA_NOWRITE) == 0 ? IOMMU_MAP_ENTRY_WRITE : 0),
	gas_flags, ma + idx, &entry);
	if (error != 0)
	break;
	if ((gas_flags & IOMMU_MF_CANSPLIT) != 0) {
	KASSERT(size >= entry->end - entry->start,
	("split increased entry size %jx %jx %jx",
	(uintmax_t)size, (uintmax_t)entry->start,
	(uintmax_t)entry->end));
	size = entry->end - entry->start;
	if (buflen1 > size)
	buflen1 = size;
	} else {
	KASSERT(entry->end - entry->start == size,
	("no split allowed %jx %jx %jx",
	(uintmax_t)size, (uintmax_t)entry->start,
	(uintmax_t)entry->end));
	}
	if (offset + buflen1 > size)
	buflen1 = size - offset;
	if (buflen1 > tag->common.maxsegsz)
	buflen1 = tag->common.maxsegsz;

	KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
	== 0,
	("alignment failed: ctx %p start 0x%jx offset %x "
	"align 0x%jx", ctx, (uintmax_t)entry->start, offset,
	(uintmax_t)tag->common.alignment));
	KASSERT(entry->end <= tag->common.lowaddr \|\|
	entry->start >= tag->common.highaddr,
	("entry placement failed: ctx %p start 0x%jx end 0x%jx "
	"lowaddr 0x%jx highaddr 0x%jx", ctx,
	(uintmax_t)entry->start, (uintmax_t)entry->end,
	(uintmax_t)tag->common.lowaddr,
	(uintmax_t)tag->common.highaddr));
	KASSERT(iommu_test_boundary(entry->start + offset, buflen1,
	tag->common.boundary),
	("boundary failed: ctx %p start 0x%jx end 0x%jx "
	"boundary 0x%jx", ctx, (uintmax_t)entry->start,
	(uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
	KASSERT(buflen1 <= tag->common.maxsegsz,
	("segment too large: ctx %p start 0x%jx end 0x%jx "
	"buflen1 0x%jx maxsegsz 0x%jx", ctx,
	(uintmax_t)entry->start, (uintmax_t)entry->end,
	(uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));

	IOMMU_DOMAIN_LOCK(domain);
	TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
	entry->flags \|= IOMMU_MAP_ENTRY_MAP;
	IOMMU_DOMAIN_UNLOCK(domain);
	TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);

	segs[seg].ds_addr = entry->start + offset;
	segs[seg].ds_len = buflen1;

	idx += OFF_TO_IDX(trunc_page(offset + buflen1));
	offset += buflen1;
	offset &= IOMMU_PAGE_MASK;
	buflen -= buflen1;
	}
	if (error == 0)
	*segp = seg;
	return (error);
	}

	static int
	iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu *tag,
	struct bus_dmamap_iommu map, vm_page_t ma, int offset, bus_size_t buflen,
	int flags, bus_dma_segment_t segs, int segp)
	{
	struct iommu_ctx *ctx;
	struct iommu_domain *domain;
	struct iommu_map_entry entry, entry1;
	struct iommu_map_entries_tailq unroll_list;
	int error;

	ctx = tag->ctx;
	domain = ctx->domain;
	atomic_add_long(&ctx->loads, 1);

	TAILQ_INIT(&unroll_list);
	error = iommu_bus_dmamap_load_something1(tag, map, ma, offset,
	buflen, flags, segs, segp, &unroll_list);
	if (error != 0) {
	/*
	* The busdma interface does not allow us to report
	* partial buffer load, so unfortunately we have to
	* revert all work done.
	*/
	IOMMU_DOMAIN_LOCK(domain);
	TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
	entry1) {
	/*
	* No entries other than what we have created
	* during the failed run might have been
	* inserted there in between, since we own ctx
	* pglock.
	*/
	TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
	TAILQ_REMOVE(&unroll_list, entry, unroll_link);
	TAILQ_INSERT_TAIL(&domain->unload_entries, entry,
	dmamap_link);
	}
	IOMMU_DOMAIN_UNLOCK(domain);
	taskqueue_enqueue(domain->iommu->delayed_taskqueue,
	&domain->unload_task);
	}

	if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
	!map->cansleep)
	error = EINPROGRESS;
	if (error == EINPROGRESS)
	iommu_bus_schedule_dmamap(domain->iommu, map);
	return (error);
	}

	static int
	iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
	struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
	bus_dma_segment_t segs, int segp)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;
	return (iommu_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
	flags, segs, segp));
	}

	static int
	iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
	vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
	int *segp)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	vm_page_t *ma, fma;
	vm_paddr_t pstart, pend, paddr;
	int error, i, ma_cnt, mflags, offset;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;
	pstart = trunc_page(buf);
	pend = round_page(buf + buflen);
	offset = buf & PAGE_MASK;
	ma_cnt = OFF_TO_IDX(pend - pstart);
	mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
	ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
	if (ma == NULL)
	return (ENOMEM);
	fma = NULL;
	for (i = 0; i < ma_cnt; i++) {
	paddr = pstart + ptoa(i);
	ma[i] = PHYS_TO_VM_PAGE(paddr);
	if (ma[i] == NULL \|\| VM_PAGE_TO_PHYS(ma[i]) != paddr) {
	/*
	* If PHYS_TO_VM_PAGE() returned NULL or the
	* vm_page was not initialized we'll use a
	* fake page.
	*/
	if (fma == NULL) {
	fma = malloc(sizeof(struct vm_page) * ma_cnt,
	M_DEVBUF, M_ZERO \| mflags);
	if (fma == NULL) {
	free(ma, M_DEVBUF);
	return (ENOMEM);
	}
	}
	vm_page_initfake(&fma[i], pstart + ptoa(i),
	VM_MEMATTR_DEFAULT);
	ma[i] = &fma[i];
	}
	}
	error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
	flags, segs, segp);
	free(fma, M_DEVBUF);
	free(ma, M_DEVBUF);
	return (error);
	}

	static int
	iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
	bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
	int *segp)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	vm_page_t *ma, fma;
	vm_paddr_t pstart, pend, paddr;
	int error, i, ma_cnt, mflags, offset;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;
	pstart = trunc_page((vm_offset_t)buf);
	pend = round_page((vm_offset_t)buf + buflen);
	offset = (vm_offset_t)buf & PAGE_MASK;
	ma_cnt = OFF_TO_IDX(pend - pstart);
	mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
	ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
	if (ma == NULL)
	return (ENOMEM);
	fma = NULL;
	for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
	if (pmap == kernel_pmap)
	paddr = pmap_kextract(pstart);
	else
	paddr = pmap_extract(pmap, pstart);
	ma[i] = PHYS_TO_VM_PAGE(paddr);
	if (ma[i] == NULL \|\| VM_PAGE_TO_PHYS(ma[i]) != paddr) {
	/*
	* If PHYS_TO_VM_PAGE() returned NULL or the
	* vm_page was not initialized we'll use a
	* fake page.
	*/
	if (fma == NULL) {
	fma = malloc(sizeof(struct vm_page) * ma_cnt,
	M_DEVBUF, M_ZERO \| mflags);
	if (fma == NULL) {
	free(ma, M_DEVBUF);
	return (ENOMEM);
	}
	}
	vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
	ma[i] = &fma[i];
	}
	}
	error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
	flags, segs, segp);
	free(ma, M_DEVBUF);
	free(fma, M_DEVBUF);
	return (error);
	}

	static void
	iommu_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
	struct memdesc mem, bus_dmamap_callback_t callback, void *callback_arg)
	{
	struct bus_dmamap_iommu *map;

	if (map1 == NULL)
	return;
	map = (struct bus_dmamap_iommu *)map1;
	map->mem = *mem;
	map->tag = (struct bus_dma_tag_iommu *)dmat;
	map->callback = callback;
	map->callback_arg = callback_arg;
	}

	static bus_dma_segment_t *
	iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
	bus_dma_segment_t *segs, int nsegs, int error)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;

	if (!map->locked) {
	KASSERT(map->cansleep,
	("map not locked and not sleepable context %p", map));

	/*
	* We are called from the delayed context. Relock the
	* driver.
	*/
	(tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
	map->locked = true;
	}

	if (segs == NULL)
	segs = tag->segments;
	return (segs);
	}

	/*
	* The limitations of busdma KPI forces the iommu to perform the actual
	* unload, consisting of the unmapping of the map entries page tables,
	* from the delayed context on i386, since page table page mapping
	* might require a sleep to be successfull. The unfortunate
	* consequence is that the DMA requests can be served some time after
	* the bus_dmamap_unload() call returned.
	*
	* On amd64, we assume that sf allocation cannot fail.
	*/
	static void
	iommu_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	struct iommu_ctx *ctx;
	struct iommu_domain *domain;
	#if defined(__amd64__)
	struct iommu_map_entries_tailq entries;
	#endif

	tag = (struct bus_dma_tag_iommu *)dmat;
	map = (struct bus_dmamap_iommu *)map1;
	ctx = tag->ctx;
	domain = ctx->domain;
	atomic_add_long(&ctx->unloads, 1);

	#if defined(__i386__)
	IOMMU_DOMAIN_LOCK(domain);
	TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link);
	IOMMU_DOMAIN_UNLOCK(domain);
	taskqueue_enqueue(domain->iommu->delayed_taskqueue,
	&domain->unload_task);
	#else /* defined(__amd64__) */
	TAILQ_INIT(&entries);
	IOMMU_DOMAIN_LOCK(domain);
	TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
	IOMMU_DOMAIN_UNLOCK(domain);
	THREAD_NO_SLEEPING();
	iommu_domain_unload(domain, &entries, false);
	THREAD_SLEEPING_OK();
	KASSERT(TAILQ_EMPTY(&entries), ("lazy iommu_ctx_unload %p", ctx));
	#endif
	}

	static void
	iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
	bus_dmasync_op_t op)
	{
	}

	struct bus_dma_impl bus_dma_iommu_impl = {
	.tag_create = iommu_bus_dma_tag_create,
	.tag_destroy = iommu_bus_dma_tag_destroy,
	.tag_set_domain = iommu_bus_dma_tag_set_domain,
	.id_mapped = iommu_bus_dma_id_mapped,
	.map_create = iommu_bus_dmamap_create,
	.map_destroy = iommu_bus_dmamap_destroy,
	.mem_alloc = iommu_bus_dmamem_alloc,
	.mem_free = iommu_bus_dmamem_free,
	.load_phys = iommu_bus_dmamap_load_phys,
	.load_buffer = iommu_bus_dmamap_load_buffer,
	.load_ma = iommu_bus_dmamap_load_ma,
	.map_waitok = iommu_bus_dmamap_waitok,
	.map_complete = iommu_bus_dmamap_complete,
	.map_unload = iommu_bus_dmamap_unload,
	.map_sync = iommu_bus_dmamap_sync,
	};

	static void
	iommu_bus_task_dmamap(void *arg, int pending)
	{
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	struct iommu_unit *unit;

	unit = arg;
	IOMMU_LOCK(unit);
	while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
	TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
	IOMMU_UNLOCK(unit);
	tag = map->tag;
	map->cansleep = true;
	map->locked = false;
	bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
	&map->mem, map->callback, map->callback_arg,
	BUS_DMA_WAITOK);
	map->cansleep = false;
	if (map->locked) {
	(tag->common.lockfunc)(tag->common.lockfuncarg,
	BUS_DMA_UNLOCK);
	} else
	map->locked = true;
	map->cansleep = false;
	IOMMU_LOCK(unit);
	}
	IOMMU_UNLOCK(unit);
	}

	static void
	iommu_bus_schedule_dmamap(struct iommu_unit unit, struct bus_dmamap_iommu map)
	{

	map->locked = false;
	IOMMU_LOCK(unit);
	TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
	IOMMU_UNLOCK(unit);
	taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
	}

	int
	iommu_init_busdma(struct iommu_unit *unit)
	{
	int error;

	unit->dma_enabled = 1;
	error = TUNABLE_INT_FETCH("hw.iommu.dma", &unit->dma_enabled);
	if (error == 0) /* compatibility */
	TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
	TAILQ_INIT(&unit->delayed_maps);
	TASK_INIT(&unit->dmamap_load_task, 0, iommu_bus_task_dmamap, unit);
	unit->delayed_taskqueue = taskqueue_create("iommu", M_WAITOK,
	taskqueue_thread_enqueue, &unit->delayed_taskqueue);
	taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
	"iommu%d busdma taskq", unit->unit);
	return (0);
	}

	void
	iommu_fini_busdma(struct iommu_unit *unit)
	{

	if (unit->delayed_taskqueue == NULL)
	return;

	taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
	taskqueue_free(unit->delayed_taskqueue);
	unit->delayed_taskqueue = NULL;
	}

	int
	-bus_dma_dmar_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1,
	+bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1,
	vm_paddr_t start, vm_size_t length, int flags)
	{
	struct bus_dma_tag_common *tc;
	struct bus_dma_tag_iommu *tag;
	struct bus_dmamap_iommu *map;
	struct iommu_ctx *ctx;
	struct iommu_domain *domain;
	struct iommu_map_entry *entry;
	vm_page_t *ma;
	vm_size_t i;
	int error;
	bool waitok;

	MPASS((start & PAGE_MASK) == 0);
	MPASS((length & PAGE_MASK) == 0);
	MPASS(length > 0);
	MPASS(start + length >= start);
	MPASS((flags & ~(BUS_DMA_NOWAIT \| BUS_DMA_NOWRITE)) == 0);

	tc = (struct bus_dma_tag_common *)dmat;
	if (tc->impl != &bus_dma_iommu_impl)
	return (0);

	tag = (struct bus_dma_tag_iommu *)dmat;
	ctx = tag->ctx;
	domain = ctx->domain;
	map = (struct bus_dmamap_iommu *)map1;
	waitok = (flags & BUS_DMA_NOWAIT) != 0;

	entry = iommu_map_alloc_entry(domain, waitok ? 0 : IOMMU_PGF_WAITOK);
	if (entry == NULL)
	return (ENOMEM);
	entry->start = start;
	entry->end = start + length;
	ma = malloc(sizeof(vm_page_t) * atop(length), M_TEMP, waitok ?
	M_WAITOK : M_NOWAIT);
	if (ma == NULL) {
	iommu_map_free_entry(domain, entry);
	return (ENOMEM);
	}
	for (i = 0; i < atop(length); i++) {
	ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
	VM_MEMATTR_DEFAULT);
	}
	error = iommu_map_region(domain, entry, IOMMU_MAP_ENTRY_READ \|
	((flags & BUS_DMA_NOWRITE) ? 0 : IOMMU_MAP_ENTRY_WRITE),
	waitok ? IOMMU_MF_CANWAIT : 0, ma);
	if (error == 0) {
	IOMMU_DOMAIN_LOCK(domain);
	TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
	entry->flags \|= IOMMU_MAP_ENTRY_MAP;
	IOMMU_DOMAIN_UNLOCK(domain);
	} else {
	iommu_domain_unload_entry(entry, true);
	}
	for (i = 0; i < atop(length); i++)
	vm_page_putfake(ma[i]);
	free(ma, M_TEMP);
	return (error);
	}
	diff --git a/sys/dev/iommu/iommu.h b/sys/dev/iommu/iommu.h
	index c3bc3d26b765..e6ad0569a9ac 100644
	--- a/sys/dev/iommu/iommu.h
	+++ b/sys/dev/iommu/iommu.h
	@@ -1,216 +1,225 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef _SYS_IOMMU_H_
	#define _SYS_IOMMU_H_

	-#include <sys/queue.h>
	-#include <sys/sysctl.h>
	-#include <sys/taskqueue.h>
	-#include <sys/tree.h>
	-#include <sys/types.h>
	-
	/* Host or physical memory address, after translation. */
	typedef uint64_t iommu_haddr_t;
	/* Guest or bus address, before translation. */
	typedef uint64_t iommu_gaddr_t;

	struct bus_dma_tag_common;
	struct iommu_map_entry;
	TAILQ_HEAD(iommu_map_entries_tailq, iommu_map_entry);

	RB_HEAD(iommu_gas_entries_tree, iommu_map_entry);
	RB_PROTOTYPE(iommu_gas_entries_tree, iommu_map_entry, rb_entry,
	iommu_gas_cmp_entries);

	struct iommu_qi_genseq {
	u_int gen;
	uint32_t seq;
	};

	struct iommu_map_entry {
	iommu_gaddr_t start;
	iommu_gaddr_t end;
	iommu_gaddr_t first; /* Least start in subtree */
	iommu_gaddr_t last; /* Greatest end in subtree */
	iommu_gaddr_t free_down; /* Max free space below the
	current R/B tree node */
	u_int flags;
	TAILQ_ENTRY(iommu_map_entry) dmamap_link; /* Link for dmamap entries */
	RB_ENTRY(iommu_map_entry) rb_entry; /* Links for domain entries */
	TAILQ_ENTRY(iommu_map_entry) unroll_link; /* Link for unroll after
	dmamap_load failure */
	struct iommu_domain *domain;
	struct iommu_qi_genseq gseq;
	};

	#define IOMMU_MAP_ENTRY_PLACE 0x0001 /* Fake entry */
	#define IOMMU_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by
	dmamap_link */
	#define IOMMU_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by
	dmamap_link */
	#define IOMMU_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */
	#define IOMMU_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */
	#define IOMMU_MAP_ENTRY_READ 0x1000 /* Read permitted */
	#define IOMMU_MAP_ENTRY_WRITE 0x2000 /* Write permitted */
	#define IOMMU_MAP_ENTRY_SNOOP 0x4000 /* Snoop */
	#define IOMMU_MAP_ENTRY_TM 0x8000 /* Transient */

	struct iommu_unit {
	struct mtx lock;
	int unit;

	int dma_enabled;

	/* Busdma delayed map load */
	struct task dmamap_load_task;
	TAILQ_HEAD(, bus_dmamap_iommu) delayed_maps;
	struct taskqueue *delayed_taskqueue;
	+
	+ /*
	+ * Bitmap of buses for which context must ignore slot:func,
	+ * duplicating the page table pointer into all context table
	+ * entries. This is a client-controlled quirk to support some
	+ * NTBs.
	+ */
	+ uint32_t buswide_ctxs[(PCI_BUSMAX + 1) / NBBY / sizeof(uint32_t)];
	};

	/*
	* Locking annotations:
	* (u) - Protected by iommu unit lock
	* (d) - Protected by domain lock
	* (c) - Immutable after initialization
	*/

	struct iommu_domain {
	struct iommu_unit iommu; / (c) */
	struct mtx lock; /* (c) */
	struct task unload_task; /* (c) */
	u_int entries_cnt; /* (d) */
	struct iommu_map_entries_tailq unload_entries; /* (d) Entries to
	unload */
	struct iommu_gas_entries_tree rb_root; /* (d) */
	iommu_gaddr_t end; /* (c) Highest address + 1 in
	the guest AS */
	struct iommu_map_entry first_place, last_place; /* (d) */
	u_int flags; /* (u) */
	};

	struct iommu_ctx {
	struct iommu_domain domain; / (c) */
	struct bus_dma_tag_iommu tag; / (c) Root tag */
	u_long loads; /* atomic updates, for stat only */
	u_long unloads; /* same */
	u_int flags; /* (u) */
	};

	/* struct iommu_ctx flags */
	#define IOMMU_CTX_FAULTED 0x0001 /* Fault was reported,
	last_fault_rec is valid */
	#define IOMMU_CTX_DISABLED 0x0002 /* Device is disabled, the
	ephemeral reference is kept
	to prevent context destruction */

	#define IOMMU_DOMAIN_GAS_INITED 0x0001
	#define IOMMU_DOMAIN_PGTBL_INITED 0x0002
	#define IOMMU_DOMAIN_IDMAP 0x0010 /* Domain uses identity
	page table */
	#define IOMMU_DOMAIN_RMRR 0x0020 /* Domain contains RMRR entry,
	cannot be turned off */

	/* Map flags */
	#define IOMMU_MF_CANWAIT 0x0001
	#define IOMMU_MF_CANSPLIT 0x0002
	#define IOMMU_MF_RMRR 0x0004

	#define IOMMU_PGF_WAITOK 0x0001
	#define IOMMU_PGF_ZERO 0x0002
	#define IOMMU_PGF_ALLOC 0x0004
	#define IOMMU_PGF_NOALLOC 0x0008
	#define IOMMU_PGF_OBJL 0x0010

	#define IOMMU_LOCK(unit) mtx_lock(&(unit)->lock)
	#define IOMMU_UNLOCK(unit) mtx_unlock(&(unit)->lock)
	#define IOMMU_ASSERT_LOCKED(unit) mtx_assert(&(unit)->lock, MA_OWNED)

	#define IOMMU_DOMAIN_LOCK(dom) mtx_lock(&(dom)->lock)
	#define IOMMU_DOMAIN_UNLOCK(dom) mtx_unlock(&(dom)->lock)
	#define IOMMU_DOMAIN_ASSERT_LOCKED(dom) mtx_assert(&(dom)->lock, MA_OWNED)

	static inline bool
	iommu_test_boundary(iommu_gaddr_t start, iommu_gaddr_t size,
	iommu_gaddr_t boundary)
	{

	if (boundary == 0)
	return (true);
	return (start + size <= ((start + boundary) & ~(boundary - 1)));
	}

	void iommu_free_ctx(struct iommu_ctx *ctx);
	void iommu_free_ctx_locked(struct iommu_unit iommu, struct iommu_ctx ctx);
	struct iommu_ctx iommu_get_ctx(struct iommu_unit , device_t dev,
	uint16_t rid, bool id_mapped, bool rmrr_init);
	struct iommu_unit *iommu_find(device_t dev, bool verbose);
	void iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free);
	void iommu_domain_unload(struct iommu_domain *domain,
	struct iommu_map_entries_tailq *entries, bool cansleep);

	struct iommu_ctx iommu_instantiate_ctx(struct iommu_unit iommu,
	device_t dev, bool rmrr);
	device_t iommu_get_requester(device_t dev, uint16_t *rid);
	int iommu_init_busdma(struct iommu_unit *unit);
	void iommu_fini_busdma(struct iommu_unit *unit);
	struct iommu_map_entry iommu_map_alloc_entry(struct iommu_domain iodom,
	u_int flags);
	void iommu_map_free_entry(struct iommu_domain , struct iommu_map_entry );
	int iommu_map(struct iommu_domain *iodom,
	const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset,
	u_int eflags, u_int flags, vm_page_t ma, struct iommu_map_entry *res);
	int iommu_map_region(struct iommu_domain *domain,
	struct iommu_map_entry entry, u_int eflags, u_int flags, vm_page_t ma);

	void iommu_gas_init_domain(struct iommu_domain *domain);
	void iommu_gas_fini_domain(struct iommu_domain *domain);
	struct iommu_map_entry iommu_gas_alloc_entry(struct iommu_domain domain,
	u_int flags);
	void iommu_gas_free_entry(struct iommu_domain *domain,
	struct iommu_map_entry *entry);
	void iommu_gas_free_space(struct iommu_domain *domain,
	struct iommu_map_entry *entry);
	int iommu_gas_map(struct iommu_domain *domain,
	const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset,
	u_int eflags, u_int flags, vm_page_t ma, struct iommu_map_entry *res);
	void iommu_gas_free_region(struct iommu_domain *domain,
	struct iommu_map_entry *entry);
	int iommu_gas_map_region(struct iommu_domain *domain,
	struct iommu_map_entry entry, u_int eflags, u_int flags, vm_page_t ma);
	int iommu_gas_reserve_region(struct iommu_domain *domain, iommu_gaddr_t start,
	iommu_gaddr_t end);

	+void iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno);
	+bool iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno);
	+
	+bool bus_dma_iommu_set_buswide(device_t dev);
	+int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map,
	+ vm_paddr_t start, vm_size_t length, int flags);
	+
	SYSCTL_DECL(_hw_iommu);

	#endif /* !_SYS_IOMMU_H_ */
	diff --git a/sys/dev/iommu/iommu_gas.c b/sys/dev/iommu/iommu_gas.c
	index 5a6e48850b5b..04d7d9667109 100644
	--- a/sys/dev/iommu/iommu_gas.c
	+++ b/sys/dev/iommu/iommu_gas.c
	@@ -1,741 +1,741 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#define RB_AUGMENT(entry) iommu_gas_augment_entry(entry)

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/malloc.h>
	#include <sys/bus.h>
	#include <sys/interrupt.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/proc.h>
	#include <sys/rwlock.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/sysctl.h>
	#include <sys/rman.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	#include <sys/vmem.h>
	-#include <dev/pci/pcivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_map.h>
	#include <vm/uma.h>
	+#include <dev/pci/pcireg.h>
	+#include <dev/pci/pcivar.h>
	+#include <dev/iommu/iommu.h>
	#include <machine/atomic.h>
	#include <machine/bus.h>
	#include <machine/md_var.h>
	#if defined(__amd64__) \|\| defined(__i386__)
	#include <machine/specialreg.h>
	#include <x86/include/busdma_impl.h>
	#include <x86/iommu/intel_reg.h>
	-#include <dev/iommu/busdma_iommu.h>
	-#include <dev/iommu/iommu.h>
	-#include <dev/pci/pcireg.h>
	#include <x86/iommu/intel_dmar.h>
	#endif
	+#include <dev/iommu/busdma_iommu.h>

	/*
	* Guest Address Space management.
	*/

	static uma_zone_t iommu_map_entry_zone;

	#ifdef INVARIANTS
	static int iommu_check_free;
	#endif

	static void
	intel_gas_init(void)
	{

	iommu_map_entry_zone = uma_zcreate("IOMMU_MAP_ENTRY",
	sizeof(struct iommu_map_entry), NULL, NULL,
	NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP);
	}
	SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL);

	struct iommu_map_entry *
	iommu_gas_alloc_entry(struct iommu_domain *domain, u_int flags)
	{
	struct iommu_map_entry *res;

	KASSERT((flags & ~(IOMMU_PGF_WAITOK)) == 0,
	("unsupported flags %x", flags));

	res = uma_zalloc(iommu_map_entry_zone, ((flags & IOMMU_PGF_WAITOK) !=
	0 ? M_WAITOK : M_NOWAIT) \| M_ZERO);
	if (res != NULL) {
	res->domain = domain;
	atomic_add_int(&domain->entries_cnt, 1);
	}
	return (res);
	}

	void
	iommu_gas_free_entry(struct iommu_domain domain, struct iommu_map_entry entry)
	{

	KASSERT(domain == (struct iommu_domain *)entry->domain,
	("mismatched free domain %p entry %p entry->domain %p", domain,
	entry, entry->domain));
	atomic_subtract_int(&domain->entries_cnt, 1);
	uma_zfree(iommu_map_entry_zone, entry);
	}

	static int
	iommu_gas_cmp_entries(struct iommu_map_entry a, struct iommu_map_entry b)
	{

	/* Last entry have zero size, so <= */
	KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)",
	a, (uintmax_t)a->start, (uintmax_t)a->end));
	KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)",
	b, (uintmax_t)b->start, (uintmax_t)b->end));
	KASSERT(a->end <= b->start \|\| b->end <= a->start \|\|
	a->end == a->start \|\| b->end == b->start,
	("overlapping entries %p (%jx, %jx) %p (%jx, %jx)",
	a, (uintmax_t)a->start, (uintmax_t)a->end,
	b, (uintmax_t)b->start, (uintmax_t)b->end));

	if (a->end < b->end)
	return (-1);
	else if (b->end < a->end)
	return (1);
	return (0);
	}

	static void
	iommu_gas_augment_entry(struct iommu_map_entry *entry)
	{
	struct iommu_map_entry *child;
	iommu_gaddr_t free_down;

	free_down = 0;
	if ((child = RB_LEFT(entry, rb_entry)) != NULL) {
	free_down = MAX(free_down, child->free_down);
	free_down = MAX(free_down, entry->start - child->last);
	entry->first = child->first;
	} else
	entry->first = entry->start;

	if ((child = RB_RIGHT(entry, rb_entry)) != NULL) {
	free_down = MAX(free_down, child->free_down);
	free_down = MAX(free_down, child->first - entry->end);
	entry->last = child->last;
	} else
	entry->last = entry->end;
	entry->free_down = free_down;
	}

	RB_GENERATE(iommu_gas_entries_tree, iommu_map_entry, rb_entry,
	iommu_gas_cmp_entries);

	#ifdef INVARIANTS
	static void
	iommu_gas_check_free(struct iommu_domain *domain)
	{
	struct iommu_map_entry entry, l, *r;
	iommu_gaddr_t v;

	RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) {
	KASSERT(domain == (struct iommu_domain *)entry->domain,
	("mismatched free domain %p entry %p entry->domain %p",
	domain, entry, entry->domain));
	l = RB_LEFT(entry, rb_entry);
	r = RB_RIGHT(entry, rb_entry);
	v = 0;
	if (l != NULL) {
	v = MAX(v, l->free_down);
	v = MAX(v, entry->start - l->last);
	}
	if (r != NULL) {
	v = MAX(v, r->free_down);
	v = MAX(v, r->first - entry->end);
	}
	MPASS(entry->free_down == v);
	}
	}
	#endif

	static bool
	iommu_gas_rb_insert(struct iommu_domain domain, struct iommu_map_entry entry)
	{
	struct iommu_map_entry *found;

	found = RB_INSERT(iommu_gas_entries_tree,
	&domain->rb_root, entry);
	return (found == NULL);
	}

	static void
	iommu_gas_rb_remove(struct iommu_domain domain, struct iommu_map_entry entry)
	{

	RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry);
	}

	void
	iommu_gas_init_domain(struct iommu_domain *domain)
	{
	struct iommu_map_entry begin, end;

	begin = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK);
	end = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK);

	IOMMU_DOMAIN_LOCK(domain);
	KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain));
	KASSERT(RB_EMPTY(&domain->rb_root),
	("non-empty entries %p", domain));

	begin->start = 0;
	begin->end = IOMMU_PAGE_SIZE;
	begin->flags = IOMMU_MAP_ENTRY_PLACE \| IOMMU_MAP_ENTRY_UNMAPPED;
	iommu_gas_rb_insert(domain, begin);

	end->start = domain->end;
	end->end = domain->end;
	end->flags = IOMMU_MAP_ENTRY_PLACE \| IOMMU_MAP_ENTRY_UNMAPPED;
	iommu_gas_rb_insert(domain, end);

	domain->first_place = begin;
	domain->last_place = end;
	domain->flags \|= IOMMU_DOMAIN_GAS_INITED;
	IOMMU_DOMAIN_UNLOCK(domain);
	}

	void
	iommu_gas_fini_domain(struct iommu_domain *domain)
	{
	struct iommu_map_entry entry, entry1;

	IOMMU_DOMAIN_ASSERT_LOCKED(domain);
	KASSERT(domain->entries_cnt == 2,
	("domain still in use %p", domain));

	entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root);
	KASSERT(entry->start == 0, ("start entry start %p", domain));
	KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain));
	KASSERT(entry->flags == IOMMU_MAP_ENTRY_PLACE,
	("start entry flags %p", domain));
	RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry);
	iommu_gas_free_entry(domain, entry);

	entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root);
	KASSERT(entry->start == domain->end, ("end entry start %p", domain));
	KASSERT(entry->end == domain->end, ("end entry end %p", domain));
	KASSERT(entry->flags == IOMMU_MAP_ENTRY_PLACE,
	("end entry flags %p", domain));
	RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry);
	iommu_gas_free_entry(domain, entry);

	RB_FOREACH_SAFE(entry, iommu_gas_entries_tree, &domain->rb_root,
	entry1) {
	KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0,
	("non-RMRR entry left %p", domain));
	RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root,
	entry);
	iommu_gas_free_entry(domain, entry);
	}
	}

	struct iommu_gas_match_args {
	struct iommu_domain *domain;
	iommu_gaddr_t size;
	int offset;
	const struct bus_dma_tag_common *common;
	u_int gas_flags;
	struct iommu_map_entry *entry;
	};

	/*
	* The interval [beg, end) is a free interval between two iommu_map_entries.
	* maxaddr is an upper bound on addresses that can be allocated. Try to
	* allocate space in the free interval, subject to the conditions expressed
	* by a, and return 'true' if and only if the allocation attempt succeeds.
	*/
	static bool
	iommu_gas_match_one(struct iommu_gas_match_args *a, iommu_gaddr_t beg,
	iommu_gaddr_t end, iommu_gaddr_t maxaddr)
	{
	iommu_gaddr_t bs, start;

	a->entry->start = roundup2(beg + IOMMU_PAGE_SIZE,
	a->common->alignment);
	if (a->entry->start + a->size > maxaddr)
	return (false);

	/* IOMMU_PAGE_SIZE to create gap after new entry. */
	if (a->entry->start < beg + IOMMU_PAGE_SIZE \|\|
	a->entry->start + a->size + a->offset + IOMMU_PAGE_SIZE > end)
	return (false);

	/* No boundary crossing. */
	if (iommu_test_boundary(a->entry->start + a->offset, a->size,
	a->common->boundary))
	return (true);

	/*
	* The start + offset to start + offset + size region crosses
	* the boundary. Check if there is enough space after the
	* next boundary after the beg.
	*/
	bs = rounddown2(a->entry->start + a->offset + a->common->boundary,
	a->common->boundary);
	start = roundup2(bs, a->common->alignment);
	/* IOMMU_PAGE_SIZE to create gap after new entry. */
	if (start + a->offset + a->size + IOMMU_PAGE_SIZE <= end &&
	start + a->offset + a->size <= maxaddr &&
	iommu_test_boundary(start + a->offset, a->size,
	a->common->boundary)) {
	a->entry->start = start;
	return (true);
	}

	/*
	* Not enough space to align at the requested boundary, or
	* boundary is smaller than the size, but allowed to split.
	* We already checked that start + size does not overlap maxaddr.
	*
	* XXXKIB. It is possible that bs is exactly at the start of
	* the next entry, then we do not have gap. Ignore for now.
	*/
	if ((a->gas_flags & IOMMU_MF_CANSPLIT) != 0) {
	a->size = bs - a->entry->start;
	return (true);
	}

	return (false);
	}

	static void
	iommu_gas_match_insert(struct iommu_gas_match_args *a)
	{
	bool found;

	/*
	* The prev->end is always aligned on the page size, which
	* causes page alignment for the entry->start too. The size
	* is checked to be multiple of the page size.
	*
	* The page sized gap is created between consequent
	* allocations to ensure that out-of-bounds accesses fault.
	*/
	a->entry->end = a->entry->start + a->size;

	found = iommu_gas_rb_insert(a->domain, a->entry);
	KASSERT(found, ("found dup %p start %jx size %jx",
	a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size));
	a->entry->flags = IOMMU_MAP_ENTRY_MAP;
	}

	static int
	iommu_gas_lowermatch(struct iommu_gas_match_args a, struct iommu_map_entry entry)
	{
	struct iommu_map_entry *child;

	child = RB_RIGHT(entry, rb_entry);
	if (child != NULL && entry->end < a->common->lowaddr &&
	iommu_gas_match_one(a, entry->end, child->first,
	a->common->lowaddr)) {
	iommu_gas_match_insert(a);
	return (0);
	}
	if (entry->free_down < a->size + a->offset + IOMMU_PAGE_SIZE)
	return (ENOMEM);
	if (entry->first >= a->common->lowaddr)
	return (ENOMEM);
	child = RB_LEFT(entry, rb_entry);
	if (child != NULL && 0 == iommu_gas_lowermatch(a, child))
	return (0);
	if (child != NULL && child->last < a->common->lowaddr &&
	iommu_gas_match_one(a, child->last, entry->start,
	a->common->lowaddr)) {
	iommu_gas_match_insert(a);
	return (0);
	}
	child = RB_RIGHT(entry, rb_entry);
	if (child != NULL && 0 == iommu_gas_lowermatch(a, child))
	return (0);
	return (ENOMEM);
	}

	static int
	iommu_gas_uppermatch(struct iommu_gas_match_args a, struct iommu_map_entry entry)
	{
	struct iommu_map_entry *child;

	if (entry->free_down < a->size + a->offset + IOMMU_PAGE_SIZE)
	return (ENOMEM);
	if (entry->last < a->common->highaddr)
	return (ENOMEM);
	child = RB_LEFT(entry, rb_entry);
	if (child != NULL && 0 == iommu_gas_uppermatch(a, child))
	return (0);
	if (child != NULL && child->last >= a->common->highaddr &&
	iommu_gas_match_one(a, child->last, entry->start,
	a->domain->end)) {
	iommu_gas_match_insert(a);
	return (0);
	}
	child = RB_RIGHT(entry, rb_entry);
	if (child != NULL && entry->end >= a->common->highaddr &&
	iommu_gas_match_one(a, entry->end, child->first,
	a->domain->end)) {
	iommu_gas_match_insert(a);
	return (0);
	}
	if (child != NULL && 0 == iommu_gas_uppermatch(a, child))
	return (0);
	return (ENOMEM);
	}

	static int
	iommu_gas_find_space(struct iommu_domain *domain,
	const struct bus_dma_tag_common *common, iommu_gaddr_t size,
	int offset, u_int flags, struct iommu_map_entry *entry)
	{
	struct iommu_gas_match_args a;
	int error;

	IOMMU_DOMAIN_ASSERT_LOCKED(domain);
	KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry));
	KASSERT((size & IOMMU_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size));

	a.domain = domain;
	a.size = size;
	a.offset = offset;
	a.common = common;
	a.gas_flags = flags;
	a.entry = entry;

	/* Handle lower region. */
	if (common->lowaddr > 0) {
	error = iommu_gas_lowermatch(&a,
	RB_ROOT(&domain->rb_root));
	if (error == 0)
	return (0);
	KASSERT(error == ENOMEM,
	("error %d from iommu_gas_lowermatch", error));
	}
	/* Handle upper region. */
	if (common->highaddr >= domain->end)
	return (ENOMEM);
	error = iommu_gas_uppermatch(&a, RB_ROOT(&domain->rb_root));
	KASSERT(error == ENOMEM,
	("error %d from iommu_gas_uppermatch", error));
	return (error);
	}

	static int
	iommu_gas_alloc_region(struct iommu_domain domain, struct iommu_map_entry entry,
	u_int flags)
	{
	struct iommu_map_entry next, prev;
	bool found;

	IOMMU_DOMAIN_ASSERT_LOCKED(domain);

	if ((entry->start & IOMMU_PAGE_MASK) != 0 \|\|
	(entry->end & IOMMU_PAGE_MASK) != 0)
	return (EINVAL);
	if (entry->start >= entry->end)
	return (EINVAL);
	if (entry->end >= domain->end)
	return (EINVAL);

	next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, entry);
	KASSERT(next != NULL, ("next must be non-null %p %jx", domain,
	(uintmax_t)entry->start));
	prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next);
	/* prev could be NULL */

	/*
	* Adapt to broken BIOSes which specify overlapping RMRR
	* entries.
	*
	* XXXKIB: this does not handle a case when prev or next
	* entries are completely covered by the current one, which
	* extends both ways.
	*/
	if (prev != NULL && prev->end > entry->start &&
	(prev->flags & IOMMU_MAP_ENTRY_PLACE) == 0) {
	if ((flags & IOMMU_MF_RMRR) == 0 \|\|
	(prev->flags & IOMMU_MAP_ENTRY_RMRR) == 0)
	return (EBUSY);
	entry->start = prev->end;
	}
	if (next->start < entry->end &&
	(next->flags & IOMMU_MAP_ENTRY_PLACE) == 0) {
	if ((flags & IOMMU_MF_RMRR) == 0 \|\|
	(next->flags & IOMMU_MAP_ENTRY_RMRR) == 0)
	return (EBUSY);
	entry->end = next->start;
	}
	if (entry->end == entry->start)
	return (0);

	if (prev != NULL && prev->end > entry->start) {
	/* This assumes that prev is the placeholder entry. */
	iommu_gas_rb_remove(domain, prev);
	prev = NULL;
	}
	if (next->start < entry->end) {
	iommu_gas_rb_remove(domain, next);
	next = NULL;
	}

	found = iommu_gas_rb_insert(domain, entry);
	KASSERT(found, ("found RMRR dup %p start %jx end %jx",
	domain, (uintmax_t)entry->start, (uintmax_t)entry->end));
	if ((flags & IOMMU_MF_RMRR) != 0)
	entry->flags = IOMMU_MAP_ENTRY_RMRR;

	#ifdef INVARIANTS
	struct iommu_map_entry ip, in;
	ip = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry);
	in = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry);
	KASSERT(prev == NULL \|\| ip == prev,
	("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)",
	entry, entry->start, entry->end, prev,
	prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end,
	ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end));
	KASSERT(next == NULL \|\| in == next,
	("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)",
	entry, entry->start, entry->end, next,
	next == NULL ? 0 : next->start, next == NULL ? 0 : next->end,
	in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end));
	#endif

	return (0);
	}

	void
	iommu_gas_free_space(struct iommu_domain domain, struct iommu_map_entry entry)
	{

	IOMMU_DOMAIN_ASSERT_LOCKED(domain);
	KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE \| IOMMU_MAP_ENTRY_RMRR \|
	IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP,
	("permanent entry %p %p", domain, entry));

	iommu_gas_rb_remove(domain, entry);
	entry->flags &= ~IOMMU_MAP_ENTRY_MAP;
	#ifdef INVARIANTS
	if (iommu_check_free)
	iommu_gas_check_free(domain);
	#endif
	}

	void
	iommu_gas_free_region(struct iommu_domain domain, struct iommu_map_entry entry)
	{
	struct iommu_map_entry next, prev;

	IOMMU_DOMAIN_ASSERT_LOCKED(domain);
	KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE \| IOMMU_MAP_ENTRY_RMRR \|
	IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR,
	("non-RMRR entry %p %p", domain, entry));

	prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry);
	next = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry);
	iommu_gas_rb_remove(domain, entry);
	entry->flags &= ~IOMMU_MAP_ENTRY_RMRR;

	if (prev == NULL)
	iommu_gas_rb_insert(domain, domain->first_place);
	if (next == NULL)
	iommu_gas_rb_insert(domain, domain->last_place);
	}

	int
	iommu_gas_map(struct iommu_domain *domain,
	const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset,
	u_int eflags, u_int flags, vm_page_t ma, struct iommu_map_entry *res)
	{
	struct iommu_map_entry *entry;
	int error;

	KASSERT((flags & ~(IOMMU_MF_CANWAIT \| IOMMU_MF_CANSPLIT)) == 0,
	("invalid flags 0x%x", flags));

	entry = iommu_gas_alloc_entry(domain,
	(flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0);
	if (entry == NULL)
	return (ENOMEM);
	IOMMU_DOMAIN_LOCK(domain);
	error = iommu_gas_find_space(domain, common, size, offset, flags,
	entry);
	if (error == ENOMEM) {
	IOMMU_DOMAIN_UNLOCK(domain);
	iommu_gas_free_entry(domain, entry);
	return (error);
	}
	#ifdef INVARIANTS
	if (iommu_check_free)
	iommu_gas_check_free(domain);
	#endif
	KASSERT(error == 0,
	("unexpected error %d from iommu_gas_find_entry", error));
	KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx",
	(uintmax_t)entry->end, (uintmax_t)domain->end));
	entry->flags \|= eflags;
	IOMMU_DOMAIN_UNLOCK(domain);

	error = domain_map_buf(domain, entry->start, entry->end - entry->start,
	ma, eflags,
	((flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0));
	if (error == ENOMEM) {
	iommu_domain_unload_entry(entry, true);
	return (error);
	}
	KASSERT(error == 0,
	("unexpected error %d from domain_map_buf", error));

	*res = entry;
	return (0);
	}

	int
	iommu_gas_map_region(struct iommu_domain domain, struct iommu_map_entry entry,
	u_int eflags, u_int flags, vm_page_t *ma)
	{
	iommu_gaddr_t start;
	int error;

	KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain,
	entry, entry->flags));
	KASSERT((flags & ~(IOMMU_MF_CANWAIT \| IOMMU_MF_RMRR)) == 0,
	("invalid flags 0x%x", flags));

	start = entry->start;
	IOMMU_DOMAIN_LOCK(domain);
	error = iommu_gas_alloc_region(domain, entry, flags);
	if (error != 0) {
	IOMMU_DOMAIN_UNLOCK(domain);
	return (error);
	}
	entry->flags \|= eflags;
	IOMMU_DOMAIN_UNLOCK(domain);
	if (entry->end == entry->start)
	return (0);

	error = domain_map_buf(domain, entry->start, entry->end - entry->start,
	ma + OFF_TO_IDX(start - entry->start), eflags,
	((flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0));
	if (error == ENOMEM) {
	iommu_domain_unload_entry(entry, false);
	return (error);
	}
	KASSERT(error == 0,
	("unexpected error %d from domain_map_buf", error));

	return (0);
	}

	int
	iommu_gas_reserve_region(struct iommu_domain *domain, iommu_gaddr_t start,
	iommu_gaddr_t end)
	{
	struct iommu_map_entry *entry;
	int error;

	entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK);
	entry->start = start;
	entry->end = end;
	IOMMU_DOMAIN_LOCK(domain);
	error = iommu_gas_alloc_region(domain, entry, IOMMU_MF_CANWAIT);
	if (error == 0)
	entry->flags \|= IOMMU_MAP_ENTRY_UNMAPPED;
	IOMMU_DOMAIN_UNLOCK(domain);
	if (error != 0)
	iommu_gas_free_entry(domain, entry);
	return (error);
	}

	struct iommu_map_entry *
	iommu_map_alloc_entry(struct iommu_domain *domain, u_int flags)
	{
	struct iommu_map_entry *res;

	res = iommu_gas_alloc_entry(domain, flags);

	return (res);
	}

	void
	iommu_map_free_entry(struct iommu_domain domain, struct iommu_map_entry entry)
	{

	iommu_gas_free_entry(domain, entry);
	}

	int
	iommu_map(struct iommu_domain *domain,
	const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset,
	u_int eflags, u_int flags, vm_page_t ma, struct iommu_map_entry *res)
	{
	int error;

	error = iommu_gas_map(domain, common, size, offset, eflags, flags,
	ma, res);

	return (error);
	}

	int
	iommu_map_region(struct iommu_domain domain, struct iommu_map_entry entry,
	u_int eflags, u_int flags, vm_page_t *ma)
	{
	int error;

	error = iommu_gas_map_region(domain, entry, eflags, flags, ma);

	return (error);
	}

	SYSCTL_NODE(_hw, OID_AUTO, iommu, CTLFLAG_RW \| CTLFLAG_MPSAFE, NULL, "");

	#ifdef INVARIANTS
	SYSCTL_INT(_hw_iommu, OID_AUTO, check_free, CTLFLAG_RWTUN,
	&iommu_check_free, 0,
	"Check the GPA RBtree for free_down and free_after validity");
	#endif
	diff --git a/sys/dev/mpr/mpr.c b/sys/dev/mpr/mpr.c
	index 863c9a59c27d..b202ea40db45 100644
	--- a/sys/dev/mpr/mpr.c
	+++ b/sys/dev/mpr/mpr.c
	@@ -1,3995 +1,3986 @@
	/*-
	* Copyright (c) 2009 Yahoo! Inc.
	* Copyright (c) 2011-2015 LSI Corp.
	* Copyright (c) 2013-2016 Avago Technologies
	* Copyright 2000-2020 Broadcom Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Broadcom Inc. (LSI) MPT-Fusion Host Adapter FreeBSD
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/* Communications core for Avago Technologies (LSI) MPT3 */

	/* TODO Move headers to mprvar */
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/selinfo.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/bio.h>
	#include <sys/malloc.h>
	#include <sys/uio.h>
	#include <sys/sysctl.h>
	#include <sys/smp.h>
	#include <sys/queue.h>
	#include <sys/kthread.h>
	#include <sys/taskqueue.h>
	#include <sys/endian.h>
	#include <sys/eventhandler.h>
	#include <sys/sbuf.h>
	#include <sys/priv.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>
	#include <sys/proc.h>

	#include <dev/pci/pcivar.h>

	#include <cam/cam.h>
	#include <cam/cam_ccb.h>
	#include <cam/scsi/scsi_all.h>

	#include <dev/mpr/mpi/mpi2_type.h>
	#include <dev/mpr/mpi/mpi2.h>
	#include <dev/mpr/mpi/mpi2_ioc.h>
	#include <dev/mpr/mpi/mpi2_sas.h>
	#include <dev/mpr/mpi/mpi2_pci.h>
	#include <dev/mpr/mpi/mpi2_cnfg.h>
	#include <dev/mpr/mpi/mpi2_init.h>
	#include <dev/mpr/mpi/mpi2_tool.h>
	#include <dev/mpr/mpr_ioctl.h>
	#include <dev/mpr/mprvar.h>
	#include <dev/mpr/mpr_table.h>
	#include <dev/mpr/mpr_sas.h>

	static int mpr_diag_reset(struct mpr_softc *sc, int sleep_flag);
	static int mpr_init_queues(struct mpr_softc *sc);
	static void mpr_resize_queues(struct mpr_softc *sc);
	static int mpr_message_unit_reset(struct mpr_softc *sc, int sleep_flag);
	static int mpr_transition_operational(struct mpr_softc *sc);
	static int mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching);
	static void mpr_iocfacts_free(struct mpr_softc *sc);
	static void mpr_startup(void *arg);
	static int mpr_send_iocinit(struct mpr_softc *sc);
	static int mpr_alloc_queues(struct mpr_softc *sc);
	static int mpr_alloc_hw_queues(struct mpr_softc *sc);
	static int mpr_alloc_replies(struct mpr_softc *sc);
	static int mpr_alloc_requests(struct mpr_softc *sc);
	static int mpr_alloc_nvme_prp_pages(struct mpr_softc *sc);
	static int mpr_attach_log(struct mpr_softc *sc);
	static __inline void mpr_complete_command(struct mpr_softc *sc,
	struct mpr_command *cm);
	static void mpr_dispatch_event(struct mpr_softc *sc, uintptr_t data,
	MPI2_EVENT_NOTIFICATION_REPLY *reply);
	static void mpr_config_complete(struct mpr_softc sc, struct mpr_command cm);
	static void mpr_periodic(void *);
	static int mpr_reregister_events(struct mpr_softc *sc);
	static void mpr_enqueue_request(struct mpr_softc sc, struct mpr_command cm);
	static int mpr_get_iocfacts(struct mpr_softc sc, MPI2_IOC_FACTS_REPLY facts);
	static int mpr_wait_db_ack(struct mpr_softc *sc, int timeout, int sleep_flag);
	static int mpr_debug_sysctl(SYSCTL_HANDLER_ARGS);
	static int mpr_dump_reqs(SYSCTL_HANDLER_ARGS);
	static void mpr_parse_debug(struct mpr_softc sc, char list);

	SYSCTL_NODE(_hw, OID_AUTO, mpr, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"MPR Driver Parameters");

	MALLOC_DEFINE(M_MPR, "mpr", "mpr driver memory");

	/*
	* Do a "Diagnostic Reset" aka a hard reset. This should get the chip out of
	* any state and back to its initialization state machine.
	*/
	static char mpt2_reset_magic[] = { 0x00, 0x0f, 0x04, 0x0b, 0x02, 0x07, 0x0d };

	/*
	* Added this union to smoothly convert le64toh cm->cm_desc.Words.
	* Compiler only supports uint64_t to be passed as an argument.
	* Otherwise it will throw this error:
	* "aggregate value used where an integer was expected"
	*/
	typedef union _reply_descriptor {
	u64 word;
	struct {
	u32 low;
	u32 high;
	} u;
	} reply_descriptor, request_descriptor;

	/* Rate limit chain-fail messages to 1 per minute */
	static struct timeval mpr_chainfail_interval = { 60, 0 };

	/*
	* sleep_flag can be either CAN_SLEEP or NO_SLEEP.
	* If this function is called from process context, it can sleep
	* and there is no harm to sleep, in case if this fuction is called
	* from Interrupt handler, we can not sleep and need NO_SLEEP flag set.
	* based on sleep flags driver will call either msleep, pause or DELAY.
	* msleep and pause are of same variant, but pause is used when mpr_mtx
	* is not hold by driver.
	*/
	static int
	mpr_diag_reset(struct mpr_softc *sc,int sleep_flag)
	{
	uint32_t reg;
	int i, error, tries = 0;
	uint8_t first_wait_done = FALSE;

	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	/* Clear any pending interrupts */
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	/*
	* Force NO_SLEEP for threads prohibited to sleep
	* e.a Thread from interrupt handler are prohibited to sleep.
	*/
	if (curthread->td_no_sleeping)
	sleep_flag = NO_SLEEP;

	mpr_dprint(sc, MPR_INIT, "sequence start, sleep_flag=%d\n", sleep_flag);
	/* Push the magic sequence */
	error = ETIMEDOUT;
	while (tries++ < 20) {
	for (i = 0; i < sizeof(mpt2_reset_magic); i++)
	mpr_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET,
	mpt2_reset_magic[i]);

	/* wait 100 msec */
	if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP)
	msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0,
	"mprdiag", hz/10);
	else if (sleep_flag == CAN_SLEEP)
	pause("mprdiag", hz/10);
	else
	DELAY(100 * 1000);

	reg = mpr_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET);
	if (reg & MPI2_DIAG_DIAG_WRITE_ENABLE) {
	error = 0;
	break;
	}
	}
	if (error) {
	mpr_dprint(sc, MPR_INIT, "sequence failed, error=%d, exit\n",
	error);
	return (error);
	}

	/* Send the actual reset. XXX need to refresh the reg? */
	reg \|= MPI2_DIAG_RESET_ADAPTER;
	mpr_dprint(sc, MPR_INIT, "sequence success, sending reset, reg= 0x%x\n",
	reg);
	mpr_regwrite(sc, MPI2_HOST_DIAGNOSTIC_OFFSET, reg);

	/* Wait up to 300 seconds in 50ms intervals */
	error = ETIMEDOUT;
	for (i = 0; i < 6000; i++) {
	/*
	* Wait 50 msec. If this is the first time through, wait 256
	* msec to satisfy Diag Reset timing requirements.
	*/
	if (first_wait_done) {
	if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP)
	msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0,
	"mprdiag", hz/20);
	else if (sleep_flag == CAN_SLEEP)
	pause("mprdiag", hz/20);
	else
	DELAY(50 * 1000);
	} else {
	DELAY(256 * 1000);
	first_wait_done = TRUE;
	}
	/*
	* Check for the RESET_ADAPTER bit to be cleared first, then
	* wait for the RESET state to be cleared, which takes a little
	* longer.
	*/
	reg = mpr_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET);
	if (reg & MPI2_DIAG_RESET_ADAPTER) {
	continue;
	}
	reg = mpr_regread(sc, MPI2_DOORBELL_OFFSET);
	if ((reg & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_RESET) {
	error = 0;
	break;
	}
	}
	if (error) {
	mpr_dprint(sc, MPR_INIT, "reset failed, error= %d, exit\n",
	error);
	return (error);
	}

	mpr_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET, 0x0);
	mpr_dprint(sc, MPR_INIT, "diag reset success, exit\n");

	return (0);
	}

	static int
	mpr_message_unit_reset(struct mpr_softc *sc, int sleep_flag)
	{
	int error;

	MPR_FUNCTRACE(sc);

	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	error = 0;
	mpr_regwrite(sc, MPI2_DOORBELL_OFFSET,
	MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET <<
	MPI2_DOORBELL_FUNCTION_SHIFT);

	if (mpr_wait_db_ack(sc, 5, sleep_flag) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"Doorbell handshake failed\n");
	error = ETIMEDOUT;
	}

	mpr_dprint(sc, MPR_INIT, "%s exit\n", __func__);
	return (error);
	}

	static int
	mpr_transition_ready(struct mpr_softc *sc)
	{
	uint32_t reg, state;
	int error, tries = 0;
	int sleep_flags;

	MPR_FUNCTRACE(sc);
	/* If we are in attach call, do not sleep */
	sleep_flags = (sc->mpr_flags & MPR_FLAGS_ATTACH_DONE)
	? CAN_SLEEP : NO_SLEEP;

	error = 0;

	mpr_dprint(sc, MPR_INIT, "%s entered, sleep_flags= %d\n",
	__func__, sleep_flags);

	while (tries++ < 1200) {
	reg = mpr_regread(sc, MPI2_DOORBELL_OFFSET);
	mpr_dprint(sc, MPR_INIT, " Doorbell= 0x%x\n", reg);

	/*
	* Ensure the IOC is ready to talk. If it's not, try
	* resetting it.
	*/
	if (reg & MPI2_DOORBELL_USED) {
	mpr_dprint(sc, MPR_INIT, " Not ready, sending diag "
	"reset\n");
	mpr_diag_reset(sc, sleep_flags);
	DELAY(50000);
	continue;
	}

	/* Is the adapter owned by another peer? */
	if ((reg & MPI2_DOORBELL_WHO_INIT_MASK) ==
	(MPI2_WHOINIT_PCI_PEER << MPI2_DOORBELL_WHO_INIT_SHIFT)) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "IOC is under the "
	"control of another peer host, aborting "
	"initialization.\n");
	error = ENXIO;
	break;
	}

	state = reg & MPI2_IOC_STATE_MASK;
	if (state == MPI2_IOC_STATE_READY) {
	/* Ready to go! */
	error = 0;
	break;
	} else if (state == MPI2_IOC_STATE_FAULT) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "IOC in fault "
	"state 0x%x, resetting\n",
	state & MPI2_DOORBELL_FAULT_CODE_MASK);
	mpr_diag_reset(sc, sleep_flags);
	} else if (state == MPI2_IOC_STATE_OPERATIONAL) {
	/* Need to take ownership */
	mpr_message_unit_reset(sc, sleep_flags);
	} else if (state == MPI2_IOC_STATE_RESET) {
	/* Wait a bit, IOC might be in transition */
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"IOC in unexpected reset state\n");
	} else {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"IOC in unknown state 0x%x\n", state);
	error = EINVAL;
	break;
	}

	/* Wait 50ms for things to settle down. */
	DELAY(50000);
	}

	if (error)
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"Cannot transition IOC to ready\n");
	mpr_dprint(sc, MPR_INIT, "%s exit\n", __func__);
	return (error);
	}

	static int
	mpr_transition_operational(struct mpr_softc *sc)
	{
	uint32_t reg, state;
	int error;

	MPR_FUNCTRACE(sc);

	error = 0;
	reg = mpr_regread(sc, MPI2_DOORBELL_OFFSET);
	mpr_dprint(sc, MPR_INIT, "%s entered, Doorbell= 0x%x\n", __func__, reg);

	state = reg & MPI2_IOC_STATE_MASK;
	if (state != MPI2_IOC_STATE_READY) {
	mpr_dprint(sc, MPR_INIT, "IOC not ready\n");
	if ((error = mpr_transition_ready(sc)) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"failed to transition ready, exit\n");
	return (error);
	}
	}

	error = mpr_send_iocinit(sc);
	mpr_dprint(sc, MPR_INIT, "%s exit\n", __func__);

	return (error);
	}

	static void
	mpr_resize_queues(struct mpr_softc *sc)
	{
	u_int reqcr, prireqcr, maxio, sges_per_frame, chain_seg_size;

	/*
	* Size the queues. Since the reply queues always need one free
	* entry, we'll deduct one reply message here. The LSI documents
	* suggest instead to add a count to the request queue, but I think
	* that it's better to deduct from reply queue.
	*/
	prireqcr = MAX(1, sc->max_prireqframes);
	prireqcr = MIN(prireqcr, sc->facts->HighPriorityCredit);

	reqcr = MAX(2, sc->max_reqframes);
	reqcr = MIN(reqcr, sc->facts->RequestCredit);

	sc->num_reqs = prireqcr + reqcr;
	sc->num_prireqs = prireqcr;
	sc->num_replies = MIN(sc->max_replyframes + sc->max_evtframes,
	sc->facts->MaxReplyDescriptorPostQueueDepth) - 1;

	/* Store the request frame size in bytes rather than as 32bit words */
	sc->reqframesz = sc->facts->IOCRequestFrameSize * 4;

	/*
	* Gen3 and beyond uses the IOCMaxChainSegmentSize from IOC Facts to
	* get the size of a Chain Frame. Previous versions use the size as a
	* Request Frame for the Chain Frame size. If IOCMaxChainSegmentSize
	* is 0, use the default value. The IOCMaxChainSegmentSize is the
	* number of 16-byte elelements that can fit in a Chain Frame, which is
	* the size of an IEEE Simple SGE.
	*/
	if (sc->facts->MsgVersion >= MPI2_VERSION_02_05) {
	chain_seg_size = htole16(sc->facts->IOCMaxChainSegmentSize);
	if (chain_seg_size == 0)
	chain_seg_size = MPR_DEFAULT_CHAIN_SEG_SIZE;
	sc->chain_frame_size = chain_seg_size *
	MPR_MAX_CHAIN_ELEMENT_SIZE;
	} else {
	sc->chain_frame_size = sc->reqframesz;
	}

	/*
	* Max IO Size is Page Size * the following:
	* ((SGEs per frame - 1 for chain element) * Max Chain Depth)
	* + 1 for no chain needed in last frame
	*
	* If user suggests a Max IO size to use, use the smaller of the
	* user's value and the calculated value as long as the user's
	* value is larger than 0. The user's value is in pages.
	*/
	sges_per_frame = sc->chain_frame_size/sizeof(MPI2_IEEE_SGE_SIMPLE64)-1;
	maxio = (sges_per_frame * sc->facts->MaxChainDepth + 1) * PAGE_SIZE;

	/*
	* If I/O size limitation requested then use it and pass up to CAM.
	* If not, use MAXPHYS as an optimization hint, but report HW limit.
	*/
	if (sc->max_io_pages > 0) {
	maxio = min(maxio, sc->max_io_pages * PAGE_SIZE);
	sc->maxio = maxio;
	} else {
	sc->maxio = maxio;
	maxio = min(maxio, MAXPHYS);
	}

	sc->num_chains = (maxio / PAGE_SIZE + sges_per_frame - 2) /
	sges_per_frame * reqcr;
	if (sc->max_chains > 0 && sc->max_chains < sc->num_chains)
	sc->num_chains = sc->max_chains;

	/*
	* Figure out the number of MSIx-based queues. If the firmware or
	* user has done something crazy and not allowed enough credit for
	* the queues to be useful then don't enable multi-queue.
	*/
	if (sc->facts->MaxMSIxVectors < 2)
	sc->msi_msgs = 1;

	if (sc->msi_msgs > 1) {
	sc->msi_msgs = MIN(sc->msi_msgs, mp_ncpus);
	sc->msi_msgs = MIN(sc->msi_msgs, sc->facts->MaxMSIxVectors);
	if (sc->num_reqs / sc->msi_msgs < 2)
	sc->msi_msgs = 1;
	}

	mpr_dprint(sc, MPR_INIT, "Sized queues to q=%d reqs=%d replies=%d\n",
	sc->msi_msgs, sc->num_reqs, sc->num_replies);
	}

	/*
	* This is called during attach and when re-initializing due to a Diag Reset.
	* IOC Facts is used to allocate many of the structures needed by the driver.
	* If called from attach, de-allocation is not required because the driver has
	* not allocated any structures yet, but if called from a Diag Reset, previously
	* allocated structures based on IOC Facts will need to be freed and re-
	* allocated bases on the latest IOC Facts.
	*/
	static int
	mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching)
	{
	int error;
	Mpi2IOCFactsReply_t saved_facts;
	uint8_t saved_mode, reallocating;

	mpr_dprint(sc, MPR_INIT\|MPR_TRACE, "%s entered\n", __func__);

	/* Save old IOC Facts and then only reallocate if Facts have changed */
	if (!attaching) {
	bcopy(sc->facts, &saved_facts, sizeof(MPI2_IOC_FACTS_REPLY));
	}

	/*
	* Get IOC Facts. In all cases throughout this function, panic if doing
	* a re-initialization and only return the error if attaching so the OS
	* can handle it.
	*/
	if ((error = mpr_get_iocfacts(sc, sc->facts)) != 0) {
	if (attaching) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "Failed to get "
	"IOC Facts with error %d, exit\n", error);
	return (error);
	} else {
	panic("%s failed to get IOC Facts with error %d\n",
	__func__, error);
	}
	}

	MPR_DPRINT_PAGE(sc, MPR_XINFO, iocfacts, sc->facts);

	snprintf(sc->fw_version, sizeof(sc->fw_version),
	"%02d.%02d.%02d.%02d",
	sc->facts->FWVersion.Struct.Major,
	sc->facts->FWVersion.Struct.Minor,
	sc->facts->FWVersion.Struct.Unit,
	sc->facts->FWVersion.Struct.Dev);

	snprintf(sc->msg_version, sizeof(sc->msg_version), "%d.%d",
	(sc->facts->MsgVersion & MPI2_IOCFACTS_MSGVERSION_MAJOR_MASK) >>
	MPI2_IOCFACTS_MSGVERSION_MAJOR_SHIFT,
	(sc->facts->MsgVersion & MPI2_IOCFACTS_MSGVERSION_MINOR_MASK) >>
	MPI2_IOCFACTS_MSGVERSION_MINOR_SHIFT);

	mpr_dprint(sc, MPR_INFO, "Firmware: %s, Driver: %s\n", sc->fw_version,
	MPR_DRIVER_VERSION);
	mpr_dprint(sc, MPR_INFO,
	"IOCCapabilities: %b\n", sc->facts->IOCCapabilities,
	"\20" "\3ScsiTaskFull" "\4DiagTrace" "\5SnapBuf" "\6ExtBuf"
	"\7EEDP" "\10BiDirTarg" "\11Multicast" "\14TransRetry" "\15IR"
	"\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc"
	"\22FastPath" "\23RDPQArray" "\24AtomicReqDesc" "\25PCIeSRIOV");

	/*
	* If the chip doesn't support event replay then a hard reset will be
	* required to trigger a full discovery. Do the reset here then
	* retransition to Ready. A hard reset might have already been done,
	* but it doesn't hurt to do it again. Only do this if attaching, not
	* for a Diag Reset.
	*/
	if (attaching && ((sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY) == 0)) {
	mpr_dprint(sc, MPR_INIT, "No event replay, resetting\n");
	mpr_diag_reset(sc, NO_SLEEP);
	if ((error = mpr_transition_ready(sc)) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "Failed to "
	"transition to ready with error %d, exit\n",
	error);
	return (error);
	}
	}

	/*
	* Set flag if IR Firmware is loaded. If the RAID Capability has
	* changed from the previous IOC Facts, log a warning, but only if
	* checking this after a Diag Reset and not during attach.
	*/
	saved_mode = sc->ir_firmware;
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID)
	sc->ir_firmware = 1;
	if (!attaching) {
	if (sc->ir_firmware != saved_mode) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "new IR/IT mode "
	"in IOC Facts does not match previous mode\n");
	}
	}

	/* Only deallocate and reallocate if relevant IOC Facts have changed */
	reallocating = FALSE;
	sc->mpr_flags &= ~MPR_FLAGS_REALLOCATED;

	if ((!attaching) &&
	((saved_facts.MsgVersion != sc->facts->MsgVersion) \|\|
	(saved_facts.HeaderVersion != sc->facts->HeaderVersion) \|\|
	(saved_facts.MaxChainDepth != sc->facts->MaxChainDepth) \|\|
	(saved_facts.RequestCredit != sc->facts->RequestCredit) \|\|
	(saved_facts.ProductID != sc->facts->ProductID) \|\|
	(saved_facts.IOCCapabilities != sc->facts->IOCCapabilities) \|\|
	(saved_facts.IOCRequestFrameSize !=
	sc->facts->IOCRequestFrameSize) \|\|
	(saved_facts.IOCMaxChainSegmentSize !=
	sc->facts->IOCMaxChainSegmentSize) \|\|
	(saved_facts.MaxTargets != sc->facts->MaxTargets) \|\|
	(saved_facts.MaxSasExpanders != sc->facts->MaxSasExpanders) \|\|
	(saved_facts.MaxEnclosures != sc->facts->MaxEnclosures) \|\|
	(saved_facts.HighPriorityCredit != sc->facts->HighPriorityCredit) \|\|
	(saved_facts.MaxReplyDescriptorPostQueueDepth !=
	sc->facts->MaxReplyDescriptorPostQueueDepth) \|\|
	(saved_facts.ReplyFrameSize != sc->facts->ReplyFrameSize) \|\|
	(saved_facts.MaxVolumes != sc->facts->MaxVolumes) \|\|
	(saved_facts.MaxPersistentEntries !=
	sc->facts->MaxPersistentEntries))) {
	reallocating = TRUE;

	/* Record that we reallocated everything */
	sc->mpr_flags \|= MPR_FLAGS_REALLOCATED;
	}

	/*
	* Some things should be done if attaching or re-allocating after a Diag
	* Reset, but are not needed after a Diag Reset if the FW has not
	* changed.
	*/
	if (attaching \|\| reallocating) {
	/*
	* Check if controller supports FW diag buffers and set flag to
	* enable each type.
	*/
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER)
	sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_TRACE].
	enabled = TRUE;
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER)
	sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_SNAPSHOT].
	enabled = TRUE;
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER)
	sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_EXTENDED].
	enabled = TRUE;

	/*
	* Set flags for some supported items.
	*/
	if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP)
	sc->eedp_enabled = TRUE;
	if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR)
	sc->control_TLR = TRUE;
	if ((sc->facts->IOCCapabilities &
	MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ) &&
	(sc->mpr_flags & MPR_FLAGS_SEA_IOC))
	sc->atomic_desc_capable = TRUE;

	mpr_resize_queues(sc);

	/*
	* Initialize all Tail Queues
	*/
	TAILQ_INIT(&sc->req_list);
	TAILQ_INIT(&sc->high_priority_req_list);
	TAILQ_INIT(&sc->chain_list);
	TAILQ_INIT(&sc->prp_page_list);
	TAILQ_INIT(&sc->tm_list);
	}

	/*
	* If doing a Diag Reset and the FW is significantly different
	* (reallocating will be set above in IOC Facts comparison), then all
	* buffers based on the IOC Facts will need to be freed before they are
	* reallocated.
	*/
	if (reallocating) {
	mpr_iocfacts_free(sc);
	mprsas_realloc_targets(sc, saved_facts.MaxTargets +
	saved_facts.MaxVolumes);
	}

	/*
	* Any deallocation has been completed. Now start reallocating
	* if needed. Will only need to reallocate if attaching or if the new
	* IOC Facts are different from the previous IOC Facts after a Diag
	* Reset. Targets have already been allocated above if needed.
	*/
	error = 0;
	while (attaching \|\| reallocating) {
	if ((error = mpr_alloc_hw_queues(sc)) != 0)
	break;
	if ((error = mpr_alloc_replies(sc)) != 0)
	break;
	if ((error = mpr_alloc_requests(sc)) != 0)
	break;
	if ((error = mpr_alloc_queues(sc)) != 0)
	break;
	break;
	}
	if (error) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Failed to alloc queues with error %d\n", error);
	mpr_free(sc);
	return (error);
	}

	/* Always initialize the queues */
	bzero(sc->free_queue, sc->fqdepth * 4);
	mpr_init_queues(sc);

	/*
	* Always get the chip out of the reset state, but only panic if not
	* attaching. If attaching and there is an error, that is handled by
	* the OS.
	*/
	error = mpr_transition_operational(sc);
	if (error != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "Failed to "
	"transition to operational with error %d\n", error);
	mpr_free(sc);
	return (error);
	}

	/*
	* Finish the queue initialization.
	* These are set here instead of in mpr_init_queues() because the
	* IOC resets these values during the state transition in
	* mpr_transition_operational(). The free index is set to 1
	* because the corresponding index in the IOC is set to 0, and the
	* IOC treats the queues as full if both are set to the same value.
	* Hence the reason that the queue can't hold all of the possible
	* replies.
	*/
	sc->replypostindex = 0;
	mpr_regwrite(sc, MPI2_REPLY_FREE_HOST_INDEX_OFFSET, sc->replyfreeindex);
	mpr_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET, 0);

	/*
	* Attach the subsystems so they can prepare their event masks.
	* XXX Should be dynamic so that IM/IR and user modules can attach
	*/
	error = 0;
	while (attaching) {
	mpr_dprint(sc, MPR_INIT, "Attaching subsystems\n");
	if ((error = mpr_attach_log(sc)) != 0)
	break;
	if ((error = mpr_attach_sas(sc)) != 0)
	break;
	if ((error = mpr_attach_user(sc)) != 0)
	break;
	break;
	}
	if (error) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Failed to attach all subsystems: error %d\n", error);
	mpr_free(sc);
	return (error);
	}

	/*
	* XXX If the number of MSI-X vectors changes during re-init, this
	* won't see it and adjust.
	*/
	if (attaching && (error = mpr_pci_setup_interrupts(sc)) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Failed to setup interrupts\n");
	mpr_free(sc);
	return (error);
	}

	return (error);
	}

	/*
	* This is called if memory is being free (during detach for example) and when
	* buffers need to be reallocated due to a Diag Reset.
	*/
	static void
	mpr_iocfacts_free(struct mpr_softc *sc)
	{
	struct mpr_command *cm;
	int i;

	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);

	if (sc->free_busaddr != 0)
	bus_dmamap_unload(sc->queues_dmat, sc->queues_map);
	if (sc->free_queue != NULL)
	bus_dmamem_free(sc->queues_dmat, sc->free_queue,
	sc->queues_map);
	if (sc->queues_dmat != NULL)
	bus_dma_tag_destroy(sc->queues_dmat);

	if (sc->chain_frames != NULL) {
	bus_dmamap_unload(sc->chain_dmat, sc->chain_map);
	bus_dmamem_free(sc->chain_dmat, sc->chain_frames,
	sc->chain_map);
	}
	if (sc->chain_dmat != NULL)
	bus_dma_tag_destroy(sc->chain_dmat);

	if (sc->sense_busaddr != 0)
	bus_dmamap_unload(sc->sense_dmat, sc->sense_map);
	if (sc->sense_frames != NULL)
	bus_dmamem_free(sc->sense_dmat, sc->sense_frames,
	sc->sense_map);
	if (sc->sense_dmat != NULL)
	bus_dma_tag_destroy(sc->sense_dmat);

	if (sc->prp_page_busaddr != 0)
	bus_dmamap_unload(sc->prp_page_dmat, sc->prp_page_map);
	if (sc->prp_pages != NULL)
	bus_dmamem_free(sc->prp_page_dmat, sc->prp_pages,
	sc->prp_page_map);
	if (sc->prp_page_dmat != NULL)
	bus_dma_tag_destroy(sc->prp_page_dmat);

	if (sc->reply_busaddr != 0)
	bus_dmamap_unload(sc->reply_dmat, sc->reply_map);
	if (sc->reply_frames != NULL)
	bus_dmamem_free(sc->reply_dmat, sc->reply_frames,
	sc->reply_map);
	if (sc->reply_dmat != NULL)
	bus_dma_tag_destroy(sc->reply_dmat);

	if (sc->req_busaddr != 0)
	bus_dmamap_unload(sc->req_dmat, sc->req_map);
	if (sc->req_frames != NULL)
	bus_dmamem_free(sc->req_dmat, sc->req_frames, sc->req_map);
	if (sc->req_dmat != NULL)
	bus_dma_tag_destroy(sc->req_dmat);

	if (sc->chains != NULL)
	free(sc->chains, M_MPR);
	if (sc->prps != NULL)
	free(sc->prps, M_MPR);
	if (sc->commands != NULL) {
	for (i = 1; i < sc->num_reqs; i++) {
	cm = &sc->commands[i];
	bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap);
	}
	free(sc->commands, M_MPR);
	}
	if (sc->buffer_dmat != NULL)
	bus_dma_tag_destroy(sc->buffer_dmat);

	mpr_pci_free_interrupts(sc);
	free(sc->queues, M_MPR);
	sc->queues = NULL;
	}

	/*
	* The terms diag reset and hard reset are used interchangeably in the MPI
	* docs to mean resetting the controller chip. In this code diag reset
	* cleans everything up, and the hard reset function just sends the reset
	* sequence to the chip. This should probably be refactored so that every
	* subsystem gets a reset notification of some sort, and can clean up
	* appropriately.
	*/
	int
	mpr_reinit(struct mpr_softc *sc)
	{
	int error;
	struct mprsas_softc *sassc;

	sassc = sc->sassc;

	MPR_FUNCTRACE(sc);

	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	mpr_dprint(sc, MPR_INIT\|MPR_INFO, "Reinitializing controller\n");
	if (sc->mpr_flags & MPR_FLAGS_DIAGRESET) {
	mpr_dprint(sc, MPR_INIT, "Reset already in progress\n");
	return 0;
	}

	/*
	* Make sure the completion callbacks can recognize they're getting
	* a NULL cm_reply due to a reset.
	*/
	sc->mpr_flags \|= MPR_FLAGS_DIAGRESET;

	/*
	* Mask interrupts here.
	*/
	mpr_dprint(sc, MPR_INIT, "Masking interrupts and resetting\n");
	mpr_mask_intr(sc);

	error = mpr_diag_reset(sc, CAN_SLEEP);
	if (error != 0) {
	panic("%s hard reset failed with error %d\n", __func__, error);
	}

	/* Restore the PCI state, including the MSI-X registers */
	mpr_pci_restore(sc);

	/* Give the I/O subsystem special priority to get itself prepared */
	mprsas_handle_reinit(sc);

	/*
	* Get IOC Facts and allocate all structures based on this information.
	* The attach function will also call mpr_iocfacts_allocate at startup.
	* If relevant values have changed in IOC Facts, this function will free
	* all of the memory based on IOC Facts and reallocate that memory.
	*/
	if ((error = mpr_iocfacts_allocate(sc, FALSE)) != 0) {
	panic("%s IOC Facts based allocation failed with error %d\n",
	__func__, error);
	}

	/*
	* Mapping structures will be re-allocated after getting IOC Page8, so
	* free these structures here.
	*/
	mpr_mapping_exit(sc);

	/*
	* The static page function currently read is IOC Page8. Others can be
	* added in future. It's possible that the values in IOC Page8 have
	* changed after a Diag Reset due to user modification, so always read
	* these. Interrupts are masked, so unmask them before getting config
	* pages.
	*/
	mpr_unmask_intr(sc);
	sc->mpr_flags &= ~MPR_FLAGS_DIAGRESET;
	mpr_base_static_config_pages(sc);

	/*
	* Some mapping info is based in IOC Page8 data, so re-initialize the
	* mapping tables.
	*/
	mpr_mapping_initialize(sc);

	/*
	* Restart will reload the event masks clobbered by the reset, and
	* then enable the port.
	*/
	mpr_reregister_events(sc);

	/* the end of discovery will release the simq, so we're done. */
	mpr_dprint(sc, MPR_INIT\|MPR_XINFO, "Finished sc %p post %u free %u\n",
	sc, sc->replypostindex, sc->replyfreeindex);
	mprsas_release_simq_reinit(sassc);
	mpr_dprint(sc, MPR_INIT, "%s exit error= %d\n", __func__, error);

	return 0;
	}

	/* Wait for the chip to ACK a word that we've put into its FIFO
	* Wait for <timeout> seconds. In single loop wait for busy loop
	* for 500 microseconds.
	* Total is [ 0.5 * (2000 * <timeout>) ] in miliseconds.
	* */
	static int
	mpr_wait_db_ack(struct mpr_softc *sc, int timeout, int sleep_flag)
	{
	u32 cntdn, count;
	u32 int_status;
	u32 doorbell;

	count = 0;
	cntdn = (sleep_flag == CAN_SLEEP) ? 1000timeout : 2000timeout;
	do {
	int_status = mpr_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);
	if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
	mpr_dprint(sc, MPR_TRACE, "%s: successful count(%d), "
	"timeout(%d)\n", __func__, count, timeout);
	return 0;
	} else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
	doorbell = mpr_regread(sc, MPI2_DOORBELL_OFFSET);
	if ((doorbell & MPI2_IOC_STATE_MASK) ==
	MPI2_IOC_STATE_FAULT) {
	mpr_dprint(sc, MPR_FAULT,
	"fault_state(0x%04x)!\n", doorbell);
	return (EFAULT);
	}
	} else if (int_status == 0xFFFFFFFF)
	goto out;

	/*
	* If it can sleep, sleep for 1 milisecond, else busy loop for
	* 0.5 milisecond
	*/
	if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP)
	msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0, "mprdba",
	hz/1000);
	else if (sleep_flag == CAN_SLEEP)
	pause("mprdba", hz/1000);
	else
	DELAY(500);
	count++;
	} while (--cntdn);

	out:
	mpr_dprint(sc, MPR_FAULT, "%s: failed due to timeout count(%d), "
	"int_status(%x)!\n", __func__, count, int_status);
	return (ETIMEDOUT);
	}

	/* Wait for the chip to signal that the next word in its FIFO can be fetched */
	static int
	mpr_wait_db_int(struct mpr_softc *sc)
	{
	int retry;

	for (retry = 0; retry < MPR_DB_MAX_WAIT; retry++) {
	if ((mpr_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET) &
	MPI2_HIS_IOC2SYS_DB_STATUS) != 0)
	return (0);
	DELAY(2000);
	}
	return (ETIMEDOUT);
	}

	/* Step through the synchronous command state machine, i.e. "Doorbell mode" */
	static int
	mpr_request_sync(struct mpr_softc sc, void req, MPI2_DEFAULT_REPLY *reply,
	int req_sz, int reply_sz, int timeout)
	{
	uint32_t *data32;
	uint16_t *data16;
	int i, count, ioc_sz, residual;
	int sleep_flags = CAN_SLEEP;

	if (curthread->td_no_sleeping)
	sleep_flags = NO_SLEEP;

	/* Step 1 */
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	/* Step 2 */
	if (mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED)
	return (EBUSY);

	/* Step 3
	* Announce that a message is coming through the doorbell. Messages
	* are pushed at 32bit words, so round up if needed.
	*/
	count = (req_sz + 3) / 4;
	mpr_regwrite(sc, MPI2_DOORBELL_OFFSET,
	(MPI2_FUNCTION_HANDSHAKE << MPI2_DOORBELL_FUNCTION_SHIFT) \|
	(count << MPI2_DOORBELL_ADD_DWORDS_SHIFT));

	/* Step 4 */
	if (mpr_wait_db_int(sc) \|\|
	(mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED) == 0) {
	mpr_dprint(sc, MPR_FAULT, "Doorbell failed to activate\n");
	return (ENXIO);
	}
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	if (mpr_wait_db_ack(sc, 5, sleep_flags) != 0) {
	mpr_dprint(sc, MPR_FAULT, "Doorbell handshake failed\n");
	return (ENXIO);
	}

	/* Step 5 */
	/* Clock out the message data synchronously in 32-bit dwords*/
	data32 = (uint32_t *)req;
	for (i = 0; i < count; i++) {
	mpr_regwrite(sc, MPI2_DOORBELL_OFFSET, htole32(data32[i]));
	if (mpr_wait_db_ack(sc, 5, sleep_flags) != 0) {
	mpr_dprint(sc, MPR_FAULT,
	"Timeout while writing doorbell\n");
	return (ENXIO);
	}
	}

	/* Step 6 */
	/* Clock in the reply in 16-bit words. The total length of the
	* message is always in the 4th byte, so clock out the first 2 words
	* manually, then loop the rest.
	*/
	data16 = (uint16_t *)reply;
	if (mpr_wait_db_int(sc) != 0) {
	mpr_dprint(sc, MPR_FAULT, "Timeout reading doorbell 0\n");
	return (ENXIO);
	}
	data16[0] =
	mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK;
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	if (mpr_wait_db_int(sc) != 0) {
	mpr_dprint(sc, MPR_FAULT, "Timeout reading doorbell 1\n");
	return (ENXIO);
	}
	data16[1] =
	mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK;
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	/* Number of 32bit words in the message */
	ioc_sz = reply->MsgLength;

	/*
	* Figure out how many 16bit words to clock in without overrunning.
	* The precision loss with dividing reply_sz can safely be
	* ignored because the messages can only be multiples of 32bits.
	*/
	residual = 0;
	count = MIN((reply_sz / 4), ioc_sz) * 2;
	if (count < ioc_sz * 2) {
	residual = ioc_sz * 2 - count;
	mpr_dprint(sc, MPR_ERROR, "Driver error, throwing away %d "
	"residual message words\n", residual);
	}

	for (i = 2; i < count; i++) {
	if (mpr_wait_db_int(sc) != 0) {
	mpr_dprint(sc, MPR_FAULT,
	"Timeout reading doorbell %d\n", i);
	return (ENXIO);
	}
	data16[i] = mpr_regread(sc, MPI2_DOORBELL_OFFSET) &
	MPI2_DOORBELL_DATA_MASK;
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	}

	/*
	* Pull out residual words that won't fit into the provided buffer.
	* This keeps the chip from hanging due to a driver programming
	* error.
	*/
	while (residual--) {
	if (mpr_wait_db_int(sc) != 0) {
	mpr_dprint(sc, MPR_FAULT, "Timeout reading doorbell\n");
	return (ENXIO);
	}
	(void)mpr_regread(sc, MPI2_DOORBELL_OFFSET);
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	}

	/* Step 7 */
	if (mpr_wait_db_int(sc) != 0) {
	mpr_dprint(sc, MPR_FAULT, "Timeout waiting to exit doorbell\n");
	return (ENXIO);
	}
	if (mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED)
	mpr_dprint(sc, MPR_FAULT, "Warning, doorbell still active\n");
	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	return (0);
	}

	static void
	mpr_enqueue_request(struct mpr_softc sc, struct mpr_command cm)
	{
	request_descriptor rd;

	MPR_FUNCTRACE(sc);
	mpr_dprint(sc, MPR_TRACE, "SMID %u cm %p ccb %p\n",
	cm->cm_desc.Default.SMID, cm, cm->cm_ccb);

	if (sc->mpr_flags & MPR_FLAGS_ATTACH_DONE && !(sc->mpr_flags &
	MPR_FLAGS_SHUTDOWN))
	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	if (++sc->io_cmds_active > sc->io_cmds_highwater)
	sc->io_cmds_highwater++;

	KASSERT(cm->cm_state == MPR_CM_STATE_BUSY, ("command not busy\n"));
	cm->cm_state = MPR_CM_STATE_INQUEUE;

	if (sc->atomic_desc_capable) {
	rd.u.low = cm->cm_desc.Words.Low;
	mpr_regwrite(sc, MPI26_ATOMIC_REQUEST_DESCRIPTOR_POST_OFFSET,
	rd.u.low);
	} else {
	rd.u.low = cm->cm_desc.Words.Low;
	rd.u.high = cm->cm_desc.Words.High;
	rd.word = htole64(rd.word);
	mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET,
	rd.u.low);
	mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET,
	rd.u.high);
	}
	}

	/*
	* Just the FACTS, ma'am.
	*/
	static int
	mpr_get_iocfacts(struct mpr_softc sc, MPI2_IOC_FACTS_REPLY facts)
	{
	MPI2_DEFAULT_REPLY *reply;
	MPI2_IOC_FACTS_REQUEST request;
	int error, req_sz, reply_sz;

	MPR_FUNCTRACE(sc);
	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	req_sz = sizeof(MPI2_IOC_FACTS_REQUEST);
	reply_sz = sizeof(MPI2_IOC_FACTS_REPLY);
	reply = (MPI2_DEFAULT_REPLY *)facts;

	bzero(&request, req_sz);
	request.Function = MPI2_FUNCTION_IOC_FACTS;
	error = mpr_request_sync(sc, &request, reply, req_sz, reply_sz, 5);

	mpr_dprint(sc, MPR_INIT, "%s exit, error= %d\n", __func__, error);
	return (error);
	}

	static int
	mpr_send_iocinit(struct mpr_softc *sc)
	{
	MPI2_IOC_INIT_REQUEST init;
	MPI2_DEFAULT_REPLY reply;
	int req_sz, reply_sz, error;
	struct timeval now;
	uint64_t time_in_msec;

	MPR_FUNCTRACE(sc);
	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	/* Do a quick sanity check on proper initialization */
	if ((sc->pqdepth == 0) \|\| (sc->fqdepth == 0) \|\| (sc->reqframesz == 0)
	\|\| (sc->replyframesz == 0)) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Driver not fully initialized for IOCInit\n");
	return (EINVAL);
	}

	req_sz = sizeof(MPI2_IOC_INIT_REQUEST);
	reply_sz = sizeof(MPI2_IOC_INIT_REPLY);
	bzero(&init, req_sz);
	bzero(&reply, reply_sz);

	/*
	* Fill in the init block. Note that most addresses are
	* deliberately in the lower 32bits of memory. This is a micro-
	* optimzation for PCI/PCIX, though it's not clear if it helps PCIe.
	*/
	init.Function = MPI2_FUNCTION_IOC_INIT;
	init.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
	init.MsgVersion = htole16(MPI2_VERSION);
	init.HeaderVersion = htole16(MPI2_HEADER_VERSION);
	init.SystemRequestFrameSize = htole16((uint16_t)(sc->reqframesz / 4));
	init.ReplyDescriptorPostQueueDepth = htole16(sc->pqdepth);
	init.ReplyFreeQueueDepth = htole16(sc->fqdepth);
	init.SenseBufferAddressHigh = 0;
	init.SystemReplyAddressHigh = 0;
	init.SystemRequestFrameBaseAddress.High = 0;
	init.SystemRequestFrameBaseAddress.Low =
	htole32((uint32_t)sc->req_busaddr);
	init.ReplyDescriptorPostQueueAddress.High = 0;
	init.ReplyDescriptorPostQueueAddress.Low =
	htole32((uint32_t)sc->post_busaddr);
	init.ReplyFreeQueueAddress.High = 0;
	init.ReplyFreeQueueAddress.Low = htole32((uint32_t)sc->free_busaddr);
	getmicrotime(&now);
	time_in_msec = (now.tv_sec * 1000 + now.tv_usec/1000);
	init.TimeStamp.High = htole32((time_in_msec >> 32) & 0xFFFFFFFF);
	init.TimeStamp.Low = htole32(time_in_msec & 0xFFFFFFFF);
	init.HostPageSize = HOST_PAGE_SIZE_4K;

	error = mpr_request_sync(sc, &init, &reply, req_sz, reply_sz, 5);
	if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
	error = ENXIO;

	mpr_dprint(sc, MPR_INIT, "IOCInit status= 0x%x\n", reply.IOCStatus);
	mpr_dprint(sc, MPR_INIT, "%s exit\n", __func__);
	return (error);
	}

	void
	mpr_memaddr_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	bus_addr_t *addr;

	addr = arg;
	*addr = segs[0].ds_addr;
	}

	void
	mpr_memaddr_wait_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	struct mpr_busdma_context *ctx;
	int need_unload, need_free;

	ctx = (struct mpr_busdma_context *)arg;
	need_unload = 0;
	need_free = 0;

	mpr_lock(ctx->softc);
	ctx->error = error;
	ctx->completed = 1;
	if ((error == 0) && (ctx->abandoned == 0)) {
	*ctx->addr = segs[0].ds_addr;
	} else {
	if (nsegs != 0)
	need_unload = 1;
	if (ctx->abandoned != 0)
	need_free = 1;
	}
	if (need_free == 0)
	wakeup(ctx);

	mpr_unlock(ctx->softc);

	if (need_unload != 0) {
	bus_dmamap_unload(ctx->buffer_dmat,
	ctx->buffer_dmamap);
	*ctx->addr = 0;
	}

	if (need_free != 0)
	free(ctx, M_MPR);
	}

	static int
	mpr_alloc_queues(struct mpr_softc *sc)
	{
	struct mpr_queue *q;
	int nq, i;

	nq = sc->msi_msgs;
	mpr_dprint(sc, MPR_INIT\|MPR_XINFO, "Allocating %d I/O queues\n", nq);

	sc->queues = malloc(sizeof(struct mpr_queue) * nq, M_MPR,
	M_NOWAIT\|M_ZERO);
	if (sc->queues == NULL)
	return (ENOMEM);

	for (i = 0; i < nq; i++) {
	q = &sc->queues[i];
	mpr_dprint(sc, MPR_INIT, "Configuring queue %d %p\n", i, q);
	q->sc = sc;
	q->qnum = i;
	}
	return (0);
	}

	static int
	mpr_alloc_hw_queues(struct mpr_softc *sc)
	{
	bus_dma_tag_template_t t;
	bus_addr_t queues_busaddr;
	uint8_t *queues;
	int qsize, fqsize, pqsize;

	/*
	* The reply free queue contains 4 byte entries in multiples of 16 and
	* aligned on a 16 byte boundary. There must always be an unused entry.
	* This queue supplies fresh reply frames for the firmware to use.
	*
	* The reply descriptor post queue contains 8 byte entries in
	* multiples of 16 and aligned on a 16 byte boundary. This queue
	* contains filled-in reply frames sent from the firmware to the host.
	*
	* These two queues are allocated together for simplicity.
	*/
	sc->fqdepth = roundup2(sc->num_replies + 1, 16);
	sc->pqdepth = roundup2(sc->num_replies + 1, 16);
	fqsize= sc->fqdepth * 4;
	pqsize = sc->pqdepth * 8;
	qsize = fqsize + pqsize;

	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.alignment = 16;
	t.lowaddr = BUS_SPACE_MAXADDR_32BIT;
	t.maxsize = t.maxsegsize = qsize;
	t.nsegments = 1;
	if (bus_dma_template_tag(&t, &sc->queues_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate queues DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->queues_dmat, (void **)&queues, BUS_DMA_NOWAIT,
	&sc->queues_map)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate queues memory\n");
	return (ENOMEM);
	}
	bzero(queues, qsize);
	bus_dmamap_load(sc->queues_dmat, sc->queues_map, queues, qsize,
	mpr_memaddr_cb, &queues_busaddr, 0);

	sc->free_queue = (uint32_t *)queues;
	sc->free_busaddr = queues_busaddr;
	sc->post_queue = (MPI2_REPLY_DESCRIPTORS_UNION *)(queues + fqsize);
	sc->post_busaddr = queues_busaddr + fqsize;
	mpr_dprint(sc, MPR_INIT, "free queue busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->free_busaddr, fqsize);
	mpr_dprint(sc, MPR_INIT, "reply queue busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->post_busaddr, pqsize);

	return (0);
	}

	static int
	mpr_alloc_replies(struct mpr_softc *sc)
	{
	bus_dma_tag_template_t t;
	int rsize, num_replies;

	/* Store the reply frame size in bytes rather than as 32bit words */
	sc->replyframesz = sc->facts->ReplyFrameSize * 4;

	/*
	* sc->num_replies should be one less than sc->fqdepth. We need to
	* allocate space for sc->fqdepth replies, but only sc->num_replies
	* replies can be used at once.
	*/
	num_replies = max(sc->fqdepth, sc->num_replies);

	rsize = sc->replyframesz * num_replies;
	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.alignment = 4;
	t.lowaddr = BUS_SPACE_MAXADDR_32BIT;
	t.maxsize = t.maxsegsize = rsize;
	t.nsegments = 1;
	if (bus_dma_template_tag(&t, &sc->reply_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate replies DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->reply_dmat, (void **)&sc->reply_frames,
	BUS_DMA_NOWAIT, &sc->reply_map)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate replies memory\n");
	return (ENOMEM);
	}
	bzero(sc->reply_frames, rsize);
	bus_dmamap_load(sc->reply_dmat, sc->reply_map, sc->reply_frames, rsize,
	mpr_memaddr_cb, &sc->reply_busaddr, 0);
	mpr_dprint(sc, MPR_INIT, "reply frames busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->reply_busaddr, rsize);

	return (0);
	}

	static void
	mpr_load_chains_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	struct mpr_softc *sc = arg;
	struct mpr_chain *chain;
	bus_size_t bo;
	int i, o, s;

	if (error != 0)
	return;

	for (i = 0, o = 0, s = 0; s < nsegs; s++) {
	for (bo = 0; bo + sc->chain_frame_size <= segs[s].ds_len;
	bo += sc->chain_frame_size) {
	chain = &sc->chains[i++];
	chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o);
	chain->chain_busaddr = segs[s].ds_addr + bo;
	o += sc->chain_frame_size;
	mpr_free_chain(sc, chain);
	}
	if (bo != segs[s].ds_len)
	o += segs[s].ds_len - bo;
	}
	sc->chain_free_lowwater = i;
	}

	static int
	mpr_alloc_requests(struct mpr_softc *sc)
	{
	bus_dma_tag_template_t t;
	struct mpr_command *cm;
	int i, rsize, nsegs;

	rsize = sc->reqframesz * sc->num_reqs;
	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.alignment = 16;
	t.lowaddr = BUS_SPACE_MAXADDR_32BIT;
	t.maxsize = t.maxsegsize = rsize;
	t.nsegments = 1;
	if (bus_dma_template_tag(&t, &sc->req_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate request DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->req_dmat, (void **)&sc->req_frames,
	BUS_DMA_NOWAIT, &sc->req_map)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate request memory\n");
	return (ENOMEM);
	}
	bzero(sc->req_frames, rsize);
	bus_dmamap_load(sc->req_dmat, sc->req_map, sc->req_frames, rsize,
	mpr_memaddr_cb, &sc->req_busaddr, 0);
	mpr_dprint(sc, MPR_INIT, "request frames busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->req_busaddr, rsize);

	sc->chains = malloc(sizeof(struct mpr_chain) * sc->num_chains, M_MPR,
	M_NOWAIT \| M_ZERO);
	if (!sc->chains) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n");
	return (ENOMEM);
	}
	rsize = sc->chain_frame_size * sc->num_chains;
	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.alignment = 16;
	t.maxsize = t.maxsegsize = rsize;
	t.nsegments = howmany(rsize, PAGE_SIZE);
	if (bus_dma_template_tag(&t, &sc->chain_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames,
	BUS_DMA_NOWAIT \| BUS_DMA_ZERO, &sc->chain_map)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n");
	return (ENOMEM);
	}
	if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames,
	rsize, mpr_load_chains_cb, sc, BUS_DMA_NOWAIT)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot load chain memory\n");
	bus_dmamem_free(sc->chain_dmat, sc->chain_frames,
	sc->chain_map);
	return (ENOMEM);
	}

	rsize = MPR_SENSE_LEN * sc->num_reqs;
	bus_dma_template_clone(&t, sc->req_dmat);
	t.maxsize = t.maxsegsize = rsize;
	if (bus_dma_template_tag(&t, &sc->sense_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate sense DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->sense_dmat, (void **)&sc->sense_frames,
	BUS_DMA_NOWAIT, &sc->sense_map)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate sense memory\n");
	return (ENOMEM);
	}
	bzero(sc->sense_frames, rsize);
	bus_dmamap_load(sc->sense_dmat, sc->sense_map, sc->sense_frames, rsize,
	mpr_memaddr_cb, &sc->sense_busaddr, 0);
	mpr_dprint(sc, MPR_INIT, "sense frames busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->sense_busaddr, rsize);

	/*
	* Allocate NVMe PRP Pages for NVMe SGL support only if the FW supports
	* these devices.
	*/
	if ((sc->facts->MsgVersion >= MPI2_VERSION_02_06) &&
	(sc->facts->ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES)) {
	if (mpr_alloc_nvme_prp_pages(sc) == ENOMEM)
	return (ENOMEM);
	}

	nsegs = (sc->maxio / PAGE_SIZE) + 1;
	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.nsegments = nsegs;
	t.flags = BUS_DMA_ALLOCNOW;
	t.lockfunc = busdma_lock_mutex;
	t.lockfuncarg = &sc->mpr_mtx;
	if (bus_dma_template_tag(&t, &sc->buffer_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate buffer DMA tag\n");
	return (ENOMEM);
	}

	/*
	* SMID 0 cannot be used as a free command per the firmware spec.
	* Just drop that command instead of risking accounting bugs.
	*/
	sc->commands = malloc(sizeof(struct mpr_command) * sc->num_reqs,
	M_MPR, M_WAITOK \| M_ZERO);
	- if (!sc->commands) {
	- mpr_dprint(sc, MPR_ERROR, "Cannot allocate command memory\n");
	- return (ENOMEM);
	- }
	for (i = 1; i < sc->num_reqs; i++) {
	cm = &sc->commands[i];
	cm->cm_req = sc->req_frames + i * sc->reqframesz;
	cm->cm_req_busaddr = sc->req_busaddr + i * sc->reqframesz;
	cm->cm_sense = &sc->sense_frames[i];
	cm->cm_sense_busaddr = sc->sense_busaddr + i * MPR_SENSE_LEN;
	cm->cm_desc.Default.SMID = i;
	cm->cm_sc = sc;
	cm->cm_state = MPR_CM_STATE_BUSY;
	TAILQ_INIT(&cm->cm_chain_list);
	TAILQ_INIT(&cm->cm_prp_page_list);
	callout_init_mtx(&cm->cm_callout, &sc->mpr_mtx, 0);

	/* XXX Is a failure here a critical problem? */
	if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap)
	== 0) {
	if (i <= sc->num_prireqs)
	mpr_free_high_priority_command(sc, cm);
	else
	mpr_free_command(sc, cm);
	} else {
	panic("failed to allocate command %d\n", i);
	sc->num_reqs = i;
	break;
	}
	}

	return (0);
	}

	/*
	* Allocate contiguous buffers for PCIe NVMe devices for building native PRPs,
	* which are scatter/gather lists for NVMe devices.
	*
	* This buffer must be contiguous due to the nature of how NVMe PRPs are built
	* and translated by FW.
	*
	* returns ENOMEM if memory could not be allocated, otherwise returns 0.
	*/
	static int
	mpr_alloc_nvme_prp_pages(struct mpr_softc *sc)
	{
	bus_dma_tag_template_t t;
	struct mpr_prp_page *prp_page;
	int PRPs_per_page, PRPs_required, pages_required;
	int rsize, i;

	/*
	* Assuming a MAX_IO_SIZE of 1MB and a PAGE_SIZE of 4k, the max number
	* of PRPs (NVMe's Scatter/Gather Element) needed per I/O is:
	* MAX_IO_SIZE / PAGE_SIZE = 256
	*
	* 1 PRP entry in main frame for PRP list pointer still leaves 255 PRPs
	* required for the remainder of the 1MB I/O. 512 PRPs can fit into one
	* page (4096 / 8 = 512), so only one page is required for each I/O.
	*
	* Each of these buffers will need to be contiguous. For simplicity,
	* only one buffer is allocated here, which has all of the space
	* required for the NVMe Queue Depth. If there are problems allocating
	* this one buffer, this function will need to change to allocate
	* individual, contiguous NVME_QDEPTH buffers.
	*
	* The real calculation will use the real max io size. Above is just an
	* example.
	*
	*/
	PRPs_required = sc->maxio / PAGE_SIZE;
	PRPs_per_page = (PAGE_SIZE / PRP_ENTRY_SIZE) - 1;
	pages_required = (PRPs_required / PRPs_per_page) + 1;

	sc->prp_buffer_size = PAGE_SIZE * pages_required;
	rsize = sc->prp_buffer_size * NVME_QDEPTH;
	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.alignment = 4;
	t.lowaddr = BUS_SPACE_MAXADDR_32BIT;
	t.maxsize = t.maxsegsize = rsize;
	t.nsegments = 1;
	if (bus_dma_template_tag(&t, &sc->prp_page_dmat)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate NVMe PRP DMA "
	"tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->prp_page_dmat, (void **)&sc->prp_pages,
	BUS_DMA_NOWAIT, &sc->prp_page_map)) {
	mpr_dprint(sc, MPR_ERROR, "Cannot allocate NVMe PRP memory\n");
	return (ENOMEM);
	}
	bzero(sc->prp_pages, rsize);
	bus_dmamap_load(sc->prp_page_dmat, sc->prp_page_map, sc->prp_pages,
	rsize, mpr_memaddr_cb, &sc->prp_page_busaddr, 0);

	sc->prps = malloc(sizeof(struct mpr_prp_page) * NVME_QDEPTH, M_MPR,
	M_WAITOK \| M_ZERO);
	for (i = 0; i < NVME_QDEPTH; i++) {
	prp_page = &sc->prps[i];
	prp_page->prp_page = (uint64_t *)(sc->prp_pages +
	i * sc->prp_buffer_size);
	prp_page->prp_page_busaddr = (uint64_t)(sc->prp_page_busaddr +
	i * sc->prp_buffer_size);
	mpr_free_prp_page(sc, prp_page);
	sc->prp_pages_free_lowwater++;
	}

	return (0);
	}

	static int
	mpr_init_queues(struct mpr_softc *sc)
	{
	int i;

	memset((uint8_t )sc->post_queue, 0xff, sc->pqdepth 8);

	/*
	* According to the spec, we need to use one less reply than we
	* have space for on the queue. So sc->num_replies (the number we
	* use) should be less than sc->fqdepth (allocated size).
	*/
	if (sc->num_replies >= sc->fqdepth)
	return (EINVAL);

	/*
	* Initialize all of the free queue entries.
	*/
	for (i = 0; i < sc->fqdepth; i++) {
	sc->free_queue[i] = sc->reply_busaddr + (i * sc->replyframesz);
	}
	sc->replyfreeindex = sc->num_replies;

	return (0);
	}

	/* Get the driver parameter tunables. Lowest priority are the driver defaults.
	* Next are the global settings, if they exist. Highest are the per-unit
	* settings, if they exist.
	*/
	void
	mpr_get_tunables(struct mpr_softc *sc)
	{
	char tmpstr[80], mpr_debug[80];

	/* XXX default to some debugging for now */
	sc->mpr_debug = MPR_INFO \| MPR_FAULT;
	sc->disable_msix = 0;
	sc->disable_msi = 0;
	sc->max_msix = MPR_MSIX_MAX;
	sc->max_chains = MPR_CHAIN_FRAMES;
	sc->max_io_pages = MPR_MAXIO_PAGES;
	sc->enable_ssu = MPR_SSU_ENABLE_SSD_DISABLE_HDD;
	sc->spinup_wait_time = DEFAULT_SPINUP_WAIT;
	sc->use_phynum = 1;
	sc->max_reqframes = MPR_REQ_FRAMES;
	sc->max_prireqframes = MPR_PRI_REQ_FRAMES;
	sc->max_replyframes = MPR_REPLY_FRAMES;
	sc->max_evtframes = MPR_EVT_REPLY_FRAMES;

	/*
	* Grab the global variables.
	*/
	bzero(mpr_debug, 80);
	if (TUNABLE_STR_FETCH("hw.mpr.debug_level", mpr_debug, 80) != 0)
	mpr_parse_debug(sc, mpr_debug);
	TUNABLE_INT_FETCH("hw.mpr.disable_msix", &sc->disable_msix);
	TUNABLE_INT_FETCH("hw.mpr.disable_msi", &sc->disable_msi);
	TUNABLE_INT_FETCH("hw.mpr.max_msix", &sc->max_msix);
	TUNABLE_INT_FETCH("hw.mpr.max_chains", &sc->max_chains);
	TUNABLE_INT_FETCH("hw.mpr.max_io_pages", &sc->max_io_pages);
	TUNABLE_INT_FETCH("hw.mpr.enable_ssu", &sc->enable_ssu);
	TUNABLE_INT_FETCH("hw.mpr.spinup_wait_time", &sc->spinup_wait_time);
	TUNABLE_INT_FETCH("hw.mpr.use_phy_num", &sc->use_phynum);
	TUNABLE_INT_FETCH("hw.mpr.max_reqframes", &sc->max_reqframes);
	TUNABLE_INT_FETCH("hw.mpr.max_prireqframes", &sc->max_prireqframes);
	TUNABLE_INT_FETCH("hw.mpr.max_replyframes", &sc->max_replyframes);
	TUNABLE_INT_FETCH("hw.mpr.max_evtframes", &sc->max_evtframes);

	/* Grab the unit-instance variables */
	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.debug_level",
	device_get_unit(sc->mpr_dev));
	bzero(mpr_debug, 80);
	if (TUNABLE_STR_FETCH(tmpstr, mpr_debug, 80) != 0)
	mpr_parse_debug(sc, mpr_debug);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.disable_msix",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->disable_msix);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.disable_msi",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->disable_msi);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_msix",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_msix);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_chains",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_chains);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_io_pages",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_io_pages);

	bzero(sc->exclude_ids, sizeof(sc->exclude_ids));
	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.exclude_ids",
	device_get_unit(sc->mpr_dev));
	TUNABLE_STR_FETCH(tmpstr, sc->exclude_ids, sizeof(sc->exclude_ids));

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.enable_ssu",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->enable_ssu);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.spinup_wait_time",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->spinup_wait_time);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.use_phy_num",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->use_phynum);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_reqframes",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_reqframes);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_prireqframes",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_prireqframes);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_replyframes",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_replyframes);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_evtframes",
	device_get_unit(sc->mpr_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_evtframes);
	}

	static void
	mpr_setup_sysctl(struct mpr_softc *sc)
	{
	struct sysctl_ctx_list *sysctl_ctx = NULL;
	struct sysctl_oid *sysctl_tree = NULL;
	char tmpstr[80], tmpstr2[80];

	/*
	* Setup the sysctl variable so the user can change the debug level
	* on the fly.
	*/
	snprintf(tmpstr, sizeof(tmpstr), "MPR controller %d",
	device_get_unit(sc->mpr_dev));
	snprintf(tmpstr2, sizeof(tmpstr2), "%d", device_get_unit(sc->mpr_dev));

	sysctl_ctx = device_get_sysctl_ctx(sc->mpr_dev);
	if (sysctl_ctx != NULL)
	sysctl_tree = device_get_sysctl_tree(sc->mpr_dev);

	if (sysctl_tree == NULL) {
	sysctl_ctx_init(&sc->sysctl_ctx);
	sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx,
	SYSCTL_STATIC_CHILDREN(_hw_mpr), OID_AUTO, tmpstr2,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, 0, tmpstr);
	if (sc->sysctl_tree == NULL)
	return;
	sysctl_ctx = &sc->sysctl_ctx;
	sysctl_tree = sc->sysctl_tree;
	}

	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "debug_level", CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_MPSAFE,
	sc, 0, mpr_debug_sysctl, "A", "mpr debug level");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "disable_msix", CTLFLAG_RD, &sc->disable_msix, 0,
	"Disable the use of MSI-X interrupts");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_msix", CTLFLAG_RD, &sc->max_msix, 0,
	"User-defined maximum number of MSIX queues");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "msix_msgs", CTLFLAG_RD, &sc->msi_msgs, 0,
	"Negotiated number of MSIX queues");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_reqframes", CTLFLAG_RD, &sc->max_reqframes, 0,
	"Total number of allocated request frames");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_prireqframes", CTLFLAG_RD, &sc->max_prireqframes, 0,
	"Total number of allocated high priority request frames");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_replyframes", CTLFLAG_RD, &sc->max_replyframes, 0,
	"Total number of allocated reply frames");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_evtframes", CTLFLAG_RD, &sc->max_evtframes, 0,
	"Total number of event frames allocated");

	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "firmware_version", CTLFLAG_RD, sc->fw_version,
	strlen(sc->fw_version), "firmware version");

	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "driver_version", CTLFLAG_RD, MPR_DRIVER_VERSION,
	strlen(MPR_DRIVER_VERSION), "driver version");

	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "msg_version", CTLFLAG_RD, sc->msg_version,
	strlen(sc->msg_version), "message interface version");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "io_cmds_active", CTLFLAG_RD,
	&sc->io_cmds_active, 0, "number of currently active commands");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "io_cmds_highwater", CTLFLAG_RD,
	&sc->io_cmds_highwater, 0, "maximum active commands seen");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "chain_free", CTLFLAG_RD,
	&sc->chain_free, 0, "number of free chain elements");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "chain_free_lowwater", CTLFLAG_RD,
	&sc->chain_free_lowwater, 0,"lowest number of free chain elements");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_chains", CTLFLAG_RD,
	&sc->max_chains, 0,"maximum chain frames that will be allocated");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_io_pages", CTLFLAG_RD,
	&sc->max_io_pages, 0,"maximum pages to allow per I/O (if <1 use "
	"IOCFacts)");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0,
	"enable SSU to SATA SSD/HDD at shutdown");

	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "chain_alloc_fail", CTLFLAG_RD,
	&sc->chain_alloc_fail, "chain allocation failures");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "spinup_wait_time", CTLFLAG_RD,
	&sc->spinup_wait_time, DEFAULT_SPINUP_WAIT, "seconds to wait for "
	"spinup after SATA ID error");

	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "dump_reqs",
	CTLTYPE_OPAQUE \| CTLFLAG_RD \| CTLFLAG_SKIP \| CTLFLAG_NEEDGIANT,
	sc, 0, mpr_dump_reqs, "I", "Dump Active Requests");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "use_phy_num", CTLFLAG_RD, &sc->use_phynum, 0,
	"Use the phy number for enumeration");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "prp_pages_free", CTLFLAG_RD,
	&sc->prp_pages_free, 0, "number of free PRP pages");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "prp_pages_free_lowwater", CTLFLAG_RD,
	&sc->prp_pages_free_lowwater, 0,"lowest number of free PRP pages");

	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "prp_page_alloc_fail", CTLFLAG_RD,
	&sc->prp_page_alloc_fail, "PRP page allocation failures");
	}

	static struct mpr_debug_string {
	char *name;
	int flag;
	} mpr_debug_strings[] = {
	{"info", MPR_INFO},
	{"fault", MPR_FAULT},
	{"event", MPR_EVENT},
	{"log", MPR_LOG},
	{"recovery", MPR_RECOVERY},
	{"error", MPR_ERROR},
	{"init", MPR_INIT},
	{"xinfo", MPR_XINFO},
	{"user", MPR_USER},
	{"mapping", MPR_MAPPING},
	{"trace", MPR_TRACE}
	};

	enum mpr_debug_level_combiner {
	COMB_NONE,
	COMB_ADD,
	COMB_SUB
	};

	static int
	mpr_debug_sysctl(SYSCTL_HANDLER_ARGS)
	{
	struct mpr_softc *sc;
	struct mpr_debug_string *string;
	struct sbuf *sbuf;
	char *buffer;
	size_t sz;
	int i, len, debug, error;

	sc = (struct mpr_softc *)arg1;

	error = sysctl_wire_old_buffer(req, 0);
	if (error != 0)
	return (error);

	sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	debug = sc->mpr_debug;

	sbuf_printf(sbuf, "%#x", debug);

	sz = sizeof(mpr_debug_strings) / sizeof(mpr_debug_strings[0]);
	for (i = 0; i < sz; i++) {
	string = &mpr_debug_strings[i];
	if (debug & string->flag)
	sbuf_printf(sbuf, ",%s", string->name);
	}

	error = sbuf_finish(sbuf);
	sbuf_delete(sbuf);

	if (error \|\| req->newptr == NULL)
	return (error);

	len = req->newlen - req->newidx;
	if (len == 0)
	return (0);

	buffer = malloc(len, M_MPR, M_ZERO\|M_WAITOK);
	error = SYSCTL_IN(req, buffer, len);

	mpr_parse_debug(sc, buffer);

	free(buffer, M_MPR);
	return (error);
	}

	static void
	mpr_parse_debug(struct mpr_softc sc, char list)
	{
	struct mpr_debug_string *string;
	enum mpr_debug_level_combiner op;
	char token, endtoken;
	size_t sz;
	int flags, i;

	if (list == NULL \|\| *list == '\0')
	return;

	if (*list == '+') {
	op = COMB_ADD;
	list++;
	} else if (*list == '-') {
	op = COMB_SUB;
	list++;
	} else
	op = COMB_NONE;
	if (*list == '\0')
	return;

	flags = 0;
	sz = sizeof(mpr_debug_strings) / sizeof(mpr_debug_strings[0]);
	while ((token = strsep(&list, ":,")) != NULL) {

	/* Handle integer flags */
	flags \|= strtol(token, &endtoken, 0);
	if (token != endtoken)
	continue;

	/* Handle text flags */
	for (i = 0; i < sz; i++) {
	string = &mpr_debug_strings[i];
	if (strcasecmp(token, string->name) == 0) {
	flags \|= string->flag;
	break;
	}
	}
	}

	switch (op) {
	case COMB_NONE:
	sc->mpr_debug = flags;
	break;
	case COMB_ADD:
	sc->mpr_debug \|= flags;
	break;
	case COMB_SUB:
	sc->mpr_debug &= (~flags);
	break;
	}
	return;
	}

	struct mpr_dumpreq_hdr {
	uint32_t smid;
	uint32_t state;
	uint32_t numframes;
	uint32_t deschi;
	uint32_t desclo;
	};

	static int
	mpr_dump_reqs(SYSCTL_HANDLER_ARGS)
	{
	struct mpr_softc *sc;
	struct mpr_chain chain, chain1;
	struct mpr_command *cm;
	struct mpr_dumpreq_hdr hdr;
	struct sbuf *sb;
	uint32_t smid, state;
	int i, numreqs, error = 0;

	sc = (struct mpr_softc *)arg1;

	if ((error = priv_check(curthread, PRIV_DRIVER)) != 0) {
	printf("priv check error %d\n", error);
	return (error);
	}

	state = MPR_CM_STATE_INQUEUE;
	smid = 1;
	numreqs = sc->num_reqs;

	if (req->newptr != NULL)
	return (EINVAL);

	if (smid == 0 \|\| smid > sc->num_reqs)
	return (EINVAL);
	if (numreqs <= 0 \|\| (numreqs + smid > sc->num_reqs))
	numreqs = sc->num_reqs;
	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);

	/* Best effort, no locking */
	for (i = smid; i < numreqs; i++) {
	cm = &sc->commands[i];
	if (cm->cm_state != state)
	continue;
	hdr.smid = i;
	hdr.state = cm->cm_state;
	hdr.numframes = 1;
	hdr.deschi = cm->cm_desc.Words.High;
	hdr.desclo = cm->cm_desc.Words.Low;
	TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,
	chain1)
	hdr.numframes++;
	sbuf_bcat(sb, &hdr, sizeof(hdr));
	sbuf_bcat(sb, cm->cm_req, 128);
	TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,
	chain1)
	sbuf_bcat(sb, chain->chain, 128);
	}

	error = sbuf_finish(sb);
	sbuf_delete(sb);
	return (error);
	}

	int
	mpr_attach(struct mpr_softc *sc)
	{
	int error;

	MPR_FUNCTRACE(sc);
	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	mtx_init(&sc->mpr_mtx, "MPR lock", NULL, MTX_DEF);
	callout_init_mtx(&sc->periodic, &sc->mpr_mtx, 0);
	callout_init_mtx(&sc->device_check_callout, &sc->mpr_mtx, 0);
	TAILQ_INIT(&sc->event_list);
	timevalclear(&sc->lastfail);

	if ((error = mpr_transition_ready(sc)) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"Failed to transition ready\n");
	return (error);
	}

	sc->facts = malloc(sizeof(MPI2_IOC_FACTS_REPLY), M_MPR,
	M_ZERO\|M_NOWAIT);
	if (!sc->facts) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT,
	"Cannot allocate memory, exit\n");
	return (ENOMEM);
	}

	/*
	* Get IOC Facts and allocate all structures based on this information.
	* A Diag Reset will also call mpr_iocfacts_allocate and re-read the IOC
	* Facts. If relevant values have changed in IOC Facts, this function
	* will free all of the memory based on IOC Facts and reallocate that
	* memory. If this fails, any allocated memory should already be freed.
	*/
	if ((error = mpr_iocfacts_allocate(sc, TRUE)) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "IOC Facts allocation "
	"failed with error %d\n", error);
	return (error);
	}

	/* Start the periodic watchdog check on the IOC Doorbell */
	mpr_periodic(sc);

	/*
	* The portenable will kick off discovery events that will drive the
	* rest of the initialization process. The CAM/SAS module will
	* hold up the boot sequence until discovery is complete.
	*/
	sc->mpr_ich.ich_func = mpr_startup;
	sc->mpr_ich.ich_arg = sc;
	if (config_intrhook_establish(&sc->mpr_ich) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Cannot establish MPR config hook\n");
	error = EINVAL;
	}

	/*
	* Allow IR to shutdown gracefully when shutdown occurs.
	*/
	sc->shutdown_eh = EVENTHANDLER_REGISTER(shutdown_final,
	mprsas_ir_shutdown, sc, SHUTDOWN_PRI_DEFAULT);

	if (sc->shutdown_eh == NULL)
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"shutdown event registration failed\n");

	mpr_setup_sysctl(sc);

	sc->mpr_flags \|= MPR_FLAGS_ATTACH_DONE;
	mpr_dprint(sc, MPR_INIT, "%s exit error= %d\n", __func__, error);

	return (error);
	}

	/* Run through any late-start handlers. */
	static void
	mpr_startup(void *arg)
	{
	struct mpr_softc *sc;

	sc = (struct mpr_softc *)arg;
	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	mpr_lock(sc);
	mpr_unmask_intr(sc);

	/* initialize device mapping tables */
	mpr_base_static_config_pages(sc);
	mpr_mapping_initialize(sc);
	mprsas_startup(sc);
	mpr_unlock(sc);

	mpr_dprint(sc, MPR_INIT, "disestablish config intrhook\n");
	config_intrhook_disestablish(&sc->mpr_ich);
	sc->mpr_ich.ich_arg = NULL;

	mpr_dprint(sc, MPR_INIT, "%s exit\n", __func__);
	}

	/* Periodic watchdog. Is called with the driver lock already held. */
	static void
	mpr_periodic(void *arg)
	{
	struct mpr_softc *sc;
	uint32_t db;

	sc = (struct mpr_softc *)arg;
	if (sc->mpr_flags & MPR_FLAGS_SHUTDOWN)
	return;

	db = mpr_regread(sc, MPI2_DOORBELL_OFFSET);
	if ((db & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
	if ((db & MPI2_DOORBELL_FAULT_CODE_MASK) ==
	IFAULT_IOP_OVER_TEMP_THRESHOLD_EXCEEDED) {
	panic("TEMPERATURE FAULT: STOPPING.");
	}
	mpr_dprint(sc, MPR_FAULT, "IOC Fault 0x%08x, Resetting\n", db);
	mpr_reinit(sc);
	}

	callout_reset(&sc->periodic, MPR_PERIODIC_DELAY * hz, mpr_periodic, sc);
	}

	static void
	mpr_log_evt_handler(struct mpr_softc *sc, uintptr_t data,
	MPI2_EVENT_NOTIFICATION_REPLY *event)
	{
	MPI2_EVENT_DATA_LOG_ENTRY_ADDED *entry;

	MPR_DPRINT_EVENT(sc, generic, event);

	switch (event->Event) {
	case MPI2_EVENT_LOG_DATA:
	mpr_dprint(sc, MPR_EVENT, "MPI2_EVENT_LOG_DATA:\n");
	if (sc->mpr_debug & MPR_EVENT)
	hexdump(event->EventData, event->EventDataLength, NULL,
	0);
	break;
	case MPI2_EVENT_LOG_ENTRY_ADDED:
	entry = (MPI2_EVENT_DATA_LOG_ENTRY_ADDED *)event->EventData;
	mpr_dprint(sc, MPR_EVENT, "MPI2_EVENT_LOG_ENTRY_ADDED event "
	"0x%x Sequence %d:\n", entry->LogEntryQualifier,
	entry->LogSequence);
	break;
	default:
	break;
	}
	return;
	}

	static int
	mpr_attach_log(struct mpr_softc *sc)
	{
	uint8_t events[16];

	bzero(events, 16);
	setbit(events, MPI2_EVENT_LOG_DATA);
	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);

	mpr_register_events(sc, events, mpr_log_evt_handler, NULL,
	&sc->mpr_log_eh);

	return (0);
	}

	static int
	mpr_detach_log(struct mpr_softc *sc)
	{

	if (sc->mpr_log_eh != NULL)
	mpr_deregister_events(sc, sc->mpr_log_eh);
	return (0);
	}

	/*
	* Free all of the driver resources and detach submodules. Should be called
	* without the lock held.
	*/
	int
	mpr_free(struct mpr_softc *sc)
	{
	int error;

	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);
	/* Turn off the watchdog */
	mpr_lock(sc);
	sc->mpr_flags \|= MPR_FLAGS_SHUTDOWN;
	mpr_unlock(sc);
	/* Lock must not be held for this */
	callout_drain(&sc->periodic);
	callout_drain(&sc->device_check_callout);

	if (((error = mpr_detach_log(sc)) != 0) \|\|
	((error = mpr_detach_sas(sc)) != 0)) {
	mpr_dprint(sc, MPR_INIT\|MPR_FAULT, "failed to detach "
	"subsystems, error= %d, exit\n", error);
	return (error);
	}

	mpr_detach_user(sc);

	/* Put the IOC back in the READY state. */
	mpr_lock(sc);
	if ((error = mpr_transition_ready(sc)) != 0) {
	mpr_unlock(sc);
	return (error);
	}
	mpr_unlock(sc);

	if (sc->facts != NULL)
	free(sc->facts, M_MPR);

	/*
	* Free all buffers that are based on IOC Facts. A Diag Reset may need
	* to free these buffers too.
	*/
	mpr_iocfacts_free(sc);

	if (sc->sysctl_tree != NULL)
	sysctl_ctx_free(&sc->sysctl_ctx);

	/* Deregister the shutdown function */
	if (sc->shutdown_eh != NULL)
	EVENTHANDLER_DEREGISTER(shutdown_final, sc->shutdown_eh);

	mtx_destroy(&sc->mpr_mtx);
	mpr_dprint(sc, MPR_INIT, "%s exit\n", __func__);

	return (0);
	}

	static __inline void
	mpr_complete_command(struct mpr_softc sc, struct mpr_command cm)
	{
	MPR_FUNCTRACE(sc);

	if (cm == NULL) {
	mpr_dprint(sc, MPR_ERROR, "Completing NULL command\n");
	return;
	}

	cm->cm_state = MPR_CM_STATE_BUSY;
	if (cm->cm_flags & MPR_CM_FLAGS_POLLED)
	cm->cm_flags \|= MPR_CM_FLAGS_COMPLETE;

	if (cm->cm_complete != NULL) {
	mpr_dprint(sc, MPR_TRACE,
	"%s cm %p calling cm_complete %p data %p reply %p\n",
	__func__, cm, cm->cm_complete, cm->cm_complete_data,
	cm->cm_reply);
	cm->cm_complete(sc, cm);
	}

	if (cm->cm_flags & MPR_CM_FLAGS_WAKEUP) {
	mpr_dprint(sc, MPR_TRACE, "waking up %p\n", cm);
	wakeup(cm);
	}

	if (sc->io_cmds_active != 0) {
	sc->io_cmds_active--;
	} else {
	mpr_dprint(sc, MPR_ERROR, "Warning: io_cmds_active is "
	"out of sync - resynching to 0\n");
	}
	}

	static void
	mpr_sas_log_info(struct mpr_softc *sc , u32 log_info)
	{
	union loginfo_type {
	u32 loginfo;
	struct {
	u32 subcode:16;
	u32 code:8;
	u32 originator:4;
	u32 bus_type:4;
	} dw;
	};
	union loginfo_type sas_loginfo;
	char *originator_str = NULL;

	sas_loginfo.loginfo = log_info;
	if (sas_loginfo.dw.bus_type != 3 /SAS/)
	return;

	/* each nexus loss loginfo */
	if (log_info == 0x31170000)
	return;

	/* eat the loginfos associated with task aborts */
	if ((log_info == 30050000) \|\| (log_info == 0x31140000) \|\|
	(log_info == 0x31130000))
	return;

	switch (sas_loginfo.dw.originator) {
	case 0:
	originator_str = "IOP";
	break;
	case 1:
	originator_str = "PL";
	break;
	case 2:
	originator_str = "IR";
	break;
	}

	mpr_dprint(sc, MPR_LOG, "log_info(0x%08x): originator(%s), "
	"code(0x%02x), sub_code(0x%04x)\n", log_info, originator_str,
	sas_loginfo.dw.code, sas_loginfo.dw.subcode);
	}

	static void
	mpr_display_reply_info(struct mpr_softc sc, uint8_t reply)
	{
	MPI2DefaultReply_t *mpi_reply;
	u16 sc_status;

	mpi_reply = (MPI2DefaultReply_t*)reply;
	sc_status = le16toh(mpi_reply->IOCStatus);
	if (sc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
	mpr_sas_log_info(sc, le32toh(mpi_reply->IOCLogInfo));
	}

	void
	mpr_intr(void *data)
	{
	struct mpr_softc *sc;
	uint32_t status;

	sc = (struct mpr_softc *)data;
	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);

	/*
	* Check interrupt status register to flush the bus. This is
	* needed for both INTx interrupts and driver-driven polling
	*/
	status = mpr_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);
	if ((status & MPI2_HIS_REPLY_DESCRIPTOR_INTERRUPT) == 0)
	return;

	mpr_lock(sc);
	mpr_intr_locked(data);
	mpr_unlock(sc);
	return;
	}

	/*
	* In theory, MSI/MSIX interrupts shouldn't need to read any registers on the
	* chip. Hopefully this theory is correct.
	*/
	void
	mpr_intr_msi(void *data)
	{
	struct mpr_softc *sc;

	sc = (struct mpr_softc *)data;
	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);
	mpr_lock(sc);
	mpr_intr_locked(data);
	mpr_unlock(sc);
	return;
	}

	/*
	* The locking is overly broad and simplistic, but easy to deal with for now.
	*/
	void
	mpr_intr_locked(void *data)
	{
	MPI2_REPLY_DESCRIPTORS_UNION *desc;
	MPI2_DIAG_RELEASE_REPLY *rel_rep;
	mpr_fw_diagnostic_buffer_t *pBuffer;
	struct mpr_softc *sc;
	uint64_t tdesc;
	struct mpr_command *cm = NULL;
	uint8_t flags;
	u_int pq;

	sc = (struct mpr_softc *)data;

	pq = sc->replypostindex;
	mpr_dprint(sc, MPR_TRACE,
	"%s sc %p starting with replypostindex %u\n",
	__func__, sc, sc->replypostindex);

	for ( ;; ) {
	cm = NULL;
	desc = &sc->post_queue[sc->replypostindex];

	/*
	* Copy and clear out the descriptor so that any reentry will
	* immediately know that this descriptor has already been
	* looked at. There is unfortunate casting magic because the
	* MPI API doesn't have a cardinal 64bit type.
	*/
	tdesc = 0xffffffffffffffff;
	tdesc = atomic_swap_64((uint64_t *)desc, tdesc);
	desc = (MPI2_REPLY_DESCRIPTORS_UNION *)&tdesc;

	flags = desc->Default.ReplyFlags &
	MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
	if ((flags == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) \|\|
	(le32toh(desc->Words.High) == 0xffffffff))
	break;

	/* increment the replypostindex now, so that event handlers
	* and cm completion handlers which decide to do a diag
	* reset can zero it without it getting incremented again
	* afterwards, and we break out of this loop on the next
	* iteration since the reply post queue has been cleared to
	* 0xFF and all descriptors look unused (which they are).
	*/
	if (++sc->replypostindex >= sc->pqdepth)
	sc->replypostindex = 0;

	switch (flags) {
	case MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS:
	case MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS:
	case MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS:
	cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)];
	KASSERT(cm->cm_state == MPR_CM_STATE_INQUEUE,
	("command not inqueue\n"));
	cm->cm_state = MPR_CM_STATE_BUSY;
	cm->cm_reply = NULL;
	break;
	case MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY:
	{
	uint32_t baddr;
	uint8_t *reply;

	/*
	* Re-compose the reply address from the address
	* sent back from the chip. The ReplyFrameAddress
	* is the lower 32 bits of the physical address of
	* particular reply frame. Convert that address to
	* host format, and then use that to provide the
	* offset against the virtual address base
	* (sc->reply_frames).
	*/
	baddr = le32toh(desc->AddressReply.ReplyFrameAddress);
	reply = sc->reply_frames +
	(baddr - ((uint32_t)sc->reply_busaddr));
	/*
	* Make sure the reply we got back is in a valid
	* range. If not, go ahead and panic here, since
	* we'll probably panic as soon as we deference the
	* reply pointer anyway.
	*/
	if ((reply < sc->reply_frames)
	\|\| (reply > (sc->reply_frames +
	(sc->fqdepth * sc->replyframesz)))) {
	printf("%s: WARNING: reply %p out of range!\n",
	__func__, reply);
	printf("%s: reply_frames %p, fqdepth %d, "
	"frame size %d\n", __func__,
	sc->reply_frames, sc->fqdepth,
	sc->replyframesz);
	printf("%s: baddr %#x,\n", __func__, baddr);
	/* LSI-TODO. See Linux Code for Graceful exit */
	panic("Reply address out of range");
	}
	if (le16toh(desc->AddressReply.SMID) == 0) {
	if (((MPI2_DEFAULT_REPLY *)reply)->Function ==
	MPI2_FUNCTION_DIAG_BUFFER_POST) {
	/*
	* If SMID is 0 for Diag Buffer Post,
	* this implies that the reply is due to
	* a release function with a status that
	* the buffer has been released. Set
	* the buffer flags accordingly.
	*/
	rel_rep =
	(MPI2_DIAG_RELEASE_REPLY *)reply;
	if ((le16toh(rel_rep->IOCStatus) &
	MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED)
	{
	pBuffer =
	&sc->fw_diag_buffer_list[
	rel_rep->BufferType];
	pBuffer->valid_data = TRUE;
	pBuffer->owned_by_firmware =
	FALSE;
	pBuffer->immediate = FALSE;
	}
	} else
	mpr_dispatch_event(sc, baddr,
	(MPI2_EVENT_NOTIFICATION_REPLY *)
	reply);
	} else {
	cm = &sc->commands[
	le16toh(desc->AddressReply.SMID)];
	if (cm->cm_state == MPR_CM_STATE_INQUEUE) {
	cm->cm_reply = reply;
	cm->cm_reply_data =
	le32toh(desc->AddressReply.
	ReplyFrameAddress);
	} else {
	mpr_dprint(sc, MPR_RECOVERY,
	"Bad state for ADDRESS_REPLY status,"
	" ignoring state %d cm %p\n",
	cm->cm_state, cm);
	}
	}
	break;
	}
	case MPI2_RPY_DESCRIPT_FLAGS_TARGETASSIST_SUCCESS:
	case MPI2_RPY_DESCRIPT_FLAGS_TARGET_COMMAND_BUFFER:
	case MPI2_RPY_DESCRIPT_FLAGS_RAID_ACCELERATOR_SUCCESS:
	default:
	/* Unhandled */
	mpr_dprint(sc, MPR_ERROR, "Unhandled reply 0x%x\n",
	desc->Default.ReplyFlags);
	cm = NULL;
	break;
	}

	if (cm != NULL) {
	// Print Error reply frame
	if (cm->cm_reply)
	mpr_display_reply_info(sc,cm->cm_reply);
	mpr_complete_command(sc, cm);
	}
	}

	if (pq != sc->replypostindex) {
	mpr_dprint(sc, MPR_TRACE, "%s sc %p writing postindex %d\n",
	__func__, sc, sc->replypostindex);
	mpr_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET,
	sc->replypostindex);
	}

	return;
	}

	static void
	mpr_dispatch_event(struct mpr_softc *sc, uintptr_t data,
	MPI2_EVENT_NOTIFICATION_REPLY *reply)
	{
	struct mpr_event_handle *eh;
	int event, handled = 0;

	event = le16toh(reply->Event);
	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
	if (isset(eh->mask, event)) {
	eh->callback(sc, data, reply);
	handled++;
	}
	}

	if (handled == 0)
	mpr_dprint(sc, MPR_EVENT, "Unhandled event 0x%x\n",
	le16toh(event));

	/*
	* This is the only place that the event/reply should be freed.
	* Anything wanting to hold onto the event data should have
	* already copied it into their own storage.
	*/
	mpr_free_reply(sc, data);
	}

	static void
	mpr_reregister_events_complete(struct mpr_softc sc, struct mpr_command cm)
	{
	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);

	if (cm->cm_reply)
	MPR_DPRINT_EVENT(sc, generic,
	(MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply);

	mpr_free_command(sc, cm);

	/* next, send a port enable */
	mprsas_startup(sc);
	}

	/*
	* For both register_events and update_events, the caller supplies a bitmap
	* of events that it _wants_. These functions then turn that into a bitmask
	* suitable for the controller.
	*/
	int
	mpr_register_events(struct mpr_softc sc, uint8_t mask,
	mpr_evt_callback_t cb, void data, struct mpr_event_handle **handle)
	{
	struct mpr_event_handle *eh;
	int error = 0;

	eh = malloc(sizeof(struct mpr_event_handle), M_MPR, M_WAITOK\|M_ZERO);
	- if (!eh) {
	- mpr_dprint(sc, MPR_EVENT\|MPR_ERROR,
	- "Cannot allocate event memory\n");
	- return (ENOMEM);
	- }
	eh->callback = cb;
	eh->data = data;
	TAILQ_INSERT_TAIL(&sc->event_list, eh, eh_list);
	if (mask != NULL)
	error = mpr_update_events(sc, eh, mask);
	*handle = eh;

	return (error);
	}

	int
	mpr_update_events(struct mpr_softc sc, struct mpr_event_handle handle,
	uint8_t *mask)
	{
	MPI2_EVENT_NOTIFICATION_REQUEST *evtreq;
	MPI2_EVENT_NOTIFICATION_REPLY *reply = NULL;
	struct mpr_command *cm = NULL;
	struct mpr_event_handle *eh;
	int error, i;

	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);

	if ((mask != NULL) && (handle != NULL))
	bcopy(mask, &handle->mask[0], 16);
	memset(sc->event_mask, 0xff, 16);

	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
	for (i = 0; i < 16; i++)
	sc->event_mask[i] &= ~eh->mask[i];
	}

	if ((cm = mpr_alloc_command(sc)) == NULL)
	return (EBUSY);
	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;
	evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
	evtreq->MsgFlags = 0;
	evtreq->SASBroadcastPrimitiveMasks = 0;
	#ifdef MPR_DEBUG_ALL_EVENTS
	{
	u_char fullmask[16];
	memset(fullmask, 0x00, 16);
	bcopy(fullmask, (uint8_t *)&evtreq->EventMasks, 16);
	}
	#else
	bcopy(sc->event_mask, (uint8_t *)&evtreq->EventMasks, 16);
	#endif
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_data = NULL;

	error = mpr_request_polled(sc, &cm);
	if (cm != NULL)
	reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply;
	if ((reply == NULL) \|\|
	(reply->IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
	error = ENXIO;

	if (reply)
	MPR_DPRINT_EVENT(sc, generic, reply);

	mpr_dprint(sc, MPR_TRACE, "%s finished error %d\n", __func__, error);

	if (cm != NULL)
	mpr_free_command(sc, cm);
	return (error);
	}

	static int
	mpr_reregister_events(struct mpr_softc *sc)
	{
	MPI2_EVENT_NOTIFICATION_REQUEST *evtreq;
	struct mpr_command *cm;
	struct mpr_event_handle *eh;
	int error, i;

	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);

	/* first, reregister events */

	memset(sc->event_mask, 0xff, 16);

	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
	for (i = 0; i < 16; i++)
	sc->event_mask[i] &= ~eh->mask[i];
	}

	if ((cm = mpr_alloc_command(sc)) == NULL)
	return (EBUSY);
	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;
	evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
	evtreq->MsgFlags = 0;
	evtreq->SASBroadcastPrimitiveMasks = 0;
	#ifdef MPR_DEBUG_ALL_EVENTS
	{
	u_char fullmask[16];
	memset(fullmask, 0x00, 16);
	bcopy(fullmask, (uint8_t *)&evtreq->EventMasks, 16);
	}
	#else
	bcopy(sc->event_mask, (uint8_t *)&evtreq->EventMasks, 16);
	#endif
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_data = NULL;
	cm->cm_complete = mpr_reregister_events_complete;

	error = mpr_map_command(sc, cm);

	mpr_dprint(sc, MPR_TRACE, "%s finished with error %d\n", __func__,
	error);
	return (error);
	}

	int
	mpr_deregister_events(struct mpr_softc sc, struct mpr_event_handle handle)
	{

	TAILQ_REMOVE(&sc->event_list, handle, eh_list);
	free(handle, M_MPR);
	return (mpr_update_events(sc, NULL, NULL));
	}

	/**
	* mpr_build_nvme_prp - This function is called for NVMe end devices to build a
	* native SGL (NVMe PRP). The native SGL is built starting in the first PRP entry
	* of the NVMe message (PRP1). If the data buffer is small enough to be described
	* entirely using PRP1, then PRP2 is not used. If needed, PRP2 is used to
	* describe a larger data buffer. If the data buffer is too large to describe
	* using the two PRP entriess inside the NVMe message, then PRP1 describes the
	* first data memory segment, and PRP2 contains a pointer to a PRP list located
	* elsewhere in memory to describe the remaining data memory segments. The PRP
	* list will be contiguous.

	* The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP
	* consists of a list of PRP entries to describe a number of noncontigous
	* physical memory segments as a single memory buffer, just as a SGL does. Note
	* however, that this function is only used by the IOCTL call, so the memory
	* given will be guaranteed to be contiguous. There is no need to translate
	* non-contiguous SGL into a PRP in this case. All PRPs will describe contiguous
	* space that is one page size each.
	*
	* Each NVMe message contains two PRP entries. The first (PRP1) either contains
	* a PRP list pointer or a PRP element, depending upon the command. PRP2 contains
	* the second PRP element if the memory being described fits within 2 PRP
	* entries, or a PRP list pointer if the PRP spans more than two entries.
	*
	* A PRP list pointer contains the address of a PRP list, structured as a linear
	* array of PRP entries. Each PRP entry in this list describes a segment of
	* physical memory.
	*
	* Each 64-bit PRP entry comprises an address and an offset field. The address
	* always points to the beginning of a PAGE_SIZE physical memory page, and the
	* offset describes where within that page the memory segment begins. Only the
	* first element in a PRP list may contain a non-zero offest, implying that all
	* memory segments following the first begin at the start of a PAGE_SIZE page.
	*
	* Each PRP element normally describes a chunck of PAGE_SIZE physical memory,
	* with exceptions for the first and last elements in the list. If the memory
	* being described by the list begins at a non-zero offset within the first page,
	* then the first PRP element will contain a non-zero offset indicating where the
	* region begins within the page. The last memory segment may end before the end
	* of the PAGE_SIZE segment, depending upon the overall size of the memory being
	* described by the PRP list.
	*
	* Since PRP entries lack any indication of size, the overall data buffer length
	* is used to determine where the end of the data memory buffer is located, and
	* how many PRP entries are required to describe it.
	*
	* Returns nothing.
	*/
	void
	mpr_build_nvme_prp(struct mpr_softc sc, struct mpr_command cm,
	Mpi26NVMeEncapsulatedRequest_t nvme_encap_request, void data,
	uint32_t data_in_sz, uint32_t data_out_sz)
	{
	int prp_size = PRP_ENTRY_SIZE;
	uint64_t prp_entry, prp1_entry, *prp2_entry;
	uint64_t prp_entry_phys, prp_page, *prp_page_phys;
	uint32_t offset, entry_len, page_mask_result, page_mask;
	bus_addr_t paddr;
	size_t length;
	struct mpr_prp_page *prp_page_info = NULL;

	/*
	* Not all commands require a data transfer. If no data, just return
	* without constructing any PRP.
	*/
	if (!data_in_sz && !data_out_sz)
	return;

	/*
	* Set pointers to PRP1 and PRP2, which are in the NVMe command. PRP1 is
	* located at a 24 byte offset from the start of the NVMe command. Then
	* set the current PRP entry pointer to PRP1.
	*/
	prp1_entry = (uint64_t *)(nvme_encap_request->NVMe_Command +
	NVME_CMD_PRP1_OFFSET);
	prp2_entry = (uint64_t *)(nvme_encap_request->NVMe_Command +
	NVME_CMD_PRP2_OFFSET);
	prp_entry = prp1_entry;

	/*
	* For the PRP entries, use the specially allocated buffer of
	* contiguous memory. PRP Page allocation failures should not happen
	* because there should be enough PRP page buffers to account for the
	* possible NVMe QDepth.
	*/
	prp_page_info = mpr_alloc_prp_page(sc);
	KASSERT(prp_page_info != NULL, ("%s: There are no PRP Pages left to be "
	"used for building a native NVMe SGL.\n", __func__));
	prp_page = (uint64_t *)prp_page_info->prp_page;
	prp_page_phys = (uint64_t *)(uintptr_t)prp_page_info->prp_page_busaddr;

	/*
	* Insert the allocated PRP page into the command's PRP page list. This
	* will be freed when the command is freed.
	*/
	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);

	/*
	* Check if we are within 1 entry of a page boundary we don't want our
	* first entry to be a PRP List entry.
	*/
	page_mask = PAGE_SIZE - 1;
	page_mask_result = (uintptr_t)((uint8_t *)prp_page + prp_size) &
	page_mask;
	if (!page_mask_result)
	{
	/* Bump up to next page boundary. */
	prp_page = (uint64_t )((uint8_t )prp_page + prp_size);
	prp_page_phys = (uint64_t )((uint8_t )prp_page_phys +
	prp_size);
	}

	/*
	* Set PRP physical pointer, which initially points to the current PRP
	* DMA memory page.
	*/
	prp_entry_phys = prp_page_phys;

	/* Get physical address and length of the data buffer. */
	paddr = (bus_addr_t)(uintptr_t)data;
	if (data_in_sz)
	length = data_in_sz;
	else
	length = data_out_sz;

	/* Loop while the length is not zero. */
	while (length)
	{
	/*
	* Check if we need to put a list pointer here if we are at page
	* boundary - prp_size (8 bytes).
	*/
	page_mask_result = (uintptr_t)((uint8_t *)prp_entry_phys +
	prp_size) & page_mask;
	if (!page_mask_result)
	{
	/*
	* This is the last entry in a PRP List, so we need to
	* put a PRP list pointer here. What this does is:
	* - bump the current memory pointer to the next
	* address, which will be the next full page.
	* - set the PRP Entry to point to that page. This is
	* now the PRP List pointer.
	* - bump the PRP Entry pointer the start of the next
	* page. Since all of this PRP memory is contiguous,
	* no need to get a new page - it's just the next
	* address.
	*/
	prp_entry_phys++;
	*prp_entry =
	htole64((uint64_t)(uintptr_t)prp_entry_phys);
	prp_entry++;
	}

	/* Need to handle if entry will be part of a page. */
	offset = (uint32_t)paddr & page_mask;
	entry_len = PAGE_SIZE - offset;

	if (prp_entry == prp1_entry)
	{
	/*
	* Must fill in the first PRP pointer (PRP1) before
	* moving on.
	*/
	*prp1_entry = htole64((uint64_t)paddr);

	/*
	* Now point to the second PRP entry within the
	* command (PRP2).
	*/
	prp_entry = prp2_entry;
	}
	else if (prp_entry == prp2_entry)
	{
	/*
	* Should the PRP2 entry be a PRP List pointer or just a
	* regular PRP pointer? If there is more than one more
	* page of data, must use a PRP List pointer.
	*/
	if (length > PAGE_SIZE)
	{
	/*
	* PRP2 will contain a PRP List pointer because
	* more PRP's are needed with this command. The
	* list will start at the beginning of the
	* contiguous buffer.
	*/
	*prp2_entry =
	htole64(
	(uint64_t)(uintptr_t)prp_entry_phys);

	/*
	* The next PRP Entry will be the start of the
	* first PRP List.
	*/
	prp_entry = prp_page;
	}
	else
	{
	/*
	* After this, the PRP Entries are complete.
	* This command uses 2 PRP's and no PRP list.
	*/
	*prp2_entry = htole64((uint64_t)paddr);
	}
	}
	else
	{
	/*
	* Put entry in list and bump the addresses.
	*
	* After PRP1 and PRP2 are filled in, this will fill in
	* all remaining PRP entries in a PRP List, one per each
	* time through the loop.
	*/
	*prp_entry = htole64((uint64_t)paddr);
	prp_entry++;
	prp_entry_phys++;
	}

	/*
	* Bump the phys address of the command's data buffer by the
	* entry_len.
	*/
	paddr += entry_len;

	/* Decrement length accounting for last partial page. */
	if (entry_len > length)
	length = 0;
	else
	length -= entry_len;
	}
	}

	/*
	* mpr_check_pcie_native_sgl - This function is called for PCIe end devices to
	* determine if the driver needs to build a native SGL. If so, that native SGL
	* is built in the contiguous buffers allocated especially for PCIe SGL
	* creation. If the driver will not build a native SGL, return TRUE and a
	* normal IEEE SGL will be built. Currently this routine supports NVMe devices
	* only.
	*
	* Returns FALSE (0) if native SGL was built, TRUE (1) if no SGL was built.
	*/
	static int
	mpr_check_pcie_native_sgl(struct mpr_softc sc, struct mpr_command cm,
	bus_dma_segment_t *segs, int segs_left)
	{
	uint32_t i, sge_dwords, length, offset, entry_len;
	uint32_t num_entries, buff_len = 0, sges_in_segment;
	uint32_t page_mask, page_mask_result, *curr_buff;
	uint32_t ptr_sgl, ptr_first_sgl, first_page_offset;
	uint32_t first_page_data_size, end_residual;
	uint64_t *msg_phys;
	bus_addr_t paddr;
	int build_native_sgl = 0, first_prp_entry;
	int prp_size = PRP_ENTRY_SIZE;
	Mpi25IeeeSgeChain64_t *main_chain_element = NULL;
	struct mpr_prp_page *prp_page_info = NULL;

	mpr_dprint(sc, MPR_TRACE, "%s\n", __func__);

	/*
	* Add up the sizes of each segment length to get the total transfer
	* size, which will be checked against the Maximum Data Transfer Size.
	* If the data transfer length exceeds the MDTS for this device, just
	* return 1 so a normal IEEE SGL will be built. F/W will break the I/O
	* up into multiple I/O's. [nvme_mdts = 0 means unlimited]
	*/
	for (i = 0; i < segs_left; i++)
	buff_len += htole32(segs[i].ds_len);
	if ((cm->cm_targ->MDTS > 0) && (buff_len > cm->cm_targ->MDTS))
	return 1;

	/* Create page_mask (to get offset within page) */
	page_mask = PAGE_SIZE - 1;

	/*
	* Check if the number of elements exceeds the max number that can be
	* put in the main message frame (H/W can only translate an SGL that
	* is contained entirely in the main message frame).
	*/
	sges_in_segment = (sc->reqframesz -
	offsetof(Mpi25SCSIIORequest_t, SGL)) / sizeof(MPI25_SGE_IO_UNION);
	if (segs_left > sges_in_segment)
	build_native_sgl = 1;
	else
	{
	/*
	* NVMe uses one PRP for each physical page (or part of physical
	* page).
	* if 4 pages or less then IEEE is OK
	* if > 5 pages then we need to build a native SGL
	* if > 4 and <= 5 pages, then check the physical address of
	* the first SG entry, then if this first size in the page
	* is >= the residual beyond 4 pages then use IEEE,
	* otherwise use native SGL
	*/
	if (buff_len > (PAGE_SIZE * 5))
	build_native_sgl = 1;
	else if ((buff_len > (PAGE_SIZE * 4)) &&
	(buff_len <= (PAGE_SIZE * 5)) )
	{
	msg_phys = (uint64_t *)(uintptr_t)segs[0].ds_addr;
	first_page_offset =
	((uint32_t)(uint64_t)(uintptr_t)msg_phys &
	page_mask);
	first_page_data_size = PAGE_SIZE - first_page_offset;
	end_residual = buff_len % PAGE_SIZE;

	/*
	* If offset into first page pushes the end of the data
	* beyond end of the 5th page, we need the extra PRP
	* list.
	*/
	if (first_page_data_size < end_residual)
	build_native_sgl = 1;

	/*
	* Check if first SG entry size is < residual beyond 4
	* pages.
	*/
	if (htole32(segs[0].ds_len) <
	(buff_len - (PAGE_SIZE * 4)))
	build_native_sgl = 1;
	}
	}

	/* check if native SGL is needed */
	if (!build_native_sgl)
	return 1;

	/*
	* Native SGL is needed.
	* Put a chain element in main message frame that points to the first
	* chain buffer.
	*
	* NOTE: The ChainOffset field must be 0 when using a chain pointer to
	* a native SGL.
	*/

	/* Set main message chain element pointer */
	main_chain_element = (pMpi25IeeeSgeChain64_t)cm->cm_sge;

	/*
	* For NVMe the chain element needs to be the 2nd SGL entry in the main
	* message.
	*/
	main_chain_element = (Mpi25IeeeSgeChain64_t *)
	((uint8_t *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));

	/*
	* For the PRP entries, use the specially allocated buffer of
	* contiguous memory. PRP Page allocation failures should not happen
	* because there should be enough PRP page buffers to account for the
	* possible NVMe QDepth.
	*/
	prp_page_info = mpr_alloc_prp_page(sc);
	KASSERT(prp_page_info != NULL, ("%s: There are no PRP Pages left to be "
	"used for building a native NVMe SGL.\n", __func__));
	curr_buff = (uint32_t *)prp_page_info->prp_page;
	msg_phys = (uint64_t *)(uintptr_t)prp_page_info->prp_page_busaddr;

	/*
	* Insert the allocated PRP page into the command's PRP page list. This
	* will be freed when the command is freed.
	*/
	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);

	/*
	* Check if we are within 1 entry of a page boundary we don't want our
	* first entry to be a PRP List entry.
	*/
	page_mask_result = (uintptr_t)((uint8_t *)curr_buff + prp_size) &
	page_mask;
	if (!page_mask_result) {
	/* Bump up to next page boundary. */
	curr_buff = (uint32_t )((uint8_t )curr_buff + prp_size);
	msg_phys = (uint64_t )((uint8_t )msg_phys + prp_size);
	}

	/* Fill in the chain element and make it an NVMe segment type. */
	main_chain_element->Address.High =
	htole32((uint32_t)((uint64_t)(uintptr_t)msg_phys >> 32));
	main_chain_element->Address.Low =
	htole32((uint32_t)(uintptr_t)msg_phys);
	main_chain_element->NextChainOffset = 0;
	main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT \|
	MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR \|
	MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;

	/* Set SGL pointer to start of contiguous PCIe buffer. */
	ptr_sgl = curr_buff;
	sge_dwords = 2;
	num_entries = 0;

	/*
	* NVMe has a very convoluted PRP format. One PRP is required for each
	* page or partial page. We need to split up OS SG entries if they are
	* longer than one page or cross a page boundary. We also have to insert
	* a PRP list pointer entry as the last entry in each physical page of
	* the PRP list.
	*
	* NOTE: The first PRP "entry" is actually placed in the first SGL entry
	* in the main message in IEEE 64 format. The 2nd entry in the main
	* message is the chain element, and the rest of the PRP entries are
	* built in the contiguous PCIe buffer.
	*/
	first_prp_entry = 1;
	ptr_first_sgl = (uint32_t *)cm->cm_sge;

	for (i = 0; i < segs_left; i++) {
	/* Get physical address and length of this SG entry. */
	paddr = segs[i].ds_addr;
	length = segs[i].ds_len;

	/*
	* Check whether a given SGE buffer lies on a non-PAGED
	* boundary if this is not the first page. If so, this is not
	* expected so have FW build the SGL.
	*/
	if ((i != 0) && (((uint32_t)paddr & page_mask) != 0)) {
	mpr_dprint(sc, MPR_ERROR, "Unaligned SGE while "
	"building NVMe PRPs, low address is 0x%x\n",
	(uint32_t)paddr);
	return 1;
	}

	/* Apart from last SGE, if any other SGE boundary is not page
	* aligned then it means that hole exists. Existence of hole
	* leads to data corruption. So fallback to IEEE SGEs.
	*/
	if (i != (segs_left - 1)) {
	if (((uint32_t)paddr + length) & page_mask) {
	mpr_dprint(sc, MPR_ERROR, "Unaligned SGE "
	"boundary while building NVMe PRPs, low "
	"address: 0x%x and length: %u\n",
	(uint32_t)paddr, length);
	return 1;
	}
	}

	/* Loop while the length is not zero. */
	while (length) {
	/*
	* Check if we need to put a list pointer here if we are
	* at page boundary - prp_size.
	*/
	page_mask_result = (uintptr_t)((uint8_t *)ptr_sgl +
	prp_size) & page_mask;
	if (!page_mask_result) {
	/*
	* Need to put a PRP list pointer here.
	*/
	msg_phys = (uint64_t )((uint8_t )msg_phys +
	prp_size);
	*ptr_sgl = htole32((uintptr_t)msg_phys);
	*(ptr_sgl+1) = htole32((uint64_t)(uintptr_t)
	msg_phys >> 32);
	ptr_sgl += sge_dwords;
	num_entries++;
	}

	/* Need to handle if entry will be part of a page. */
	offset = (uint32_t)paddr & page_mask;
	entry_len = PAGE_SIZE - offset;
	if (first_prp_entry) {
	/*
	* Put IEEE entry in first SGE in main message.
	* (Simple element, System addr, not end of
	* list.)
	*/
	*ptr_first_sgl = htole32((uint32_t)paddr);
	*(ptr_first_sgl + 1) =
	htole32((uint32_t)((uint64_t)paddr >> 32));
	*(ptr_first_sgl + 2) = htole32(entry_len);
	*(ptr_first_sgl + 3) = 0;

	/* No longer the first PRP entry. */
	first_prp_entry = 0;
	} else {
	/* Put entry in list. */
	*ptr_sgl = htole32((uint32_t)paddr);
	*(ptr_sgl + 1) =
	htole32((uint32_t)((uint64_t)paddr >> 32));

	/* Bump ptr_sgl, msg_phys, and num_entries. */
	ptr_sgl += sge_dwords;
	msg_phys = (uint64_t )((uint8_t )msg_phys +
	prp_size);
	num_entries++;
	}

	/* Bump the phys address by the entry_len. */
	paddr += entry_len;

	/* Decrement length accounting for last partial page. */
	if (entry_len > length)
	length = 0;
	else
	length -= entry_len;
	}
	}

	/* Set chain element Length. */
	main_chain_element->Length = htole32(num_entries * prp_size);

	/* Return 0, indicating we built a native SGL. */
	return 0;
	}

	/*
	* Add a chain element as the next SGE for the specified command.
	* Reset cm_sge and cm_sgesize to indicate all the available space. Chains are
	* only required for IEEE commands. Therefore there is no code for commands
	* that have the MPR_CM_FLAGS_SGE_SIMPLE flag set (and those commands
	* shouldn't be requesting chains).
	*/
	static int
	mpr_add_chain(struct mpr_command *cm, int segsleft)
	{
	struct mpr_softc *sc = cm->cm_sc;
	MPI2_REQUEST_HEADER *req;
	MPI25_IEEE_SGE_CHAIN64 *ieee_sgc;
	struct mpr_chain *chain;
	int sgc_size, current_segs, rem_segs, segs_per_frame;
	uint8_t next_chain_offset = 0;

	/*
	* Fail if a command is requesting a chain for SIMPLE SGE's. For SAS3
	* only IEEE commands should be requesting chains. Return some error
	* code other than 0.
	*/
	if (cm->cm_flags & MPR_CM_FLAGS_SGE_SIMPLE) {
	mpr_dprint(sc, MPR_ERROR, "A chain element cannot be added to "
	"an MPI SGL.\n");
	return(ENOBUFS);
	}

	sgc_size = sizeof(MPI25_IEEE_SGE_CHAIN64);
	if (cm->cm_sglsize < sgc_size)
	panic("MPR: Need SGE Error Code\n");

	chain = mpr_alloc_chain(cm->cm_sc);
	if (chain == NULL)
	return (ENOBUFS);

	/*
	* Note: a double-linked list is used to make it easier to walk for
	* debugging.
	*/
	TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link);

	/*
	* Need to know if the number of frames left is more than 1 or not. If
	* more than 1 frame is required, NextChainOffset will need to be set,
	* which will just be the last segment of the frame.
	*/
	rem_segs = 0;
	if (cm->cm_sglsize < (sgc_size * segsleft)) {
	/*
	* rem_segs is the number of segements remaining after the
	* segments that will go into the current frame. Since it is
	* known that at least one more frame is required, account for
	* the chain element. To know if more than one more frame is
	* required, just check if there will be a remainder after using
	* the current frame (with this chain) and the next frame. If
	* so the NextChainOffset must be the last element of the next
	* frame.
	*/
	current_segs = (cm->cm_sglsize / sgc_size) - 1;
	rem_segs = segsleft - current_segs;
	segs_per_frame = sc->chain_frame_size / sgc_size;
	if (rem_segs > segs_per_frame) {
	next_chain_offset = segs_per_frame - 1;
	}
	}
	ieee_sgc = &((MPI25_SGE_IO_UNION *)cm->cm_sge)->IeeeChain;
	ieee_sgc->Length = next_chain_offset ?
	htole32((uint32_t)sc->chain_frame_size) :
	htole32((uint32_t)rem_segs * (uint32_t)sgc_size);
	ieee_sgc->NextChainOffset = next_chain_offset;
	ieee_sgc->Flags = (MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT \|
	MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
	ieee_sgc->Address.Low = htole32(chain->chain_busaddr);
	ieee_sgc->Address.High = htole32(chain->chain_busaddr >> 32);
	cm->cm_sge = &((MPI25_SGE_IO_UNION *)chain->chain)->IeeeSimple;
	req = (MPI2_REQUEST_HEADER *)cm->cm_req;
	req->ChainOffset = (sc->chain_frame_size - sgc_size) >> 4;

	cm->cm_sglsize = sc->chain_frame_size;
	return (0);
	}

	/*
	* Add one scatter-gather element to the scatter-gather list for a command.
	* Maintain cm_sglsize and cm_sge as the remaining size and pointer to the
	* next SGE to fill in, respectively. In Gen3, the MPI SGL does not have a
	* chain, so don't consider any chain additions.
	*/
	int
	mpr_push_sge(struct mpr_command cm, MPI2_SGE_SIMPLE64 sge, size_t len,
	int segsleft)
	{
	uint32_t saved_buf_len, saved_address_low, saved_address_high;
	u32 sge_flags;

	/*
	* case 1: >=1 more segment, no room for anything (error)
	* case 2: 1 more segment and enough room for it
	*/

	if (cm->cm_sglsize < (segsleft * sizeof(MPI2_SGE_SIMPLE64))) {
	mpr_dprint(cm->cm_sc, MPR_ERROR,
	"%s: warning: Not enough room for MPI SGL in frame.\n",
	__func__);
	return(ENOBUFS);
	}

	KASSERT(segsleft == 1,
	("segsleft cannot be more than 1 for an MPI SGL; segsleft = %d\n",
	segsleft));

	/*
	* There is one more segment left to add for the MPI SGL and there is
	* enough room in the frame to add it. This is the normal case because
	* MPI SGL's don't have chains, otherwise something is wrong.
	*
	* If this is a bi-directional request, need to account for that
	* here. Save the pre-filled sge values. These will be used
	* either for the 2nd SGL or for a single direction SGL. If
	* cm_out_len is non-zero, this is a bi-directional request, so
	* fill in the OUT SGL first, then the IN SGL, otherwise just
	* fill in the IN SGL. Note that at this time, when filling in
	* 2 SGL's for a bi-directional request, they both use the same
	* DMA buffer (same cm command).
	*/
	saved_buf_len = sge->FlagsLength & 0x00FFFFFF;
	saved_address_low = sge->Address.Low;
	saved_address_high = sge->Address.High;
	if (cm->cm_out_len) {
	sge->FlagsLength = cm->cm_out_len \|
	((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_SGE_FLAGS_END_OF_BUFFER \|
	MPI2_SGE_FLAGS_HOST_TO_IOC \|
	MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
	MPI2_SGE_FLAGS_SHIFT);
	cm->cm_sglsize -= len;
	/* Endian Safe code */
	sge_flags = sge->FlagsLength;
	sge->FlagsLength = htole32(sge_flags);
	sge->Address.High = htole32(sge->Address.High);
	sge->Address.Low = htole32(sge->Address.Low);
	bcopy(sge, cm->cm_sge, len);
	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);
	}
	sge->FlagsLength = saved_buf_len \|
	((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_SGE_FLAGS_END_OF_BUFFER \|
	MPI2_SGE_FLAGS_LAST_ELEMENT \|
	MPI2_SGE_FLAGS_END_OF_LIST \|
	MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
	MPI2_SGE_FLAGS_SHIFT);
	if (cm->cm_flags & MPR_CM_FLAGS_DATAIN) {
	sge->FlagsLength \|=
	((uint32_t)(MPI2_SGE_FLAGS_IOC_TO_HOST) <<
	MPI2_SGE_FLAGS_SHIFT);
	} else {
	sge->FlagsLength \|=
	((uint32_t)(MPI2_SGE_FLAGS_HOST_TO_IOC) <<
	MPI2_SGE_FLAGS_SHIFT);
	}
	sge->Address.Low = saved_address_low;
	sge->Address.High = saved_address_high;

	cm->cm_sglsize -= len;
	/* Endian Safe code */
	sge_flags = sge->FlagsLength;
	sge->FlagsLength = htole32(sge_flags);
	sge->Address.High = htole32(sge->Address.High);
	sge->Address.Low = htole32(sge->Address.Low);
	bcopy(sge, cm->cm_sge, len);
	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);
	return (0);
	}

	/*
	* Add one IEEE scatter-gather element (chain or simple) to the IEEE scatter-
	* gather list for a command. Maintain cm_sglsize and cm_sge as the
	* remaining size and pointer to the next SGE to fill in, respectively.
	*/
	int
	mpr_push_ieee_sge(struct mpr_command cm, void sgep, int segsleft)
	{
	MPI2_IEEE_SGE_SIMPLE64 *sge = sgep;
	int error, ieee_sge_size = sizeof(MPI25_SGE_IO_UNION);
	uint32_t saved_buf_len, saved_address_low, saved_address_high;
	uint32_t sge_length;

	/*
	* case 1: No room for chain or segment (error).
	* case 2: Two or more segments left but only room for chain.
	* case 3: Last segment and room for it, so set flags.
	*/

	/*
	* There should be room for at least one element, or there is a big
	* problem.
	*/
	if (cm->cm_sglsize < ieee_sge_size)
	panic("MPR: Need SGE Error Code\n");

	if ((segsleft >= 2) && (cm->cm_sglsize < (ieee_sge_size * 2))) {
	if ((error = mpr_add_chain(cm, segsleft)) != 0)
	return (error);
	}

	if (segsleft == 1) {
	/*
	* If this is a bi-directional request, need to account for that
	* here. Save the pre-filled sge values. These will be used
	* either for the 2nd SGL or for a single direction SGL. If
	* cm_out_len is non-zero, this is a bi-directional request, so
	* fill in the OUT SGL first, then the IN SGL, otherwise just
	* fill in the IN SGL. Note that at this time, when filling in
	* 2 SGL's for a bi-directional request, they both use the same
	* DMA buffer (same cm command).
	*/
	saved_buf_len = sge->Length;
	saved_address_low = sge->Address.Low;
	saved_address_high = sge->Address.High;
	if (cm->cm_out_len) {
	sge->Length = cm->cm_out_len;
	sge->Flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
	cm->cm_sglsize -= ieee_sge_size;
	/* Endian Safe code */
	sge_length = sge->Length;
	sge->Length = htole32(sge_length);
	sge->Address.High = htole32(sge->Address.High);
	sge->Address.Low = htole32(sge->Address.Low);
	bcopy(sgep, cm->cm_sge, ieee_sge_size);
	cm->cm_sge =
	(MPI25_SGE_IO_UNION *)((uintptr_t)cm->cm_sge +
	ieee_sge_size);
	}
	sge->Length = saved_buf_len;
	sge->Flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR \|
	MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
	sge->Address.Low = saved_address_low;
	sge->Address.High = saved_address_high;
	}

	cm->cm_sglsize -= ieee_sge_size;
	/* Endian Safe code */
	sge_length = sge->Length;
	sge->Length = htole32(sge_length);
	sge->Address.High = htole32(sge->Address.High);
	sge->Address.Low = htole32(sge->Address.Low);
	bcopy(sgep, cm->cm_sge, ieee_sge_size);
	cm->cm_sge = (MPI25_SGE_IO_UNION *)((uintptr_t)cm->cm_sge +
	ieee_sge_size);
	return (0);
	}

	/*
	* Add one dma segment to the scatter-gather list for a command.
	*/
	int
	mpr_add_dmaseg(struct mpr_command *cm, vm_paddr_t pa, size_t len, u_int flags,
	int segsleft)
	{
	MPI2_SGE_SIMPLE64 sge;
	MPI2_IEEE_SGE_SIMPLE64 ieee_sge;

	if (!(cm->cm_flags & MPR_CM_FLAGS_SGE_SIMPLE)) {
	ieee_sge.Flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR);
	ieee_sge.Length = len;
	mpr_from_u64(pa, &ieee_sge.Address);

	return (mpr_push_ieee_sge(cm, &ieee_sge, segsleft));
	} else {
	/*
	* This driver always uses 64-bit address elements for
	* simplicity.
	*/
	flags \|= MPI2_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_SGE_FLAGS_64_BIT_ADDRESSING;
	/* Set Endian safe macro in mpr_push_sge */
	sge.FlagsLength = len \| (flags << MPI2_SGE_FLAGS_SHIFT);
	mpr_from_u64(pa, &sge.Address);

	return (mpr_push_sge(cm, &sge, sizeof sge, segsleft));
	}
	}

	static void
	mpr_data_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	struct mpr_softc *sc;
	struct mpr_command *cm;
	u_int i, dir, sflags;

	cm = (struct mpr_command *)arg;
	sc = cm->cm_sc;

	/*
	* In this case, just print out a warning and let the chip tell the
	* user they did the wrong thing.
	*/
	if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) {
	mpr_dprint(sc, MPR_ERROR, "%s: warning: busdma returned %d "
	"segments, more than the %d allowed\n", __func__, nsegs,
	cm->cm_max_segs);
	}

	/*
	* Set up DMA direction flags. Bi-directional requests are also handled
	* here. In that case, both direction flags will be set.
	*/
	sflags = 0;
	if (cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) {
	/*
	* We have to add a special case for SMP passthrough, there
	* is no easy way to generically handle it. The first
	* S/G element is used for the command (therefore the
	* direction bit needs to be set). The second one is used
	* for the reply. We'll leave it to the caller to make
	* sure we only have two buffers.
	*/
	/*
	* Even though the busdma man page says it doesn't make
	* sense to have both direction flags, it does in this case.
	* We have one s/g element being accessed in each direction.
	*/
	dir = BUS_DMASYNC_PREWRITE \| BUS_DMASYNC_PREREAD;

	/*
	* Set the direction flag on the first buffer in the SMP
	* passthrough request. We'll clear it for the second one.
	*/
	sflags \|= MPI2_SGE_FLAGS_DIRECTION \|
	MPI2_SGE_FLAGS_END_OF_BUFFER;
	} else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT) {
	sflags \|= MPI2_SGE_FLAGS_HOST_TO_IOC;
	dir = BUS_DMASYNC_PREWRITE;
	} else
	dir = BUS_DMASYNC_PREREAD;

	/* Check if a native SG list is needed for an NVMe PCIe device. */
	if (cm->cm_targ && cm->cm_targ->is_nvme &&
	mpr_check_pcie_native_sgl(sc, cm, segs, nsegs) == 0) {
	/* A native SG list was built, skip to end. */
	goto out;
	}

	for (i = 0; i < nsegs; i++) {
	if ((cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) && (i != 0)) {
	sflags &= ~MPI2_SGE_FLAGS_DIRECTION;
	}
	error = mpr_add_dmaseg(cm, segs[i].ds_addr, segs[i].ds_len,
	sflags, nsegs - i);
	if (error != 0) {
	/* Resource shortage, roll back! */
	if (ratecheck(&sc->lastfail, &mpr_chainfail_interval))
	mpr_dprint(sc, MPR_INFO, "Out of chain frames, "
	"consider increasing hw.mpr.max_chains.\n");
	cm->cm_flags \|= MPR_CM_FLAGS_CHAIN_FAILED;
	mpr_complete_command(sc, cm);
	return;
	}
	}

	out:
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
	mpr_enqueue_request(sc, cm);

	return;
	}

	static void
	mpr_data_cb2(void arg, bus_dma_segment_t segs, int nsegs, bus_size_t mapsize,
	int error)
	{
	mpr_data_cb(arg, segs, nsegs, error);
	}

	/*
	* This is the routine to enqueue commands ansynchronously.
	* Note that the only error path here is from bus_dmamap_load(), which can
	* return EINPROGRESS if it is waiting for resources. Other than this, it's
	* assumed that if you have a command in-hand, then you have enough credits
	* to use it.
	*/
	int
	mpr_map_command(struct mpr_softc sc, struct mpr_command cm)
	{
	int error = 0;

	if (cm->cm_flags & MPR_CM_FLAGS_USE_UIO) {
	error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap,
	&cm->cm_uio, mpr_data_cb2, cm, 0);
	} else if (cm->cm_flags & MPR_CM_FLAGS_USE_CCB) {
	error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap,
	cm->cm_data, mpr_data_cb, cm, 0);
	} else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) {
	error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap,
	cm->cm_data, cm->cm_length, mpr_data_cb, cm, 0);
	} else {
	/* Add a zero-length element as needed */
	if (cm->cm_sge != NULL)
	mpr_add_dmaseg(cm, 0, 0, 0, 1);
	mpr_enqueue_request(sc, cm);
	}

	return (error);
	}

	/*
	* This is the routine to enqueue commands synchronously. An error of
	* EINPROGRESS from mpr_map_command() is ignored since the command will
	* be executed and enqueued automatically. Other errors come from msleep().
	*/
	int
	mpr_wait_command(struct mpr_softc sc, struct mpr_command *cmp, int timeout,
	int sleep_flag)
	{
	int error, rc;
	struct timeval cur_time, start_time;
	struct mpr_command cm = cmp;

	if (sc->mpr_flags & MPR_FLAGS_DIAGRESET)
	return EBUSY;

	cm->cm_complete = NULL;
	cm->cm_flags \|= (MPR_CM_FLAGS_WAKEUP + MPR_CM_FLAGS_POLLED);
	error = mpr_map_command(sc, cm);
	if ((error != 0) && (error != EINPROGRESS))
	return (error);

	// Check for context and wait for 50 mSec at a time until time has
	// expired or the command has finished. If msleep can't be used, need
	// to poll.
	if (curthread->td_no_sleeping)
	sleep_flag = NO_SLEEP;
	getmicrouptime(&start_time);
	if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP) {
	error = msleep(cm, &sc->mpr_mtx, 0, "mprwait", timeout*hz);
	if (error == EWOULDBLOCK) {
	/*
	* Record the actual elapsed time in the case of a
	* timeout for the message below.
	*/
	getmicrouptime(&cur_time);
	timevalsub(&cur_time, &start_time);
	}
	} else {
	while ((cm->cm_flags & MPR_CM_FLAGS_COMPLETE) == 0) {
	mpr_intr_locked(sc);
	if (sleep_flag == CAN_SLEEP)
	pause("mprwait", hz/20);
	else
	DELAY(50000);

	getmicrouptime(&cur_time);
	timevalsub(&cur_time, &start_time);
	if (cur_time.tv_sec > timeout) {
	error = EWOULDBLOCK;
	break;
	}
	}
	}

	if (error == EWOULDBLOCK) {
	if (cm->cm_timeout_handler == NULL) {
	mpr_dprint(sc, MPR_FAULT, "Calling Reinit from %s, timeout=%d,"
	" elapsed=%jd\n", __func__, timeout,
	(intmax_t)cur_time.tv_sec);
	rc = mpr_reinit(sc);
	mpr_dprint(sc, MPR_FAULT, "Reinit %s\n", (rc == 0) ? "success" :
	"failed");
	} else
	cm->cm_timeout_handler(sc, cm);
	if (sc->mpr_flags & MPR_FLAGS_REALLOCATED) {
	/*
	* Tell the caller that we freed the command in a
	* reinit.
	*/
	*cmp = NULL;
	}
	error = ETIMEDOUT;
	}
	return (error);
	}

	/*
	* This is the routine to enqueue a command synchonously and poll for
	* completion. Its use should be rare.
	*/
	int
	mpr_request_polled(struct mpr_softc sc, struct mpr_command *cmp)
	{
	int error, rc;
	struct timeval cur_time, start_time;
	struct mpr_command cm = cmp;

	error = 0;

	cm->cm_flags \|= MPR_CM_FLAGS_POLLED;
	cm->cm_complete = NULL;
	mpr_map_command(sc, cm);

	getmicrouptime(&start_time);
	while ((cm->cm_flags & MPR_CM_FLAGS_COMPLETE) == 0) {
	mpr_intr_locked(sc);

	if (mtx_owned(&sc->mpr_mtx))
	msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0,
	"mprpoll", hz/20);
	else
	pause("mprpoll", hz/20);

	/*
	* Check for real-time timeout and fail if more than 60 seconds.
	*/
	getmicrouptime(&cur_time);
	timevalsub(&cur_time, &start_time);
	if (cur_time.tv_sec > 60) {
	mpr_dprint(sc, MPR_FAULT, "polling failed\n");
	error = ETIMEDOUT;
	break;
	}
	}
	cm->cm_state = MPR_CM_STATE_BUSY;
	if (error) {
	mpr_dprint(sc, MPR_FAULT, "Calling Reinit from %s\n", __func__);
	rc = mpr_reinit(sc);
	mpr_dprint(sc, MPR_FAULT, "Reinit %s\n", (rc == 0) ? "success" :
	"failed");

	if (sc->mpr_flags & MPR_FLAGS_REALLOCATED) {
	/*
	* Tell the caller that we freed the command in a
	* reinit.
	*/
	*cmp = NULL;
	}
	}
	return (error);
	}

	/*
	* The MPT driver had a verbose interface for config pages. In this driver,
	* reduce it to much simpler terms, similar to the Linux driver.
	*/
	int
	mpr_read_config_page(struct mpr_softc sc, struct mpr_config_params params)
	{
	MPI2_CONFIG_REQUEST *req;
	struct mpr_command *cm;
	int error;

	if (sc->mpr_flags & MPR_FLAGS_BUSY) {
	return (EBUSY);
	}

	cm = mpr_alloc_command(sc);
	if (cm == NULL) {
	return (EBUSY);
	}

	req = (MPI2_CONFIG_REQUEST *)cm->cm_req;
	req->Function = MPI2_FUNCTION_CONFIG;
	req->Action = params->action;
	req->SGLFlags = 0;
	req->ChainOffset = 0;
	req->PageAddress = params->page_address;
	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
	MPI2_CONFIG_EXTENDED_PAGE_HEADER *hdr;

	hdr = &params->hdr.Ext;
	req->ExtPageType = hdr->ExtPageType;
	req->ExtPageLength = hdr->ExtPageLength;
	req->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
	req->Header.PageLength = 0; /* Must be set to zero */
	req->Header.PageNumber = hdr->PageNumber;
	req->Header.PageVersion = hdr->PageVersion;
	} else {
	MPI2_CONFIG_PAGE_HEADER *hdr;

	hdr = &params->hdr.Struct;
	req->Header.PageType = hdr->PageType;
	req->Header.PageNumber = hdr->PageNumber;
	req->Header.PageLength = hdr->PageLength;
	req->Header.PageVersion = hdr->PageVersion;
	}

	cm->cm_data = params->buffer;
	cm->cm_length = params->length;
	if (cm->cm_data != NULL) {
	cm->cm_sge = &req->PageBufferSGE;
	cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
	cm->cm_flags = MPR_CM_FLAGS_SGE_SIMPLE \| MPR_CM_FLAGS_DATAIN;
	} else
	cm->cm_sge = NULL;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;

	cm->cm_complete_data = params;
	if (params->callback != NULL) {
	cm->cm_complete = mpr_config_complete;
	return (mpr_map_command(sc, cm));
	} else {
	error = mpr_wait_command(sc, &cm, 0, CAN_SLEEP);
	if (error) {
	mpr_dprint(sc, MPR_FAULT,
	"Error %d reading config page\n", error);
	if (cm != NULL)
	mpr_free_command(sc, cm);
	return (error);
	}
	mpr_config_complete(sc, cm);
	}

	return (0);
	}

	int
	mpr_write_config_page(struct mpr_softc sc, struct mpr_config_params params)
	{
	return (EINVAL);
	}

	static void
	mpr_config_complete(struct mpr_softc sc, struct mpr_command cm)
	{
	MPI2_CONFIG_REPLY *reply;
	struct mpr_config_params *params;

	MPR_FUNCTRACE(sc);
	params = cm->cm_complete_data;

	if (cm->cm_data != NULL) {
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
	BUS_DMASYNC_POSTREAD);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
	}

	/*
	* XXX KDM need to do more error recovery? This results in the
	* device in question not getting probed.
	*/
	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	params->status = MPI2_IOCSTATUS_BUSY;
	goto done;
	}

	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
	if (reply == NULL) {
	params->status = MPI2_IOCSTATUS_BUSY;
	goto done;
	}
	params->status = reply->IOCStatus;
	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
	params->hdr.Ext.ExtPageType = reply->ExtPageType;
	params->hdr.Ext.ExtPageLength = reply->ExtPageLength;
	params->hdr.Ext.PageType = reply->Header.PageType;
	params->hdr.Ext.PageNumber = reply->Header.PageNumber;
	params->hdr.Ext.PageVersion = reply->Header.PageVersion;
	} else {
	params->hdr.Struct.PageType = reply->Header.PageType;
	params->hdr.Struct.PageNumber = reply->Header.PageNumber;
	params->hdr.Struct.PageLength = reply->Header.PageLength;
	params->hdr.Struct.PageVersion = reply->Header.PageVersion;
	}

	done:
	mpr_free_command(sc, cm);
	if (params->callback != NULL)
	params->callback(sc, params);

	return;
	}
	diff --git a/sys/dev/mpr/mpr_sas.c b/sys/dev/mpr/mpr_sas.c
	index 64eed15ea5a6..0e31f0c4d21e 100644
	--- a/sys/dev/mpr/mpr_sas.c
	+++ b/sys/dev/mpr/mpr_sas.c
	@@ -1,3563 +1,3548 @@
	/*-
	* Copyright (c) 2009 Yahoo! Inc.
	* Copyright (c) 2011-2015 LSI Corp.
	* Copyright (c) 2013-2016 Avago Technologies
	* Copyright 2000-2020 Broadcom Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Broadcom Inc. (LSI) MPT-Fusion Host Adapter FreeBSD
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/* Communications core for Avago Technologies (LSI) MPT3 */

	/* TODO Move headers to mprvar */
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/selinfo.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/bio.h>
	#include <sys/malloc.h>
	#include <sys/uio.h>
	#include <sys/sysctl.h>
	#include <sys/endian.h>
	#include <sys/queue.h>
	#include <sys/kthread.h>
	#include <sys/taskqueue.h>
	#include <sys/sbuf.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>

	#include <machine/stdarg.h>

	#include <cam/cam.h>
	#include <cam/cam_ccb.h>
	#include <cam/cam_debug.h>
	#include <cam/cam_sim.h>
	#include <cam/cam_xpt_sim.h>
	#include <cam/cam_xpt_periph.h>
	#include <cam/cam_periph.h>
	#include <cam/scsi/scsi_all.h>
	#include <cam/scsi/scsi_message.h>
	#include <cam/scsi/smp_all.h>

	#include <dev/nvme/nvme.h>

	#include <dev/mpr/mpi/mpi2_type.h>
	#include <dev/mpr/mpi/mpi2.h>
	#include <dev/mpr/mpi/mpi2_ioc.h>
	#include <dev/mpr/mpi/mpi2_sas.h>
	#include <dev/mpr/mpi/mpi2_pci.h>
	#include <dev/mpr/mpi/mpi2_cnfg.h>
	#include <dev/mpr/mpi/mpi2_init.h>
	#include <dev/mpr/mpi/mpi2_tool.h>
	#include <dev/mpr/mpr_ioctl.h>
	#include <dev/mpr/mprvar.h>
	#include <dev/mpr/mpr_table.h>
	#include <dev/mpr/mpr_sas.h>

	#define MPRSAS_DISCOVERY_TIMEOUT 20
	#define MPRSAS_MAX_DISCOVERY_TIMEOUTS 10 /* 200 seconds */

	/*
	* static array to check SCSI OpCode for EEDP protection bits
	*/
	#define PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
	#define PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
	#define PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
	static uint8_t op_code_prot[256] = {
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	};

	MALLOC_DEFINE(M_MPRSAS, "MPRSAS", "MPR SAS memory");

	static void mprsas_remove_device(struct mpr_softc , struct mpr_command );
	static void mprsas_remove_complete(struct mpr_softc , struct mpr_command );
	static void mprsas_action(struct cam_sim sim, union ccb ccb);
	static void mprsas_poll(struct cam_sim *sim);
	static void mprsas_scsiio_timeout(void *data);
	static void mprsas_abort_complete(struct mpr_softc sc, struct mpr_command cm);
	static void mprsas_action_scsiio(struct mprsas_softc , union ccb );
	static void mprsas_scsiio_complete(struct mpr_softc , struct mpr_command );
	static void mprsas_action_resetdev(struct mprsas_softc , union ccb );
	static void mprsas_resetdev_complete(struct mpr_softc , struct mpr_command );
	static int mprsas_send_abort(struct mpr_softc sc, struct mpr_command tm,
	struct mpr_command *cm);
	static void mprsas_async(void *callback_arg, uint32_t code,
	struct cam_path path, void arg);
	static int mprsas_send_portenable(struct mpr_softc *sc);
	static void mprsas_portenable_complete(struct mpr_softc *sc,
	struct mpr_command *cm);

	static void mprsas_smpio_complete(struct mpr_softc sc, struct mpr_command cm);
	static void mprsas_send_smpcmd(struct mprsas_softc sassc, union ccb ccb,
	uint64_t sasaddr);
	static void mprsas_action_smpio(struct mprsas_softc sassc, union ccb ccb);

	struct mprsas_target *
	mprsas_find_target_by_handle(struct mprsas_softc *sassc, int start,
	uint16_t handle)
	{
	struct mprsas_target *target;
	int i;

	for (i = start; i < sassc->maxtargets; i++) {
	target = &sassc->targets[i];
	if (target->handle == handle)
	return (target);
	}

	return (NULL);
	}

	/* we need to freeze the simq during attach and diag reset, to avoid failing
	* commands before device handles have been found by discovery. Since
	* discovery involves reading config pages and possibly sending commands,
	* discovery actions may continue even after we receive the end of discovery
	* event, so refcount discovery actions instead of assuming we can unfreeze
	* the simq when we get the event.
	*/
	void
	mprsas_startup_increment(struct mprsas_softc *sassc)
	{
	MPR_FUNCTRACE(sassc->sc);

	if ((sassc->flags & MPRSAS_IN_STARTUP) != 0) {
	if (sassc->startup_refcount++ == 0) {
	/* just starting, freeze the simq */
	mpr_dprint(sassc->sc, MPR_INIT,
	"%s freezing simq\n", __func__);
	xpt_hold_boot();
	xpt_freeze_simq(sassc->sim, 1);
	}
	mpr_dprint(sassc->sc, MPR_INIT, "%s refcount %u\n", __func__,
	sassc->startup_refcount);
	}
	}

	void
	mprsas_release_simq_reinit(struct mprsas_softc *sassc)
	{
	if (sassc->flags & MPRSAS_QUEUE_FROZEN) {
	sassc->flags &= ~MPRSAS_QUEUE_FROZEN;
	xpt_release_simq(sassc->sim, 1);
	mpr_dprint(sassc->sc, MPR_INFO, "Unfreezing SIM queue\n");
	}
	}

	void
	mprsas_startup_decrement(struct mprsas_softc *sassc)
	{
	MPR_FUNCTRACE(sassc->sc);

	if ((sassc->flags & MPRSAS_IN_STARTUP) != 0) {
	if (--sassc->startup_refcount == 0) {
	/* finished all discovery-related actions, release
	* the simq and rescan for the latest topology.
	*/
	mpr_dprint(sassc->sc, MPR_INIT,
	"%s releasing simq\n", __func__);
	sassc->flags &= ~MPRSAS_IN_STARTUP;
	xpt_release_simq(sassc->sim, 1);
	xpt_release_boot();
	}
	mpr_dprint(sassc->sc, MPR_INIT, "%s refcount %u\n", __func__,
	sassc->startup_refcount);
	}
	}

	/*
	* The firmware requires us to stop sending commands when we're doing task
	* management.
	* use.
	* XXX The logic for serializing the device has been made lazy and moved to
	* mprsas_prepare_for_tm().
	*/
	struct mpr_command *
	mprsas_alloc_tm(struct mpr_softc *sc)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpr_command *tm;

	MPR_FUNCTRACE(sc);
	tm = mpr_alloc_high_priority_command(sc);
	if (tm == NULL)
	return (NULL);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
	return tm;
	}

	void
	mprsas_free_tm(struct mpr_softc sc, struct mpr_command tm)
	{
	int target_id = 0xFFFFFFFF;

	MPR_FUNCTRACE(sc);
	if (tm == NULL)
	return;

	/*
	* For TM's the devq is frozen for the device. Unfreeze it here and
	* free the resources used for freezing the devq. Must clear the
	* INRESET flag as well or scsi I/O will not work.
	*/
	if (tm->cm_targ != NULL) {
	tm->cm_targ->flags &= ~MPRSAS_TARGET_INRESET;
	target_id = tm->cm_targ->tid;
	}
	if (tm->cm_ccb) {
	mpr_dprint(sc, MPR_INFO, "Unfreezing devq for target ID %d\n",
	target_id);
	xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
	xpt_free_path(tm->cm_ccb->ccb_h.path);
	xpt_free_ccb(tm->cm_ccb);
	}

	mpr_free_high_priority_command(sc, tm);
	}

	void
	mprsas_rescan_target(struct mpr_softc sc, struct mprsas_target targ)
	{
	struct mprsas_softc *sassc = sc->sassc;
	path_id_t pathid;
	target_id_t targetid;
	union ccb *ccb;

	MPR_FUNCTRACE(sc);
	pathid = cam_sim_path(sassc->sim);
	if (targ == NULL)
	targetid = CAM_TARGET_WILDCARD;
	else
	targetid = targ - sassc->targets;

	/*
	* Allocate a CCB and schedule a rescan.
	*/
	ccb = xpt_alloc_ccb_nowait();
	if (ccb == NULL) {
	mpr_dprint(sc, MPR_ERROR, "unable to alloc CCB for rescan\n");
	return;
	}

	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
	CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
	mpr_dprint(sc, MPR_ERROR, "unable to create path for rescan\n");
	xpt_free_ccb(ccb);
	return;
	}

	if (targetid == CAM_TARGET_WILDCARD)
	ccb->ccb_h.func_code = XPT_SCAN_BUS;
	else
	ccb->ccb_h.func_code = XPT_SCAN_TGT;

	mpr_dprint(sc, MPR_TRACE, "%s targetid %u\n", __func__, targetid);
	xpt_rescan(ccb);
	}

	static void
	mprsas_log_command(struct mpr_command cm, u_int level, const char fmt, ...)
	{
	struct sbuf sb;
	va_list ap;
	char str[224];
	char path_str[64];

	if (cm == NULL)
	return;

	/* No need to be in here if debugging isn't enabled */
	if ((cm->cm_sc->mpr_debug & level) == 0)
	return;

	sbuf_new(&sb, str, sizeof(str), 0);

	va_start(ap, fmt);

	if (cm->cm_ccb != NULL) {
	xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
	sizeof(path_str));
	sbuf_cat(&sb, path_str);
	if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
	scsi_command_string(&cm->cm_ccb->csio, &sb);
	sbuf_printf(&sb, "length %d ",
	cm->cm_ccb->csio.dxfer_len);
	}
	} else {
	sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
	cam_sim_name(cm->cm_sc->sassc->sim),
	cam_sim_unit(cm->cm_sc->sassc->sim),
	cam_sim_bus(cm->cm_sc->sassc->sim),
	cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
	cm->cm_lun);
	}

	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
	sbuf_vprintf(&sb, fmt, ap);
	sbuf_finish(&sb);
	mpr_print_field(cm->cm_sc, "%s", sbuf_data(&sb));

	va_end(ap);
	}

	static void
	mprsas_remove_volume(struct mpr_softc sc, struct mpr_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	struct mprsas_target *targ;
	uint16_t handle;

	MPR_FUNCTRACE(sc);

	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
	targ = tm->cm_targ;

	if (reply == NULL) {
	/* XXX retry the remove after the diag reset completes? */
	mpr_dprint(sc, MPR_FAULT, "%s NULL reply resetting device "
	"0x%04x\n", __func__, handle);
	mprsas_free_tm(sc, tm);
	return;
	}

	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) {
	mpr_dprint(sc, MPR_ERROR, "IOCStatus = 0x%x while resetting "
	"device 0x%x\n", le16toh(reply->IOCStatus), handle);
	}

	mpr_dprint(sc, MPR_XINFO, "Reset aborted %u commands\n",
	le32toh(reply->TerminationCount));
	mpr_free_reply(sc, tm->cm_reply_data);
	tm->cm_reply = NULL; /* Ensures the reply won't get re-freed */

	mpr_dprint(sc, MPR_XINFO, "clearing target %u handle 0x%04x\n",
	targ->tid, handle);

	/*
	* Don't clear target if remove fails because things will get confusing.
	* Leave the devname and sasaddr intact so that we know to avoid reusing
	* this target id if possible, and so we can assign the same target id
	* to this device if it comes back in the future.
	*/
	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	targ = tm->cm_targ;
	targ->handle = 0x0;
	targ->encl_handle = 0x0;
	targ->encl_level_valid = 0x0;
	targ->encl_level = 0x0;
	targ->connector_name[0] = ' ';
	targ->connector_name[1] = ' ';
	targ->connector_name[2] = ' ';
	targ->connector_name[3] = ' ';
	targ->encl_slot = 0x0;
	targ->exp_dev_handle = 0x0;
	targ->phy_num = 0x0;
	targ->linkrate = 0x0;
	targ->devinfo = 0x0;
	targ->flags = 0x0;
	targ->scsi_req_desc_type = 0;
	}

	mprsas_free_tm(sc, tm);
	}


	/*
	* No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
	* Otherwise Volume Delete is same as Bare Drive Removal.
	*/
	void
	mprsas_prepare_volume_remove(struct mprsas_softc *sassc, uint16_t handle)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpr_softc *sc;
	struct mpr_command *cm;
	struct mprsas_target *targ = NULL;

	MPR_FUNCTRACE(sassc->sc);
	sc = sassc->sc;

	targ = mprsas_find_target_by_handle(sassc, 0, handle);
	if (targ == NULL) {
	/* FIXME: what is the action? */
	/* We don't know about this device? */
	mpr_dprint(sc, MPR_ERROR,
	"%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
	return;
	}

	targ->flags \|= MPRSAS_TARGET_INREMOVAL;

	cm = mprsas_alloc_tm(sc);
	if (cm == NULL) {
	mpr_dprint(sc, MPR_ERROR,
	"%s: command alloc failure\n", __func__);
	return;
	}

	mprsas_rescan_target(sc, targ);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
	req->DevHandle = targ->handle;
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;

	if (!targ->is_nvme \|\| sc->custom_nvme_tm_handling) {
	/* SAS Hard Link Reset / SATA Link Reset */
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
	} else {
	/* PCIe Protocol Level Reset*/
	req->MsgFlags =
	MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
	}

	cm->cm_targ = targ;
	cm->cm_data = NULL;
	cm->cm_complete = mprsas_remove_volume;
	cm->cm_complete_data = (void *)(uintptr_t)handle;

	mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n",
	__func__, targ->tid);
	mprsas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);

	mpr_map_command(sc, cm);
	}

	/*
	* The firmware performs debounce on the link to avoid transient link errors
	* and false removals. When it does decide that link has been lost and a
	* device needs to go away, it expects that the host will perform a target reset
	* and then an op remove. The reset has the side-effect of aborting any
	* outstanding requests for the device, which is required for the op-remove to
	* succeed. It's not clear if the host should check for the device coming back
	* alive after the reset.
	*/
	void
	mprsas_prepare_remove(struct mprsas_softc *sassc, uint16_t handle)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpr_softc *sc;
	struct mpr_command *tm;
	struct mprsas_target *targ = NULL;

	MPR_FUNCTRACE(sassc->sc);

	sc = sassc->sc;

	targ = mprsas_find_target_by_handle(sassc, 0, handle);
	if (targ == NULL) {
	/* FIXME: what is the action? */
	/* We don't know about this device? */
	mpr_dprint(sc, MPR_ERROR, "%s : invalid handle 0x%x \n",
	__func__, handle);
	return;
	}

	targ->flags \|= MPRSAS_TARGET_INREMOVAL;

	tm = mprsas_alloc_tm(sc);
	if (tm == NULL) {
	mpr_dprint(sc, MPR_ERROR, "%s: command alloc failure\n",
	__func__);
	return;
	}

	mprsas_rescan_target(sc, targ);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	memset(req, 0, sizeof(*req));
	req->DevHandle = htole16(targ->handle);
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;

	/* SAS Hard Link Reset / SATA Link Reset */
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;

	tm->cm_targ = targ;
	tm->cm_data = NULL;
	tm->cm_complete = mprsas_remove_device;
	tm->cm_complete_data = (void *)(uintptr_t)handle;

	mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n",
	__func__, targ->tid);
	mprsas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);

	mpr_map_command(sc, tm);
	}

	static void
	mprsas_remove_device(struct mpr_softc sc, struct mpr_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
	struct mprsas_target *targ;
	uint16_t handle;

	MPR_FUNCTRACE(sc);

	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for remove of "
	"handle %#04x! This should not happen!\n", __func__,
	tm->cm_flags, handle);
	}

	if (reply == NULL) {
	/* XXX retry the remove after the diag reset completes? */
	mpr_dprint(sc, MPR_FAULT, "%s NULL reply resetting device "
	"0x%04x\n", __func__, handle);
	mprsas_free_tm(sc, tm);
	return;
	}

	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) {
	mpr_dprint(sc, MPR_ERROR, "IOCStatus = 0x%x while resetting "
	"device 0x%x\n", le16toh(reply->IOCStatus), handle);
	}

	mpr_dprint(sc, MPR_XINFO, "Reset aborted %u commands\n",
	le32toh(reply->TerminationCount));
	mpr_free_reply(sc, tm->cm_reply_data);
	tm->cm_reply = NULL; /* Ensures the reply won't get re-freed */

	/* Reuse the existing command */
	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
	memset(req, 0, sizeof(*req));
	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
	req->DevHandle = htole16(handle);
	tm->cm_data = NULL;
	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	tm->cm_complete = mprsas_remove_complete;
	tm->cm_complete_data = (void *)(uintptr_t)handle;

	/*
	* Wait to send the REMOVE_DEVICE until all the commands have cleared.
	* They should be aborted or time out and we'll kick thus off there
	* if so.
	*/
	if (TAILQ_FIRST(&targ->commands) == NULL) {
	mpr_dprint(sc, MPR_INFO, "No pending commands: starting remove_device\n");
	mpr_map_command(sc, tm);
	targ->pending_remove_tm = NULL;
	} else {
	targ->pending_remove_tm = tm;
	}

	mpr_dprint(sc, MPR_INFO, "clearing target %u handle 0x%04x\n",
	targ->tid, handle);
	if (targ->encl_level_valid) {
	mpr_dprint(sc, MPR_INFO, "At enclosure level %d, slot %d, "
	"connector name (%4s)\n", targ->encl_level, targ->encl_slot,
	targ->connector_name);
	}
	}

	static void
	mprsas_remove_complete(struct mpr_softc sc, struct mpr_command tm)
	{
	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
	uint16_t handle;
	struct mprsas_target *targ;
	struct mprsas_lun *lun;

	MPR_FUNCTRACE(sc);

	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;

	targ = tm->cm_targ;

	/*
	* At this point, we should have no pending commands for the target.
	* The remove target has just completed.
	*/
	KASSERT(TAILQ_FIRST(&targ->commands) == NULL,
	("%s: no commands should be pending\n", __func__));

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_XINFO, "%s: cm_flags = %#x for remove of "
	"handle %#04x! This should not happen!\n", __func__,
	tm->cm_flags, handle);
	mprsas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	/* most likely a chip reset */
	mpr_dprint(sc, MPR_FAULT, "%s NULL reply removing device "
	"0x%04x\n", __func__, handle);
	mprsas_free_tm(sc, tm);
	return;
	}

	mpr_dprint(sc, MPR_XINFO, "%s on handle 0x%04x, IOCStatus= 0x%x\n",
	__func__, handle, le16toh(reply->IOCStatus));

	/*
	* Don't clear target if remove fails because things will get confusing.
	* Leave the devname and sasaddr intact so that we know to avoid reusing
	* this target id if possible, and so we can assign the same target id
	* to this device if it comes back in the future.
	*/
	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	targ->handle = 0x0;
	targ->encl_handle = 0x0;
	targ->encl_level_valid = 0x0;
	targ->encl_level = 0x0;
	targ->connector_name[0] = ' ';
	targ->connector_name[1] = ' ';
	targ->connector_name[2] = ' ';
	targ->connector_name[3] = ' ';
	targ->encl_slot = 0x0;
	targ->exp_dev_handle = 0x0;
	targ->phy_num = 0x0;
	targ->linkrate = 0x0;
	targ->devinfo = 0x0;
	targ->flags = 0x0;
	targ->scsi_req_desc_type = 0;

	while (!SLIST_EMPTY(&targ->luns)) {
	lun = SLIST_FIRST(&targ->luns);
	SLIST_REMOVE_HEAD(&targ->luns, lun_link);
	free(lun, M_MPR);
	}
	}

	mprsas_free_tm(sc, tm);
	}

	static int
	mprsas_register_events(struct mpr_softc *sc)
	{
	uint8_t events[16];

	bzero(events, 16);
	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
	setbit(events, MPI2_EVENT_IR_VOLUME);
	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
	setbit(events, MPI2_EVENT_TEMP_THRESHOLD);
	setbit(events, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
	if (sc->facts->MsgVersion >= MPI2_VERSION_02_06) {
	setbit(events, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
	if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) {
	setbit(events, MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_PCIE_ENUMERATION);
	setbit(events, MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
	}
	}

	mpr_register_events(sc, events, mprsas_evt_handler, NULL,
	&sc->sassc->mprsas_eh);

	return (0);
	}

	int
	mpr_attach_sas(struct mpr_softc *sc)
	{
	struct mprsas_softc *sassc;
	cam_status status;
	int unit, error = 0, reqs;

	MPR_FUNCTRACE(sc);
	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);

	sassc = malloc(sizeof(struct mprsas_softc), M_MPR, M_WAITOK\|M_ZERO);
	- if (!sassc) {
	- mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	- "Cannot allocate SAS subsystem memory\n");
	- return (ENOMEM);
	- }

	/*
	* XXX MaxTargets could change during a reinit. Since we don't
	* resize the targets[] array during such an event, cache the value
	* of MaxTargets here so that we don't get into trouble later. This
	* should move into the reinit logic.
	*/
	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
	sassc->targets = malloc(sizeof(struct mprsas_target) *
	sassc->maxtargets, M_MPR, M_WAITOK\|M_ZERO);
	- if (!sassc->targets) {
	- mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	- "Cannot allocate SAS target memory\n");
	- free(sassc, M_MPR);
	- return (ENOMEM);
	- }
	sc->sassc = sassc;
	sassc->sc = sc;

	reqs = sc->num_reqs - sc->num_prireqs - 1;
	if ((sassc->devq = cam_simq_alloc(reqs)) == NULL) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR, "Cannot allocate SIMQ\n");
	error = ENOMEM;
	goto out;
	}

	unit = device_get_unit(sc->mpr_dev);
	sassc->sim = cam_sim_alloc(mprsas_action, mprsas_poll, "mpr", sassc,
	unit, &sc->mpr_mtx, reqs, reqs, sassc->devq);
	if (sassc->sim == NULL) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR, "Cannot allocate SIM\n");
	error = EINVAL;
	goto out;
	}

	TAILQ_INIT(&sassc->ev_queue);

	/* Initialize taskqueue for Event Handling */
	TASK_INIT(&sassc->ev_task, 0, mprsas_firmware_event_work, sc);
	sassc->ev_tq = taskqueue_create("mpr_taskq", M_NOWAIT \| M_ZERO,
	taskqueue_thread_enqueue, &sassc->ev_tq);
	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
	device_get_nameunit(sc->mpr_dev));

	mpr_lock(sc);

	/*
	* XXX There should be a bus for every port on the adapter, but since
	* we're just going to fake the topology for now, we'll pretend that
	* everything is just a target on a single bus.
	*/
	if ((error = xpt_bus_register(sassc->sim, sc->mpr_dev, 0)) != 0) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Error %d registering SCSI bus\n", error);
	mpr_unlock(sc);
	goto out;
	}

	/*
	* Assume that discovery events will start right away.
	*
	* Hold off boot until discovery is complete.
	*/
	sassc->flags \|= MPRSAS_IN_STARTUP \| MPRSAS_IN_DISCOVERY;
	sc->sassc->startup_refcount = 0;
	mprsas_startup_increment(sassc);

	callout_init(&sassc->discovery_callout, 1 /mpsafe/);

	/*
	* Register for async events so we can determine the EEDP
	* capabilities of devices.
	*/
	status = xpt_create_path(&sassc->path, /periph/NULL,
	cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
	CAM_LUN_WILDCARD);
	if (status != CAM_REQ_CMP) {
	mpr_dprint(sc, MPR_INIT\|MPR_ERROR,
	"Error %#x creating sim path\n", status);
	sassc->path = NULL;
	} else {
	int event;

	event = AC_ADVINFO_CHANGED \| AC_FOUND_DEVICE;
	status = xpt_register_async(event, mprsas_async, sc,
	sassc->path);

	if (status != CAM_REQ_CMP) {
	mpr_dprint(sc, MPR_ERROR,
	"Error %#x registering async handler for "
	"AC_ADVINFO_CHANGED events\n", status);
	xpt_free_path(sassc->path);
	sassc->path = NULL;
	}
	}
	if (status != CAM_REQ_CMP) {
	/*
	* EEDP use is the exception, not the rule.
	* Warn the user, but do not fail to attach.
	*/
	mpr_printf(sc, "EEDP capabilities disabled.\n");
	}

	mpr_unlock(sc);

	mprsas_register_events(sc);
	out:
	if (error)
	mpr_detach_sas(sc);

	mpr_dprint(sc, MPR_INIT, "%s exit, error= %d\n", __func__, error);
	return (error);
	}

	int
	mpr_detach_sas(struct mpr_softc *sc)
	{
	struct mprsas_softc *sassc;
	struct mprsas_lun lun, lun_tmp;
	struct mprsas_target *targ;
	int i;

	MPR_FUNCTRACE(sc);

	if (sc->sassc == NULL)
	return (0);

	sassc = sc->sassc;
	mpr_deregister_events(sc, sassc->mprsas_eh);

	/*
	* Drain and free the event handling taskqueue with the lock
	* unheld so that any parallel processing tasks drain properly
	* without deadlocking.
	*/
	if (sassc->ev_tq != NULL)
	taskqueue_free(sassc->ev_tq);

	/* Make sure CAM doesn't wedge if we had to bail out early. */
	mpr_lock(sc);

	while (sassc->startup_refcount != 0)
	mprsas_startup_decrement(sassc);

	/* Deregister our async handler */
	if (sassc->path != NULL) {
	xpt_register_async(0, mprsas_async, sc, sassc->path);
	xpt_free_path(sassc->path);
	sassc->path = NULL;
	}

	if (sassc->flags & MPRSAS_IN_STARTUP)
	xpt_release_simq(sassc->sim, 1);

	if (sassc->sim != NULL) {
	xpt_bus_deregister(cam_sim_path(sassc->sim));
	cam_sim_free(sassc->sim, FALSE);
	}

	mpr_unlock(sc);

	if (sassc->devq != NULL)
	cam_simq_free(sassc->devq);

	for (i = 0; i < sassc->maxtargets; i++) {
	targ = &sassc->targets[i];
	SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
	free(lun, M_MPR);
	}
	}
	free(sassc->targets, M_MPR);
	free(sassc, M_MPR);
	sc->sassc = NULL;

	return (0);
	}

	void
	mprsas_discovery_end(struct mprsas_softc *sassc)
	{
	struct mpr_softc *sc = sassc->sc;

	MPR_FUNCTRACE(sc);

	if (sassc->flags & MPRSAS_DISCOVERY_TIMEOUT_PENDING)
	callout_stop(&sassc->discovery_callout);

	/*
	* After discovery has completed, check the mapping table for any
	* missing devices and update their missing counts. Only do this once
	* whenever the driver is initialized so that missing counts aren't
	* updated unnecessarily. Note that just because discovery has
	* completed doesn't mean that events have been processed yet. The
	* check_devices function is a callout timer that checks if ALL devices
	* are missing. If so, it will wait a little longer for events to
	* complete and keep resetting itself until some device in the mapping
	* table is not missing, meaning that event processing has started.
	*/
	if (sc->track_mapping_events) {
	mpr_dprint(sc, MPR_XINFO \| MPR_MAPPING, "Discovery has "
	"completed. Check for missing devices in the mapping "
	"table.\n");
	callout_reset(&sc->device_check_callout,
	MPR_MISSING_CHECK_DELAY * hz, mpr_mapping_check_devices,
	sc);
	}
	}

	static void
	mprsas_action(struct cam_sim sim, union ccb ccb)
	{
	struct mprsas_softc *sassc;

	sassc = cam_sim_softc(sim);

	MPR_FUNCTRACE(sassc->sc);
	mpr_dprint(sassc->sc, MPR_TRACE, "ccb func_code 0x%x\n",
	ccb->ccb_h.func_code);
	mtx_assert(&sassc->sc->mpr_mtx, MA_OWNED);

	switch (ccb->ccb_h.func_code) {
	case XPT_PATH_INQ:
	{
	struct ccb_pathinq *cpi = &ccb->cpi;
	struct mpr_softc *sc = sassc->sc;

	cpi->version_num = 1;
	cpi->hba_inquiry = PI_SDTR_ABLE\|PI_TAG_ABLE\|PI_WIDE_16;
	cpi->target_sprt = 0;
	cpi->hba_misc = PIM_NOBUSRESET \| PIM_UNMAPPED \| PIM_NOSCAN;
	cpi->hba_eng_cnt = 0;
	cpi->max_target = sassc->maxtargets - 1;
	cpi->max_lun = 255;

	/*
	* initiator_id is set here to an ID outside the set of valid
	* target IDs (including volumes).
	*/
	cpi->initiator_id = sassc->maxtargets;
	strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
	strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
	strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
	cpi->unit_number = cam_sim_unit(sim);
	cpi->bus_id = cam_sim_bus(sim);
	/*
	* XXXSLM-I think this needs to change based on config page or
	* something instead of hardcoded to 150000.
	*/
	cpi->base_transfer_speed = 150000;
	cpi->transport = XPORT_SAS;
	cpi->transport_version = 0;
	cpi->protocol = PROTO_SCSI;
	cpi->protocol_version = SCSI_REV_SPC;
	cpi->maxio = sc->maxio;
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	}
	case XPT_GET_TRAN_SETTINGS:
	{
	struct ccb_trans_settings *cts;
	struct ccb_trans_settings_sas *sas;
	struct ccb_trans_settings_scsi *scsi;
	struct mprsas_target *targ;

	cts = &ccb->cts;
	sas = &cts->xport_specific.sas;
	scsi = &cts->proto_specific.scsi;

	KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_GET_TRAN_SETTINGS\n",
	cts->ccb_h.target_id));
	targ = &sassc->targets[cts->ccb_h.target_id];
	if (targ->handle == 0x0) {
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	break;
	}

	cts->protocol_version = SCSI_REV_SPC2;
	cts->transport = XPORT_SAS;
	cts->transport_version = 0;

	sas->valid = CTS_SAS_VALID_SPEED;
	switch (targ->linkrate) {
	case 0x08:
	sas->bitrate = 150000;
	break;
	case 0x09:
	sas->bitrate = 300000;
	break;
	case 0x0a:
	sas->bitrate = 600000;
	break;
	case 0x0b:
	sas->bitrate = 1200000;
	break;
	default:
	sas->valid = 0;
	}

	cts->protocol = PROTO_SCSI;
	scsi->valid = CTS_SCSI_VALID_TQ;
	scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;

	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	}
	case XPT_CALC_GEOMETRY:
	cam_calc_geometry(&ccb->ccg, /extended/1);
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	case XPT_RESET_DEV:
	mpr_dprint(sassc->sc, MPR_XINFO, "mprsas_action "
	"XPT_RESET_DEV\n");
	mprsas_action_resetdev(sassc, ccb);
	return;
	case XPT_RESET_BUS:
	case XPT_ABORT:
	case XPT_TERM_IO:
	mpr_dprint(sassc->sc, MPR_XINFO, "mprsas_action faking success "
	"for abort or reset\n");
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	case XPT_SCSI_IO:
	mprsas_action_scsiio(sassc, ccb);
	return;
	case XPT_SMP_IO:
	mprsas_action_smpio(sassc, ccb);
	return;
	default:
	mprsas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
	break;
	}
	xpt_done(ccb);

	}

	static void
	mprsas_announce_reset(struct mpr_softc *sc, uint32_t ac_code,
	target_id_t target_id, lun_id_t lun_id)
	{
	path_id_t path_id = cam_sim_path(sc->sassc->sim);
	struct cam_path *path;

	mpr_dprint(sc, MPR_XINFO, "%s code %x target %d lun %jx\n", __func__,
	ac_code, target_id, (uintmax_t)lun_id);

	if (xpt_create_path(&path, NULL,
	path_id, target_id, lun_id) != CAM_REQ_CMP) {
	mpr_dprint(sc, MPR_ERROR, "unable to create path for reset "
	"notification\n");
	return;
	}

	xpt_async(ac_code, path, NULL);
	xpt_free_path(path);
	}

	static void
	mprsas_complete_all_commands(struct mpr_softc *sc)
	{
	struct mpr_command *cm;
	int i;
	int completed;

	MPR_FUNCTRACE(sc);
	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	/* complete all commands with a NULL reply */
	for (i = 1; i < sc->num_reqs; i++) {
	cm = &sc->commands[i];
	if (cm->cm_state == MPR_CM_STATE_FREE)
	continue;

	cm->cm_state = MPR_CM_STATE_BUSY;
	cm->cm_reply = NULL;
	completed = 0;

	if (cm->cm_flags & MPR_CM_FLAGS_SATA_ID_TIMEOUT) {
	MPASS(cm->cm_data);
	free(cm->cm_data, M_MPR);
	cm->cm_data = NULL;
	}

	if (cm->cm_flags & MPR_CM_FLAGS_POLLED)
	cm->cm_flags \|= MPR_CM_FLAGS_COMPLETE;

	if (cm->cm_complete != NULL) {
	mprsas_log_command(cm, MPR_RECOVERY,
	"completing cm %p state %x ccb %p for diag reset\n",
	cm, cm->cm_state, cm->cm_ccb);
	cm->cm_complete(sc, cm);
	completed = 1;
	} else if (cm->cm_flags & MPR_CM_FLAGS_WAKEUP) {
	mprsas_log_command(cm, MPR_RECOVERY,
	"waking up cm %p state %x ccb %p for diag reset\n",
	cm, cm->cm_state, cm->cm_ccb);
	wakeup(cm);
	completed = 1;
	}

	if ((completed == 0) && (cm->cm_state != MPR_CM_STATE_FREE)) {
	/* this should never happen, but if it does, log */
	mprsas_log_command(cm, MPR_RECOVERY,
	"cm %p state %x flags 0x%x ccb %p during diag "
	"reset\n", cm, cm->cm_state, cm->cm_flags,
	cm->cm_ccb);
	}
	}

	sc->io_cmds_active = 0;
	}

	void
	mprsas_handle_reinit(struct mpr_softc *sc)
	{
	int i;

	/* Go back into startup mode and freeze the simq, so that CAM
	* doesn't send any commands until after we've rediscovered all
	* targets and found the proper device handles for them.
	*
	* After the reset, portenable will trigger discovery, and after all
	* discovery-related activities have finished, the simq will be
	* released.
	*/
	mpr_dprint(sc, MPR_INIT, "%s startup\n", __func__);
	sc->sassc->flags \|= MPRSAS_IN_STARTUP;
	sc->sassc->flags \|= MPRSAS_IN_DISCOVERY;
	mprsas_startup_increment(sc->sassc);

	/* notify CAM of a bus reset */
	mprsas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
	CAM_LUN_WILDCARD);

	/* complete and cleanup after all outstanding commands */
	mprsas_complete_all_commands(sc);

	mpr_dprint(sc, MPR_INIT, "%s startup %u after command completion\n",
	__func__, sc->sassc->startup_refcount);

	/* zero all the target handles, since they may change after the
	* reset, and we have to rediscover all the targets and use the new
	* handles.
	*/
	for (i = 0; i < sc->sassc->maxtargets; i++) {
	if (sc->sassc->targets[i].outstanding != 0)
	mpr_dprint(sc, MPR_INIT, "target %u outstanding %u\n",
	i, sc->sassc->targets[i].outstanding);
	sc->sassc->targets[i].handle = 0x0;
	sc->sassc->targets[i].exp_dev_handle = 0x0;
	sc->sassc->targets[i].outstanding = 0;
	sc->sassc->targets[i].flags = MPRSAS_TARGET_INDIAGRESET;
	}
	}
	static void
	mprsas_tm_timeout(void *data)
	{
	struct mpr_command *tm = data;
	struct mpr_softc *sc = tm->cm_sc;

	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	mprsas_log_command(tm, MPR_INFO\|MPR_RECOVERY, "task mgmt %p timed "
	"out\n", tm);

	KASSERT(tm->cm_state == MPR_CM_STATE_INQUEUE,
	("command not inqueue\n"));

	tm->cm_state = MPR_CM_STATE_BUSY;
	mpr_reinit(sc);
	}

	static void
	mprsas_logical_unit_reset_complete(struct mpr_softc sc, struct mpr_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	unsigned int cm_count = 0;
	struct mpr_command *cm;
	struct mprsas_target *targ;

	callout_stop(&tm->cm_callout);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_RECOVERY\|MPR_ERROR,
	"%s: cm_flags = %#x for LUN reset! "
	"This should not happen!\n", __func__, tm->cm_flags);
	mprsas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	mpr_dprint(sc, MPR_RECOVERY, "NULL reset reply for tm %p\n",
	tm);
	if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
	/* this completion was due to a reset, just cleanup */
	mpr_dprint(sc, MPR_RECOVERY, "Hardware undergoing "
	"reset, ignoring NULL LUN reset reply\n");
	targ->tm = NULL;
	mprsas_free_tm(sc, tm);
	}
	else {
	/* we should have gotten a reply. */
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY, "NULL reply on "
	"LUN reset attempt, resetting controller\n");
	mpr_reinit(sc);
	}
	return;
	}

	mpr_dprint(sc, MPR_RECOVERY,
	"logical unit reset status 0x%x code 0x%x count %u\n",
	le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
	le32toh(reply->TerminationCount));

	/*
	* See if there are any outstanding commands for this LUN.
	* This could be made more efficient by using a per-LU data
	* structure of some sort.
	*/
	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
	if (cm->cm_lun == tm->cm_lun)
	cm_count++;
	}

	if (cm_count == 0) {
	mpr_dprint(sc, MPR_RECOVERY\|MPR_INFO,
	"Finished recovery after LUN reset for target %u\n",
	targ->tid);

	mprsas_announce_reset(sc, AC_SENT_BDR, targ->tid,
	tm->cm_lun);

	/*
	* We've finished recovery for this logical unit. check and
	* see if some other logical unit has a timedout command
	* that needs to be processed.
	*/
	cm = TAILQ_FIRST(&targ->timedout_commands);
	if (cm) {
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"More commands to abort for target %u\n", targ->tid);
	mprsas_send_abort(sc, tm, cm);
	} else {
	targ->tm = NULL;
	mprsas_free_tm(sc, tm);
	}
	} else {
	/* if we still have commands for this LUN, the reset
	* effectively failed, regardless of the status reported.
	* Escalate to a target reset.
	*/
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"logical unit reset complete for target %u, but still "
	"have %u command(s), sending target reset\n", targ->tid,
	cm_count);
	if (!targ->is_nvme \|\| sc->custom_nvme_tm_handling)
	mprsas_send_reset(sc, tm,
	MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
	else
	mpr_reinit(sc);
	}
	}

	static void
	mprsas_target_reset_complete(struct mpr_softc sc, struct mpr_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mprsas_target *targ;

	callout_stop(&tm->cm_callout);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for target "
	"reset! This should not happen!\n", __func__, tm->cm_flags);
	mprsas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	mpr_dprint(sc, MPR_RECOVERY,
	"NULL target reset reply for tm %p TaskMID %u\n",
	tm, le16toh(req->TaskMID));
	if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
	/* this completion was due to a reset, just cleanup */
	mpr_dprint(sc, MPR_RECOVERY, "Hardware undergoing "
	"reset, ignoring NULL target reset reply\n");
	targ->tm = NULL;
	mprsas_free_tm(sc, tm);
	}
	else {
	/* we should have gotten a reply. */
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY, "NULL reply on "
	"target reset attempt, resetting controller\n");
	mpr_reinit(sc);
	}
	return;
	}

	mpr_dprint(sc, MPR_RECOVERY,
	"target reset status 0x%x code 0x%x count %u\n",
	le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
	le32toh(reply->TerminationCount));

	if (targ->outstanding == 0) {
	/*
	* We've finished recovery for this target and all
	* of its logical units.
	*/
	mpr_dprint(sc, MPR_RECOVERY\|MPR_INFO,
	"Finished reset recovery for target %u\n", targ->tid);

	mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
	CAM_LUN_WILDCARD);

	targ->tm = NULL;
	mprsas_free_tm(sc, tm);
	} else {
	/*
	* After a target reset, if this target still has
	* outstanding commands, the reset effectively failed,
	* regardless of the status reported. escalate.
	*/
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"Target reset complete for target %u, but still have %u "
	"command(s), resetting controller\n", targ->tid,
	targ->outstanding);
	mpr_reinit(sc);
	}
	}

	#define MPR_RESET_TIMEOUT 30

	int
	mprsas_send_reset(struct mpr_softc sc, struct mpr_command tm, uint8_t type)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mprsas_target *target;
	int err, timeout;

	target = tm->cm_targ;
	if (target->handle == 0) {
	mpr_dprint(sc, MPR_ERROR, "%s null devhandle for target_id "
	"%d\n", __func__, target->tid);
	return -1;
	}

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = htole16(target->handle);
	req->TaskType = type;

	if (!target->is_nvme \|\| sc->custom_nvme_tm_handling) {
	timeout = MPR_RESET_TIMEOUT;
	/*
	* Target reset method =
	* SAS Hard Link Reset / SATA Link Reset
	*/
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
	} else {
	timeout = (target->controller_reset_timeout) ? (
	target->controller_reset_timeout) : (MPR_RESET_TIMEOUT);
	/* PCIe Protocol Level Reset*/
	req->MsgFlags =
	MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
	}

	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
	/* XXX Need to handle invalid LUNs */
	MPR_SET_LUN(req->LUN, tm->cm_lun);
	tm->cm_targ->logical_unit_resets++;
	mpr_dprint(sc, MPR_RECOVERY\|MPR_INFO,
	"Sending logical unit reset to target %u lun %d\n",
	target->tid, tm->cm_lun);
	tm->cm_complete = mprsas_logical_unit_reset_complete;
	mprsas_prepare_for_tm(sc, tm, target, tm->cm_lun);
	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
	tm->cm_targ->target_resets++;
	mpr_dprint(sc, MPR_RECOVERY\|MPR_INFO,
	"Sending target reset to target %u\n", target->tid);
	tm->cm_complete = mprsas_target_reset_complete;
	mprsas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
	}
	else {
	mpr_dprint(sc, MPR_ERROR, "unexpected reset type 0x%x\n", type);
	return -1;
	}

	if (target->encl_level_valid) {
	mpr_dprint(sc, MPR_RECOVERY\|MPR_INFO,
	"At enclosure level %d, slot %d, connector name (%4s)\n",
	target->encl_level, target->encl_slot,
	target->connector_name);
	}

	tm->cm_data = NULL;
	tm->cm_complete_data = (void *)tm;

	callout_reset(&tm->cm_callout, timeout * hz,
	mprsas_tm_timeout, tm);

	err = mpr_map_command(sc, tm);
	if (err)
	mpr_dprint(sc, MPR_ERROR\|MPR_RECOVERY,
	"error %d sending reset type %u\n", err, type);

	return err;
	}


	static void
	mprsas_abort_complete(struct mpr_softc sc, struct mpr_command tm)
	{
	struct mpr_command *cm;
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mprsas_target *targ;

	callout_stop(&tm->cm_callout);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_RECOVERY\|MPR_ERROR,
	"cm_flags = %#x for abort %p TaskMID %u!\n",
	tm->cm_flags, tm, le16toh(req->TaskMID));
	mprsas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	mpr_dprint(sc, MPR_RECOVERY,
	"NULL abort reply for tm %p TaskMID %u\n",
	tm, le16toh(req->TaskMID));
	if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
	/* this completion was due to a reset, just cleanup */
	mpr_dprint(sc, MPR_RECOVERY, "Hardware undergoing "
	"reset, ignoring NULL abort reply\n");
	targ->tm = NULL;
	mprsas_free_tm(sc, tm);
	} else {
	/* we should have gotten a reply. */
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY, "NULL reply on "
	"abort attempt, resetting controller\n");
	mpr_reinit(sc);
	}
	return;
	}

	mpr_dprint(sc, MPR_RECOVERY,
	"abort TaskMID %u status 0x%x code 0x%x count %u\n",
	le16toh(req->TaskMID),
	le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
	le32toh(reply->TerminationCount));

	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
	if (cm == NULL) {
	/*
	* if there are no more timedout commands, we're done with
	* error recovery for this target.
	*/
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"Finished abort recovery for target %u\n", targ->tid);
	targ->tm = NULL;
	mprsas_free_tm(sc, tm);
	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
	/* abort success, but we have more timedout commands to abort */
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"Continuing abort recovery for target %u\n", targ->tid);
	mprsas_send_abort(sc, tm, cm);
	} else {
	/*
	* we didn't get a command completion, so the abort
	* failed as far as we're concerned. escalate.
	*/
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"Abort failed for target %u, sending logical unit reset\n",
	targ->tid);

	mprsas_send_reset(sc, tm,
	MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
	}
	}

	#define MPR_ABORT_TIMEOUT 5

	static int
	mprsas_send_abort(struct mpr_softc sc, struct mpr_command tm,
	struct mpr_command *cm)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mprsas_target *targ;
	int err, timeout;

	targ = cm->cm_targ;
	if (targ->handle == 0) {
	mpr_dprint(sc, MPR_ERROR\|MPR_RECOVERY,
	"%s null devhandle for target_id %d\n",
	__func__, cm->cm_ccb->ccb_h.target_id);
	return -1;
	}

	mprsas_log_command(cm, MPR_RECOVERY\|MPR_INFO,
	"Aborting command %p\n", cm);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = htole16(targ->handle);
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;

	/* XXX Need to handle invalid LUNs */
	MPR_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);

	req->TaskMID = htole16(cm->cm_desc.Default.SMID);

	tm->cm_data = NULL;
	tm->cm_complete = mprsas_abort_complete;
	tm->cm_complete_data = (void *)tm;
	tm->cm_targ = cm->cm_targ;
	tm->cm_lun = cm->cm_lun;

	if (!targ->is_nvme \|\| sc->custom_nvme_tm_handling)
	timeout = MPR_ABORT_TIMEOUT;
	else
	timeout = sc->nvme_abort_timeout;

	callout_reset(&tm->cm_callout, timeout * hz,
	mprsas_tm_timeout, tm);

	targ->aborts++;

	mprsas_prepare_for_tm(sc, tm, targ, tm->cm_lun);

	err = mpr_map_command(sc, tm);
	if (err)
	mpr_dprint(sc, MPR_ERROR\|MPR_RECOVERY,
	"error %d sending abort for cm %p SMID %u\n",
	err, cm, req->TaskMID);
	return err;
	}

	static void
	mprsas_scsiio_timeout(void *data)
	{
	sbintime_t elapsed, now;
	union ccb *ccb;
	struct mpr_softc *sc;
	struct mpr_command *cm;
	struct mprsas_target *targ;

	cm = (struct mpr_command *)data;
	sc = cm->cm_sc;
	ccb = cm->cm_ccb;
	now = sbinuptime();

	MPR_FUNCTRACE(sc);
	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	mpr_dprint(sc, MPR_XINFO\|MPR_RECOVERY, "Timeout checking cm %p\n", cm);

	/*
	* Run the interrupt handler to make sure it's not pending. This
	* isn't perfect because the command could have already completed
	* and been re-used, though this is unlikely.
	*/
	mpr_intr_locked(sc);
	if (cm->cm_flags & MPR_CM_FLAGS_ON_RECOVERY) {
	mprsas_log_command(cm, MPR_XINFO,
	"SCSI command %p almost timed out\n", cm);
	return;
	}

	if (cm->cm_ccb == NULL) {
	mpr_dprint(sc, MPR_ERROR, "command timeout with NULL ccb\n");
	return;
	}

	targ = cm->cm_targ;
	targ->timeouts++;

	elapsed = now - ccb->ccb_h.qos.sim_data;
	mprsas_log_command(cm, MPR_INFO\|MPR_RECOVERY,
	"Command timeout on target %u(0x%04x), %d set, %d.%d elapsed\n",
	targ->tid, targ->handle, ccb->ccb_h.timeout,
	sbintime_getsec(elapsed), elapsed & 0xffffffff);
	if (targ->encl_level_valid) {
	mpr_dprint(sc, MPR_INFO\|MPR_RECOVERY,
	"At enclosure level %d, slot %d, connector name (%4s)\n",
	targ->encl_level, targ->encl_slot, targ->connector_name);
	}

	/* XXX first, check the firmware state, to see if it's still
	* operational. if not, do a diag reset.
	*/
	mprsas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
	cm->cm_flags \|= MPR_CM_FLAGS_ON_RECOVERY \| MPR_CM_FLAGS_TIMEDOUT;
	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);

	if (targ->tm != NULL) {
	/* target already in recovery, just queue up another
	* timedout command to be processed later.
	*/
	mpr_dprint(sc, MPR_RECOVERY, "queued timedout cm %p for "
	"processing by tm %p\n", cm, targ->tm);
	}
	else if ((targ->tm = mprsas_alloc_tm(sc)) != NULL) {

	/* start recovery by aborting the first timedout command */
	mpr_dprint(sc, MPR_RECOVERY\|MPR_INFO,
	"Sending abort to target %u for SMID %d\n", targ->tid,
	cm->cm_desc.Default.SMID);
	mpr_dprint(sc, MPR_RECOVERY, "timedout cm %p allocated tm %p\n",
	cm, targ->tm);
	mprsas_send_abort(sc, targ->tm, cm);
	}
	else {
	/* XXX queue this target up for recovery once a TM becomes
	* available. The firmware only has a limited number of
	* HighPriority credits for the high priority requests used
	* for task management, and we ran out.
	*
	* Isilon: don't worry about this for now, since we have
	* more credits than disks in an enclosure, and limit
	* ourselves to one TM per target for recovery.
	*/
	mpr_dprint(sc, MPR_ERROR\|MPR_RECOVERY,
	"timedout cm %p failed to allocate a tm\n", cm);
	}
	}

	/**
	* mprsas_build_nvme_unmap - Build Native NVMe DSM command equivalent
	* to SCSI Unmap.
	* Return 0 - for success,
	* 1 - to immediately return back the command with success status to CAM
	* negative value - to fallback to firmware path i.e. issue scsi unmap
	* to FW without any translation.
	*/
	static int
	mprsas_build_nvme_unmap(struct mpr_softc sc, struct mpr_command cm,
	union ccb ccb, struct mprsas_target targ)
	{
	Mpi26NVMeEncapsulatedRequest_t *req = NULL;
	struct ccb_scsiio *csio;
	struct unmap_parm_list *plist;
	struct nvme_dsm_range *nvme_dsm_ranges = NULL;
	struct nvme_command *c;
	int i, res;
	uint16_t ndesc, list_len, data_length;
	struct mpr_prp_page *prp_page_info;
	uint64_t nvme_dsm_ranges_dma_handle;

	csio = &ccb->csio;
	list_len = (scsiio_cdb_ptr(csio)[7] << 8 \| scsiio_cdb_ptr(csio)[8]);
	if (!list_len) {
	mpr_dprint(sc, MPR_ERROR, "Parameter list length is Zero\n");
	return -EINVAL;
	}

	plist = malloc(csio->dxfer_len, M_MPR, M_ZERO\|M_NOWAIT);
	if (!plist) {
	mpr_dprint(sc, MPR_ERROR, "Unable to allocate memory to "
	"save UNMAP data\n");
	return -ENOMEM;
	}

	/* Copy SCSI unmap data to a local buffer */
	bcopy(csio->data_ptr, plist, csio->dxfer_len);

	/* return back the unmap command to CAM with success status,
	* if number of descripts is zero.
	*/
	ndesc = be16toh(plist->unmap_blk_desc_data_len) >> 4;
	if (!ndesc) {
	mpr_dprint(sc, MPR_XINFO, "Number of descriptors in "
	"UNMAP cmd is Zero\n");
	res = 1;
	goto out;
	}

	data_length = ndesc * sizeof(struct nvme_dsm_range);
	if (data_length > targ->MDTS) {
	mpr_dprint(sc, MPR_ERROR, "data length: %d is greater than "
	"Device's MDTS: %d\n", data_length, targ->MDTS);
	res = -EINVAL;
	goto out;
	}

	prp_page_info = mpr_alloc_prp_page(sc);
	KASSERT(prp_page_info != NULL, ("%s: There is no PRP Page for "
	"UNMAP command.\n", __func__));

	/*
	* Insert the allocated PRP page into the command's PRP page list. This
	* will be freed when the command is freed.
	*/
	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);

	nvme_dsm_ranges = (struct nvme_dsm_range *)prp_page_info->prp_page;
	nvme_dsm_ranges_dma_handle = prp_page_info->prp_page_busaddr;

	bzero(nvme_dsm_ranges, data_length);

	/* Convert SCSI unmap's descriptor data to NVMe DSM specific Range data
	* for each descriptors contained in SCSI UNMAP data.
	*/
	for (i = 0; i < ndesc; i++) {
	nvme_dsm_ranges[i].length =
	htole32(be32toh(plist->desc[i].nlb));
	nvme_dsm_ranges[i].starting_lba =
	htole64(be64toh(plist->desc[i].slba));
	nvme_dsm_ranges[i].attributes = 0;
	}

	/* Build MPI2.6's NVMe Encapsulated Request Message */
	req = (Mpi26NVMeEncapsulatedRequest_t *)cm->cm_req;
	bzero(req, sizeof(*req));
	req->DevHandle = htole16(targ->handle);
	req->Function = MPI2_FUNCTION_NVME_ENCAPSULATED;
	req->Flags = MPI26_NVME_FLAGS_WRITE;
	req->ErrorResponseBaseAddress.High =
	htole32((uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32));
	req->ErrorResponseBaseAddress.Low =
	htole32(cm->cm_sense_busaddr);
	req->ErrorResponseAllocationLength =
	htole16(sizeof(struct nvme_completion));
	req->EncapsulatedCommandLength =
	htole16(sizeof(struct nvme_command));
	req->DataLength = htole32(data_length);

	/* Build NVMe DSM command */
	c = (struct nvme_command *) req->NVMe_Command;
	c->opc = NVME_OPC_DATASET_MANAGEMENT;
	c->nsid = htole32(csio->ccb_h.target_lun + 1);
	c->cdw10 = htole32(ndesc - 1);
	c->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE);

	cm->cm_length = data_length;
	cm->cm_data = NULL;

	cm->cm_complete = mprsas_scsiio_complete;
	cm->cm_complete_data = ccb;
	cm->cm_targ = targ;
	cm->cm_lun = csio->ccb_h.target_lun;
	cm->cm_ccb = ccb;

	cm->cm_desc.Default.RequestFlags =
	MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;

	csio->ccb_h.qos.sim_data = sbinuptime();
	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
	mprsas_scsiio_timeout, cm, 0);

	targ->issued++;
	targ->outstanding++;
	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
	ccb->ccb_h.status \|= CAM_SIM_QUEUED;

	mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n",
	__func__, cm, ccb, targ->outstanding);

	mpr_build_nvme_prp(sc, cm, req,
	(void *)(uintptr_t)nvme_dsm_ranges_dma_handle, 0, data_length);
	mpr_map_command(sc, cm);

	out:
	free(plist, M_MPR);
	return 0;
	}

	static void
	mprsas_action_scsiio(struct mprsas_softc sassc, union ccb ccb)
	{
	MPI2_SCSI_IO_REQUEST *req;
	struct ccb_scsiio *csio;
	struct mpr_softc *sc;
	struct mprsas_target *targ;
	struct mprsas_lun *lun;
	struct mpr_command *cm;
	uint8_t i, lba_byte, *ref_tag_addr, scsi_opcode;
	uint16_t eedp_flags;
	uint32_t mpi_control;
	int rc;

	sc = sassc->sc;
	MPR_FUNCTRACE(sc);
	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	csio = &ccb->csio;
	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_SCSI_IO\n",
	csio->ccb_h.target_id));
	targ = &sassc->targets[csio->ccb_h.target_id];
	mpr_dprint(sc, MPR_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
	if (targ->handle == 0x0) {
	mpr_dprint(sc, MPR_ERROR, "%s NULL handle for target %u\n",
	__func__, csio->ccb_h.target_id);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}
	if (targ->flags & MPR_TARGET_FLAGS_RAID_COMPONENT) {
	mpr_dprint(sc, MPR_ERROR, "%s Raid component no SCSI IO "
	"supported %u\n", __func__, csio->ccb_h.target_id);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}
	/*
	* Sometimes, it is possible to get a command that is not "In
	* Progress" and was actually aborted by the upper layer. Check for
	* this here and complete the command without error.
	*/
	if (mprsas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
	mpr_dprint(sc, MPR_TRACE, "%s Command is not in progress for "
	"target %u\n", __func__, csio->ccb_h.target_id);
	xpt_done(ccb);
	return;
	}
	/*
	* If devinfo is 0 this will be a volume. In that case don't tell CAM
	* that the volume has timed out. We want volumes to be enumerated
	* until they are deleted/removed, not just failed. In either event,
	* we're removing the target due to a firmware event telling us
	* the device is now gone (as opposed to some transient event). Since
	* we're opting to remove failed devices from the OS's view, we need
	* to propagate that status up the stack.
	*/
	if (targ->flags & MPRSAS_TARGET_INREMOVAL) {
	if (targ->devinfo == 0)
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}

	if ((sc->mpr_flags & MPR_FLAGS_SHUTDOWN) != 0) {
	mpr_dprint(sc, MPR_INFO, "%s shutting down\n", __func__);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}

	/*
	* If target has a reset in progress, freeze the devq and return. The
	* devq will be released when the TM reset is finished.
	*/
	if (targ->flags & MPRSAS_TARGET_INRESET) {
	ccb->ccb_h.status = CAM_BUSY \| CAM_DEV_QFRZN;
	mpr_dprint(sc, MPR_INFO, "%s: Freezing devq for target ID %d\n",
	__func__, targ->tid);
	xpt_freeze_devq(ccb->ccb_h.path, 1);
	xpt_done(ccb);
	return;
	}

	cm = mpr_alloc_command(sc);
	if (cm == NULL \|\| (sc->mpr_flags & MPR_FLAGS_DIAGRESET)) {
	if (cm != NULL) {
	mpr_free_command(sc, cm);
	}
	if ((sassc->flags & MPRSAS_QUEUE_FROZEN) == 0) {
	xpt_freeze_simq(sassc->sim, 1);
	sassc->flags \|= MPRSAS_QUEUE_FROZEN;
	}
	ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
	ccb->ccb_h.status \|= CAM_REQUEUE_REQ;
	xpt_done(ccb);
	return;
	}

	/* For NVME device's issue UNMAP command directly to NVME drives by
	* constructing equivalent native NVMe DataSetManagement command.
	*/
	scsi_opcode = scsiio_cdb_ptr(csio)[0];
	if (scsi_opcode == UNMAP &&
	targ->is_nvme &&
	(csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
	rc = mprsas_build_nvme_unmap(sc, cm, ccb, targ);
	if (rc == 1) { /* return command to CAM with success status */
	mpr_free_command(sc, cm);
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	xpt_done(ccb);
	return;
	} else if (!rc) /* Issued NVMe Encapsulated Request Message */
	return;
	}

	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
	bzero(req, sizeof(*req));
	req->DevHandle = htole16(targ->handle);
	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
	req->MsgFlags = 0;
	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
	req->SenseBufferLength = MPR_SENSE_LEN;
	req->SGLFlags = 0;
	req->ChainOffset = 0;
	req->SGLOffset0 = 24; /* 32bit word offset to the SGL */
	req->SGLOffset1= 0;
	req->SGLOffset2= 0;
	req->SGLOffset3= 0;
	req->SkipCount = 0;
	req->DataLength = htole32(csio->dxfer_len);
	req->BidirectionalDataLength = 0;
	req->IoFlags = htole16(csio->cdb_len);
	req->EEDPFlags = 0;

	/* Note: BiDirectional transfers are not supported */
	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
	case CAM_DIR_IN:
	mpi_control = MPI2_SCSIIO_CONTROL_READ;
	cm->cm_flags \|= MPR_CM_FLAGS_DATAIN;
	break;
	case CAM_DIR_OUT:
	mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
	cm->cm_flags \|= MPR_CM_FLAGS_DATAOUT;
	break;
	case CAM_DIR_NONE:
	default:
	mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
	break;
	}

	if (csio->cdb_len == 32)
	mpi_control \|= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
	/*
	* It looks like the hardware doesn't require an explicit tag
	* number for each transaction. SAM Task Management not supported
	* at the moment.
	*/
	switch (csio->tag_action) {
	case MSG_HEAD_OF_Q_TAG:
	mpi_control \|= MPI2_SCSIIO_CONTROL_HEADOFQ;
	break;
	case MSG_ORDERED_Q_TAG:
	mpi_control \|= MPI2_SCSIIO_CONTROL_ORDEREDQ;
	break;
	case MSG_ACA_TASK:
	mpi_control \|= MPI2_SCSIIO_CONTROL_ACAQ;
	break;
	case CAM_TAG_ACTION_NONE:
	case MSG_SIMPLE_Q_TAG:
	default:
	mpi_control \|= MPI2_SCSIIO_CONTROL_SIMPLEQ;
	break;
	}
	mpi_control \|= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
	req->Control = htole32(mpi_control);

	if (MPR_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
	mpr_free_command(sc, cm);
	mprsas_set_ccbstatus(ccb, CAM_LUN_INVALID);
	xpt_done(ccb);
	return;
	}

	if (csio->ccb_h.flags & CAM_CDB_POINTER)
	bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
	else {
	KASSERT(csio->cdb_len <= IOCDBLEN,
	("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER "
	"is not set", csio->cdb_len));
	bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
	}
	req->IoFlags = htole16(csio->cdb_len);

	/*
	* Check if EEDP is supported and enabled. If it is then check if the
	* SCSI opcode could be using EEDP. If so, make sure the LUN exists and
	* is formatted for EEDP support. If all of this is true, set CDB up
	* for EEDP transfer.
	*/
	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
	if (sc->eedp_enabled && eedp_flags) {
	SLIST_FOREACH(lun, &targ->luns, lun_link) {
	if (lun->lun_id == csio->ccb_h.target_lun) {
	break;
	}
	}

	if ((lun != NULL) && (lun->eedp_formatted)) {
	req->EEDPBlockSize = htole16(lun->eedp_block_size);
	eedp_flags \|= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG \|
	MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG \|
	MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
	if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC) {
	eedp_flags \|=
	MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
	}
	req->EEDPFlags = htole16(eedp_flags);

	/*
	* If CDB less than 32, fill in Primary Ref Tag with
	* low 4 bytes of LBA. If CDB is 32, tag stuff is
	* already there. Also, set protection bit. FreeBSD
	* currently does not support CDBs bigger than 16, but
	* the code doesn't hurt, and will be here for the
	* future.
	*/
	if (csio->cdb_len != 32) {
	lba_byte = (csio->cdb_len == 16) ? 6 : 2;
	ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
	PrimaryReferenceTag;
	for (i = 0; i < 4; i++) {
	*ref_tag_addr =
	req->CDB.CDB32[lba_byte + i];
	ref_tag_addr++;
	}
	req->CDB.EEDP32.PrimaryReferenceTag =
	htole32(req->
	CDB.EEDP32.PrimaryReferenceTag);
	req->CDB.EEDP32.PrimaryApplicationTagMask =
	0xFFFF;
	req->CDB.CDB32[1] =
	(req->CDB.CDB32[1] & 0x1F) \| 0x20;
	} else {
	eedp_flags \|=
	MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
	req->EEDPFlags = htole16(eedp_flags);
	req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
	0x1F) \| 0x20;
	}
	}
	}

	cm->cm_length = csio->dxfer_len;
	if (cm->cm_length != 0) {
	cm->cm_data = ccb;
	cm->cm_flags \|= MPR_CM_FLAGS_USE_CCB;
	} else {
	cm->cm_data = NULL;
	}
	cm->cm_sge = &req->SGL;
	cm->cm_sglsize = (32 - 24) * 4;
	cm->cm_complete = mprsas_scsiio_complete;
	cm->cm_complete_data = ccb;
	cm->cm_targ = targ;
	cm->cm_lun = csio->ccb_h.target_lun;
	cm->cm_ccb = ccb;
	/*
	* If using FP desc type, need to set a bit in IoFlags (SCSI IO is 0)
	* and set descriptor type.
	*/
	if (targ->scsi_req_desc_type ==
	MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO) {
	req->IoFlags \|= MPI25_SCSIIO_IOFLAGS_FAST_PATH;
	cm->cm_desc.FastPathSCSIIO.RequestFlags =
	MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
	if (!sc->atomic_desc_capable) {
	cm->cm_desc.FastPathSCSIIO.DevHandle =
	htole16(targ->handle);
	}
	} else {
	cm->cm_desc.SCSIIO.RequestFlags =
	MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
	if (!sc->atomic_desc_capable)
	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
	}

	csio->ccb_h.qos.sim_data = sbinuptime();
	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
	mprsas_scsiio_timeout, cm, 0);

	targ->issued++;
	targ->outstanding++;
	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
	ccb->ccb_h.status \|= CAM_SIM_QUEUED;

	mprsas_log_command(cm, MPR_XINFO, "%s cm %p ccb %p outstanding %u\n",
	__func__, cm, ccb, targ->outstanding);

	mpr_map_command(sc, cm);
	return;
	}

	/**
	* mpr_sc_failed_io_info - translated non-succesfull SCSI_IO request
	*/
	static void
	mpr_sc_failed_io_info(struct mpr_softc sc, struct ccb_scsiio csio,
	Mpi2SCSIIOReply_t mpi_reply, struct mprsas_target targ)
	{
	u32 response_info;
	u8 *response_bytes;
	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
	MPI2_IOCSTATUS_MASK;
	u8 scsi_state = mpi_reply->SCSIState;
	u8 scsi_status = mpi_reply->SCSIStatus;
	char *desc_ioc_state = NULL;
	char *desc_scsi_status = NULL;
	u32 log_info = le32toh(mpi_reply->IOCLogInfo);

	if (log_info == 0x31170000)
	return;

	desc_ioc_state = mpr_describe_table(mpr_iocstatus_string,
	ioc_status);
	desc_scsi_status = mpr_describe_table(mpr_scsi_status_string,
	scsi_status);

	mpr_dprint(sc, MPR_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
	le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
	if (targ->encl_level_valid) {
	mpr_dprint(sc, MPR_XINFO, "At enclosure level %d, slot %d, "
	"connector name (%4s)\n", targ->encl_level, targ->encl_slot,
	targ->connector_name);
	}

	/*
	* We can add more detail about underflow data here
	* TO-DO
	*/
	mpr_dprint(sc, MPR_XINFO, "\tscsi_status(%s)(0x%02x), "
	"scsi_state %b\n", desc_scsi_status, scsi_status,
	scsi_state, "\20" "\1AutosenseValid" "\2AutosenseFailed"
	"\3NoScsiStatus" "\4Terminated" "\5Response InfoValid");

	if (sc->mpr_debug & MPR_XINFO &&
	scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
	mpr_dprint(sc, MPR_XINFO, "-> Sense Buffer Data : Start :\n");
	scsi_sense_print(csio);
	mpr_dprint(sc, MPR_XINFO, "-> Sense Buffer Data : End :\n");
	}

	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
	response_info = le32toh(mpi_reply->ResponseInfo);
	response_bytes = (u8 *)&response_info;
	mpr_dprint(sc, MPR_XINFO, "response code(0x%01x): %s\n",
	response_bytes[0],
	mpr_describe_table(mpr_scsi_taskmgmt_string,
	response_bytes[0]));
	}
	}

	/** mprsas_nvme_trans_status_code
	*
	* Convert Native NVMe command error status to
	* equivalent SCSI error status.
	*
	* Returns appropriate scsi_status
	*/
	static u8
	mprsas_nvme_trans_status_code(uint16_t nvme_status,
	struct mpr_command *cm)
	{
	u8 status = MPI2_SCSI_STATUS_GOOD;
	int skey, asc, ascq;
	union ccb *ccb = cm->cm_complete_data;
	int returned_sense_len;
	uint8_t sct, sc;

	sct = NVME_STATUS_GET_SCT(nvme_status);
	sc = NVME_STATUS_GET_SC(nvme_status);

	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_NO_SENSE;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;

	switch (sct) {
	case NVME_SCT_GENERIC:
	switch (sc) {
	case NVME_SC_SUCCESS:
	status = MPI2_SCSI_STATUS_GOOD;
	skey = SSD_KEY_NO_SENSE;
	asc = SCSI_ASC_NO_SENSE;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_INVALID_OPCODE:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_ILLEGAL_COMMAND;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_INVALID_FIELD:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_INVALID_CDB;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_DATA_TRANSFER_ERROR:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_NO_SENSE;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_ABORTED_POWER_LOSS:
	status = MPI2_SCSI_STATUS_TASK_ABORTED;
	skey = SSD_KEY_ABORTED_COMMAND;
	asc = SCSI_ASC_WARNING;
	ascq = SCSI_ASCQ_POWER_LOSS_EXPECTED;
	break;
	case NVME_SC_INTERNAL_DEVICE_ERROR:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_HARDWARE_ERROR;
	asc = SCSI_ASC_INTERNAL_TARGET_FAILURE;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_ABORTED_BY_REQUEST:
	case NVME_SC_ABORTED_SQ_DELETION:
	case NVME_SC_ABORTED_FAILED_FUSED:
	case NVME_SC_ABORTED_MISSING_FUSED:
	status = MPI2_SCSI_STATUS_TASK_ABORTED;
	skey = SSD_KEY_ABORTED_COMMAND;
	asc = SCSI_ASC_NO_SENSE;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
	ascq = SCSI_ASCQ_INVALID_LUN_ID;
	break;
	case NVME_SC_LBA_OUT_OF_RANGE:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_ILLEGAL_BLOCK;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_CAPACITY_EXCEEDED:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_NO_SENSE;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_NAMESPACE_NOT_READY:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_NOT_READY;
	asc = SCSI_ASC_LUN_NOT_READY;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	}
	break;
	case NVME_SCT_COMMAND_SPECIFIC:
	switch (sc) {
	case NVME_SC_INVALID_FORMAT:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_FORMAT_COMMAND_FAILED;
	ascq = SCSI_ASCQ_FORMAT_COMMAND_FAILED;
	break;
	case NVME_SC_CONFLICTING_ATTRIBUTES:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_INVALID_CDB;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	}
	break;
	case NVME_SCT_MEDIA_ERROR:
	switch (sc) {
	case NVME_SC_WRITE_FAULTS:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_UNRECOVERED_READ_ERROR:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_UNRECOVERED_READ_ERROR;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_GUARD_CHECK_ERROR:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED;
	ascq = SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED;
	break;
	case NVME_SC_APPLICATION_TAG_CHECK_ERROR:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED;
	ascq = SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED;
	break;
	case NVME_SC_REFERENCE_TAG_CHECK_ERROR:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MEDIUM_ERROR;
	asc = SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED;
	ascq = SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED;
	break;
	case NVME_SC_COMPARE_FAILURE:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_MISCOMPARE;
	asc = SCSI_ASC_MISCOMPARE_DURING_VERIFY;
	ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
	break;
	case NVME_SC_ACCESS_DENIED:
	status = MPI2_SCSI_STATUS_CHECK_CONDITION;
	skey = SSD_KEY_ILLEGAL_REQUEST;
	asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
	ascq = SCSI_ASCQ_INVALID_LUN_ID;
	break;
	}
	break;
	}

	returned_sense_len = sizeof(struct scsi_sense_data);
	if (returned_sense_len < ccb->csio.sense_len)
	ccb->csio.sense_resid = ccb->csio.sense_len -
	returned_sense_len;
	else
	ccb->csio.sense_resid = 0;

	scsi_set_sense_data(&ccb->csio.sense_data, SSD_TYPE_FIXED,
	1, skey, asc, ascq, SSD_ELEM_NONE);
	ccb->ccb_h.status \|= CAM_AUTOSNS_VALID;

	return status;
	}

	/** mprsas_complete_nvme_unmap
	*
	* Complete native NVMe command issued using NVMe Encapsulated
	* Request Message.
	*/
	static u8
	mprsas_complete_nvme_unmap(struct mpr_softc sc, struct mpr_command cm)
	{
	Mpi26NVMeEncapsulatedErrorReply_t *mpi_reply;
	struct nvme_completion *nvme_completion = NULL;
	u8 scsi_status = MPI2_SCSI_STATUS_GOOD;

	mpi_reply =(Mpi26NVMeEncapsulatedErrorReply_t *)cm->cm_reply;
	if (le16toh(mpi_reply->ErrorResponseCount)){
	nvme_completion = (struct nvme_completion *)cm->cm_sense;
	scsi_status = mprsas_nvme_trans_status_code(
	nvme_completion->status, cm);
	}
	return scsi_status;
	}

	static void
	mprsas_scsiio_complete(struct mpr_softc sc, struct mpr_command cm)
	{
	MPI2_SCSI_IO_REPLY *rep;
	union ccb *ccb;
	struct ccb_scsiio *csio;
	struct mprsas_softc *sassc;
	struct scsi_vpd_supported_page_list *vpd_list = NULL;
	u8 TLR_bits, TLR_on, scsi_cdb;
	int dir = 0, i;
	u16 alloc_len;
	struct mprsas_target *target;
	target_id_t target_id;

	MPR_FUNCTRACE(sc);
	mpr_dprint(sc, MPR_TRACE,
	"cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
	cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
	cm->cm_targ->outstanding);

	callout_stop(&cm->cm_callout);
	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	sassc = sc->sassc;
	ccb = cm->cm_complete_data;
	csio = &ccb->csio;
	target_id = csio->ccb_h.target_id;
	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
	/*
	* XXX KDM if the chain allocation fails, does it matter if we do
	* the sync and unload here? It is simpler to do it in every case,
	* assuming it doesn't cause problems.
	*/
	if (cm->cm_data != NULL) {
	if (cm->cm_flags & MPR_CM_FLAGS_DATAIN)
	dir = BUS_DMASYNC_POSTREAD;
	else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT)
	dir = BUS_DMASYNC_POSTWRITE;
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
	}

	cm->cm_targ->completed++;
	cm->cm_targ->outstanding--;
	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
	ccb->ccb_h.status &= ~(CAM_STATUS_MASK \| CAM_SIM_QUEUED);

	if (cm->cm_flags & MPR_CM_FLAGS_ON_RECOVERY) {
	TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
	KASSERT(cm->cm_state == MPR_CM_STATE_BUSY,
	("Not busy for CM_FLAGS_TIMEDOUT: %d\n", cm->cm_state));
	cm->cm_flags &= ~MPR_CM_FLAGS_ON_RECOVERY;
	if (cm->cm_reply != NULL)
	mprsas_log_command(cm, MPR_RECOVERY,
	"completed timedout cm %p ccb %p during recovery "
	"ioc %x scsi %x state %x xfer %u\n", cm, cm->cm_ccb,
	le16toh(rep->IOCStatus), rep->SCSIStatus,
	rep->SCSIState, le32toh(rep->TransferCount));
	else
	mprsas_log_command(cm, MPR_RECOVERY,
	"completed timedout cm %p ccb %p during recovery\n",
	cm, cm->cm_ccb);
	} else if (cm->cm_targ->tm != NULL) {
	if (cm->cm_reply != NULL)
	mprsas_log_command(cm, MPR_RECOVERY,
	"completed cm %p ccb %p during recovery "
	"ioc %x scsi %x state %x xfer %u\n",
	cm, cm->cm_ccb, le16toh(rep->IOCStatus),
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	else
	mprsas_log_command(cm, MPR_RECOVERY,
	"completed cm %p ccb %p during recovery\n",
	cm, cm->cm_ccb);
	} else if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0) {
	mprsas_log_command(cm, MPR_RECOVERY,
	"reset completed cm %p ccb %p\n", cm, cm->cm_ccb);
	}

	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	/*
	* We ran into an error after we tried to map the command,
	* so we're getting a callback without queueing the command
	* to the hardware. So we set the status here, and it will
	* be retained below. We'll go through the "fast path",
	* because there can be no reply when we haven't actually
	* gone out to the hardware.
	*/
	mprsas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);

	/*
	* Currently the only error included in the mask is
	* MPR_CM_FLAGS_CHAIN_FAILED, which means we're out of
	* chain frames. We need to freeze the queue until we get
	* a command that completed without this error, which will
	* hopefully have some chain frames attached that we can
	* use. If we wanted to get smarter about it, we would
	* only unfreeze the queue in this condition when we're
	* sure that we're getting some chain frames back. That's
	* probably unnecessary.
	*/
	if ((sassc->flags & MPRSAS_QUEUE_FROZEN) == 0) {
	xpt_freeze_simq(sassc->sim, 1);
	sassc->flags \|= MPRSAS_QUEUE_FROZEN;
	mpr_dprint(sc, MPR_XINFO, "Error sending command, "
	"freezing SIM queue\n");
	}
	}

	/*
	* Point to the SCSI CDB, which is dependent on the CAM_CDB_POINTER
	* flag, and use it in a few places in the rest of this function for
	* convenience. Use the macro if available.
	*/
	scsi_cdb = scsiio_cdb_ptr(csio);

	/*
	* If this is a Start Stop Unit command and it was issued by the driver
	* during shutdown, decrement the refcount to account for all of the
	* commands that were sent. All SSU commands should be completed before
	* shutdown completes, meaning SSU_refcount will be 0 after SSU_started
	* is TRUE.
	*/
	if (sc->SSU_started && (scsi_cdb[0] == START_STOP_UNIT)) {
	mpr_dprint(sc, MPR_INFO, "Decrementing SSU count.\n");
	sc->SSU_refcount--;
	}

	/* Take the fast path to completion */
	if (cm->cm_reply == NULL) {
	if (mprsas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
	if ((sc->mpr_flags & MPR_FLAGS_DIAGRESET) != 0)
	mprsas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
	else {
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	csio->scsi_status = SCSI_STATUS_OK;
	}
	if (sassc->flags & MPRSAS_QUEUE_FROZEN) {
	ccb->ccb_h.status \|= CAM_RELEASE_SIMQ;
	sassc->flags &= ~MPRSAS_QUEUE_FROZEN;
	mpr_dprint(sc, MPR_XINFO,
	"Unfreezing SIM queue\n");
	}
	}

	/*
	* There are two scenarios where the status won't be
	* CAM_REQ_CMP. The first is if MPR_CM_FLAGS_ERROR_MASK is
	* set, the second is in the MPR_FLAGS_DIAGRESET above.
	*/
	if (mprsas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
	/*
	* Freeze the dev queue so that commands are
	* executed in the correct order after error
	* recovery.
	*/
	ccb->ccb_h.status \|= CAM_DEV_QFRZN;
	xpt_freeze_devq(ccb->ccb_h.path, /count/ 1);
	}
	mpr_free_command(sc, cm);
	xpt_done(ccb);
	return;
	}

	target = &sassc->targets[target_id];
	if (scsi_cdb[0] == UNMAP &&
	target->is_nvme &&
	(csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
	rep->SCSIStatus = mprsas_complete_nvme_unmap(sc, cm);
	csio->scsi_status = rep->SCSIStatus;
	}

	mprsas_log_command(cm, MPR_XINFO,
	"ioc %x scsi %x state %x xfer %u\n",
	le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));

	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
	csio->resid = cm->cm_length - le32toh(rep->TransferCount);
	/* FALLTHROUGH */
	case MPI2_IOCSTATUS_SUCCESS:
	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
	if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
	mprsas_log_command(cm, MPR_XINFO, "recovered error\n");

	/* Completion failed at the transport level. */
	if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS \|
	MPI2_SCSI_STATE_TERMINATED)) {
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	break;
	}

	/* In a modern packetized environment, an autosense failure
	* implies that there's not much else that can be done to
	* recover the command.
	*/
	if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
	mprsas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
	break;
	}

	/*
	* CAM doesn't care about SAS Response Info data, but if this is
	* the state check if TLR should be done. If not, clear the
	* TLR_bits for the target.
	*/
	if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
	((le32toh(rep->ResponseInfo) & MPI2_SCSI_RI_MASK_REASONCODE)
	== MPR_SCSI_RI_INVALID_FRAME)) {
	sc->mapping_table[target_id].TLR_bits =
	(u8)MPI2_SCSIIO_CONTROL_NO_TLR;
	}

	/*
	* Intentionally override the normal SCSI status reporting
	* for these two cases. These are likely to happen in a
	* multi-initiator environment, and we want to make sure that
	* CAM retries these commands rather than fail them.
	*/
	if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) \|\|
	(rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
	mprsas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
	break;
	}

	/* Handle normal status and sense */
	csio->scsi_status = rep->SCSIStatus;
	if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mprsas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);

	if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
	int sense_len, returned_sense_len;

	returned_sense_len = min(le32toh(rep->SenseCount),
	sizeof(struct scsi_sense_data));
	if (returned_sense_len < csio->sense_len)
	csio->sense_resid = csio->sense_len -
	returned_sense_len;
	else
	csio->sense_resid = 0;

	sense_len = min(returned_sense_len,
	csio->sense_len - csio->sense_resid);
	bzero(&csio->sense_data, sizeof(csio->sense_data));
	bcopy(cm->cm_sense, &csio->sense_data, sense_len);
	ccb->ccb_h.status \|= CAM_AUTOSNS_VALID;
	}

	/*
	* Check if this is an INQUIRY command. If it's a VPD inquiry,
	* and it's page code 0 (Supported Page List), and there is
	* inquiry data, and this is for a sequential access device, and
	* the device is an SSP target, and TLR is supported by the
	* controller, turn the TLR_bits value ON if page 0x90 is
	* supported.
	*/
	if ((scsi_cdb[0] == INQUIRY) &&
	(scsi_cdb[1] & SI_EVPD) &&
	(scsi_cdb[2] == SVPD_SUPPORTED_PAGE_LIST) &&
	((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
	(csio->data_ptr != NULL) &&
	((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
	(sc->control_TLR) &&
	(sc->mapping_table[target_id].device_info &
	MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
	vpd_list = (struct scsi_vpd_supported_page_list *)
	csio->data_ptr;
	TLR_bits = &sc->mapping_table[target_id].TLR_bits;
	*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
	TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
	alloc_len = ((u16)scsi_cdb[3] << 8) + scsi_cdb[4];
	alloc_len -= csio->resid;
	for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
	if (vpd_list->list[i] == 0x90) {
	*TLR_bits = TLR_on;
	break;
	}
	}
	}

	/*
	* If this is a SATA direct-access end device, mark it so that
	* a SCSI StartStopUnit command will be sent to it when the
	* driver is being shutdown.
	*/
	if ((scsi_cdb[0] == INQUIRY) &&
	(csio->data_ptr != NULL) &&
	((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
	(sc->mapping_table[target_id].device_info &
	MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
	((sc->mapping_table[target_id].device_info &
	MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
	MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
	target = &sassc->targets[target_id];
	target->supports_SSU = TRUE;
	mpr_dprint(sc, MPR_XINFO, "Target %d supports SSU\n",
	target_id);
	}
	break;
	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
	/*
	* If devinfo is 0 this will be a volume. In that case don't
	* tell CAM that the volume is not there. We want volumes to
	* be enumerated until they are deleted/removed, not just
	* failed.
	*/
	if (cm->cm_targ->devinfo == 0)
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	break;
	case MPI2_IOCSTATUS_INVALID_SGL:
	mpr_print_scsiio_cmd(sc, cm);
	mprsas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
	break;
	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
	/*
	* This is one of the responses that comes back when an I/O
	* has been aborted. If it is because of a timeout that we
	* initiated, just set the status to CAM_CMD_TIMEOUT.
	* Otherwise set it to CAM_REQ_ABORTED. The effect on the
	* command is the same (it gets retried, subject to the
	* retry counter), the only difference is what gets printed
	* on the console.
	*/
	if (cm->cm_flags & MPR_CM_FLAGS_TIMEDOUT)
	mprsas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
	else
	mprsas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
	break;
	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
	/* resid is ignored for this condition */
	csio->resid = 0;
	mprsas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
	break;
	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
	/*
	* These can sometimes be transient transport-related
	* errors, and sometimes persistent drive-related errors.
	* We used to retry these without decrementing the retry
	* count by returning CAM_REQUEUE_REQ. Unfortunately, if
	* we hit a persistent drive problem that returns one of
	* these error codes, we would retry indefinitely. So,
	* return CAM_REQ_CMP_ERROR so that we decrement the retry
	* count and avoid infinite retries. We're taking the
	* potential risk of flagging false failures in the event
	* of a topology-related error (e.g. a SAS expander problem
	* causes a command addressed to a drive to fail), but
	* avoiding getting into an infinite retry loop. However,
	* if we get them while were moving a device, we should
	* fail the request as 'not there' because the device
	* is effectively gone.
	*/
	if (cm->cm_targ->flags & MPRSAS_TARGET_INREMOVAL)
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	else
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	mpr_dprint(sc, MPR_INFO,
	"Controller reported %s tgt %u SMID %u loginfo %x%s\n",
	mpr_describe_table(mpr_iocstatus_string,
	le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK),
	target_id, cm->cm_desc.Default.SMID,
	le32toh(rep->IOCLogInfo),
	(cm->cm_targ->flags & MPRSAS_TARGET_INREMOVAL) ? " departing" : "");
	mpr_dprint(sc, MPR_XINFO,
	"SCSIStatus %x SCSIState %x xfercount %u\n",
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	break;
	case MPI2_IOCSTATUS_INVALID_FUNCTION:
	case MPI2_IOCSTATUS_INTERNAL_ERROR:
	case MPI2_IOCSTATUS_INVALID_VPID:
	case MPI2_IOCSTATUS_INVALID_FIELD:
	case MPI2_IOCSTATUS_INVALID_STATE:
	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
	default:
	mprsas_log_command(cm, MPR_XINFO,
	"completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
	le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	csio->resid = cm->cm_length;

	if (scsi_cdb[0] == UNMAP &&
	target->is_nvme &&
	(csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR)
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);

	break;
	}

	mpr_sc_failed_io_info(sc, csio, rep, cm->cm_targ);

	if (sassc->flags & MPRSAS_QUEUE_FROZEN) {
	ccb->ccb_h.status \|= CAM_RELEASE_SIMQ;
	sassc->flags &= ~MPRSAS_QUEUE_FROZEN;
	mpr_dprint(sc, MPR_XINFO, "Command completed, unfreezing SIM "
	"queue\n");
	}

	if (mprsas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
	ccb->ccb_h.status \|= CAM_DEV_QFRZN;
	xpt_freeze_devq(ccb->ccb_h.path, /count/ 1);
	}

	/*
	* Check to see if we're removing the device. If so, and this is the
	* last command on the queue, proceed with the deferred removal of the
	* device. Note, for removing a volume, this won't trigger because
	* pending_remove_tm will be NULL.
	*/
	if (cm->cm_targ->flags & MPRSAS_TARGET_INREMOVAL) {
	if (TAILQ_FIRST(&cm->cm_targ->commands) == NULL &&
	cm->cm_targ->pending_remove_tm != NULL) {
	mpr_dprint(sc, MPR_INFO, "Last pending command complete: starting remove_device\n");
	mpr_map_command(sc, cm->cm_targ->pending_remove_tm);
	cm->cm_targ->pending_remove_tm = NULL;
	}
	}

	mpr_free_command(sc, cm);
	xpt_done(ccb);
	}

	static void
	mprsas_smpio_complete(struct mpr_softc sc, struct mpr_command cm)
	{
	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
	MPI2_SMP_PASSTHROUGH_REQUEST *req;
	uint64_t sasaddr;
	union ccb *ccb;

	ccb = cm->cm_complete_data;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and SMP
	* commands require two S/G elements only. That should be handled
	* in the standard request size.
	*/
	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x on SMP "
	"request!\n", __func__, cm->cm_flags);
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
	if (rpl == NULL) {
	mpr_dprint(sc, MPR_ERROR, "%s: NULL cm_reply!\n", __func__);
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
	sasaddr = le32toh(req->SASAddress.Low);
	sasaddr \|= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;

	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS \|\|
	rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
	mpr_dprint(sc, MPR_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
	__func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	mpr_dprint(sc, MPR_XINFO, "%s: SMP request to SAS address %#jx "
	"completed successfully\n", __func__, (uintmax_t)sasaddr);

	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mprsas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);

	bailout:
	/*
	* We sync in both directions because we had DMAs in the S/G list
	* in both directions.
	*/
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
	mpr_free_command(sc, cm);
	xpt_done(ccb);
	}

	static void
	mprsas_send_smpcmd(struct mprsas_softc sassc, union ccb ccb, uint64_t sasaddr)
	{
	struct mpr_command *cm;
	uint8_t request, response;
	MPI2_SMP_PASSTHROUGH_REQUEST *req;
	struct mpr_softc *sc;
	struct sglist *sg;
	int error;

	sc = sassc->sc;
	sg = NULL;
	error = 0;

	switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
	case CAM_DATA_PADDR:
	case CAM_DATA_SG_PADDR:
	/*
	* XXX We don't yet support physical addresses here.
	*/
	mpr_dprint(sc, MPR_ERROR, "%s: physical addresses not "
	"supported\n", __func__);
	mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID);
	xpt_done(ccb);
	return;
	case CAM_DATA_SG:
	/*
	* The chip does not support more than one buffer for the
	* request or response.
	*/
	if ((ccb->smpio.smp_request_sglist_cnt > 1)
	\|\| (ccb->smpio.smp_response_sglist_cnt > 1)) {
	mpr_dprint(sc, MPR_ERROR, "%s: multiple request or "
	"response buffer segments not supported for SMP\n",
	__func__);
	mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID);
	xpt_done(ccb);
	return;
	}

	/*
	* The CAM_SCATTER_VALID flag was originally implemented
	* for the XPT_SCSI_IO CCB, which only has one data pointer.
	* We have two. So, just take that flag to mean that we
	* might have S/G lists, and look at the S/G segment count
	* to figure out whether that is the case for each individual
	* buffer.
	*/
	if (ccb->smpio.smp_request_sglist_cnt != 0) {
	bus_dma_segment_t *req_sg;

	req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
	request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
	} else
	request = ccb->smpio.smp_request;

	if (ccb->smpio.smp_response_sglist_cnt != 0) {
	bus_dma_segment_t *rsp_sg;

	rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
	response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
	} else
	response = ccb->smpio.smp_response;
	break;
	case CAM_DATA_VADDR:
	request = ccb->smpio.smp_request;
	response = ccb->smpio.smp_response;
	break;
	default:
	mprsas_set_ccbstatus(ccb, CAM_REQ_INVALID);
	xpt_done(ccb);
	return;
	}

	cm = mpr_alloc_command(sc);
	if (cm == NULL) {
	mpr_dprint(sc, MPR_ERROR, "%s: cannot allocate command\n",
	__func__);
	mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
	xpt_done(ccb);
	return;
	}

	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
	bzero(req, sizeof(*req));
	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;

	/* Allow the chip to use any route to this SAS address. */
	req->PhysicalPort = 0xff;

	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
	req->SGLFlags =
	MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE \| MPI2_SGLFLAGS_SGL_TYPE_MPI;

	mpr_dprint(sc, MPR_XINFO, "%s: sending SMP request to SAS address "
	"%#jx\n", __func__, (uintmax_t)sasaddr);

	mpr_init_sge(cm, req, &req->SGL);

	/*
	* Set up a uio to pass into mpr_map_command(). This allows us to
	* do one map command, and one busdma call in there.
	*/
	cm->cm_uio.uio_iov = cm->cm_iovec;
	cm->cm_uio.uio_iovcnt = 2;
	cm->cm_uio.uio_segflg = UIO_SYSSPACE;

	/*
	* The read/write flag isn't used by busdma, but set it just in
	* case. This isn't exactly accurate, either, since we're going in
	* both directions.
	*/
	cm->cm_uio.uio_rw = UIO_WRITE;

	cm->cm_iovec[0].iov_base = request;
	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
	cm->cm_iovec[1].iov_base = response;
	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;

	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
	cm->cm_iovec[1].iov_len;

	/*
	* Trigger a warning message in mpr_data_cb() for the user if we
	* wind up exceeding two S/G segments. The chip expects one
	* segment for the request and another for the response.
	*/
	cm->cm_max_segs = 2;

	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete = mprsas_smpio_complete;
	cm->cm_complete_data = ccb;

	/*
	* Tell the mapping code that we're using a uio, and that this is
	* an SMP passthrough request. There is a little special-case
	* logic there (in mpr_data_cb()) to handle the bidirectional
	* transfer.
	*/
	cm->cm_flags \|= MPR_CM_FLAGS_USE_UIO \| MPR_CM_FLAGS_SMP_PASS \|
	MPR_CM_FLAGS_DATAIN \| MPR_CM_FLAGS_DATAOUT;

	/* The chip data format is little endian. */
	req->SASAddress.High = htole32(sasaddr >> 32);
	req->SASAddress.Low = htole32(sasaddr);

	/*
	* XXX Note that we don't have a timeout/abort mechanism here.
	* From the manual, it looks like task management requests only
	* work for SCSI IO and SATA passthrough requests. We may need to
	* have a mechanism to retry requests in the event of a chip reset
	* at least. Hopefully the chip will insure that any errors short
	* of that are relayed back to the driver.
	*/
	error = mpr_map_command(sc, cm);
	if ((error != 0) && (error != EINPROGRESS)) {
	mpr_dprint(sc, MPR_ERROR, "%s: error %d returned from "
	"mpr_map_command()\n", __func__, error);
	goto bailout_error;
	}

	return;

	bailout_error:
	mpr_free_command(sc, cm);
	mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
	xpt_done(ccb);
	return;
	}

	static void
	mprsas_action_smpio(struct mprsas_softc sassc, union ccb ccb)
	{
	struct mpr_softc *sc;
	struct mprsas_target *targ;
	uint64_t sasaddr = 0;

	sc = sassc->sc;

	/*
	* Make sure the target exists.
	*/
	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
	targ = &sassc->targets[ccb->ccb_h.target_id];
	if (targ->handle == 0x0) {
	mpr_dprint(sc, MPR_ERROR, "%s: target %d does not exist!\n",
	__func__, ccb->ccb_h.target_id);
	mprsas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
	xpt_done(ccb);
	return;
	}

	/*
	* If this device has an embedded SMP target, we'll talk to it
	* directly.
	* figure out what the expander's address is.
	*/
	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
	sasaddr = targ->sasaddr;

	/*
	* If we don't have a SAS address for the expander yet, try
	* grabbing it from the page 0x83 information cached in the
	* transport layer for this target. LSI expanders report the
	* expander SAS address as the port-associated SAS address in
	* Inquiry VPD page 0x83. Maxim expanders don't report it in page
	* 0x83.
	*
	* XXX KDM disable this for now, but leave it commented out so that
	* it is obvious that this is another possible way to get the SAS
	* address.
	*
	* The parent handle method below is a little more reliable, and
	* the other benefit is that it works for devices other than SES
	* devices. So you can send a SMP request to a da(4) device and it
	* will get routed to the expander that device is attached to.
	* (Assuming the da(4) device doesn't contain an SMP target...)
	*/
	#if 0
	if (sasaddr == 0)
	sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
	#endif

	/*
	* If we still don't have a SAS address for the expander, look for
	* the parent device of this device, which is probably the expander.
	*/
	if (sasaddr == 0) {
	#ifdef OLD_MPR_PROBE
	struct mprsas_target *parent_target;
	#endif

	if (targ->parent_handle == 0x0) {
	mpr_dprint(sc, MPR_ERROR, "%s: handle %d does not have "
	"a valid parent handle!\n", __func__, targ->handle);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}
	#ifdef OLD_MPR_PROBE
	parent_target = mprsas_find_target_by_handle(sassc, 0,
	targ->parent_handle);

	if (parent_target == NULL) {
	mpr_dprint(sc, MPR_ERROR, "%s: handle %d does not have "
	"a valid parent target!\n", __func__, targ->handle);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}

	if ((parent_target->devinfo &
	MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
	mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent %d "
	"does not have an SMP target!\n", __func__,
	targ->handle, parent_target->handle);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}

	sasaddr = parent_target->sasaddr;
	#else /* OLD_MPR_PROBE */
	if ((targ->parent_devinfo &
	MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
	mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent %d "
	"does not have an SMP target!\n", __func__,
	targ->handle, targ->parent_handle);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;

	}
	if (targ->parent_sasaddr == 0x0) {
	mpr_dprint(sc, MPR_ERROR, "%s: handle %d parent handle "
	"%d does not have a valid SAS address!\n", __func__,
	targ->handle, targ->parent_handle);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}

	sasaddr = targ->parent_sasaddr;
	#endif /* OLD_MPR_PROBE */

	}

	if (sasaddr == 0) {
	mpr_dprint(sc, MPR_INFO, "%s: unable to find SAS address for "
	"handle %d\n", __func__, targ->handle);
	mprsas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}
	mprsas_send_smpcmd(sassc, ccb, sasaddr);

	return;

	bailout:
	xpt_done(ccb);

	}

	static void
	mprsas_action_resetdev(struct mprsas_softc sassc, union ccb ccb)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpr_softc *sc;
	struct mpr_command *tm;
	struct mprsas_target *targ;

	MPR_FUNCTRACE(sassc->sc);
	mtx_assert(&sassc->sc->mpr_mtx, MA_OWNED);

	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets, ("Target %d out of "
	"bounds in XPT_RESET_DEV\n", ccb->ccb_h.target_id));
	sc = sassc->sc;
	tm = mprsas_alloc_tm(sc);
	if (tm == NULL) {
	mpr_dprint(sc, MPR_ERROR, "command alloc failure in "
	"mprsas_action_resetdev\n");
	mprsas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
	xpt_done(ccb);
	return;
	}

	targ = &sassc->targets[ccb->ccb_h.target_id];
	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = htole16(targ->handle);
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;

	if (!targ->is_nvme \|\| sc->custom_nvme_tm_handling) {
	/* SAS Hard Link Reset / SATA Link Reset */
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
	} else {
	/* PCIe Protocol Level Reset*/
	req->MsgFlags =
	MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
	}

	tm->cm_data = NULL;
	tm->cm_complete = mprsas_resetdev_complete;
	tm->cm_complete_data = ccb;

	mpr_dprint(sc, MPR_INFO, "%s: Sending reset for target ID %d\n",
	__func__, targ->tid);
	tm->cm_targ = targ;

	mprsas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
	mpr_map_command(sc, tm);
	}

	static void
	mprsas_resetdev_complete(struct mpr_softc sc, struct mpr_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
	union ccb *ccb;

	MPR_FUNCTRACE(sc);
	mtx_assert(&sc->mpr_mtx, MA_OWNED);

	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	ccb = tm->cm_complete_data;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;

	mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for reset of "
	"handle %#04x! This should not happen!\n", __func__,
	tm->cm_flags, req->DevHandle);
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	mpr_dprint(sc, MPR_XINFO, "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n",
	__func__, le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));

	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP);
	mprsas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
	CAM_LUN_WILDCARD);
	}
	else
	mprsas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);

	bailout:

	mprsas_free_tm(sc, tm);
	xpt_done(ccb);
	}

	static void
	mprsas_poll(struct cam_sim *sim)
	{
	struct mprsas_softc *sassc;

	sassc = cam_sim_softc(sim);

	if (sassc->sc->mpr_debug & MPR_TRACE) {
	/* frequent debug messages during a panic just slow
	* everything down too much.
	*/
	mpr_dprint(sassc->sc, MPR_XINFO, "%s clearing MPR_TRACE\n",
	__func__);
	sassc->sc->mpr_debug &= ~MPR_TRACE;
	}

	mpr_intr_locked(sassc->sc);
	}

	static void
	mprsas_async(void callback_arg, uint32_t code, struct cam_path path,
	void *arg)
	{
	struct mpr_softc *sc;

	sc = (struct mpr_softc *)callback_arg;

	switch (code) {
	case AC_ADVINFO_CHANGED: {
	struct mprsas_target *target;
	struct mprsas_softc *sassc;
	struct scsi_read_capacity_data_long rcap_buf;
	struct ccb_dev_advinfo cdai;
	struct mprsas_lun *lun;
	lun_id_t lunid;
	int found_lun;
	uintptr_t buftype;

	buftype = (uintptr_t)arg;

	found_lun = 0;
	sassc = sc->sassc;

	/*
	* We're only interested in read capacity data changes.
	*/
	if (buftype != CDAI_TYPE_RCAPLONG)
	break;

	/*
	* We should have a handle for this, but check to make sure.
	*/
	KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
	("Target %d out of bounds in mprsas_async\n",
	xpt_path_target_id(path)));
	target = &sassc->targets[xpt_path_target_id(path)];
	if (target->handle == 0)
	break;

	lunid = xpt_path_lun_id(path);

	SLIST_FOREACH(lun, &target->luns, lun_link) {
	if (lun->lun_id == lunid) {
	found_lun = 1;
	break;
	}
	}

	if (found_lun == 0) {
	lun = malloc(sizeof(struct mprsas_lun), M_MPR,
	M_NOWAIT \| M_ZERO);
	if (lun == NULL) {
	mpr_dprint(sc, MPR_ERROR, "Unable to alloc "
	"LUN for EEDP support.\n");
	break;
	}
	lun->lun_id = lunid;
	SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
	}

	bzero(&rcap_buf, sizeof(rcap_buf));
	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
	cdai.ccb_h.flags = CAM_DIR_IN;
	cdai.buftype = CDAI_TYPE_RCAPLONG;
	cdai.flags = CDAI_FLAG_NONE;
	cdai.bufsiz = sizeof(rcap_buf);
	cdai.buf = (uint8_t *)&rcap_buf;
	xpt_action((union ccb *)&cdai);
	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
	cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);

	if ((mprsas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
	&& (rcap_buf.prot & SRC16_PROT_EN)) {
	switch (rcap_buf.prot & SRC16_P_TYPE) {
	case SRC16_PTYPE_1:
	case SRC16_PTYPE_3:
	lun->eedp_formatted = TRUE;
	lun->eedp_block_size =
	scsi_4btoul(rcap_buf.length);
	break;
	case SRC16_PTYPE_2:
	default:
	lun->eedp_formatted = FALSE;
	lun->eedp_block_size = 0;
	break;
	}
	} else {
	lun->eedp_formatted = FALSE;
	lun->eedp_block_size = 0;
	}
	break;
	}
	case AC_FOUND_DEVICE:
	default:
	break;
	}
	}

	/*
	* Set the INRESET flag for this target so that no I/O will be sent to
	* the target until the reset has completed. If an I/O request does
	* happen, the devq will be frozen. The CCB holds the path which is
	* used to release the devq. The devq is released and the CCB is freed
	* when the TM completes.
	*/
	void
	mprsas_prepare_for_tm(struct mpr_softc sc, struct mpr_command tm,
	struct mprsas_target *target, lun_id_t lun_id)
	{
	union ccb *ccb;
	path_id_t path_id;

	ccb = xpt_alloc_ccb_nowait();
	if (ccb) {
	path_id = cam_sim_path(sc->sassc->sim);
	if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
	target->tid, lun_id) != CAM_REQ_CMP) {
	xpt_free_ccb(ccb);
	} else {
	tm->cm_ccb = ccb;
	tm->cm_targ = target;
	target->flags \|= MPRSAS_TARGET_INRESET;
	}
	}
	}

	int
	mprsas_startup(struct mpr_softc *sc)
	{
	/*
	* Send the port enable message and set the wait_for_port_enable flag.
	* This flag helps to keep the simq frozen until all discovery events
	* are processed.
	*/
	sc->wait_for_port_enable = 1;
	mprsas_send_portenable(sc);
	return (0);
	}

	static int
	mprsas_send_portenable(struct mpr_softc *sc)
	{
	MPI2_PORT_ENABLE_REQUEST *request;
	struct mpr_command *cm;

	MPR_FUNCTRACE(sc);

	if ((cm = mpr_alloc_command(sc)) == NULL)
	return (EBUSY);
	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
	request->Function = MPI2_FUNCTION_PORT_ENABLE;
	request->MsgFlags = 0;
	request->VP_ID = 0;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete = mprsas_portenable_complete;
	cm->cm_data = NULL;
	cm->cm_sge = NULL;

	mpr_map_command(sc, cm);
	mpr_dprint(sc, MPR_XINFO,
	"mpr_send_portenable finished cm %p req %p complete %p\n",
	cm, cm->cm_req, cm->cm_complete);
	return (0);
	}

	static void
	mprsas_portenable_complete(struct mpr_softc sc, struct mpr_command cm)
	{
	MPI2_PORT_ENABLE_REPLY *reply;
	struct mprsas_softc *sassc;

	MPR_FUNCTRACE(sc);
	sassc = sc->sassc;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* port enable commands don't have S/G lists.
	*/
	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {
	mpr_dprint(sc, MPR_ERROR, "%s: cm_flags = %#x for port enable! "
	"This should not happen!\n", __func__, cm->cm_flags);
	}

	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
	if (reply == NULL)
	mpr_dprint(sc, MPR_FAULT, "Portenable NULL reply\n");
	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS)
	mpr_dprint(sc, MPR_FAULT, "Portenable failed\n");

	mpr_free_command(sc, cm);
	/*
	* Done waiting for port enable to complete. Decrement the refcount.
	* If refcount is 0, discovery is complete and a rescan of the bus can
	* take place.
	*/
	sc->wait_for_port_enable = 0;
	sc->port_enable_complete = 1;
	wakeup(&sc->port_enable_complete);
	mprsas_startup_decrement(sassc);
	}

	int
	mprsas_check_id(struct mprsas_softc *sassc, int id)
	{
	struct mpr_softc *sc = sassc->sc;
	char *ids;
	char *name;

	ids = &sc->exclude_ids[0];
	while((name = strsep(&ids, ",")) != NULL) {
	if (name[0] == '\0')
	continue;
	if (strtol(name, NULL, 0) == (long)id)
	return (1);
	}

	return (0);
	}

	void
	mprsas_realloc_targets(struct mpr_softc *sc, int maxtargets)
	{
	struct mprsas_softc *sassc;
	struct mprsas_lun lun, lun_tmp;
	struct mprsas_target *targ;
	int i;

	sassc = sc->sassc;
	/*
	* The number of targets is based on IOC Facts, so free all of
	* the allocated LUNs for each target and then the target buffer
	* itself.
	*/
	for (i=0; i< maxtargets; i++) {
	targ = &sassc->targets[i];
	SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
	free(lun, M_MPR);
	}
	}
	free(sassc->targets, M_MPR);

	sassc->targets = malloc(sizeof(struct mprsas_target) * maxtargets,
	M_MPR, M_WAITOK\|M_ZERO);
	- if (!sassc->targets) {
	- panic("%s failed to alloc targets with error %d\n",
	- __func__, ENOMEM);
	- }
	}
	diff --git a/sys/dev/mpr/mpr_user.c b/sys/dev/mpr/mpr_user.c
	index 308b559c7172..751741bcc4f6 100644
	--- a/sys/dev/mpr/mpr_user.c
	+++ b/sys/dev/mpr/mpr_user.c
	@@ -1,2613 +1,2606 @@
	/*-
	* Copyright (c) 2008 Yahoo!, Inc.
	* All rights reserved.
	* Written by: John Baldwin <jhb@FreeBSD.org>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the author nor the names of any co-contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Broadcom Inc. (LSI) MPT-Fusion Host Adapter FreeBSD userland interface
	*/
	/*-
	* Copyright (c) 2011-2015 LSI Corp.
	* Copyright (c) 2013-2016 Avago Technologies
	* Copyright 2000-2020 Broadcom Inc.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Broadcom Inc. (LSI) MPT-Fusion Host Adapter FreeBSD
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/* TODO Move headers to mprvar */
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/selinfo.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/bio.h>
	#include <sys/abi_compat.h>
	#include <sys/malloc.h>
	#include <sys/uio.h>
	#include <sys/sysctl.h>
	#include <sys/ioccom.h>
	#include <sys/endian.h>
	#include <sys/queue.h>
	#include <sys/kthread.h>
	#include <sys/taskqueue.h>
	#include <sys/proc.h>
	#include <sys/sysent.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>

	#include <cam/cam.h>
	#include <cam/cam_ccb.h>

	#include <dev/mpr/mpi/mpi2_type.h>
	#include <dev/mpr/mpi/mpi2.h>
	#include <dev/mpr/mpi/mpi2_ioc.h>
	#include <dev/mpr/mpi/mpi2_cnfg.h>
	#include <dev/mpr/mpi/mpi2_init.h>
	#include <dev/mpr/mpi/mpi2_tool.h>
	#include <dev/mpr/mpi/mpi2_pci.h>
	#include <dev/mpr/mpr_ioctl.h>
	#include <dev/mpr/mprvar.h>
	#include <dev/mpr/mpr_table.h>
	#include <dev/mpr/mpr_sas.h>
	#include <dev/pci/pcivar.h>
	#include <dev/pci/pcireg.h>

	static d_open_t mpr_open;
	static d_close_t mpr_close;
	static d_ioctl_t mpr_ioctl_devsw;

	static struct cdevsw mpr_cdevsw = {
	.d_version = D_VERSION,
	.d_flags = 0,
	.d_open = mpr_open,
	.d_close = mpr_close,
	.d_ioctl = mpr_ioctl_devsw,
	.d_name = "mpr",
	};

	typedef int (mpr_user_f)(struct mpr_command , struct mpr_usr_command );
	static mpr_user_f mpi_pre_ioc_facts;
	static mpr_user_f mpi_pre_port_facts;
	static mpr_user_f mpi_pre_fw_download;
	static mpr_user_f mpi_pre_fw_upload;
	static mpr_user_f mpi_pre_sata_passthrough;
	static mpr_user_f mpi_pre_smp_passthrough;
	static mpr_user_f mpi_pre_config;
	static mpr_user_f mpi_pre_sas_io_unit_control;

	static int mpr_user_read_cfg_header(struct mpr_softc *,
	struct mpr_cfg_page_req *);
	static int mpr_user_read_cfg_page(struct mpr_softc *,
	struct mpr_cfg_page_req , void );
	static int mpr_user_read_extcfg_header(struct mpr_softc *,
	struct mpr_ext_cfg_page_req *);
	static int mpr_user_read_extcfg_page(struct mpr_softc *,
	struct mpr_ext_cfg_page_req , void );
	static int mpr_user_write_cfg_page(struct mpr_softc *,
	struct mpr_cfg_page_req , void );
	static int mpr_user_setup_request(struct mpr_command *,
	struct mpr_usr_command *);
	static int mpr_user_command(struct mpr_softc , struct mpr_usr_command );

	static int mpr_user_pass_thru(struct mpr_softc sc, mpr_pass_thru_t data);
	static void mpr_user_get_adapter_data(struct mpr_softc *sc,
	mpr_adapter_data_t *data);
	static void mpr_user_read_pci_info(struct mpr_softc sc, mpr_pci_info_t data);
	static uint8_t mpr_get_fw_diag_buffer_number(struct mpr_softc *sc,
	uint32_t unique_id);
	static int mpr_post_fw_diag_buffer(struct mpr_softc *sc,
	mpr_fw_diagnostic_buffer_t pBuffer, uint32_t return_code);
	static int mpr_release_fw_diag_buffer(struct mpr_softc *sc,
	mpr_fw_diagnostic_buffer_t pBuffer, uint32_t return_code,
	uint32_t diag_type);
	static int mpr_diag_register(struct mpr_softc *sc,
	mpr_fw_diag_register_t diag_register, uint32_t return_code);
	static int mpr_diag_unregister(struct mpr_softc *sc,
	mpr_fw_diag_unregister_t diag_unregister, uint32_t return_code);
	static int mpr_diag_query(struct mpr_softc sc, mpr_fw_diag_query_t diag_query,
	uint32_t *return_code);
	static int mpr_diag_read_buffer(struct mpr_softc *sc,
	mpr_diag_read_buffer_t diag_read_buffer, uint8_t ioctl_buf,
	uint32_t *return_code);
	static int mpr_diag_release(struct mpr_softc *sc,
	mpr_fw_diag_release_t diag_release, uint32_t return_code);
	static int mpr_do_diag_action(struct mpr_softc *sc, uint32_t action,
	uint8_t diag_action, uint32_t length, uint32_t return_code);
	static int mpr_user_diag_action(struct mpr_softc sc, mpr_diag_action_t data);
	static void mpr_user_event_query(struct mpr_softc sc, mpr_event_query_t data);
	static void mpr_user_event_enable(struct mpr_softc *sc,
	mpr_event_enable_t *data);
	static int mpr_user_event_report(struct mpr_softc *sc,
	mpr_event_report_t *data);
	static int mpr_user_reg_access(struct mpr_softc sc, mpr_reg_access_t data);
	static int mpr_user_btdh(struct mpr_softc sc, mpr_btdh_mapping_t data);

	static MALLOC_DEFINE(M_MPRUSER, "mpr_user", "Buffers for mpr(4) ioctls");

	/*
	* MPI functions that support IEEE SGLs for SAS3.
	*/
	static uint8_t ieee_sgl_func_list[] = {
	MPI2_FUNCTION_SCSI_IO_REQUEST,
	MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH,
	MPI2_FUNCTION_SMP_PASSTHROUGH,
	MPI2_FUNCTION_SATA_PASSTHROUGH,
	MPI2_FUNCTION_FW_UPLOAD,
	MPI2_FUNCTION_FW_DOWNLOAD,
	MPI2_FUNCTION_TARGET_ASSIST,
	MPI2_FUNCTION_TARGET_STATUS_SEND,
	MPI2_FUNCTION_TOOLBOX
	};

	int
	mpr_attach_user(struct mpr_softc *sc)
	{
	int unit;

	unit = device_get_unit(sc->mpr_dev);
	sc->mpr_cdev = make_dev(&mpr_cdevsw, unit, UID_ROOT, GID_OPERATOR, 0640,
	"mpr%d", unit);

	if (sc->mpr_cdev == NULL)
	return (ENOMEM);

	sc->mpr_cdev->si_drv1 = sc;
	return (0);
	}

	void
	mpr_detach_user(struct mpr_softc *sc)
	{

	/* XXX: do a purge of pending requests? */
	if (sc->mpr_cdev != NULL)
	destroy_dev(sc->mpr_cdev);
	}

	static int
	mpr_open(struct cdev dev, int flags, int fmt, struct thread td)
	{

	return (0);
	}

	static int
	mpr_close(struct cdev dev, int flags, int fmt, struct thread td)
	{

	return (0);
	}

	static int
	mpr_user_read_cfg_header(struct mpr_softc *sc,
	struct mpr_cfg_page_req *page_req)
	{
	MPI2_CONFIG_PAGE_HEADER *hdr;
	struct mpr_config_params params;
	int error;

	hdr = &params.hdr.Struct;
	params.action = MPI2_CONFIG_ACTION_PAGE_HEADER;
	params.page_address = le32toh(page_req->page_address);
	hdr->PageVersion = 0;
	hdr->PageLength = 0;
	hdr->PageNumber = page_req->header.PageNumber;
	hdr->PageType = page_req->header.PageType;
	params.buffer = NULL;
	params.length = 0;
	params.callback = NULL;

	if ((error = mpr_read_config_page(sc, &params)) != 0) {
	/*
	* Leave the request. Without resetting the chip, it's
	* still owned by it and we'll just get into trouble
	* freeing it now. Mark it as abandoned so that if it
	* shows up later it can be freed.
	*/
	mpr_printf(sc, "read_cfg_header timed out\n");
	return (ETIMEDOUT);
	}

	page_req->ioc_status = htole16(params.status);
	if ((page_req->ioc_status & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	bcopy(hdr, &page_req->header, sizeof(page_req->header));
	}

	return (0);
	}

	static int
	mpr_user_read_cfg_page(struct mpr_softc sc, struct mpr_cfg_page_req page_req,
	void *buf)
	{
	MPI2_CONFIG_PAGE_HEADER reqhdr, hdr;
	struct mpr_config_params params;
	int error;

	reqhdr = buf;
	hdr = &params.hdr.Struct;
	hdr->PageVersion = reqhdr->PageVersion;
	hdr->PageLength = reqhdr->PageLength;
	hdr->PageNumber = reqhdr->PageNumber;
	hdr->PageType = reqhdr->PageType & MPI2_CONFIG_PAGETYPE_MASK;
	params.action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
	params.page_address = le32toh(page_req->page_address);
	params.buffer = buf;
	params.length = le32toh(page_req->len);
	params.callback = NULL;

	if ((error = mpr_read_config_page(sc, &params)) != 0) {
	mpr_printf(sc, "mpr_user_read_cfg_page timed out\n");
	return (ETIMEDOUT);
	}

	page_req->ioc_status = htole16(params.status);
	return (0);
	}

	static int
	mpr_user_read_extcfg_header(struct mpr_softc *sc,
	struct mpr_ext_cfg_page_req *ext_page_req)
	{
	MPI2_CONFIG_EXTENDED_PAGE_HEADER *hdr;
	struct mpr_config_params params;
	int error;

	hdr = &params.hdr.Ext;
	params.action = MPI2_CONFIG_ACTION_PAGE_HEADER;
	hdr->PageVersion = ext_page_req->header.PageVersion;
	hdr->PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
	hdr->ExtPageLength = 0;
	hdr->PageNumber = ext_page_req->header.PageNumber;
	hdr->ExtPageType = ext_page_req->header.ExtPageType;
	params.page_address = le32toh(ext_page_req->page_address);
	params.buffer = NULL;
	params.length = 0;
	params.callback = NULL;

	if ((error = mpr_read_config_page(sc, &params)) != 0) {
	/*
	* Leave the request. Without resetting the chip, it's
	* still owned by it and we'll just get into trouble
	* freeing it now. Mark it as abandoned so that if it
	* shows up later it can be freed.
	*/
	mpr_printf(sc, "mpr_user_read_extcfg_header timed out\n");
	return (ETIMEDOUT);
	}

	ext_page_req->ioc_status = htole16(params.status);
	if ((ext_page_req->ioc_status & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	ext_page_req->header.PageVersion = hdr->PageVersion;
	ext_page_req->header.PageNumber = hdr->PageNumber;
	ext_page_req->header.PageType = hdr->PageType;
	ext_page_req->header.ExtPageLength = hdr->ExtPageLength;
	ext_page_req->header.ExtPageType = hdr->ExtPageType;
	}

	return (0);
	}

	static int
	mpr_user_read_extcfg_page(struct mpr_softc *sc,
	struct mpr_ext_cfg_page_req ext_page_req, void buf)
	{
	MPI2_CONFIG_EXTENDED_PAGE_HEADER reqhdr, hdr;
	struct mpr_config_params params;
	int error;

	reqhdr = buf;
	hdr = &params.hdr.Ext;
	params.action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
	params.page_address = le32toh(ext_page_req->page_address);
	hdr->PageVersion = reqhdr->PageVersion;
	hdr->PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
	hdr->PageNumber = reqhdr->PageNumber;
	hdr->ExtPageType = reqhdr->ExtPageType;
	hdr->ExtPageLength = reqhdr->ExtPageLength;
	params.buffer = buf;
	params.length = le32toh(ext_page_req->len);
	params.callback = NULL;

	if ((error = mpr_read_config_page(sc, &params)) != 0) {
	mpr_printf(sc, "mpr_user_read_extcfg_page timed out\n");
	return (ETIMEDOUT);
	}

	ext_page_req->ioc_status = htole16(params.status);
	return (0);
	}

	static int
	mpr_user_write_cfg_page(struct mpr_softc *sc,
	struct mpr_cfg_page_req page_req, void buf)
	{
	MPI2_CONFIG_PAGE_HEADER reqhdr, hdr;
	struct mpr_config_params params;
	u_int hdr_attr;
	int error;

	reqhdr = buf;
	hdr = &params.hdr.Struct;
	hdr_attr = reqhdr->PageType & MPI2_CONFIG_PAGEATTR_MASK;
	if (hdr_attr != MPI2_CONFIG_PAGEATTR_CHANGEABLE &&
	hdr_attr != MPI2_CONFIG_PAGEATTR_PERSISTENT) {
	mpr_printf(sc, "page type 0x%x not changeable\n",
	reqhdr->PageType & MPI2_CONFIG_PAGETYPE_MASK);
	return (EINVAL);
	}

	/*
	* There isn't any point in restoring stripped out attributes
	* if you then mask them going down to issue the request.
	*/

	hdr->PageVersion = reqhdr->PageVersion;
	hdr->PageLength = reqhdr->PageLength;
	hdr->PageNumber = reqhdr->PageNumber;
	hdr->PageType = reqhdr->PageType;
	params.action = MPI2_CONFIG_ACTION_PAGE_WRITE_CURRENT;
	params.page_address = le32toh(page_req->page_address);
	params.buffer = buf;
	params.length = le32toh(page_req->len);
	params.callback = NULL;

	if ((error = mpr_write_config_page(sc, &params)) != 0) {
	mpr_printf(sc, "mpr_write_cfg_page timed out\n");
	return (ETIMEDOUT);
	}

	page_req->ioc_status = htole16(params.status);
	return (0);
	}

	void
	mpr_init_sge(struct mpr_command cm, void req, void *sge)
	{
	int off, space;

	space = (int)cm->cm_sc->reqframesz;
	off = (uintptr_t)sge - (uintptr_t)req;

	KASSERT(off < space, ("bad pointers %p %p, off %d, space %d",
	req, sge, off, space));

	cm->cm_sge = sge;
	cm->cm_sglsize = space - off;
	}

	/*
	* Prepare the mpr_command for an IOC_FACTS request.
	*/
	static int
	mpi_pre_ioc_facts(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI2_IOC_FACTS_REQUEST req = (void )cm->cm_req;
	MPI2_IOC_FACTS_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	cm->cm_sge = NULL;
	cm->cm_sglsize = 0;
	return (0);
	}

	/*
	* Prepare the mpr_command for a PORT_FACTS request.
	*/
	static int
	mpi_pre_port_facts(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI2_PORT_FACTS_REQUEST req = (void )cm->cm_req;
	MPI2_PORT_FACTS_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	cm->cm_sge = NULL;
	cm->cm_sglsize = 0;
	return (0);
	}

	/*
	* Prepare the mpr_command for a FW_DOWNLOAD request.
	*/
	static int
	mpi_pre_fw_download(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI25_FW_DOWNLOAD_REQUEST req = (void )cm->cm_req;
	MPI2_FW_DOWNLOAD_REPLY *rpl;
	int error;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	if (cmd->len == 0)
	return (EINVAL);

	error = copyin(cmd->buf, cm->cm_data, cmd->len);
	if (error != 0)
	return (error);

	mpr_init_sge(cm, req, &req->SGL);

	/*
	* For now, the F/W image must be provided in a single request.
	*/
	if ((req->MsgFlags & MPI2_FW_DOWNLOAD_MSGFLGS_LAST_SEGMENT) == 0)
	return (EINVAL);
	if (req->TotalImageSize != cmd->len)
	return (EINVAL);

	req->ImageOffset = 0;
	req->ImageSize = cmd->len;

	cm->cm_flags \|= MPR_CM_FLAGS_DATAOUT;

	return (mpr_push_ieee_sge(cm, &req->SGL, 0));
	}

	/*
	* Prepare the mpr_command for a FW_UPLOAD request.
	*/
	static int
	mpi_pre_fw_upload(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI25_FW_UPLOAD_REQUEST req = (void )cm->cm_req;
	MPI2_FW_UPLOAD_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpr_init_sge(cm, req, &req->SGL);
	if (cmd->len == 0) {
	/* Perhaps just asking what the size of the fw is? */
	return (0);
	}

	req->ImageOffset = 0;
	req->ImageSize = cmd->len;

	cm->cm_flags \|= MPR_CM_FLAGS_DATAIN;

	return (mpr_push_ieee_sge(cm, &req->SGL, 0));
	}

	/*
	* Prepare the mpr_command for a SATA_PASSTHROUGH request.
	*/
	static int
	mpi_pre_sata_passthrough(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI2_SATA_PASSTHROUGH_REQUEST req = (void )cm->cm_req;
	MPI2_SATA_PASSTHROUGH_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpr_init_sge(cm, req, &req->SGL);
	return (0);
	}

	/*
	* Prepare the mpr_command for a SMP_PASSTHROUGH request.
	*/
	static int
	mpi_pre_smp_passthrough(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI2_SMP_PASSTHROUGH_REQUEST req = (void )cm->cm_req;
	MPI2_SMP_PASSTHROUGH_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpr_init_sge(cm, req, &req->SGL);
	return (0);
	}

	/*
	* Prepare the mpr_command for a CONFIG request.
	*/
	static int
	mpi_pre_config(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI2_CONFIG_REQUEST req = (void )cm->cm_req;
	MPI2_CONFIG_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpr_init_sge(cm, req, &req->PageBufferSGE);
	return (0);
	}

	/*
	* Prepare the mpr_command for a SAS_IO_UNIT_CONTROL request.
	*/
	static int
	mpi_pre_sas_io_unit_control(struct mpr_command *cm,
	struct mpr_usr_command *cmd)
	{

	cm->cm_sge = NULL;
	cm->cm_sglsize = 0;
	return (0);
	}

	/*
	* A set of functions to prepare an mpr_command for the various
	* supported requests.
	*/
	struct mpr_user_func {
	U8 Function;
	mpr_user_f *f_pre;
	} mpr_user_func_list[] = {
	{ MPI2_FUNCTION_IOC_FACTS, mpi_pre_ioc_facts },
	{ MPI2_FUNCTION_PORT_FACTS, mpi_pre_port_facts },
	{ MPI2_FUNCTION_FW_DOWNLOAD, mpi_pre_fw_download },
	{ MPI2_FUNCTION_FW_UPLOAD, mpi_pre_fw_upload },
	{ MPI2_FUNCTION_SATA_PASSTHROUGH, mpi_pre_sata_passthrough },
	{ MPI2_FUNCTION_SMP_PASSTHROUGH, mpi_pre_smp_passthrough},
	{ MPI2_FUNCTION_CONFIG, mpi_pre_config},
	{ MPI2_FUNCTION_SAS_IO_UNIT_CONTROL, mpi_pre_sas_io_unit_control },
	{ 0xFF, NULL } /* list end */
	};

	static int
	mpr_user_setup_request(struct mpr_command cm, struct mpr_usr_command cmd)
	{
	MPI2_REQUEST_HEADER hdr = (MPI2_REQUEST_HEADER )cm->cm_req;
	struct mpr_user_func *f;

	for (f = mpr_user_func_list; f->f_pre != NULL; f++) {
	if (hdr->Function == f->Function)
	return (f->f_pre(cm, cmd));
	}
	return (EINVAL);
	}

	static int
	mpr_user_command(struct mpr_softc sc, struct mpr_usr_command cmd)
	{
	MPI2_REQUEST_HEADER *hdr;
	MPI2_DEFAULT_REPLY *rpl = NULL;
	void *buf = NULL;
	struct mpr_command *cm = NULL;
	int err = 0;
	int sz;

	mpr_lock(sc);
	cm = mpr_alloc_command(sc);

	if (cm == NULL) {
	mpr_printf(sc, "%s: no mpr requests\n", __func__);
	err = ENOMEM;
	goto RetFree;
	}
	mpr_unlock(sc);

	hdr = (MPI2_REQUEST_HEADER *)cm->cm_req;

	mpr_dprint(sc, MPR_USER, "%s: req %p %d rpl %p %d\n", __func__,
	cmd->req, cmd->req_len, cmd->rpl, cmd->rpl_len);

	if (cmd->req_len > (int)sc->reqframesz) {
	err = EINVAL;
	goto RetFreeUnlocked;
	}
	err = copyin(cmd->req, hdr, cmd->req_len);
	if (err != 0)
	goto RetFreeUnlocked;

	mpr_dprint(sc, MPR_USER, "%s: Function %02X MsgFlags %02X\n", __func__,
	hdr->Function, hdr->MsgFlags);

	if (cmd->len > 0) {
	buf = malloc(cmd->len, M_MPRUSER, M_WAITOK\|M_ZERO);
	cm->cm_data = buf;
	cm->cm_length = cmd->len;
	} else {
	cm->cm_data = NULL;
	cm->cm_length = 0;
	}

	cm->cm_flags = MPR_CM_FLAGS_SGE_SIMPLE;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;

	err = mpr_user_setup_request(cm, cmd);
	if (err == EINVAL) {
	mpr_printf(sc, "%s: unsupported parameter or unsupported "
	"function in request (function = 0x%X)\n", __func__,
	hdr->Function);
	}
	if (err != 0)
	goto RetFreeUnlocked;

	mpr_lock(sc);
	err = mpr_wait_command(sc, &cm, 30, CAN_SLEEP);

	if (err \|\| (cm == NULL)) {
	mpr_printf(sc, "%s: invalid request: error %d\n",
	__func__, err);
	goto RetFree;
	}

	if (cm != NULL)
	rpl = (MPI2_DEFAULT_REPLY *)cm->cm_reply;
	if (rpl != NULL)
	sz = rpl->MsgLength * 4;
	else
	sz = 0;

	if (sz > cmd->rpl_len) {
	mpr_printf(sc, "%s: user reply buffer (%d) smaller than "
	"returned buffer (%d)\n", __func__, cmd->rpl_len, sz);
	sz = cmd->rpl_len;
	}

	mpr_unlock(sc);
	copyout(rpl, cmd->rpl, sz);
	if (buf != NULL)
	copyout(buf, cmd->buf, cmd->len);
	mpr_dprint(sc, MPR_USER, "%s: reply size %d\n", __func__, sz);

	RetFreeUnlocked:
	mpr_lock(sc);
	RetFree:
	if (cm != NULL)
	mpr_free_command(sc, cm);
	mpr_unlock(sc);
	if (buf != NULL)
	free(buf, M_MPRUSER);
	return (err);
	}

	static int
	mpr_user_pass_thru(struct mpr_softc sc, mpr_pass_thru_t data)
	{
	MPI2_REQUEST_HEADER *hdr, tmphdr;
	MPI2_DEFAULT_REPLY *rpl;
	Mpi26NVMeEncapsulatedErrorReply_t *nvme_error_reply = NULL;
	Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request = NULL;
	struct mpr_command *cm = NULL;
	int i, err = 0, dir = 0, sz;
	uint8_t tool, function = 0;
	u_int sense_len;
	struct mprsas_target *targ = NULL;

	/*
	* Only allow one passthru command at a time. Use the MPR_FLAGS_BUSY
	* bit to denote that a passthru is being processed.
	*/
	mpr_lock(sc);
	if (sc->mpr_flags & MPR_FLAGS_BUSY) {
	mpr_dprint(sc, MPR_USER, "%s: Only one passthru command "
	"allowed at a single time.", __func__);
	mpr_unlock(sc);
	return (EBUSY);
	}
	sc->mpr_flags \|= MPR_FLAGS_BUSY;
	mpr_unlock(sc);

	/*
	* Do some validation on data direction. Valid cases are:
	* 1) DataSize is 0 and direction is NONE
	* 2) DataSize is non-zero and one of:
	* a) direction is READ or
	* b) direction is WRITE or
	* c) direction is BOTH and DataOutSize is non-zero
	* If valid and the direction is BOTH, change the direction to READ.
	* if valid and the direction is not BOTH, make sure DataOutSize is 0.
	*/
	if (((data->DataSize == 0) &&
	(data->DataDirection == MPR_PASS_THRU_DIRECTION_NONE)) \|\|
	((data->DataSize != 0) &&
	((data->DataDirection == MPR_PASS_THRU_DIRECTION_READ) \|\|
	(data->DataDirection == MPR_PASS_THRU_DIRECTION_WRITE) \|\|
	((data->DataDirection == MPR_PASS_THRU_DIRECTION_BOTH) &&
	(data->DataOutSize != 0))))) {
	if (data->DataDirection == MPR_PASS_THRU_DIRECTION_BOTH)
	data->DataDirection = MPR_PASS_THRU_DIRECTION_READ;
	else
	data->DataOutSize = 0;
	} else {
	err = EINVAL;
	goto RetFreeUnlocked;
	}

	mpr_dprint(sc, MPR_USER, "%s: req 0x%jx %d rpl 0x%jx %d "
	"data in 0x%jx %d data out 0x%jx %d data dir %d\n", __func__,
	data->PtrRequest, data->RequestSize, data->PtrReply,
	data->ReplySize, data->PtrData, data->DataSize,
	data->PtrDataOut, data->DataOutSize, data->DataDirection);

	/*
	* copy in the header so we know what we're dealing with before we
	* commit to allocating a command for it.
	*/
	err = copyin(PTRIN(data->PtrRequest), &tmphdr, data->RequestSize);
	if (err != 0)
	goto RetFreeUnlocked;

	if (data->RequestSize > (int)sc->reqframesz) {
	err = EINVAL;
	goto RetFreeUnlocked;
	}

	function = tmphdr.Function;
	mpr_dprint(sc, MPR_USER, "%s: Function %02X MsgFlags %02X\n", __func__,
	function, tmphdr.MsgFlags);

	/*
	* Handle a passthru TM request.
	*/
	if (function == MPI2_FUNCTION_SCSI_TASK_MGMT) {
	MPI2_SCSI_TASK_MANAGE_REQUEST *task;

	mpr_lock(sc);
	cm = mprsas_alloc_tm(sc);
	if (cm == NULL) {
	err = EINVAL;
	goto Ret;
	}

	/* Copy the header in. Only a small fixup is needed. */
	task = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
	bcopy(&tmphdr, task, data->RequestSize);
	task->TaskMID = cm->cm_desc.Default.SMID;

	cm->cm_data = NULL;
	cm->cm_complete = NULL;
	cm->cm_complete_data = NULL;

	targ = mprsas_find_target_by_handle(sc->sassc, 0,
	task->DevHandle);
	if (targ == NULL) {
	mpr_dprint(sc, MPR_INFO,
	"%s %d : invalid handle for requested TM 0x%x \n",
	__func__, __LINE__, task->DevHandle);
	err = 1;
	} else {
	mprsas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
	err = mpr_wait_command(sc, &cm, 30, CAN_SLEEP);
	}

	if (err != 0) {
	err = EIO;
	mpr_dprint(sc, MPR_FAULT, "%s: task management failed",
	__func__);
	}
	/*
	* Copy the reply data and sense data to user space.
	*/
	if ((cm != NULL) && (cm->cm_reply != NULL)) {
	rpl = (MPI2_DEFAULT_REPLY *)cm->cm_reply;
	sz = rpl->MsgLength * 4;

	if (sz > data->ReplySize) {
	mpr_printf(sc, "%s: user reply buffer (%d) "
	"smaller than returned buffer (%d)\n",
	__func__, data->ReplySize, sz);
	}
	mpr_unlock(sc);
	copyout(cm->cm_reply, PTRIN(data->PtrReply),
	data->ReplySize);
	mpr_lock(sc);
	}
	mprsas_free_tm(sc, cm);
	goto Ret;
	}

	mpr_lock(sc);
	cm = mpr_alloc_command(sc);

	if (cm == NULL) {
	mpr_printf(sc, "%s: no mpr requests\n", __func__);
	err = ENOMEM;
	goto Ret;
	}
	mpr_unlock(sc);

	hdr = (MPI2_REQUEST_HEADER *)cm->cm_req;
	bcopy(&tmphdr, hdr, data->RequestSize);

	/*
	* Do some checking to make sure the IOCTL request contains a valid
	* request. Then set the SGL info.
	*/
	mpr_init_sge(cm, hdr, (void )((uint8_t )hdr + data->RequestSize));

	/*
	* Set up for read, write or both. From check above, DataOutSize will
	* be 0 if direction is READ or WRITE, but it will have some non-zero
	* value if the direction is BOTH. So, just use the biggest size to get
	* the cm_data buffer size. If direction is BOTH, 2 SGLs need to be set
	* up; the first is for the request and the second will contain the
	* response data. cm_out_len needs to be set here and this will be used
	* when the SGLs are set up.
	*/
	cm->cm_data = NULL;
	cm->cm_length = MAX(data->DataSize, data->DataOutSize);
	cm->cm_out_len = data->DataOutSize;
	cm->cm_flags = 0;
	if (cm->cm_length != 0) {
	cm->cm_data = malloc(cm->cm_length, M_MPRUSER, M_WAITOK \|
	M_ZERO);
	cm->cm_flags = MPR_CM_FLAGS_DATAIN;
	if (data->DataOutSize) {
	cm->cm_flags \|= MPR_CM_FLAGS_DATAOUT;
	err = copyin(PTRIN(data->PtrDataOut),
	cm->cm_data, data->DataOutSize);
	} else if (data->DataDirection ==
	MPR_PASS_THRU_DIRECTION_WRITE) {
	cm->cm_flags = MPR_CM_FLAGS_DATAOUT;
	err = copyin(PTRIN(data->PtrData),
	cm->cm_data, data->DataSize);
	}
	if (err != 0)
	mpr_dprint(sc, MPR_FAULT, "%s: failed to copy IOCTL "
	"data from user space\n", __func__);
	}
	/*
	* Set this flag only if processing a command that does not need an
	* IEEE SGL. The CLI Tool within the Toolbox uses IEEE SGLs, so clear
	* the flag only for that tool if processing a Toolbox function.
	*/
	cm->cm_flags \|= MPR_CM_FLAGS_SGE_SIMPLE;
	for (i = 0; i < sizeof (ieee_sgl_func_list); i++) {
	if (function == ieee_sgl_func_list[i]) {
	if (function == MPI2_FUNCTION_TOOLBOX)
	{
	tool = (uint8_t)hdr->FunctionDependent1;
	if (tool != MPI2_TOOLBOX_DIAGNOSTIC_CLI_TOOL)
	break;
	}
	cm->cm_flags &= ~MPR_CM_FLAGS_SGE_SIMPLE;
	break;
	}
	}
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;

	if (function == MPI2_FUNCTION_NVME_ENCAPSULATED) {
	nvme_encap_request =
	(Mpi26NVMeEncapsulatedRequest_t *)cm->cm_req;
	cm->cm_desc.Default.RequestFlags =
	MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;

	/*
	* Get the Physical Address of the sense buffer.
	* Save the user's Error Response buffer address and use that
	* field to hold the sense buffer address.
	* Clear the internal sense buffer, which will potentially hold
	* the Completion Queue Entry on return, or 0 if no Entry.
	* Build the PRPs and set direction bits.
	* Send the request.
	*/
	cm->nvme_error_response =
	(uint64_t *)(uintptr_t)(((uint64_t)nvme_encap_request->
	ErrorResponseBaseAddress.High << 32) \|
	(uint64_t)nvme_encap_request->
	ErrorResponseBaseAddress.Low);
	nvme_encap_request->ErrorResponseBaseAddress.High =
	htole32((uint32_t)((uint64_t)cm->cm_sense_busaddr >> 32));
	nvme_encap_request->ErrorResponseBaseAddress.Low =
	htole32(cm->cm_sense_busaddr);
	memset(cm->cm_sense, 0, NVME_ERROR_RESPONSE_SIZE);
	mpr_build_nvme_prp(sc, cm, nvme_encap_request, cm->cm_data,
	data->DataSize, data->DataOutSize);
	}

	/*
	* Set up Sense buffer and SGL offset for IO passthru. SCSI IO request
	* uses SCSI IO or Fast Path SCSI IO descriptor.
	*/
	if ((function == MPI2_FUNCTION_SCSI_IO_REQUEST) \|\|
	(function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH)) {
	MPI2_SCSI_IO_REQUEST *scsi_io_req;

	scsi_io_req = (MPI2_SCSI_IO_REQUEST *)hdr;
	/*
	* Put SGE for data and data_out buffer at the end of
	* scsi_io_request message header (64 bytes in total).
	* Following above SGEs, the residual space will be used by
	* sense data.
	*/
	scsi_io_req->SenseBufferLength = (uint8_t)(data->RequestSize -
	64);
	scsi_io_req->SenseBufferLowAddress =
	htole32(cm->cm_sense_busaddr);

	/*
	* Set SGLOffset0 value. This is the number of dwords that SGL
	* is offset from the beginning of MPI2_SCSI_IO_REQUEST struct.
	*/
	scsi_io_req->SGLOffset0 = 24;

	/*
	* Setup descriptor info. RAID passthrough must use the
	* default request descriptor which is already set, so if this
	* is a SCSI IO request, change the descriptor to SCSI IO or
	* Fast Path SCSI IO. Also, if this is a SCSI IO request,
	* handle the reply in the mprsas_scsio_complete function.
	*/
	if (function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
	targ = mprsas_find_target_by_handle(sc->sassc, 0,
	scsi_io_req->DevHandle);

	if (!targ) {
	printf("No Target found for handle %d\n",
	scsi_io_req->DevHandle);
	err = EINVAL;
	goto RetFreeUnlocked;
	}

	if (targ->scsi_req_desc_type ==
	MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO) {
	cm->cm_desc.FastPathSCSIIO.RequestFlags =
	MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
	if (!sc->atomic_desc_capable) {
	cm->cm_desc.FastPathSCSIIO.DevHandle =
	scsi_io_req->DevHandle;
	}
	scsi_io_req->IoFlags \|=
	MPI25_SCSIIO_IOFLAGS_FAST_PATH;
	} else {
	cm->cm_desc.SCSIIO.RequestFlags =
	MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
	if (!sc->atomic_desc_capable) {
	cm->cm_desc.SCSIIO.DevHandle =
	scsi_io_req->DevHandle;
	}
	}

	/*
	* Make sure the DevHandle is not 0 because this is a
	* likely error.
	*/
	if (scsi_io_req->DevHandle == 0) {
	err = EINVAL;
	goto RetFreeUnlocked;
	}
	}
	}

	mpr_lock(sc);

	err = mpr_wait_command(sc, &cm, 30, CAN_SLEEP);

	if (err \|\| (cm == NULL)) {
	mpr_printf(sc, "%s: invalid request: error %d\n", __func__,
	err);
	goto RetFree;
	}

	/*
	* Sync the DMA data, if any. Then copy the data to user space.
	*/
	if (cm->cm_data != NULL) {
	if (cm->cm_flags & MPR_CM_FLAGS_DATAIN)
	dir = BUS_DMASYNC_POSTREAD;
	else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT)
	dir = BUS_DMASYNC_POSTWRITE;
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);

	if (cm->cm_flags & MPR_CM_FLAGS_DATAIN) {
	mpr_unlock(sc);
	err = copyout(cm->cm_data,
	PTRIN(data->PtrData), data->DataSize);
	mpr_lock(sc);
	if (err != 0)
	mpr_dprint(sc, MPR_FAULT, "%s: failed to copy "
	"IOCTL data to user space\n", __func__);
	}
	}

	/*
	* Copy the reply data and sense data to user space.
	*/
	if (cm->cm_reply != NULL) {
	rpl = (MPI2_DEFAULT_REPLY *)cm->cm_reply;
	sz = rpl->MsgLength * 4;

	if (sz > data->ReplySize) {
	mpr_printf(sc, "%s: user reply buffer (%d) smaller "
	"than returned buffer (%d)\n", __func__,
	data->ReplySize, sz);
	}
	mpr_unlock(sc);
	copyout(cm->cm_reply, PTRIN(data->PtrReply), data->ReplySize);
	mpr_lock(sc);

	if ((function == MPI2_FUNCTION_SCSI_IO_REQUEST) \|\|
	(function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH)) {
	if (((MPI2_SCSI_IO_REPLY *)rpl)->SCSIState &
	MPI2_SCSI_STATE_AUTOSENSE_VALID) {
	sense_len =
	MIN((le32toh(((MPI2_SCSI_IO_REPLY *)rpl)->
	SenseCount)), sizeof(struct
	scsi_sense_data));
	mpr_unlock(sc);
	copyout(cm->cm_sense, (PTRIN(data->PtrReply +
	sizeof(MPI2_SCSI_IO_REPLY))), sense_len);
	mpr_lock(sc);
	}
	}

	/*
	* Copy out the NVMe Error Reponse to user. The Error Response
	* buffer is given by the user, but a sense buffer is used to
	* get that data from the IOC. The user's
	* ErrorResponseBaseAddress is saved in the
	* 'nvme_error_response' field before the command because that
	* field is set to a sense buffer. When the command is
	* complete, the Error Response data from the IOC is copied to
	* that user address after it is checked for validity.
	* Also note that 'sense' buffers are not defined for
	* NVMe commands. Sense terminalogy is only used here so that
	* the same IOCTL structure and sense buffers can be used for
	* NVMe.
	*/
	if (function == MPI2_FUNCTION_NVME_ENCAPSULATED) {
	if (cm->nvme_error_response == NULL) {
	mpr_dprint(sc, MPR_INFO, "NVMe Error Response "
	"buffer is NULL. Response data will not be "
	"returned.\n");
	mpr_unlock(sc);
	goto RetFreeUnlocked;
	}

	nvme_error_reply =
	(Mpi26NVMeEncapsulatedErrorReply_t *)cm->cm_reply;
	sz = MIN(le32toh(nvme_error_reply->ErrorResponseCount),
	NVME_ERROR_RESPONSE_SIZE);
	mpr_unlock(sc);
	copyout(cm->cm_sense,
	(PTRIN(data->PtrReply +
	sizeof(MPI2_SCSI_IO_REPLY))), sz);
	mpr_lock(sc);
	}
	}
	mpr_unlock(sc);

	RetFreeUnlocked:
	mpr_lock(sc);

	RetFree:
	if (cm != NULL) {
	if (cm->cm_data)
	free(cm->cm_data, M_MPRUSER);
	mpr_free_command(sc, cm);
	}
	Ret:
	sc->mpr_flags &= ~MPR_FLAGS_BUSY;
	mpr_unlock(sc);

	return (err);
	}

	static void
	mpr_user_get_adapter_data(struct mpr_softc sc, mpr_adapter_data_t data)
	{
	Mpi2ConfigReply_t mpi_reply;
	Mpi2BiosPage3_t config_page;

	/*
	* Use the PCI interface functions to get the Bus, Device, and Function
	* information.
	*/
	data->PciInformation.u.bits.BusNumber = pci_get_bus(sc->mpr_dev);
	data->PciInformation.u.bits.DeviceNumber = pci_get_slot(sc->mpr_dev);
	data->PciInformation.u.bits.FunctionNumber =
	pci_get_function(sc->mpr_dev);

	/*
	* Get the FW version that should already be saved in IOC Facts.
	*/
	data->MpiFirmwareVersion = sc->facts->FWVersion.Word;

	/*
	* General device info.
	*/
	if (sc->mpr_flags & MPR_FLAGS_GEN35_IOC)
	data->AdapterType = MPRIOCTL_ADAPTER_TYPE_SAS35;
	else
	data->AdapterType = MPRIOCTL_ADAPTER_TYPE_SAS3;
	data->PCIDeviceHwId = pci_get_device(sc->mpr_dev);
	data->PCIDeviceHwRev = pci_read_config(sc->mpr_dev, PCIR_REVID, 1);
	data->SubSystemId = pci_get_subdevice(sc->mpr_dev);
	data->SubsystemVendorId = pci_get_subvendor(sc->mpr_dev);

	/*
	* Get the driver version.
	*/
	strcpy((char *)&data->DriverVersion[0], MPR_DRIVER_VERSION);

	/*
	* Need to get BIOS Config Page 3 for the BIOS Version.
	*/
	data->BiosVersion = 0;
	mpr_lock(sc);
	if (mpr_config_get_bios_pg3(sc, &mpi_reply, &config_page))
	printf("%s: Error while retrieving BIOS Version\n", __func__);
	else
	data->BiosVersion = config_page.BiosVersion;
	mpr_unlock(sc);
	}

	static void
	mpr_user_read_pci_info(struct mpr_softc sc, mpr_pci_info_t data)
	{
	int i;

	/*
	* Use the PCI interface functions to get the Bus, Device, and Function
	* information.
	*/
	data->BusNumber = pci_get_bus(sc->mpr_dev);
	data->DeviceNumber = pci_get_slot(sc->mpr_dev);
	data->FunctionNumber = pci_get_function(sc->mpr_dev);

	/*
	* Now get the interrupt vector and the pci header. The vector can
	* only be 0 right now. The header is the first 256 bytes of config
	* space.
	*/
	data->InterruptVector = 0;
	for (i = 0; i < sizeof (data->PciHeader); i++) {
	data->PciHeader[i] = pci_read_config(sc->mpr_dev, i, 1);
	}
	}

	static uint8_t
	mpr_get_fw_diag_buffer_number(struct mpr_softc *sc, uint32_t unique_id)
	{
	uint8_t index;

	for (index = 0; index < MPI2_DIAG_BUF_TYPE_COUNT; index++) {
	if (sc->fw_diag_buffer_list[index].unique_id == unique_id) {
	return (index);
	}
	}

	return (MPR_FW_DIAGNOSTIC_UID_NOT_FOUND);
	}

	static int
	mpr_post_fw_diag_buffer(struct mpr_softc *sc,
	mpr_fw_diagnostic_buffer_t pBuffer, uint32_t return_code)
	{
	MPI2_DIAG_BUFFER_POST_REQUEST *req;
	MPI2_DIAG_BUFFER_POST_REPLY *reply;
	struct mpr_command *cm = NULL;
	int i, status;

	/*
	* If buffer is not enabled, just leave.
	*/
	*return_code = MPR_FW_DIAG_ERROR_POST_FAILED;
	if (!pBuffer->enabled) {
	return (MPR_DIAG_FAILURE);
	}

	/*
	* Clear some flags initially.
	*/
	pBuffer->force_release = FALSE;
	pBuffer->valid_data = FALSE;
	pBuffer->owned_by_firmware = FALSE;

	/*
	* Get a command.
	*/
	cm = mpr_alloc_command(sc);
	if (cm == NULL) {
	mpr_printf(sc, "%s: no mpr requests\n", __func__);
	return (MPR_DIAG_FAILURE);
	}

	/*
	* Build the request for releasing the FW Diag Buffer and send it.
	*/
	req = (MPI2_DIAG_BUFFER_POST_REQUEST *)cm->cm_req;
	req->Function = MPI2_FUNCTION_DIAG_BUFFER_POST;
	req->BufferType = pBuffer->buffer_type;
	req->ExtendedType = pBuffer->extended_type;
	req->BufferLength = pBuffer->size;
	for (i = 0; i < (sizeof(req->ProductSpecific) / 4); i++)
	req->ProductSpecific[i] = pBuffer->product_specific[i];
	mpr_from_u64(sc->fw_diag_busaddr, &req->BufferAddress);
	cm->cm_data = NULL;
	cm->cm_length = 0;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete_data = NULL;

	/*
	* Send command synchronously.
	*/
	status = mpr_wait_command(sc, &cm, 30, CAN_SLEEP);
	if (status \|\| (cm == NULL)) {
	mpr_printf(sc, "%s: invalid request: error %d\n", __func__,
	status);
	status = MPR_DIAG_FAILURE;
	goto done;
	}

	/*
	* Process POST reply.
	*/
	reply = (MPI2_DIAG_BUFFER_POST_REPLY *)cm->cm_reply;
	if (reply == NULL) {
	mpr_printf(sc, "%s: reply is NULL, probably due to "
	"reinitialization", __func__);
	status = MPR_DIAG_FAILURE;
	goto done;
	}

	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) {
	status = MPR_DIAG_FAILURE;
	mpr_dprint(sc, MPR_FAULT, "%s: post of FW Diag Buffer failed "
	"with IOCStatus = 0x%x, IOCLogInfo = 0x%x and "
	"TransferLength = 0x%x\n", __func__,
	le16toh(reply->IOCStatus), le32toh(reply->IOCLogInfo),
	le32toh(reply->TransferLength));
	goto done;
	}

	/*
	* Post was successful.
	*/
	pBuffer->valid_data = TRUE;
	pBuffer->owned_by_firmware = TRUE;
	*return_code = MPR_FW_DIAG_ERROR_SUCCESS;
	status = MPR_DIAG_SUCCESS;

	done:
	if (cm != NULL)
	mpr_free_command(sc, cm);
	return (status);
	}

	static int
	mpr_release_fw_diag_buffer(struct mpr_softc *sc,
	mpr_fw_diagnostic_buffer_t pBuffer, uint32_t return_code,
	uint32_t diag_type)
	{
	MPI2_DIAG_RELEASE_REQUEST *req;
	MPI2_DIAG_RELEASE_REPLY *reply;
	struct mpr_command *cm = NULL;
	int status;

	/*
	* If buffer is not enabled, just leave.
	*/
	*return_code = MPR_FW_DIAG_ERROR_RELEASE_FAILED;
	if (!pBuffer->enabled) {
	mpr_dprint(sc, MPR_USER, "%s: This buffer type is not "
	"supported by the IOC", __func__);
	return (MPR_DIAG_FAILURE);
	}

	/*
	* Clear some flags initially.
	*/
	pBuffer->force_release = FALSE;
	pBuffer->valid_data = FALSE;
	pBuffer->owned_by_firmware = FALSE;

	/*
	* Get a command.
	*/
	cm = mpr_alloc_command(sc);
	if (cm == NULL) {
	mpr_printf(sc, "%s: no mpr requests\n", __func__);
	return (MPR_DIAG_FAILURE);
	}

	/*
	* Build the request for releasing the FW Diag Buffer and send it.
	*/
	req = (MPI2_DIAG_RELEASE_REQUEST *)cm->cm_req;
	req->Function = MPI2_FUNCTION_DIAG_RELEASE;
	req->BufferType = pBuffer->buffer_type;
	cm->cm_data = NULL;
	cm->cm_length = 0;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete_data = NULL;

	/*
	* Send command synchronously.
	*/
	status = mpr_wait_command(sc, &cm, 30, CAN_SLEEP);
	if (status \|\| (cm == NULL)) {
	mpr_printf(sc, "%s: invalid request: error %d\n", __func__,
	status);
	status = MPR_DIAG_FAILURE;
	goto done;
	}

	/*
	* Process RELEASE reply.
	*/
	reply = (MPI2_DIAG_RELEASE_REPLY *)cm->cm_reply;
	if (reply == NULL) {
	mpr_printf(sc, "%s: reply is NULL, probably due to "
	"reinitialization", __func__);
	status = MPR_DIAG_FAILURE;
	goto done;
	}
	if (((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) \|\| pBuffer->owned_by_firmware) {
	status = MPR_DIAG_FAILURE;
	mpr_dprint(sc, MPR_FAULT, "%s: release of FW Diag Buffer "
	"failed with IOCStatus = 0x%x and IOCLogInfo = 0x%x\n",
	__func__, le16toh(reply->IOCStatus),
	le32toh(reply->IOCLogInfo));
	goto done;
	}

	/*
	* Release was successful.
	*/
	*return_code = MPR_FW_DIAG_ERROR_SUCCESS;
	status = MPR_DIAG_SUCCESS;

	/*
	* If this was for an UNREGISTER diag type command, clear the unique ID.
	*/
	if (diag_type == MPR_FW_DIAG_TYPE_UNREGISTER) {
	pBuffer->unique_id = MPR_FW_DIAG_INVALID_UID;
	}

	done:
	if (cm != NULL)
	mpr_free_command(sc, cm);

	return (status);
	}

	static int
	mpr_diag_register(struct mpr_softc sc, mpr_fw_diag_register_t diag_register,
	uint32_t *return_code)
	{
	bus_dma_tag_template_t t;
	mpr_fw_diagnostic_buffer_t *pBuffer;
	struct mpr_busdma_context *ctx;
	uint8_t extended_type, buffer_type, i;
	uint32_t buffer_size;
	uint32_t unique_id;
	int status;
	int error;

	extended_type = diag_register->ExtendedType;
	buffer_type = diag_register->BufferType;
	buffer_size = diag_register->RequestedBufferSize;
	unique_id = diag_register->UniqueId;
	ctx = NULL;
	error = 0;

	/*
	* Check for valid buffer type
	*/
	if (buffer_type >= MPI2_DIAG_BUF_TYPE_COUNT) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	return (MPR_DIAG_FAILURE);
	}

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should not be found. If it is, the ID is already in use.
	*/
	i = mpr_get_fw_diag_buffer_number(sc, unique_id);
	pBuffer = &sc->fw_diag_buffer_list[buffer_type];
	if (i != MPR_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}

	/*
	* The buffer's unique ID should not be registered yet, and the given
	* unique ID cannot be 0.
	*/
	if ((pBuffer->unique_id != MPR_FW_DIAG_INVALID_UID) \|\|
	(unique_id == MPR_FW_DIAG_INVALID_UID)) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}

	/*
	* If this buffer is already posted as immediate, just change owner.
	*/
	if (pBuffer->immediate && pBuffer->owned_by_firmware &&
	(pBuffer->unique_id == MPR_FW_DIAG_INVALID_UID)) {
	pBuffer->immediate = FALSE;
	pBuffer->unique_id = unique_id;
	return (MPR_DIAG_SUCCESS);
	}

	/*
	* Post a new buffer after checking if it's enabled. The DMA buffer
	* that is allocated will be contiguous (nsegments = 1).
	*/
	if (!pBuffer->enabled) {
	*return_code = MPR_FW_DIAG_ERROR_NO_BUFFER;
	return (MPR_DIAG_FAILURE);
	}
	bus_dma_template_init(&t, sc->mpr_parent_dmat);
	t.lowaddr = BUS_SPACE_MAXADDR_32BIT;
	t.maxsize = t.maxsegsize = buffer_size;
	t.nsegments = 1;
	if (bus_dma_template_tag(&t, &sc->fw_diag_dmat)) {
	mpr_dprint(sc, MPR_ERROR,
	"Cannot allocate FW diag buffer DMA tag\n");
	*return_code = MPR_FW_DIAG_ERROR_NO_BUFFER;
	status = MPR_DIAG_FAILURE;
	goto bailout;
	}
	if (bus_dmamem_alloc(sc->fw_diag_dmat, (void **)&sc->fw_diag_buffer,
	BUS_DMA_NOWAIT, &sc->fw_diag_map)) {
	mpr_dprint(sc, MPR_ERROR,
	"Cannot allocate FW diag buffer memory\n");
	*return_code = MPR_FW_DIAG_ERROR_NO_BUFFER;
	status = MPR_DIAG_FAILURE;
	goto bailout;
	}
	bzero(sc->fw_diag_buffer, buffer_size);

	ctx = malloc(sizeof(*ctx), M_MPR, M_WAITOK \| M_ZERO);
	- if (ctx == NULL) {
	- device_printf(sc->mpr_dev, "%s: context malloc failed\n",
	- __func__);
	- *return_code = MPR_FW_DIAG_ERROR_NO_BUFFER;
	- status = MPR_DIAG_FAILURE;
	- goto bailout;
	- }
	ctx->addr = &sc->fw_diag_busaddr;
	ctx->buffer_dmat = sc->fw_diag_dmat;
	ctx->buffer_dmamap = sc->fw_diag_map;
	ctx->softc = sc;
	error = bus_dmamap_load(sc->fw_diag_dmat, sc->fw_diag_map,
	sc->fw_diag_buffer, buffer_size, mpr_memaddr_wait_cb,
	ctx, 0);
	if (error == EINPROGRESS) {

	/* XXX KDM */
	device_printf(sc->mpr_dev, "%s: Deferred bus_dmamap_load\n",
	__func__);
	/*
	* Wait for the load to complete. If we're interrupted,
	* bail out.
	*/
	mpr_lock(sc);
	if (ctx->completed == 0) {
	error = msleep(ctx, &sc->mpr_mtx, PCATCH, "mprwait", 0);
	if (error != 0) {
	/*
	* We got an error from msleep(9). This is
	* most likely due to a signal. Tell
	* mpr_memaddr_wait_cb() that we've abandoned
	* the context, so it needs to clean up when
	* it is called.
	*/
	ctx->abandoned = 1;

	/* The callback will free this memory */
	ctx = NULL;
	mpr_unlock(sc);

	device_printf(sc->mpr_dev, "Cannot "
	"bus_dmamap_load FW diag buffer, error = "
	"%d returned from msleep\n", error);
	*return_code = MPR_FW_DIAG_ERROR_NO_BUFFER;
	status = MPR_DIAG_FAILURE;
	goto bailout;
	}
	}
	mpr_unlock(sc);
	}

	if ((error != 0) \|\| (ctx->error != 0)) {
	device_printf(sc->mpr_dev, "Cannot bus_dmamap_load FW diag "
	"buffer, %serror = %d\n", error ? "" : "callback ",
	error ? error : ctx->error);
	*return_code = MPR_FW_DIAG_ERROR_NO_BUFFER;
	status = MPR_DIAG_FAILURE;
	goto bailout;
	}

	bus_dmamap_sync(sc->fw_diag_dmat, sc->fw_diag_map, BUS_DMASYNC_PREREAD);

	pBuffer->size = buffer_size;

	/*
	* Copy the given info to the diag buffer and post the buffer.
	*/
	pBuffer->buffer_type = buffer_type;
	pBuffer->immediate = FALSE;
	if (buffer_type == MPI2_DIAG_BUF_TYPE_TRACE) {
	for (i = 0; i < (sizeof (pBuffer->product_specific) / 4);
	i++) {
	pBuffer->product_specific[i] =
	diag_register->ProductSpecific[i];
	}
	}
	pBuffer->extended_type = extended_type;
	pBuffer->unique_id = unique_id;
	status = mpr_post_fw_diag_buffer(sc, pBuffer, return_code);

	bailout:

	/*
	* In case there was a failure, free the DMA buffer.
	*/
	if (status == MPR_DIAG_FAILURE) {
	if (sc->fw_diag_busaddr != 0) {
	bus_dmamap_unload(sc->fw_diag_dmat, sc->fw_diag_map);
	sc->fw_diag_busaddr = 0;
	}
	if (sc->fw_diag_buffer != NULL) {
	bus_dmamem_free(sc->fw_diag_dmat, sc->fw_diag_buffer,
	sc->fw_diag_map);
	sc->fw_diag_buffer = NULL;
	}
	if (sc->fw_diag_dmat != NULL) {
	bus_dma_tag_destroy(sc->fw_diag_dmat);
	sc->fw_diag_dmat = NULL;
	}
	}

	if (ctx != NULL)
	free(ctx, M_MPR);

	return (status);
	}

	static int
	mpr_diag_unregister(struct mpr_softc *sc,
	mpr_fw_diag_unregister_t diag_unregister, uint32_t return_code)
	{
	mpr_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i;
	uint32_t unique_id;
	int status;

	unique_id = diag_unregister->UniqueId;

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should be there.
	*/
	i = mpr_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPR_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}

	pBuffer = &sc->fw_diag_buffer_list[i];

	/*
	* Try to release the buffer from FW before freeing it. If release
	* fails, don't free the DMA buffer in case FW tries to access it
	* later. If buffer is not owned by firmware, can't release it.
	*/
	if (!pBuffer->owned_by_firmware) {
	status = MPR_DIAG_SUCCESS;
	} else {
	status = mpr_release_fw_diag_buffer(sc, pBuffer, return_code,
	MPR_FW_DIAG_TYPE_UNREGISTER);
	}

	/*
	* At this point, return the current status no matter what happens with
	* the DMA buffer.
	*/
	pBuffer->unique_id = MPR_FW_DIAG_INVALID_UID;
	if (status == MPR_DIAG_SUCCESS) {
	if (sc->fw_diag_busaddr != 0) {
	bus_dmamap_unload(sc->fw_diag_dmat, sc->fw_diag_map);
	sc->fw_diag_busaddr = 0;
	}
	if (sc->fw_diag_buffer != NULL) {
	bus_dmamem_free(sc->fw_diag_dmat, sc->fw_diag_buffer,
	sc->fw_diag_map);
	sc->fw_diag_buffer = NULL;
	}
	if (sc->fw_diag_dmat != NULL) {
	bus_dma_tag_destroy(sc->fw_diag_dmat);
	sc->fw_diag_dmat = NULL;
	}
	}

	return (status);
	}

	static int
	mpr_diag_query(struct mpr_softc sc, mpr_fw_diag_query_t diag_query,
	uint32_t *return_code)
	{
	mpr_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i;
	uint32_t unique_id;

	unique_id = diag_query->UniqueId;

	/*
	* If ID is valid, query on ID.
	* If ID is invalid, query on buffer type.
	*/
	if (unique_id == MPR_FW_DIAG_INVALID_UID) {
	i = diag_query->BufferType;
	if (i >= MPI2_DIAG_BUF_TYPE_COUNT) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}
	} else {
	i = mpr_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPR_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}
	}

	/*
	* Fill query structure with the diag buffer info.
	*/
	pBuffer = &sc->fw_diag_buffer_list[i];
	diag_query->BufferType = pBuffer->buffer_type;
	diag_query->ExtendedType = pBuffer->extended_type;
	if (diag_query->BufferType == MPI2_DIAG_BUF_TYPE_TRACE) {
	for (i = 0; i < (sizeof(diag_query->ProductSpecific) / 4);
	i++) {
	diag_query->ProductSpecific[i] =
	pBuffer->product_specific[i];
	}
	}
	diag_query->TotalBufferSize = pBuffer->size;
	diag_query->DriverAddedBufferSize = 0;
	diag_query->UniqueId = pBuffer->unique_id;
	diag_query->ApplicationFlags = 0;
	diag_query->DiagnosticFlags = 0;

	/*
	* Set/Clear application flags
	*/
	if (pBuffer->immediate) {
	diag_query->ApplicationFlags &= ~MPR_FW_DIAG_FLAG_APP_OWNED;
	} else {
	diag_query->ApplicationFlags \|= MPR_FW_DIAG_FLAG_APP_OWNED;
	}
	if (pBuffer->valid_data \|\| pBuffer->owned_by_firmware) {
	diag_query->ApplicationFlags \|= MPR_FW_DIAG_FLAG_BUFFER_VALID;
	} else {
	diag_query->ApplicationFlags &= ~MPR_FW_DIAG_FLAG_BUFFER_VALID;
	}
	if (pBuffer->owned_by_firmware) {
	diag_query->ApplicationFlags \|=
	MPR_FW_DIAG_FLAG_FW_BUFFER_ACCESS;
	} else {
	diag_query->ApplicationFlags &=
	~MPR_FW_DIAG_FLAG_FW_BUFFER_ACCESS;
	}

	return (MPR_DIAG_SUCCESS);
	}

	static int
	mpr_diag_read_buffer(struct mpr_softc *sc,
	mpr_diag_read_buffer_t diag_read_buffer, uint8_t ioctl_buf,
	uint32_t *return_code)
	{
	mpr_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i, *pData;
	uint32_t unique_id;
	int status;

	unique_id = diag_read_buffer->UniqueId;

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should be there.
	*/
	i = mpr_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPR_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}

	pBuffer = &sc->fw_diag_buffer_list[i];

	/*
	* Make sure requested read is within limits
	*/
	if (diag_read_buffer->StartingOffset + diag_read_buffer->BytesToRead >
	pBuffer->size) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	return (MPR_DIAG_FAILURE);
	}

	/* Sync the DMA map before we copy to userland. */
	bus_dmamap_sync(sc->fw_diag_dmat, sc->fw_diag_map,
	BUS_DMASYNC_POSTREAD);

	/*
	* Copy the requested data from DMA to the diag_read_buffer. The DMA
	* buffer that was allocated is one contiguous buffer.
	*/
	pData = (uint8_t *)(sc->fw_diag_buffer +
	diag_read_buffer->StartingOffset);
	if (copyout(pData, ioctl_buf, diag_read_buffer->BytesToRead) != 0)
	return (MPR_DIAG_FAILURE);
	diag_read_buffer->Status = 0;

	/*
	* Set or clear the Force Release flag.
	*/
	if (pBuffer->force_release) {
	diag_read_buffer->Flags \|= MPR_FW_DIAG_FLAG_FORCE_RELEASE;
	} else {
	diag_read_buffer->Flags &= ~MPR_FW_DIAG_FLAG_FORCE_RELEASE;
	}

	/*
	* If buffer is to be reregistered, make sure it's not already owned by
	* firmware first.
	*/
	status = MPR_DIAG_SUCCESS;
	if (!pBuffer->owned_by_firmware) {
	if (diag_read_buffer->Flags & MPR_FW_DIAG_FLAG_REREGISTER) {
	status = mpr_post_fw_diag_buffer(sc, pBuffer,
	return_code);
	}
	}

	return (status);
	}

	static int
	mpr_diag_release(struct mpr_softc sc, mpr_fw_diag_release_t diag_release,
	uint32_t *return_code)
	{
	mpr_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i;
	uint32_t unique_id;
	int status;

	unique_id = diag_release->UniqueId;

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should be there.
	*/
	i = mpr_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPR_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPR_FW_DIAG_ERROR_INVALID_UID;
	return (MPR_DIAG_FAILURE);
	}

	pBuffer = &sc->fw_diag_buffer_list[i];

	/*
	* If buffer is not owned by firmware, it's already been released.
	*/
	if (!pBuffer->owned_by_firmware) {
	*return_code = MPR_FW_DIAG_ERROR_ALREADY_RELEASED;
	return (MPR_DIAG_FAILURE);
	}

	/*
	* Release the buffer.
	*/
	status = mpr_release_fw_diag_buffer(sc, pBuffer, return_code,
	MPR_FW_DIAG_TYPE_RELEASE);
	return (status);
	}

	static int
	mpr_do_diag_action(struct mpr_softc sc, uint32_t action, uint8_t diag_action,
	uint32_t length, uint32_t *return_code)
	{
	mpr_fw_diag_register_t diag_register;
	mpr_fw_diag_unregister_t diag_unregister;
	mpr_fw_diag_query_t diag_query;
	mpr_diag_read_buffer_t diag_read_buffer;
	mpr_fw_diag_release_t diag_release;
	int status = MPR_DIAG_SUCCESS;
	uint32_t original_return_code;

	original_return_code = *return_code;
	*return_code = MPR_FW_DIAG_ERROR_SUCCESS;

	switch (action) {
	case MPR_FW_DIAG_TYPE_REGISTER:
	if (!length) {
	*return_code =
	MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPR_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_register,
	sizeof(diag_register)) != 0)
	return (MPR_DIAG_FAILURE);
	status = mpr_diag_register(sc, &diag_register,
	return_code);
	break;

	case MPR_FW_DIAG_TYPE_UNREGISTER:
	if (length < sizeof(diag_unregister)) {
	*return_code =
	MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPR_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_unregister,
	sizeof(diag_unregister)) != 0)
	return (MPR_DIAG_FAILURE);
	status = mpr_diag_unregister(sc, &diag_unregister,
	return_code);
	break;

	case MPR_FW_DIAG_TYPE_QUERY:
	if (length < sizeof (diag_query)) {
	*return_code =
	MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPR_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_query, sizeof(diag_query))
	!= 0)
	return (MPR_DIAG_FAILURE);
	status = mpr_diag_query(sc, &diag_query, return_code);
	if (status == MPR_DIAG_SUCCESS)
	if (copyout(&diag_query, diag_action,
	sizeof (diag_query)) != 0)
	return (MPR_DIAG_FAILURE);
	break;

	case MPR_FW_DIAG_TYPE_READ_BUFFER:
	if (copyin(diag_action, &diag_read_buffer,
	sizeof(diag_read_buffer)) != 0)
	return (MPR_DIAG_FAILURE);
	if (length < diag_read_buffer.BytesToRead) {
	*return_code =
	MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPR_DIAG_FAILURE;
	break;
	}
	status = mpr_diag_read_buffer(sc, &diag_read_buffer,
	PTRIN(diag_read_buffer.PtrDataBuffer),
	return_code);
	if (status == MPR_DIAG_SUCCESS) {
	if (copyout(&diag_read_buffer, diag_action,
	sizeof(diag_read_buffer) -
	sizeof(diag_read_buffer.PtrDataBuffer)) !=
	0)
	return (MPR_DIAG_FAILURE);
	}
	break;

	case MPR_FW_DIAG_TYPE_RELEASE:
	if (length < sizeof(diag_release)) {
	*return_code =
	MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPR_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_release,
	sizeof(diag_release)) != 0)
	return (MPR_DIAG_FAILURE);
	status = mpr_diag_release(sc, &diag_release,
	return_code);
	break;

	default:
	*return_code = MPR_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPR_DIAG_FAILURE;
	break;
	}

	if ((status == MPR_DIAG_FAILURE) &&
	(original_return_code == MPR_FW_DIAG_NEW) &&
	(*return_code != MPR_FW_DIAG_ERROR_SUCCESS))
	status = MPR_DIAG_SUCCESS;

	return (status);
	}

	static int
	mpr_user_diag_action(struct mpr_softc sc, mpr_diag_action_t data)
	{
	int status;

	/*
	* Only allow one diag action at one time.
	*/
	if (sc->mpr_flags & MPR_FLAGS_BUSY) {
	mpr_dprint(sc, MPR_USER, "%s: Only one FW diag command "
	"allowed at a single time.", __func__);
	return (EBUSY);
	}
	sc->mpr_flags \|= MPR_FLAGS_BUSY;

	/*
	* Send diag action request
	*/
	if (data->Action == MPR_FW_DIAG_TYPE_REGISTER \|\|
	data->Action == MPR_FW_DIAG_TYPE_UNREGISTER \|\|
	data->Action == MPR_FW_DIAG_TYPE_QUERY \|\|
	data->Action == MPR_FW_DIAG_TYPE_READ_BUFFER \|\|
	data->Action == MPR_FW_DIAG_TYPE_RELEASE) {
	status = mpr_do_diag_action(sc, data->Action,
	PTRIN(data->PtrDiagAction), data->Length,
	&data->ReturnCode);
	} else
	status = EINVAL;

	sc->mpr_flags &= ~MPR_FLAGS_BUSY;
	return (status);
	}

	/*
	* Copy the event recording mask and the event queue size out. For
	* clarification, the event recording mask (events_to_record) is not the same
	* thing as the event mask (event_mask). events_to_record has a bit set for
	* every event type that is to be recorded by the driver, and event_mask has a
	* bit cleared for every event that is allowed into the driver from the IOC.
	* They really have nothing to do with each other.
	*/
	static void
	mpr_user_event_query(struct mpr_softc sc, mpr_event_query_t data)
	{
	uint8_t i;

	mpr_lock(sc);
	data->Entries = MPR_EVENT_QUEUE_SIZE;

	for (i = 0; i < 4; i++) {
	data->Types[i] = sc->events_to_record[i];
	}
	mpr_unlock(sc);
	}

	/*
	* Set the driver's event mask according to what's been given. See
	* mpr_user_event_query for explanation of the event recording mask and the IOC
	* event mask. It's the app's responsibility to enable event logging by setting
	* the bits in events_to_record. Initially, no events will be logged.
	*/
	static void
	mpr_user_event_enable(struct mpr_softc sc, mpr_event_enable_t data)
	{
	uint8_t i;

	mpr_lock(sc);
	for (i = 0; i < 4; i++) {
	sc->events_to_record[i] = data->Types[i];
	}
	mpr_unlock(sc);
	}

	/*
	* Copy out the events that have been recorded, up to the max events allowed.
	*/
	static int
	mpr_user_event_report(struct mpr_softc sc, mpr_event_report_t data)
	{
	int status = 0;
	uint32_t size;

	mpr_lock(sc);
	size = data->Size;
	if ((size >= sizeof(sc->recorded_events)) && (status == 0)) {
	mpr_unlock(sc);
	if (copyout((void *)sc->recorded_events,
	PTRIN(data->PtrEvents), size) != 0)
	status = EFAULT;
	mpr_lock(sc);
	} else {
	/*
	* data->Size value is not large enough to copy event data.
	*/
	status = EFAULT;
	}

	/*
	* Change size value to match the number of bytes that were copied.
	*/
	if (status == 0)
	data->Size = sizeof(sc->recorded_events);
	mpr_unlock(sc);

	return (status);
	}

	/*
	* Record events into the driver from the IOC if they are not masked.
	*/
	void
	mprsas_record_event(struct mpr_softc *sc,
	MPI2_EVENT_NOTIFICATION_REPLY *event_reply)
	{
	uint32_t event;
	int i, j;
	uint16_t event_data_len;
	boolean_t sendAEN = FALSE;

	event = event_reply->Event;

	/*
	* Generate a system event to let anyone who cares know that a
	* LOG_ENTRY_ADDED event has occurred. This is sent no matter what the
	* event mask is set to.
	*/
	if (event == MPI2_EVENT_LOG_ENTRY_ADDED) {
	sendAEN = TRUE;
	}

	/*
	* Record the event only if its corresponding bit is set in
	* events_to_record. event_index is the index into recorded_events and
	* event_number is the overall number of an event being recorded since
	* start-of-day. event_index will roll over; event_number will never
	* roll over.
	*/
	i = (uint8_t)(event / 32);
	j = (uint8_t)(event % 32);
	if ((i < 4) && ((1 << j) & sc->events_to_record[i])) {
	i = sc->event_index;
	sc->recorded_events[i].Type = event;
	sc->recorded_events[i].Number = ++sc->event_number;
	bzero(sc->recorded_events[i].Data, MPR_MAX_EVENT_DATA_LENGTH *
	4);
	event_data_len = event_reply->EventDataLength;

	if (event_data_len > 0) {
	/*
	* Limit data to size in m_event entry
	*/
	if (event_data_len > MPR_MAX_EVENT_DATA_LENGTH) {
	event_data_len = MPR_MAX_EVENT_DATA_LENGTH;
	}
	for (j = 0; j < event_data_len; j++) {
	sc->recorded_events[i].Data[j] =
	event_reply->EventData[j];
	}

	/*
	* check for index wrap-around
	*/
	if (++i == MPR_EVENT_QUEUE_SIZE) {
	i = 0;
	}
	sc->event_index = (uint8_t)i;

	/*
	* Set flag to send the event.
	*/
	sendAEN = TRUE;
	}
	}

	/*
	* Generate a system event if flag is set to let anyone who cares know
	* that an event has occurred.
	*/
	if (sendAEN) {
	//SLM-how to send a system event (see kqueue, kevent)
	// (void) ddi_log_sysevent(mpt->m_dip, DDI_VENDOR_LSI, "MPT_SAS",
	// "SAS", NULL, NULL, DDI_NOSLEEP);
	}
	}

	static int
	mpr_user_reg_access(struct mpr_softc sc, mpr_reg_access_t data)
	{
	int status = 0;

	switch (data->Command) {
	/*
	* IO access is not supported.
	*/
	case REG_IO_READ:
	case REG_IO_WRITE:
	mpr_dprint(sc, MPR_USER, "IO access is not supported. "
	"Use memory access.");
	status = EINVAL;
	break;

	case REG_MEM_READ:
	data->RegData = mpr_regread(sc, data->RegOffset);
	break;

	case REG_MEM_WRITE:
	mpr_regwrite(sc, data->RegOffset, data->RegData);
	break;

	default:
	status = EINVAL;
	break;
	}

	return (status);
	}

	static int
	mpr_user_btdh(struct mpr_softc sc, mpr_btdh_mapping_t data)
	{
	uint8_t bt2dh = FALSE;
	uint8_t dh2bt = FALSE;
	uint16_t dev_handle, bus, target;

	bus = data->Bus;
	target = data->TargetID;
	dev_handle = data->DevHandle;

	/*
	* When DevHandle is 0xFFFF and Bus/Target are not 0xFFFF, use Bus/
	* Target to get DevHandle. When Bus/Target are 0xFFFF and DevHandle is
	* not 0xFFFF, use DevHandle to get Bus/Target. Anything else is
	* invalid.
	*/
	if ((bus == 0xFFFF) && (target == 0xFFFF) && (dev_handle != 0xFFFF))
	dh2bt = TRUE;
	if ((dev_handle == 0xFFFF) && (bus != 0xFFFF) && (target != 0xFFFF))
	bt2dh = TRUE;
	if (!dh2bt && !bt2dh)
	return (EINVAL);

	/*
	* Only handle bus of 0. Make sure target is within range.
	*/
	if (bt2dh) {
	if (bus != 0)
	return (EINVAL);

	if (target > sc->max_devices) {
	mpr_dprint(sc, MPR_XINFO, "Target ID is out of range "
	"for Bus/Target to DevHandle mapping.");
	return (EINVAL);
	}
	dev_handle = sc->mapping_table[target].dev_handle;
	if (dev_handle)
	data->DevHandle = dev_handle;
	} else {
	bus = 0;
	target = mpr_mapping_get_tid_from_handle(sc, dev_handle);
	data->Bus = bus;
	data->TargetID = target;
	}

	return (0);
	}

	static int
	mpr_ioctl(struct cdev dev, u_long cmd, void arg, int flag,
	struct thread *td)
	{
	struct mpr_softc *sc;
	struct mpr_cfg_page_req *page_req;
	struct mpr_ext_cfg_page_req *ext_page_req;
	void *mpr_page;
	int error, msleep_ret;

	mpr_page = NULL;
	sc = dev->si_drv1;
	page_req = (void *)arg;
	ext_page_req = (void *)arg;

	switch (cmd) {
	case MPRIO_READ_CFG_HEADER:
	mpr_lock(sc);
	error = mpr_user_read_cfg_header(sc, page_req);
	mpr_unlock(sc);
	break;
	case MPRIO_READ_CFG_PAGE:
	mpr_page = malloc(page_req->len, M_MPRUSER, M_WAITOK \| M_ZERO);
	error = copyin(page_req->buf, mpr_page,
	sizeof(MPI2_CONFIG_PAGE_HEADER));
	if (error)
	break;
	mpr_lock(sc);
	error = mpr_user_read_cfg_page(sc, page_req, mpr_page);
	mpr_unlock(sc);
	if (error)
	break;
	error = copyout(mpr_page, page_req->buf, page_req->len);
	break;
	case MPRIO_READ_EXT_CFG_HEADER:
	mpr_lock(sc);
	error = mpr_user_read_extcfg_header(sc, ext_page_req);
	mpr_unlock(sc);
	break;
	case MPRIO_READ_EXT_CFG_PAGE:
	mpr_page = malloc(ext_page_req->len, M_MPRUSER,
	M_WAITOK \| M_ZERO);
	error = copyin(ext_page_req->buf, mpr_page,
	sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER));
	if (error)
	break;
	mpr_lock(sc);
	error = mpr_user_read_extcfg_page(sc, ext_page_req, mpr_page);
	mpr_unlock(sc);
	if (error)
	break;
	error = copyout(mpr_page, ext_page_req->buf, ext_page_req->len);
	break;
	case MPRIO_WRITE_CFG_PAGE:
	mpr_page = malloc(page_req->len, M_MPRUSER, M_WAITOK\|M_ZERO);
	error = copyin(page_req->buf, mpr_page, page_req->len);
	if (error)
	break;
	mpr_lock(sc);
	error = mpr_user_write_cfg_page(sc, page_req, mpr_page);
	mpr_unlock(sc);
	break;
	case MPRIO_MPR_COMMAND:
	error = mpr_user_command(sc, (struct mpr_usr_command *)arg);
	break;
	case MPTIOCTL_PASS_THRU:
	/*
	* The user has requested to pass through a command to be
	* executed by the MPT firmware. Call our routine which does
	* this. Only allow one passthru IOCTL at one time.
	*/
	error = mpr_user_pass_thru(sc, (mpr_pass_thru_t *)arg);
	break;
	case MPTIOCTL_GET_ADAPTER_DATA:
	/*
	* The user has requested to read adapter data. Call our
	* routine which does this.
	*/
	error = 0;
	mpr_user_get_adapter_data(sc, (mpr_adapter_data_t *)arg);
	break;
	case MPTIOCTL_GET_PCI_INFO:
	/*
	* The user has requested to read pci info. Call
	* our routine which does this.
	*/
	mpr_lock(sc);
	error = 0;
	mpr_user_read_pci_info(sc, (mpr_pci_info_t *)arg);
	mpr_unlock(sc);
	break;
	case MPTIOCTL_RESET_ADAPTER:
	mpr_lock(sc);
	sc->port_enable_complete = 0;
	uint32_t reinit_start = time_uptime;
	error = mpr_reinit(sc);
	/* Sleep for 300 second. */
	msleep_ret = msleep(&sc->port_enable_complete, &sc->mpr_mtx,
	PRIBIO, "mpr_porten", 300 * hz);
	mpr_unlock(sc);
	if (msleep_ret)
	printf("Port Enable did not complete after Diag "
	"Reset msleep error %d.\n", msleep_ret);
	else
	mpr_dprint(sc, MPR_USER, "Hard Reset with Port Enable "
	"completed in %d seconds.\n",
	(uint32_t)(time_uptime - reinit_start));
	break;
	case MPTIOCTL_DIAG_ACTION:
	/*
	* The user has done a diag buffer action. Call our routine
	* which does this. Only allow one diag action at one time.
	*/
	mpr_lock(sc);
	error = mpr_user_diag_action(sc, (mpr_diag_action_t *)arg);
	mpr_unlock(sc);
	break;
	case MPTIOCTL_EVENT_QUERY:
	/*
	* The user has done an event query. Call our routine which does
	* this.
	*/
	error = 0;
	mpr_user_event_query(sc, (mpr_event_query_t *)arg);
	break;
	case MPTIOCTL_EVENT_ENABLE:
	/*
	* The user has done an event enable. Call our routine which
	* does this.
	*/
	error = 0;
	mpr_user_event_enable(sc, (mpr_event_enable_t *)arg);
	break;
	case MPTIOCTL_EVENT_REPORT:
	/*
	* The user has done an event report. Call our routine which
	* does this.
	*/
	error = mpr_user_event_report(sc, (mpr_event_report_t *)arg);
	break;
	case MPTIOCTL_REG_ACCESS:
	/*
	* The user has requested register access. Call our routine
	* which does this.
	*/
	mpr_lock(sc);
	error = mpr_user_reg_access(sc, (mpr_reg_access_t *)arg);
	mpr_unlock(sc);
	break;
	case MPTIOCTL_BTDH_MAPPING:
	/*
	* The user has requested to translate a bus/target to a
	* DevHandle or a DevHandle to a bus/target. Call our routine
	* which does this.
	*/
	error = mpr_user_btdh(sc, (mpr_btdh_mapping_t *)arg);
	break;
	default:
	error = ENOIOCTL;
	break;
	}

	if (mpr_page != NULL)
	free(mpr_page, M_MPRUSER);

	return (error);
	}

	#ifdef COMPAT_FREEBSD32

	struct mpr_cfg_page_req32 {
	MPI2_CONFIG_PAGE_HEADER header;
	uint32_t page_address;
	uint32_t buf;
	int len;
	uint16_t ioc_status;
	};

	struct mpr_ext_cfg_page_req32 {
	MPI2_CONFIG_EXTENDED_PAGE_HEADER header;
	uint32_t page_address;
	uint32_t buf;
	int len;
	uint16_t ioc_status;
	};

	struct mpr_raid_action32 {
	uint8_t action;
	uint8_t volume_bus;
	uint8_t volume_id;
	uint8_t phys_disk_num;
	uint32_t action_data_word;
	uint32_t buf;
	int len;
	uint32_t volume_status;
	uint32_t action_data[4];
	uint16_t action_status;
	uint16_t ioc_status;
	uint8_t write;
	};

	struct mpr_usr_command32 {
	uint32_t req;
	uint32_t req_len;
	uint32_t rpl;
	uint32_t rpl_len;
	uint32_t buf;
	int len;
	uint32_t flags;
	};

	#define MPRIO_READ_CFG_HEADER32 _IOWR('M', 200, struct mpr_cfg_page_req32)
	#define MPRIO_READ_CFG_PAGE32 _IOWR('M', 201, struct mpr_cfg_page_req32)
	#define MPRIO_READ_EXT_CFG_HEADER32 _IOWR('M', 202, struct mpr_ext_cfg_page_req32)
	#define MPRIO_READ_EXT_CFG_PAGE32 _IOWR('M', 203, struct mpr_ext_cfg_page_req32)
	#define MPRIO_WRITE_CFG_PAGE32 _IOWR('M', 204, struct mpr_cfg_page_req32)
	#define MPRIO_RAID_ACTION32 _IOWR('M', 205, struct mpr_raid_action32)
	#define MPRIO_MPR_COMMAND32 _IOWR('M', 210, struct mpr_usr_command32)

	static int
	mpr_ioctl32(struct cdev dev, u_long cmd32, void _arg, int flag,
	struct thread *td)
	{
	struct mpr_cfg_page_req32 *page32 = _arg;
	struct mpr_ext_cfg_page_req32 *ext32 = _arg;
	struct mpr_raid_action32 *raid32 = _arg;
	struct mpr_usr_command32 *user32 = _arg;
	union {
	struct mpr_cfg_page_req page;
	struct mpr_ext_cfg_page_req ext;
	struct mpr_raid_action raid;
	struct mpr_usr_command user;
	} arg;
	u_long cmd;
	int error;

	switch (cmd32) {
	case MPRIO_READ_CFG_HEADER32:
	case MPRIO_READ_CFG_PAGE32:
	case MPRIO_WRITE_CFG_PAGE32:
	if (cmd32 == MPRIO_READ_CFG_HEADER32)
	cmd = MPRIO_READ_CFG_HEADER;
	else if (cmd32 == MPRIO_READ_CFG_PAGE32)
	cmd = MPRIO_READ_CFG_PAGE;
	else
	cmd = MPRIO_WRITE_CFG_PAGE;
	CP(*page32, arg.page, header);
	CP(*page32, arg.page, page_address);
	PTRIN_CP(*page32, arg.page, buf);
	CP(*page32, arg.page, len);
	CP(*page32, arg.page, ioc_status);
	break;

	case MPRIO_READ_EXT_CFG_HEADER32:
	case MPRIO_READ_EXT_CFG_PAGE32:
	if (cmd32 == MPRIO_READ_EXT_CFG_HEADER32)
	cmd = MPRIO_READ_EXT_CFG_HEADER;
	else
	cmd = MPRIO_READ_EXT_CFG_PAGE;
	CP(*ext32, arg.ext, header);
	CP(*ext32, arg.ext, page_address);
	PTRIN_CP(*ext32, arg.ext, buf);
	CP(*ext32, arg.ext, len);
	CP(*ext32, arg.ext, ioc_status);
	break;

	case MPRIO_RAID_ACTION32:
	cmd = MPRIO_RAID_ACTION;
	CP(*raid32, arg.raid, action);
	CP(*raid32, arg.raid, volume_bus);
	CP(*raid32, arg.raid, volume_id);
	CP(*raid32, arg.raid, phys_disk_num);
	CP(*raid32, arg.raid, action_data_word);
	PTRIN_CP(*raid32, arg.raid, buf);
	CP(*raid32, arg.raid, len);
	CP(*raid32, arg.raid, volume_status);
	bcopy(raid32->action_data, arg.raid.action_data,
	sizeof arg.raid.action_data);
	CP(*raid32, arg.raid, ioc_status);
	CP(*raid32, arg.raid, write);
	break;

	case MPRIO_MPR_COMMAND32:
	cmd = MPRIO_MPR_COMMAND;
	PTRIN_CP(*user32, arg.user, req);
	CP(*user32, arg.user, req_len);
	PTRIN_CP(*user32, arg.user, rpl);
	CP(*user32, arg.user, rpl_len);
	PTRIN_CP(*user32, arg.user, buf);
	CP(*user32, arg.user, len);
	CP(*user32, arg.user, flags);
	break;
	default:
	return (ENOIOCTL);
	}

	error = mpr_ioctl(dev, cmd, &arg, flag, td);
	if (error == 0 && (cmd32 & IOC_OUT) != 0) {
	switch (cmd32) {
	case MPRIO_READ_CFG_HEADER32:
	case MPRIO_READ_CFG_PAGE32:
	case MPRIO_WRITE_CFG_PAGE32:
	CP(arg.page, *page32, header);
	CP(arg.page, *page32, page_address);
	PTROUT_CP(arg.page, *page32, buf);
	CP(arg.page, *page32, len);
	CP(arg.page, *page32, ioc_status);
	break;

	case MPRIO_READ_EXT_CFG_HEADER32:
	case MPRIO_READ_EXT_CFG_PAGE32:
	CP(arg.ext, *ext32, header);
	CP(arg.ext, *ext32, page_address);
	PTROUT_CP(arg.ext, *ext32, buf);
	CP(arg.ext, *ext32, len);
	CP(arg.ext, *ext32, ioc_status);
	break;

	case MPRIO_RAID_ACTION32:
	CP(arg.raid, *raid32, action);
	CP(arg.raid, *raid32, volume_bus);
	CP(arg.raid, *raid32, volume_id);
	CP(arg.raid, *raid32, phys_disk_num);
	CP(arg.raid, *raid32, action_data_word);
	PTROUT_CP(arg.raid, *raid32, buf);
	CP(arg.raid, *raid32, len);
	CP(arg.raid, *raid32, volume_status);
	bcopy(arg.raid.action_data, raid32->action_data,
	sizeof arg.raid.action_data);
	CP(arg.raid, *raid32, ioc_status);
	CP(arg.raid, *raid32, write);
	break;

	case MPRIO_MPR_COMMAND32:
	PTROUT_CP(arg.user, *user32, req);
	CP(arg.user, *user32, req_len);
	PTROUT_CP(arg.user, *user32, rpl);
	CP(arg.user, *user32, rpl_len);
	PTROUT_CP(arg.user, *user32, buf);
	CP(arg.user, *user32, len);
	CP(arg.user, *user32, flags);
	break;
	}
	}

	return (error);
	}
	#endif /* COMPAT_FREEBSD32 */

	static int
	mpr_ioctl_devsw(struct cdev *dev, u_long com, caddr_t arg, int flag,
	struct thread *td)
	{
	#ifdef COMPAT_FREEBSD32
	if (SV_CURPROC_FLAG(SV_ILP32))
	return (mpr_ioctl32(dev, com, arg, flag, td));
	#endif
	return (mpr_ioctl(dev, com, arg, flag, td));
	}
	diff --git a/sys/dev/mps/mps.c b/sys/dev/mps/mps.c
	index 9802296f06f8..1adbf58c9e17 100644
	--- a/sys/dev/mps/mps.c
	+++ b/sys/dev/mps/mps.c
	@@ -1,3281 +1,3273 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2009 Yahoo! Inc.
	* Copyright (c) 2011-2015 LSI Corp.
	* Copyright (c) 2013-2015 Avago Technologies
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/* Communications core for Avago Technologies (LSI) MPT2 */

	/* TODO Move headers to mpsvar */
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/selinfo.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/bio.h>
	#include <sys/malloc.h>
	#include <sys/uio.h>
	#include <sys/sysctl.h>
	#include <sys/smp.h>
	#include <sys/queue.h>
	#include <sys/kthread.h>
	#include <sys/taskqueue.h>
	#include <sys/endian.h>
	#include <sys/eventhandler.h>
	#include <sys/sbuf.h>
	#include <sys/priv.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>
	#include <sys/proc.h>

	#include <dev/pci/pcivar.h>

	#include <cam/cam.h>
	#include <cam/scsi/scsi_all.h>

	#include <dev/mps/mpi/mpi2_type.h>
	#include <dev/mps/mpi/mpi2.h>
	#include <dev/mps/mpi/mpi2_ioc.h>
	#include <dev/mps/mpi/mpi2_sas.h>
	#include <dev/mps/mpi/mpi2_cnfg.h>
	#include <dev/mps/mpi/mpi2_init.h>
	#include <dev/mps/mpi/mpi2_tool.h>
	#include <dev/mps/mps_ioctl.h>
	#include <dev/mps/mpsvar.h>
	#include <dev/mps/mps_table.h>

	static int mps_diag_reset(struct mps_softc *sc, int sleep_flag);
	static int mps_init_queues(struct mps_softc *sc);
	static void mps_resize_queues(struct mps_softc *sc);
	static int mps_message_unit_reset(struct mps_softc *sc, int sleep_flag);
	static int mps_transition_operational(struct mps_softc *sc);
	static int mps_iocfacts_allocate(struct mps_softc *sc, uint8_t attaching);
	static void mps_iocfacts_free(struct mps_softc *sc);
	static void mps_startup(void *arg);
	static int mps_send_iocinit(struct mps_softc *sc);
	static int mps_alloc_queues(struct mps_softc *sc);
	static int mps_alloc_hw_queues(struct mps_softc *sc);
	static int mps_alloc_replies(struct mps_softc *sc);
	static int mps_alloc_requests(struct mps_softc *sc);
	static int mps_attach_log(struct mps_softc *sc);
	static __inline void mps_complete_command(struct mps_softc *sc,
	struct mps_command *cm);
	static void mps_dispatch_event(struct mps_softc *sc, uintptr_t data,
	MPI2_EVENT_NOTIFICATION_REPLY *reply);
	static void mps_config_complete(struct mps_softc sc, struct mps_command cm);
	static void mps_periodic(void *);
	static int mps_reregister_events(struct mps_softc *sc);
	static void mps_enqueue_request(struct mps_softc sc, struct mps_command cm);
	static int mps_get_iocfacts(struct mps_softc sc, MPI2_IOC_FACTS_REPLY facts);
	static int mps_wait_db_ack(struct mps_softc *sc, int timeout, int sleep_flag);
	static int mps_debug_sysctl(SYSCTL_HANDLER_ARGS);
	static int mps_dump_reqs(SYSCTL_HANDLER_ARGS);
	static void mps_parse_debug(struct mps_softc sc, char list);

	SYSCTL_NODE(_hw, OID_AUTO, mps, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"MPS Driver Parameters");

	MALLOC_DEFINE(M_MPT2, "mps", "mpt2 driver memory");
	MALLOC_DECLARE(M_MPSUSER);

	/*
	* Do a "Diagnostic Reset" aka a hard reset. This should get the chip out of
	* any state and back to its initialization state machine.
	*/
	static char mpt2_reset_magic[] = { 0x00, 0x0f, 0x04, 0x0b, 0x02, 0x07, 0x0d };

	/* Added this union to smoothly convert le64toh cm->cm_desc.Words.
	* Compiler only support unint64_t to be passed as argument.
	* Otherwise it will throw below error
	* "aggregate value used where an integer was expected"
	*/

	typedef union _reply_descriptor {
	u64 word;
	struct {
	u32 low;
	u32 high;
	} u;
	}reply_descriptor,address_descriptor;

	/* Rate limit chain-fail messages to 1 per minute */
	static struct timeval mps_chainfail_interval = { 60, 0 };

	/*
	* sleep_flag can be either CAN_SLEEP or NO_SLEEP.
	* If this function is called from process context, it can sleep
	* and there is no harm to sleep, in case if this fuction is called
	* from Interrupt handler, we can not sleep and need NO_SLEEP flag set.
	* based on sleep flags driver will call either msleep, pause or DELAY.
	* msleep and pause are of same variant, but pause is used when mps_mtx
	* is not hold by driver.
	*
	*/
	static int
	mps_diag_reset(struct mps_softc *sc,int sleep_flag)
	{
	uint32_t reg;
	int i, error, tries = 0;
	uint8_t first_wait_done = FALSE;

	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	/* Clear any pending interrupts */
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	/*
	* Force NO_SLEEP for threads prohibited to sleep
	* e.a Thread from interrupt handler are prohibited to sleep.
	*/
	if (curthread->td_no_sleeping != 0)
	sleep_flag = NO_SLEEP;

	mps_dprint(sc, MPS_INIT, "sequence start, sleep_flag= %d\n", sleep_flag);

	/* Push the magic sequence */
	error = ETIMEDOUT;
	while (tries++ < 20) {
	for (i = 0; i < sizeof(mpt2_reset_magic); i++)
	mps_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET,
	mpt2_reset_magic[i]);
	/* wait 100 msec */
	if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP)
	msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0,
	"mpsdiag", hz/10);
	else if (sleep_flag == CAN_SLEEP)
	pause("mpsdiag", hz/10);
	else
	DELAY(100 * 1000);

	reg = mps_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET);
	if (reg & MPI2_DIAG_DIAG_WRITE_ENABLE) {
	error = 0;
	break;
	}
	}
	if (error) {
	mps_dprint(sc, MPS_INIT, "sequence failed, error=%d, exit\n",
	error);
	return (error);
	}

	/* Send the actual reset. XXX need to refresh the reg? */
	reg \|= MPI2_DIAG_RESET_ADAPTER;
	mps_dprint(sc, MPS_INIT, "sequence success, sending reset, reg= 0x%x\n",
	reg);
	mps_regwrite(sc, MPI2_HOST_DIAGNOSTIC_OFFSET, reg);

	/* Wait up to 300 seconds in 50ms intervals */
	error = ETIMEDOUT;
	for (i = 0; i < 6000; i++) {
	/*
	* Wait 50 msec. If this is the first time through, wait 256
	* msec to satisfy Diag Reset timing requirements.
	*/
	if (first_wait_done) {
	if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP)
	msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0,
	"mpsdiag", hz/20);
	else if (sleep_flag == CAN_SLEEP)
	pause("mpsdiag", hz/20);
	else
	DELAY(50 * 1000);
	} else {
	DELAY(256 * 1000);
	first_wait_done = TRUE;
	}
	/*
	* Check for the RESET_ADAPTER bit to be cleared first, then
	* wait for the RESET state to be cleared, which takes a little
	* longer.
	*/
	reg = mps_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET);
	if (reg & MPI2_DIAG_RESET_ADAPTER) {
	continue;
	}
	reg = mps_regread(sc, MPI2_DOORBELL_OFFSET);
	if ((reg & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_RESET) {
	error = 0;
	break;
	}
	}
	if (error) {
	mps_dprint(sc, MPS_INIT, "reset failed, error= %d, exit\n",
	error);
	return (error);
	}

	mps_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET, 0x0);
	mps_dprint(sc, MPS_INIT, "diag reset success, exit\n");

	return (0);
	}

	static int
	mps_message_unit_reset(struct mps_softc *sc, int sleep_flag)
	{
	int error;

	MPS_FUNCTRACE(sc);

	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	error = 0;
	mps_regwrite(sc, MPI2_DOORBELL_OFFSET,
	MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET <<
	MPI2_DOORBELL_FUNCTION_SHIFT);

	if (mps_wait_db_ack(sc, 5, sleep_flag) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT,
	"Doorbell handshake failed\n");
	error = ETIMEDOUT;
	}

	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);
	return (error);
	}

	static int
	mps_transition_ready(struct mps_softc *sc)
	{
	uint32_t reg, state;
	int error, tries = 0;
	int sleep_flags;

	MPS_FUNCTRACE(sc);
	/* If we are in attach call, do not sleep */
	sleep_flags = (sc->mps_flags & MPS_FLAGS_ATTACH_DONE)
	? CAN_SLEEP:NO_SLEEP;
	error = 0;

	mps_dprint(sc, MPS_INIT, "%s entered, sleep_flags= %d\n",
	__func__, sleep_flags);

	while (tries++ < 1200) {
	reg = mps_regread(sc, MPI2_DOORBELL_OFFSET);
	mps_dprint(sc, MPS_INIT, " Doorbell= 0x%x\n", reg);

	/*
	* Ensure the IOC is ready to talk. If it's not, try
	* resetting it.
	*/
	if (reg & MPI2_DOORBELL_USED) {
	mps_dprint(sc, MPS_INIT, " Not ready, sending diag "
	"reset\n");
	mps_diag_reset(sc, sleep_flags);
	DELAY(50000);
	continue;
	}

	/* Is the adapter owned by another peer? */
	if ((reg & MPI2_DOORBELL_WHO_INIT_MASK) ==
	(MPI2_WHOINIT_PCI_PEER << MPI2_DOORBELL_WHO_INIT_SHIFT)) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "IOC is under the "
	"control of another peer host, aborting "
	"initialization.\n");
	error = ENXIO;
	break;
	}

	state = reg & MPI2_IOC_STATE_MASK;
	if (state == MPI2_IOC_STATE_READY) {
	/* Ready to go! */
	error = 0;
	break;
	} else if (state == MPI2_IOC_STATE_FAULT) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "IOC in fault "
	"state 0x%x, resetting\n",
	state & MPI2_DOORBELL_FAULT_CODE_MASK);
	mps_diag_reset(sc, sleep_flags);
	} else if (state == MPI2_IOC_STATE_OPERATIONAL) {
	/* Need to take ownership */
	mps_message_unit_reset(sc, sleep_flags);
	} else if (state == MPI2_IOC_STATE_RESET) {
	/* Wait a bit, IOC might be in transition */
	mps_dprint(sc, MPS_INIT\|MPS_FAULT,
	"IOC in unexpected reset state\n");
	} else {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT,
	"IOC in unknown state 0x%x\n", state);
	error = EINVAL;
	break;
	}

	/* Wait 50ms for things to settle down. */
	DELAY(50000);
	}

	if (error)
	mps_dprint(sc, MPS_INIT\|MPS_FAULT,
	"Cannot transition IOC to ready\n");
	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);

	return (error);
	}

	static int
	mps_transition_operational(struct mps_softc *sc)
	{
	uint32_t reg, state;
	int error;

	MPS_FUNCTRACE(sc);

	error = 0;
	reg = mps_regread(sc, MPI2_DOORBELL_OFFSET);
	mps_dprint(sc, MPS_INIT, "%s entered, Doorbell= 0x%x\n", __func__, reg);

	state = reg & MPI2_IOC_STATE_MASK;
	if (state != MPI2_IOC_STATE_READY) {
	mps_dprint(sc, MPS_INIT, "IOC not ready\n");
	if ((error = mps_transition_ready(sc)) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT,
	"failed to transition ready, exit\n");
	return (error);
	}
	}

	error = mps_send_iocinit(sc);
	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);

	return (error);
	}

	static void
	mps_resize_queues(struct mps_softc *sc)
	{
	u_int reqcr, prireqcr, maxio, sges_per_frame;

	/*
	* Size the queues. Since the reply queues always need one free
	* entry, we'll deduct one reply message here. The LSI documents
	* suggest instead to add a count to the request queue, but I think
	* that it's better to deduct from reply queue.
	*/
	prireqcr = MAX(1, sc->max_prireqframes);
	prireqcr = MIN(prireqcr, sc->facts->HighPriorityCredit);

	reqcr = MAX(2, sc->max_reqframes);
	reqcr = MIN(reqcr, sc->facts->RequestCredit);

	sc->num_reqs = prireqcr + reqcr;
	sc->num_prireqs = prireqcr;
	sc->num_replies = MIN(sc->max_replyframes + sc->max_evtframes,
	sc->facts->MaxReplyDescriptorPostQueueDepth) - 1;

	/* Store the request frame size in bytes rather than as 32bit words */
	sc->reqframesz = sc->facts->IOCRequestFrameSize * 4;

	/*
	* Max IO Size is Page Size * the following:
	* ((SGEs per frame - 1 for chain element) * Max Chain Depth)
	* + 1 for no chain needed in last frame
	*
	* If user suggests a Max IO size to use, use the smaller of the
	* user's value and the calculated value as long as the user's
	* value is larger than 0. The user's value is in pages.
	*/
	sges_per_frame = sc->reqframesz / sizeof(MPI2_SGE_SIMPLE64) - 1;
	maxio = (sges_per_frame * sc->facts->MaxChainDepth + 1) * PAGE_SIZE;

	/*
	* If I/O size limitation requested, then use it and pass up to CAM.
	* If not, use MAXPHYS as an optimization hint, but report HW limit.
	*/
	if (sc->max_io_pages > 0) {
	maxio = min(maxio, sc->max_io_pages * PAGE_SIZE);
	sc->maxio = maxio;
	} else {
	sc->maxio = maxio;
	maxio = min(maxio, MAXPHYS);
	}

	sc->num_chains = (maxio / PAGE_SIZE + sges_per_frame - 2) /
	sges_per_frame * reqcr;
	if (sc->max_chains > 0 && sc->max_chains < sc->num_chains)
	sc->num_chains = sc->max_chains;

	/*
	* Figure out the number of MSIx-based queues. If the firmware or
	* user has done something crazy and not allowed enough credit for
	* the queues to be useful then don't enable multi-queue.
	*/
	if (sc->facts->MaxMSIxVectors < 2)
	sc->msi_msgs = 1;

	if (sc->msi_msgs > 1) {
	sc->msi_msgs = MIN(sc->msi_msgs, mp_ncpus);
	sc->msi_msgs = MIN(sc->msi_msgs, sc->facts->MaxMSIxVectors);
	if (sc->num_reqs / sc->msi_msgs < 2)
	sc->msi_msgs = 1;
	}

	mps_dprint(sc, MPS_INIT, "Sized queues to q=%d reqs=%d replies=%d\n",
	sc->msi_msgs, sc->num_reqs, sc->num_replies);
	}

	/*
	* This is called during attach and when re-initializing due to a Diag Reset.
	* IOC Facts is used to allocate many of the structures needed by the driver.
	* If called from attach, de-allocation is not required because the driver has
	* not allocated any structures yet, but if called from a Diag Reset, previously
	* allocated structures based on IOC Facts will need to be freed and re-
	* allocated bases on the latest IOC Facts.
	*/
	static int
	mps_iocfacts_allocate(struct mps_softc *sc, uint8_t attaching)
	{
	int error;
	Mpi2IOCFactsReply_t saved_facts;
	uint8_t saved_mode, reallocating;

	mps_dprint(sc, MPS_INIT\|MPS_TRACE, "%s entered\n", __func__);

	/* Save old IOC Facts and then only reallocate if Facts have changed */
	if (!attaching) {
	bcopy(sc->facts, &saved_facts, sizeof(MPI2_IOC_FACTS_REPLY));
	}

	/*
	* Get IOC Facts. In all cases throughout this function, panic if doing
	* a re-initialization and only return the error if attaching so the OS
	* can handle it.
	*/
	if ((error = mps_get_iocfacts(sc, sc->facts)) != 0) {
	if (attaching) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "Failed to get "
	"IOC Facts with error %d, exit\n", error);
	return (error);
	} else {
	panic("%s failed to get IOC Facts with error %d\n",
	__func__, error);
	}
	}

	MPS_DPRINT_PAGE(sc, MPS_XINFO, iocfacts, sc->facts);

	snprintf(sc->fw_version, sizeof(sc->fw_version),
	"%02d.%02d.%02d.%02d",
	sc->facts->FWVersion.Struct.Major,
	sc->facts->FWVersion.Struct.Minor,
	sc->facts->FWVersion.Struct.Unit,
	sc->facts->FWVersion.Struct.Dev);

	snprintf(sc->msg_version, sizeof(sc->msg_version), "%d.%d",
	(sc->facts->MsgVersion & MPI2_IOCFACTS_MSGVERSION_MAJOR_MASK) >>
	MPI2_IOCFACTS_MSGVERSION_MAJOR_SHIFT,
	(sc->facts->MsgVersion & MPI2_IOCFACTS_MSGVERSION_MINOR_MASK) >>
	MPI2_IOCFACTS_MSGVERSION_MINOR_SHIFT);

	mps_dprint(sc, MPS_INFO, "Firmware: %s, Driver: %s\n", sc->fw_version,
	MPS_DRIVER_VERSION);
	mps_dprint(sc, MPS_INFO, "IOCCapabilities: %b\n",
	sc->facts->IOCCapabilities,
	"\20" "\3ScsiTaskFull" "\4DiagTrace" "\5SnapBuf" "\6ExtBuf"
	"\7EEDP" "\10BiDirTarg" "\11Multicast" "\14TransRetry" "\15IR"
	"\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc");

	/*
	* If the chip doesn't support event replay then a hard reset will be
	* required to trigger a full discovery. Do the reset here then
	* retransition to Ready. A hard reset might have already been done,
	* but it doesn't hurt to do it again. Only do this if attaching, not
	* for a Diag Reset.
	*/
	if (attaching && ((sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY) == 0)) {
	mps_dprint(sc, MPS_INIT, "No event replay, reseting\n");
	mps_diag_reset(sc, NO_SLEEP);
	if ((error = mps_transition_ready(sc)) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "Failed to "
	"transition to ready with error %d, exit\n",
	error);
	return (error);
	}
	}

	/*
	* Set flag if IR Firmware is loaded. If the RAID Capability has
	* changed from the previous IOC Facts, log a warning, but only if
	* checking this after a Diag Reset and not during attach.
	*/
	saved_mode = sc->ir_firmware;
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID)
	sc->ir_firmware = 1;
	if (!attaching) {
	if (sc->ir_firmware != saved_mode) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "new IR/IT mode "
	"in IOC Facts does not match previous mode\n");
	}
	}

	/* Only deallocate and reallocate if relevant IOC Facts have changed */
	reallocating = FALSE;
	sc->mps_flags &= ~MPS_FLAGS_REALLOCATED;

	if ((!attaching) &&
	((saved_facts.MsgVersion != sc->facts->MsgVersion) \|\|
	(saved_facts.HeaderVersion != sc->facts->HeaderVersion) \|\|
	(saved_facts.MaxChainDepth != sc->facts->MaxChainDepth) \|\|
	(saved_facts.RequestCredit != sc->facts->RequestCredit) \|\|
	(saved_facts.ProductID != sc->facts->ProductID) \|\|
	(saved_facts.IOCCapabilities != sc->facts->IOCCapabilities) \|\|
	(saved_facts.IOCRequestFrameSize !=
	sc->facts->IOCRequestFrameSize) \|\|
	(saved_facts.MaxTargets != sc->facts->MaxTargets) \|\|
	(saved_facts.MaxSasExpanders != sc->facts->MaxSasExpanders) \|\|
	(saved_facts.MaxEnclosures != sc->facts->MaxEnclosures) \|\|
	(saved_facts.HighPriorityCredit != sc->facts->HighPriorityCredit) \|\|
	(saved_facts.MaxReplyDescriptorPostQueueDepth !=
	sc->facts->MaxReplyDescriptorPostQueueDepth) \|\|
	(saved_facts.ReplyFrameSize != sc->facts->ReplyFrameSize) \|\|
	(saved_facts.MaxVolumes != sc->facts->MaxVolumes) \|\|
	(saved_facts.MaxPersistentEntries !=
	sc->facts->MaxPersistentEntries))) {
	reallocating = TRUE;

	/* Record that we reallocated everything */
	sc->mps_flags \|= MPS_FLAGS_REALLOCATED;
	}

	/*
	* Some things should be done if attaching or re-allocating after a Diag
	* Reset, but are not needed after a Diag Reset if the FW has not
	* changed.
	*/
	if (attaching \|\| reallocating) {
	/*
	* Check if controller supports FW diag buffers and set flag to
	* enable each type.
	*/
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER)
	sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_TRACE].
	enabled = TRUE;
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER)
	sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_SNAPSHOT].
	enabled = TRUE;
	if (sc->facts->IOCCapabilities &
	MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER)
	sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_EXTENDED].
	enabled = TRUE;

	/*
	* Set flag if EEDP is supported and if TLR is supported.
	*/
	if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP)
	sc->eedp_enabled = TRUE;
	if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR)
	sc->control_TLR = TRUE;

	mps_resize_queues(sc);

	/*
	* Initialize all Tail Queues
	*/
	TAILQ_INIT(&sc->req_list);
	TAILQ_INIT(&sc->high_priority_req_list);
	TAILQ_INIT(&sc->chain_list);
	TAILQ_INIT(&sc->tm_list);
	}

	/*
	* If doing a Diag Reset and the FW is significantly different
	* (reallocating will be set above in IOC Facts comparison), then all
	* buffers based on the IOC Facts will need to be freed before they are
	* reallocated.
	*/
	if (reallocating) {
	mps_iocfacts_free(sc);
	mpssas_realloc_targets(sc, saved_facts.MaxTargets +
	saved_facts.MaxVolumes);
	}

	/*
	* Any deallocation has been completed. Now start reallocating
	* if needed. Will only need to reallocate if attaching or if the new
	* IOC Facts are different from the previous IOC Facts after a Diag
	* Reset. Targets have already been allocated above if needed.
	*/
	error = 0;
	while (attaching \|\| reallocating) {
	if ((error = mps_alloc_hw_queues(sc)) != 0)
	break;
	if ((error = mps_alloc_replies(sc)) != 0)
	break;
	if ((error = mps_alloc_requests(sc)) != 0)
	break;
	if ((error = mps_alloc_queues(sc)) != 0)
	break;

	break;
	}
	if (error) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT,
	"Failed to alloc queues with error %d\n", error);
	mps_free(sc);
	return (error);
	}

	/* Always initialize the queues */
	bzero(sc->free_queue, sc->fqdepth * 4);
	mps_init_queues(sc);

	/*
	* Always get the chip out of the reset state, but only panic if not
	* attaching. If attaching and there is an error, that is handled by
	* the OS.
	*/
	error = mps_transition_operational(sc);
	if (error != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "Failed to "
	"transition to operational with error %d\n", error);
	mps_free(sc);
	return (error);
	}

	/*
	* Finish the queue initialization.
	* These are set here instead of in mps_init_queues() because the
	* IOC resets these values during the state transition in
	* mps_transition_operational(). The free index is set to 1
	* because the corresponding index in the IOC is set to 0, and the
	* IOC treats the queues as full if both are set to the same value.
	* Hence the reason that the queue can't hold all of the possible
	* replies.
	*/
	sc->replypostindex = 0;
	mps_regwrite(sc, MPI2_REPLY_FREE_HOST_INDEX_OFFSET, sc->replyfreeindex);
	mps_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET, 0);

	/*
	* Attach the subsystems so they can prepare their event masks.
	* XXX Should be dynamic so that IM/IR and user modules can attach
	*/
	error = 0;
	while (attaching) {
	mps_dprint(sc, MPS_INIT, "Attaching subsystems\n");
	if ((error = mps_attach_log(sc)) != 0)
	break;
	if ((error = mps_attach_sas(sc)) != 0)
	break;
	if ((error = mps_attach_user(sc)) != 0)
	break;
	break;
	}
	if (error) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "Failed to attach all "
	"subsystems: error %d\n", error);
	mps_free(sc);
	return (error);
	}

	/*
	* XXX If the number of MSI-X vectors changes during re-init, this
	* won't see it and adjust.
	*/
	if (attaching && (error = mps_pci_setup_interrupts(sc)) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "Failed to setup "
	"interrupts\n");
	mps_free(sc);
	return (error);
	}

	/*
	* Set flag if this is a WD controller. This shouldn't ever change, but
	* reset it after a Diag Reset, just in case.
	*/
	sc->WD_available = FALSE;
	if (pci_get_device(sc->mps_dev) == MPI2_MFGPAGE_DEVID_SSS6200)
	sc->WD_available = TRUE;

	return (error);
	}

	/*
	* This is called if memory is being free (during detach for example) and when
	* buffers need to be reallocated due to a Diag Reset.
	*/
	static void
	mps_iocfacts_free(struct mps_softc *sc)
	{
	struct mps_command *cm;
	int i;

	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);

	if (sc->free_busaddr != 0)
	bus_dmamap_unload(sc->queues_dmat, sc->queues_map);
	if (sc->free_queue != NULL)
	bus_dmamem_free(sc->queues_dmat, sc->free_queue,
	sc->queues_map);
	if (sc->queues_dmat != NULL)
	bus_dma_tag_destroy(sc->queues_dmat);

	if (sc->chain_frames != NULL) {
	bus_dmamap_unload(sc->chain_dmat, sc->chain_map);
	bus_dmamem_free(sc->chain_dmat, sc->chain_frames,
	sc->chain_map);
	}
	if (sc->chain_dmat != NULL)
	bus_dma_tag_destroy(sc->chain_dmat);

	if (sc->sense_busaddr != 0)
	bus_dmamap_unload(sc->sense_dmat, sc->sense_map);
	if (sc->sense_frames != NULL)
	bus_dmamem_free(sc->sense_dmat, sc->sense_frames,
	sc->sense_map);
	if (sc->sense_dmat != NULL)
	bus_dma_tag_destroy(sc->sense_dmat);

	if (sc->reply_busaddr != 0)
	bus_dmamap_unload(sc->reply_dmat, sc->reply_map);
	if (sc->reply_frames != NULL)
	bus_dmamem_free(sc->reply_dmat, sc->reply_frames,
	sc->reply_map);
	if (sc->reply_dmat != NULL)
	bus_dma_tag_destroy(sc->reply_dmat);

	if (sc->req_busaddr != 0)
	bus_dmamap_unload(sc->req_dmat, sc->req_map);
	if (sc->req_frames != NULL)
	bus_dmamem_free(sc->req_dmat, sc->req_frames, sc->req_map);
	if (sc->req_dmat != NULL)
	bus_dma_tag_destroy(sc->req_dmat);

	if (sc->chains != NULL)
	free(sc->chains, M_MPT2);
	if (sc->commands != NULL) {
	for (i = 1; i < sc->num_reqs; i++) {
	cm = &sc->commands[i];
	bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap);
	}
	free(sc->commands, M_MPT2);
	}
	if (sc->buffer_dmat != NULL)
	bus_dma_tag_destroy(sc->buffer_dmat);

	mps_pci_free_interrupts(sc);
	free(sc->queues, M_MPT2);
	sc->queues = NULL;
	}

	/*
	* The terms diag reset and hard reset are used interchangeably in the MPI
	* docs to mean resetting the controller chip. In this code diag reset
	* cleans everything up, and the hard reset function just sends the reset
	* sequence to the chip. This should probably be refactored so that every
	* subsystem gets a reset notification of some sort, and can clean up
	* appropriately.
	*/
	int
	mps_reinit(struct mps_softc *sc)
	{
	int error;
	struct mpssas_softc *sassc;

	sassc = sc->sassc;

	MPS_FUNCTRACE(sc);

	mtx_assert(&sc->mps_mtx, MA_OWNED);

	mps_dprint(sc, MPS_INIT\|MPS_INFO, "Reinitializing controller\n");
	if (sc->mps_flags & MPS_FLAGS_DIAGRESET) {
	mps_dprint(sc, MPS_INIT, "Reset already in progress\n");
	return 0;
	}

	/* make sure the completion callbacks can recognize they're getting
	* a NULL cm_reply due to a reset.
	*/
	sc->mps_flags \|= MPS_FLAGS_DIAGRESET;

	/*
	* Mask interrupts here.
	*/
	mps_dprint(sc, MPS_INIT, "masking interrupts and resetting\n");
	mps_mask_intr(sc);

	error = mps_diag_reset(sc, CAN_SLEEP);
	if (error != 0) {
	/* XXXSL No need to panic here */
	panic("%s hard reset failed with error %d\n",
	__func__, error);
	}

	/* Restore the PCI state, including the MSI-X registers */
	mps_pci_restore(sc);

	/* Give the I/O subsystem special priority to get itself prepared */
	mpssas_handle_reinit(sc);

	/*
	* Get IOC Facts and allocate all structures based on this information.
	* The attach function will also call mps_iocfacts_allocate at startup.
	* If relevant values have changed in IOC Facts, this function will free
	* all of the memory based on IOC Facts and reallocate that memory.
	*/
	if ((error = mps_iocfacts_allocate(sc, FALSE)) != 0) {
	panic("%s IOC Facts based allocation failed with error %d\n",
	__func__, error);
	}

	/*
	* Mapping structures will be re-allocated after getting IOC Page8, so
	* free these structures here.
	*/
	mps_mapping_exit(sc);

	/*
	* The static page function currently read is IOC Page8. Others can be
	* added in future. It's possible that the values in IOC Page8 have
	* changed after a Diag Reset due to user modification, so always read
	* these. Interrupts are masked, so unmask them before getting config
	* pages.
	*/
	mps_unmask_intr(sc);
	sc->mps_flags &= ~MPS_FLAGS_DIAGRESET;
	mps_base_static_config_pages(sc);

	/*
	* Some mapping info is based in IOC Page8 data, so re-initialize the
	* mapping tables.
	*/
	mps_mapping_initialize(sc);

	/*
	* Restart will reload the event masks clobbered by the reset, and
	* then enable the port.
	*/
	mps_reregister_events(sc);

	/* the end of discovery will release the simq, so we're done. */
	mps_dprint(sc, MPS_INIT\|MPS_XINFO, "Finished sc %p post %u free %u\n",
	sc, sc->replypostindex, sc->replyfreeindex);

	mpssas_release_simq_reinit(sassc);
	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);

	return 0;
	}

	/* Wait for the chip to ACK a word that we've put into its FIFO
	* Wait for <timeout> seconds. In single loop wait for busy loop
	* for 500 microseconds.
	* Total is [ 0.5 * (2000 * <timeout>) ] in miliseconds.
	* */
	static int
	mps_wait_db_ack(struct mps_softc *sc, int timeout, int sleep_flag)
	{

	u32 cntdn, count;
	u32 int_status;
	u32 doorbell;

	count = 0;
	cntdn = (sleep_flag == CAN_SLEEP) ? 1000timeout : 2000timeout;
	do {
	int_status = mps_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);
	if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
	mps_dprint(sc, MPS_TRACE,
	"%s: successful count(%d), timeout(%d)\n",
	__func__, count, timeout);
	return 0;
	} else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
	doorbell = mps_regread(sc, MPI2_DOORBELL_OFFSET);
	if ((doorbell & MPI2_IOC_STATE_MASK) ==
	MPI2_IOC_STATE_FAULT) {
	mps_dprint(sc, MPS_FAULT,
	"fault_state(0x%04x)!\n", doorbell);
	return (EFAULT);
	}
	} else if (int_status == 0xFFFFFFFF)
	goto out;

	/* If it can sleep, sleep for 1 milisecond, else busy loop for
	* 0.5 milisecond */
	if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP)
	msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0,
	"mpsdba", hz/1000);
	else if (sleep_flag == CAN_SLEEP)
	pause("mpsdba", hz/1000);
	else
	DELAY(500);
	count++;
	} while (--cntdn);

	out:
	mps_dprint(sc, MPS_FAULT, "%s: failed due to timeout count(%d), "
	"int_status(%x)!\n", __func__, count, int_status);
	return (ETIMEDOUT);

	}

	/* Wait for the chip to signal that the next word in its FIFO can be fetched */
	static int
	mps_wait_db_int(struct mps_softc *sc)
	{
	int retry;

	for (retry = 0; retry < MPS_DB_MAX_WAIT; retry++) {
	if ((mps_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET) &
	MPI2_HIS_IOC2SYS_DB_STATUS) != 0)
	return (0);
	DELAY(2000);
	}
	return (ETIMEDOUT);
	}

	/* Step through the synchronous command state machine, i.e. "Doorbell mode" */
	static int
	mps_request_sync(struct mps_softc sc, void req, MPI2_DEFAULT_REPLY *reply,
	int req_sz, int reply_sz, int timeout)
	{
	uint32_t *data32;
	uint16_t *data16;
	int i, count, ioc_sz, residual;
	int sleep_flags = CAN_SLEEP;

	if (curthread->td_no_sleeping != 0)
	sleep_flags = NO_SLEEP;

	/* Step 1 */
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	/* Step 2 */
	if (mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED)
	return (EBUSY);

	/* Step 3
	* Announce that a message is coming through the doorbell. Messages
	* are pushed at 32bit words, so round up if needed.
	*/
	count = (req_sz + 3) / 4;
	mps_regwrite(sc, MPI2_DOORBELL_OFFSET,
	(MPI2_FUNCTION_HANDSHAKE << MPI2_DOORBELL_FUNCTION_SHIFT) \|
	(count << MPI2_DOORBELL_ADD_DWORDS_SHIFT));

	/* Step 4 */
	if (mps_wait_db_int(sc) \|\|
	(mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED) == 0) {
	mps_dprint(sc, MPS_FAULT, "Doorbell failed to activate\n");
	return (ENXIO);
	}
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	if (mps_wait_db_ack(sc, 5, sleep_flags) != 0) {
	mps_dprint(sc, MPS_FAULT, "Doorbell handshake failed\n");
	return (ENXIO);
	}

	/* Step 5 */
	/* Clock out the message data synchronously in 32-bit dwords*/
	data32 = (uint32_t *)req;
	for (i = 0; i < count; i++) {
	mps_regwrite(sc, MPI2_DOORBELL_OFFSET, htole32(data32[i]));
	if (mps_wait_db_ack(sc, 5, sleep_flags) != 0) {
	mps_dprint(sc, MPS_FAULT,
	"Timeout while writing doorbell\n");
	return (ENXIO);
	}
	}

	/* Step 6 */
	/* Clock in the reply in 16-bit words. The total length of the
	* message is always in the 4th byte, so clock out the first 2 words
	* manually, then loop the rest.
	*/
	data16 = (uint16_t *)reply;
	if (mps_wait_db_int(sc) != 0) {
	mps_dprint(sc, MPS_FAULT, "Timeout reading doorbell 0\n");
	return (ENXIO);
	}
	data16[0] =
	mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK;
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	if (mps_wait_db_int(sc) != 0) {
	mps_dprint(sc, MPS_FAULT, "Timeout reading doorbell 1\n");
	return (ENXIO);
	}
	data16[1] =
	mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK;
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	/* Number of 32bit words in the message */
	ioc_sz = reply->MsgLength;

	/*
	* Figure out how many 16bit words to clock in without overrunning.
	* The precision loss with dividing reply_sz can safely be
	* ignored because the messages can only be multiples of 32bits.
	*/
	residual = 0;
	count = MIN((reply_sz / 4), ioc_sz) * 2;
	if (count < ioc_sz * 2) {
	residual = ioc_sz * 2 - count;
	mps_dprint(sc, MPS_ERROR, "Driver error, throwing away %d "
	"residual message words\n", residual);
	}

	for (i = 2; i < count; i++) {
	if (mps_wait_db_int(sc) != 0) {
	mps_dprint(sc, MPS_FAULT,
	"Timeout reading doorbell %d\n", i);
	return (ENXIO);
	}
	data16[i] = mps_regread(sc, MPI2_DOORBELL_OFFSET) &
	MPI2_DOORBELL_DATA_MASK;
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	}

	/*
	* Pull out residual words that won't fit into the provided buffer.
	* This keeps the chip from hanging due to a driver programming
	* error.
	*/
	while (residual--) {
	if (mps_wait_db_int(sc) != 0) {
	mps_dprint(sc, MPS_FAULT,
	"Timeout reading doorbell\n");
	return (ENXIO);
	}
	(void)mps_regread(sc, MPI2_DOORBELL_OFFSET);
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);
	}

	/* Step 7 */
	if (mps_wait_db_int(sc) != 0) {
	mps_dprint(sc, MPS_FAULT, "Timeout waiting to exit doorbell\n");
	return (ENXIO);
	}
	if (mps_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED)
	mps_dprint(sc, MPS_FAULT, "Warning, doorbell still active\n");
	mps_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0);

	return (0);
	}

	static void
	mps_enqueue_request(struct mps_softc sc, struct mps_command cm)
	{
	reply_descriptor rd;
	MPS_FUNCTRACE(sc);
	mps_dprint(sc, MPS_TRACE, "SMID %u cm %p ccb %p\n",
	cm->cm_desc.Default.SMID, cm, cm->cm_ccb);

	if (sc->mps_flags & MPS_FLAGS_ATTACH_DONE && !(sc->mps_flags & MPS_FLAGS_SHUTDOWN))
	mtx_assert(&sc->mps_mtx, MA_OWNED);

	if (++sc->io_cmds_active > sc->io_cmds_highwater)
	sc->io_cmds_highwater++;
	rd.u.low = cm->cm_desc.Words.Low;
	rd.u.high = cm->cm_desc.Words.High;
	rd.word = htole64(rd.word);

	KASSERT(cm->cm_state == MPS_CM_STATE_BUSY, ("command not busy\n"));
	cm->cm_state = MPS_CM_STATE_INQUEUE;

	/* TODO-We may need to make below regwrite atomic */
	mps_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET,
	rd.u.low);
	mps_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET,
	rd.u.high);
	}

	/*
	* Just the FACTS, ma'am.
	*/
	static int
	mps_get_iocfacts(struct mps_softc sc, MPI2_IOC_FACTS_REPLY facts)
	{
	MPI2_DEFAULT_REPLY *reply;
	MPI2_IOC_FACTS_REQUEST request;
	int error, req_sz, reply_sz;

	MPS_FUNCTRACE(sc);
	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	req_sz = sizeof(MPI2_IOC_FACTS_REQUEST);
	reply_sz = sizeof(MPI2_IOC_FACTS_REPLY);
	reply = (MPI2_DEFAULT_REPLY *)facts;

	bzero(&request, req_sz);
	request.Function = MPI2_FUNCTION_IOC_FACTS;
	error = mps_request_sync(sc, &request, reply, req_sz, reply_sz, 5);
	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);

	return (error);
	}

	static int
	mps_send_iocinit(struct mps_softc *sc)
	{
	MPI2_IOC_INIT_REQUEST init;
	MPI2_DEFAULT_REPLY reply;
	int req_sz, reply_sz, error;
	struct timeval now;
	uint64_t time_in_msec;

	MPS_FUNCTRACE(sc);
	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	/* Do a quick sanity check on proper initialization */
	if ((sc->pqdepth == 0) \|\| (sc->fqdepth == 0) \|\| (sc->reqframesz == 0)
	\|\| (sc->replyframesz == 0)) {
	mps_dprint(sc, MPS_INIT\|MPS_ERROR,
	"Driver not fully initialized for IOCInit\n");
	return (EINVAL);
	}

	req_sz = sizeof(MPI2_IOC_INIT_REQUEST);
	reply_sz = sizeof(MPI2_IOC_INIT_REPLY);
	bzero(&init, req_sz);
	bzero(&reply, reply_sz);

	/*
	* Fill in the init block. Note that most addresses are
	* deliberately in the lower 32bits of memory. This is a micro-
	* optimzation for PCI/PCIX, though it's not clear if it helps PCIe.
	*/
	init.Function = MPI2_FUNCTION_IOC_INIT;
	init.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
	init.MsgVersion = htole16(MPI2_VERSION);
	init.HeaderVersion = htole16(MPI2_HEADER_VERSION);
	init.SystemRequestFrameSize = htole16((uint16_t)(sc->reqframesz / 4));
	init.ReplyDescriptorPostQueueDepth = htole16(sc->pqdepth);
	init.ReplyFreeQueueDepth = htole16(sc->fqdepth);
	init.SenseBufferAddressHigh = 0;
	init.SystemReplyAddressHigh = 0;
	init.SystemRequestFrameBaseAddress.High = 0;
	init.SystemRequestFrameBaseAddress.Low = htole32((uint32_t)sc->req_busaddr);
	init.ReplyDescriptorPostQueueAddress.High = 0;
	init.ReplyDescriptorPostQueueAddress.Low = htole32((uint32_t)sc->post_busaddr);
	init.ReplyFreeQueueAddress.High = 0;
	init.ReplyFreeQueueAddress.Low = htole32((uint32_t)sc->free_busaddr);
	getmicrotime(&now);
	time_in_msec = (now.tv_sec * 1000 + now.tv_usec/1000);
	init.TimeStamp.High = htole32((time_in_msec >> 32) & 0xFFFFFFFF);
	init.TimeStamp.Low = htole32(time_in_msec & 0xFFFFFFFF);

	error = mps_request_sync(sc, &init, &reply, req_sz, reply_sz, 5);
	if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
	error = ENXIO;

	mps_dprint(sc, MPS_INIT, "IOCInit status= 0x%x\n", reply.IOCStatus);
	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);
	return (error);
	}

	void
	mps_memaddr_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	bus_addr_t *addr;

	addr = arg;
	*addr = segs[0].ds_addr;
	}

	void
	mps_memaddr_wait_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	struct mps_busdma_context *ctx;
	int need_unload, need_free;

	ctx = (struct mps_busdma_context *)arg;
	need_unload = 0;
	need_free = 0;

	mps_lock(ctx->softc);
	ctx->error = error;
	ctx->completed = 1;
	if ((error == 0) && (ctx->abandoned == 0)) {
	*ctx->addr = segs[0].ds_addr;
	} else {
	if (nsegs != 0)
	need_unload = 1;
	if (ctx->abandoned != 0)
	need_free = 1;
	}
	if (need_free == 0)
	wakeup(ctx);

	mps_unlock(ctx->softc);

	if (need_unload != 0) {
	bus_dmamap_unload(ctx->buffer_dmat,
	ctx->buffer_dmamap);
	*ctx->addr = 0;
	}

	if (need_free != 0)
	free(ctx, M_MPSUSER);
	}

	static int
	mps_alloc_queues(struct mps_softc *sc)
	{
	struct mps_queue *q;
	u_int nq, i;

	nq = sc->msi_msgs;
	mps_dprint(sc, MPS_INIT\|MPS_XINFO, "Allocating %d I/O queues\n", nq);

	sc->queues = malloc(sizeof(struct mps_queue) * nq, M_MPT2,
	M_NOWAIT\|M_ZERO);
	if (sc->queues == NULL)
	return (ENOMEM);

	for (i = 0; i < nq; i++) {
	q = &sc->queues[i];
	mps_dprint(sc, MPS_INIT, "Configuring queue %d %p\n", i, q);
	q->sc = sc;
	q->qnum = i;
	}

	return (0);
	}

	static int
	mps_alloc_hw_queues(struct mps_softc *sc)
	{
	bus_addr_t queues_busaddr;
	uint8_t *queues;
	int qsize, fqsize, pqsize;

	/*
	* The reply free queue contains 4 byte entries in multiples of 16 and
	* aligned on a 16 byte boundary. There must always be an unused entry.
	* This queue supplies fresh reply frames for the firmware to use.
	*
	* The reply descriptor post queue contains 8 byte entries in
	* multiples of 16 and aligned on a 16 byte boundary. This queue
	* contains filled-in reply frames sent from the firmware to the host.
	*
	* These two queues are allocated together for simplicity.
	*/
	sc->fqdepth = roundup2(sc->num_replies + 1, 16);
	sc->pqdepth = roundup2(sc->num_replies + 1, 16);
	fqsize= sc->fqdepth * 4;
	pqsize = sc->pqdepth * 8;
	qsize = fqsize + pqsize;

	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	16, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	qsize, /* maxsize */
	1, /* nsegments */
	qsize, /* maxsegsize */
	0, /* flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->queues_dmat)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate queues DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->queues_dmat, (void **)&queues, BUS_DMA_NOWAIT,
	&sc->queues_map)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate queues memory\n");
	return (ENOMEM);
	}
	bzero(queues, qsize);
	bus_dmamap_load(sc->queues_dmat, sc->queues_map, queues, qsize,
	mps_memaddr_cb, &queues_busaddr, 0);

	sc->free_queue = (uint32_t *)queues;
	sc->free_busaddr = queues_busaddr;
	sc->post_queue = (MPI2_REPLY_DESCRIPTORS_UNION *)(queues + fqsize);
	sc->post_busaddr = queues_busaddr + fqsize;
	mps_dprint(sc, MPS_INIT, "free queue busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->free_busaddr, fqsize);
	mps_dprint(sc, MPS_INIT, "reply queue busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->post_busaddr, pqsize);

	return (0);
	}

	static int
	mps_alloc_replies(struct mps_softc *sc)
	{
	int rsize, num_replies;

	/* Store the reply frame size in bytes rather than as 32bit words */
	sc->replyframesz = sc->facts->ReplyFrameSize * 4;

	/*
	* sc->num_replies should be one less than sc->fqdepth. We need to
	* allocate space for sc->fqdepth replies, but only sc->num_replies
	* replies can be used at once.
	*/
	num_replies = max(sc->fqdepth, sc->num_replies);

	rsize = sc->replyframesz * num_replies;
	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	4, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	rsize, /* maxsize */
	1, /* nsegments */
	rsize, /* maxsegsize */
	0, /* flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->reply_dmat)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate replies DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->reply_dmat, (void **)&sc->reply_frames,
	BUS_DMA_NOWAIT, &sc->reply_map)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate replies memory\n");
	return (ENOMEM);
	}
	bzero(sc->reply_frames, rsize);
	bus_dmamap_load(sc->reply_dmat, sc->reply_map, sc->reply_frames, rsize,
	mps_memaddr_cb, &sc->reply_busaddr, 0);

	mps_dprint(sc, MPS_INIT, "reply frames busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->reply_busaddr, rsize);

	return (0);
	}

	static void
	mps_load_chains_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	struct mps_softc *sc = arg;
	struct mps_chain *chain;
	bus_size_t bo;
	int i, o, s;

	if (error != 0)
	return;

	for (i = 0, o = 0, s = 0; s < nsegs; s++) {
	for (bo = 0; bo + sc->reqframesz <= segs[s].ds_len;
	bo += sc->reqframesz) {
	chain = &sc->chains[i++];
	chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o);
	chain->chain_busaddr = segs[s].ds_addr + bo;
	o += sc->reqframesz;
	mps_free_chain(sc, chain);
	}
	if (bo != segs[s].ds_len)
	o += segs[s].ds_len - bo;
	}
	sc->chain_free_lowwater = i;
	}

	static int
	mps_alloc_requests(struct mps_softc *sc)
	{
	struct mps_command *cm;
	int i, rsize, nsegs;

	rsize = sc->reqframesz * sc->num_reqs;
	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	16, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	rsize, /* maxsize */
	1, /* nsegments */
	rsize, /* maxsegsize */
	0, /* flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->req_dmat)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate request DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->req_dmat, (void **)&sc->req_frames,
	BUS_DMA_NOWAIT, &sc->req_map)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate request memory\n");
	return (ENOMEM);
	}
	bzero(sc->req_frames, rsize);
	bus_dmamap_load(sc->req_dmat, sc->req_map, sc->req_frames, rsize,
	mps_memaddr_cb, &sc->req_busaddr, 0);
	mps_dprint(sc, MPS_INIT, "request frames busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->req_busaddr, rsize);

	sc->chains = malloc(sizeof(struct mps_chain) * sc->num_chains, M_MPT2,
	M_NOWAIT \| M_ZERO);
	if (!sc->chains) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate chain memory\n");
	return (ENOMEM);
	}
	rsize = sc->reqframesz * sc->num_chains;
	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	16, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	rsize, /* maxsize */
	howmany(rsize, PAGE_SIZE), /* nsegments */
	rsize, /* maxsegsize */
	0, /* flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->chain_dmat)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate chain DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames,
	BUS_DMA_NOWAIT \| BUS_DMA_ZERO, &sc->chain_map)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate chain memory\n");
	return (ENOMEM);
	}
	if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames,
	rsize, mps_load_chains_cb, sc, BUS_DMA_NOWAIT)) {
	mps_dprint(sc, MPS_ERROR, "Cannot load chain memory\n");
	bus_dmamem_free(sc->chain_dmat, sc->chain_frames,
	sc->chain_map);
	return (ENOMEM);
	}

	rsize = MPS_SENSE_LEN * sc->num_reqs;
	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	1, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	rsize, /* maxsize */
	1, /* nsegments */
	rsize, /* maxsegsize */
	0, /* flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->sense_dmat)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate sense DMA tag\n");
	return (ENOMEM);
	}
	if (bus_dmamem_alloc(sc->sense_dmat, (void **)&sc->sense_frames,
	BUS_DMA_NOWAIT, &sc->sense_map)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate sense memory\n");
	return (ENOMEM);
	}
	bzero(sc->sense_frames, rsize);
	bus_dmamap_load(sc->sense_dmat, sc->sense_map, sc->sense_frames, rsize,
	mps_memaddr_cb, &sc->sense_busaddr, 0);
	mps_dprint(sc, MPS_INIT, "sense frames busaddr= %#016jx size= %d\n",
	(uintmax_t)sc->sense_busaddr, rsize);

	nsegs = (sc->maxio / PAGE_SIZE) + 1;
	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	1, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
	nsegs, /* nsegments */
	BUS_SPACE_MAXSIZE_24BIT,/* maxsegsize */
	BUS_DMA_ALLOCNOW, /* flags */
	busdma_lock_mutex, /* lockfunc */
	&sc->mps_mtx, /* lockarg */
	&sc->buffer_dmat)) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate buffer DMA tag\n");
	return (ENOMEM);
	}

	/*
	* SMID 0 cannot be used as a free command per the firmware spec.
	* Just drop that command instead of risking accounting bugs.
	*/
	sc->commands = malloc(sizeof(struct mps_command) * sc->num_reqs,
	M_MPT2, M_WAITOK \| M_ZERO);
	- if(!sc->commands) {
	- mps_dprint(sc, MPS_ERROR, "Cannot allocate command memory\n");
	- return (ENOMEM);
	- }
	for (i = 1; i < sc->num_reqs; i++) {
	cm = &sc->commands[i];
	cm->cm_req = sc->req_frames + i * sc->reqframesz;
	cm->cm_req_busaddr = sc->req_busaddr + i * sc->reqframesz;
	cm->cm_sense = &sc->sense_frames[i];
	cm->cm_sense_busaddr = sc->sense_busaddr + i * MPS_SENSE_LEN;
	cm->cm_desc.Default.SMID = i;
	cm->cm_sc = sc;
	cm->cm_state = MPS_CM_STATE_BUSY;
	TAILQ_INIT(&cm->cm_chain_list);
	callout_init_mtx(&cm->cm_callout, &sc->mps_mtx, 0);

	/* XXX Is a failure here a critical problem? */
	if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap) == 0)
	if (i <= sc->num_prireqs)
	mps_free_high_priority_command(sc, cm);
	else
	mps_free_command(sc, cm);
	else {
	panic("failed to allocate command %d\n", i);
	sc->num_reqs = i;
	break;
	}
	}

	return (0);
	}

	static int
	mps_init_queues(struct mps_softc *sc)
	{
	int i;

	memset((uint8_t )sc->post_queue, 0xff, sc->pqdepth 8);

	/*
	* According to the spec, we need to use one less reply than we
	* have space for on the queue. So sc->num_replies (the number we
	* use) should be less than sc->fqdepth (allocated size).
	*/
	if (sc->num_replies >= sc->fqdepth)
	return (EINVAL);

	/*
	* Initialize all of the free queue entries.
	*/
	for (i = 0; i < sc->fqdepth; i++)
	sc->free_queue[i] = sc->reply_busaddr + (i * sc->replyframesz);
	sc->replyfreeindex = sc->num_replies;

	return (0);
	}

	/* Get the driver parameter tunables. Lowest priority are the driver defaults.
	* Next are the global settings, if they exist. Highest are the per-unit
	* settings, if they exist.
	*/
	void
	mps_get_tunables(struct mps_softc *sc)
	{
	char tmpstr[80], mps_debug[80];

	/* XXX default to some debugging for now */
	sc->mps_debug = MPS_INFO\|MPS_FAULT;
	sc->disable_msix = 0;
	sc->disable_msi = 0;
	sc->max_msix = MPS_MSIX_MAX;
	sc->max_chains = MPS_CHAIN_FRAMES;
	sc->max_io_pages = MPS_MAXIO_PAGES;
	sc->enable_ssu = MPS_SSU_ENABLE_SSD_DISABLE_HDD;
	sc->spinup_wait_time = DEFAULT_SPINUP_WAIT;
	sc->use_phynum = 1;
	sc->max_reqframes = MPS_REQ_FRAMES;
	sc->max_prireqframes = MPS_PRI_REQ_FRAMES;
	sc->max_replyframes = MPS_REPLY_FRAMES;
	sc->max_evtframes = MPS_EVT_REPLY_FRAMES;

	/*
	* Grab the global variables.
	*/
	bzero(mps_debug, 80);
	if (TUNABLE_STR_FETCH("hw.mps.debug_level", mps_debug, 80) != 0)
	mps_parse_debug(sc, mps_debug);
	TUNABLE_INT_FETCH("hw.mps.disable_msix", &sc->disable_msix);
	TUNABLE_INT_FETCH("hw.mps.disable_msi", &sc->disable_msi);
	TUNABLE_INT_FETCH("hw.mps.max_msix", &sc->max_msix);
	TUNABLE_INT_FETCH("hw.mps.max_chains", &sc->max_chains);
	TUNABLE_INT_FETCH("hw.mps.max_io_pages", &sc->max_io_pages);
	TUNABLE_INT_FETCH("hw.mps.enable_ssu", &sc->enable_ssu);
	TUNABLE_INT_FETCH("hw.mps.spinup_wait_time", &sc->spinup_wait_time);
	TUNABLE_INT_FETCH("hw.mps.use_phy_num", &sc->use_phynum);
	TUNABLE_INT_FETCH("hw.mps.max_reqframes", &sc->max_reqframes);
	TUNABLE_INT_FETCH("hw.mps.max_prireqframes", &sc->max_prireqframes);
	TUNABLE_INT_FETCH("hw.mps.max_replyframes", &sc->max_replyframes);
	TUNABLE_INT_FETCH("hw.mps.max_evtframes", &sc->max_evtframes);

	/* Grab the unit-instance variables */
	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.debug_level",
	device_get_unit(sc->mps_dev));
	bzero(mps_debug, 80);
	if (TUNABLE_STR_FETCH(tmpstr, mps_debug, 80) != 0)
	mps_parse_debug(sc, mps_debug);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.disable_msix",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->disable_msix);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.disable_msi",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->disable_msi);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_msix",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_msix);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_chains",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_chains);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_io_pages",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_io_pages);

	bzero(sc->exclude_ids, sizeof(sc->exclude_ids));
	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.exclude_ids",
	device_get_unit(sc->mps_dev));
	TUNABLE_STR_FETCH(tmpstr, sc->exclude_ids, sizeof(sc->exclude_ids));

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.enable_ssu",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->enable_ssu);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.spinup_wait_time",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->spinup_wait_time);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.use_phy_num",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->use_phynum);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_reqframes",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_reqframes);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_prireqframes",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_prireqframes);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_replyframes",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_replyframes);

	snprintf(tmpstr, sizeof(tmpstr), "dev.mps.%d.max_evtframes",
	device_get_unit(sc->mps_dev));
	TUNABLE_INT_FETCH(tmpstr, &sc->max_evtframes);

	}

	static void
	mps_setup_sysctl(struct mps_softc *sc)
	{
	struct sysctl_ctx_list *sysctl_ctx = NULL;
	struct sysctl_oid *sysctl_tree = NULL;
	char tmpstr[80], tmpstr2[80];

	/*
	* Setup the sysctl variable so the user can change the debug level
	* on the fly.
	*/
	snprintf(tmpstr, sizeof(tmpstr), "MPS controller %d",
	device_get_unit(sc->mps_dev));
	snprintf(tmpstr2, sizeof(tmpstr2), "%d", device_get_unit(sc->mps_dev));

	sysctl_ctx = device_get_sysctl_ctx(sc->mps_dev);
	if (sysctl_ctx != NULL)
	sysctl_tree = device_get_sysctl_tree(sc->mps_dev);

	if (sysctl_tree == NULL) {
	sysctl_ctx_init(&sc->sysctl_ctx);
	sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx,
	SYSCTL_STATIC_CHILDREN(_hw_mps), OID_AUTO, tmpstr2,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, 0, tmpstr);
	if (sc->sysctl_tree == NULL)
	return;
	sysctl_ctx = &sc->sysctl_ctx;
	sysctl_tree = sc->sysctl_tree;
	}

	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "debug_level", CTLTYPE_STRING \| CTLFLAG_RW \|CTLFLAG_MPSAFE,
	sc, 0, mps_debug_sysctl, "A", "mps debug level");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "disable_msix", CTLFLAG_RD, &sc->disable_msix, 0,
	"Disable the use of MSI-X interrupts");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "disable_msi", CTLFLAG_RD, &sc->disable_msi, 0,
	"Disable the use of MSI interrupts");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_msix", CTLFLAG_RD, &sc->max_msix, 0,
	"User-defined maximum number of MSIX queues");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "msix_msgs", CTLFLAG_RD, &sc->msi_msgs, 0,
	"Negotiated number of MSIX queues");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_reqframes", CTLFLAG_RD, &sc->max_reqframes, 0,
	"Total number of allocated request frames");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_prireqframes", CTLFLAG_RD, &sc->max_prireqframes, 0,
	"Total number of allocated high priority request frames");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_replyframes", CTLFLAG_RD, &sc->max_replyframes, 0,
	"Total number of allocated reply frames");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_evtframes", CTLFLAG_RD, &sc->max_evtframes, 0,
	"Total number of event frames allocated");

	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "firmware_version", CTLFLAG_RD, sc->fw_version,
	strlen(sc->fw_version), "firmware version");

	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "driver_version", CTLFLAG_RD, MPS_DRIVER_VERSION,
	strlen(MPS_DRIVER_VERSION), "driver version");

	SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "msg_version", CTLFLAG_RD, sc->msg_version,
	strlen(sc->msg_version), "message interface version");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "io_cmds_active", CTLFLAG_RD,
	&sc->io_cmds_active, 0, "number of currently active commands");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "io_cmds_highwater", CTLFLAG_RD,
	&sc->io_cmds_highwater, 0, "maximum active commands seen");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "chain_free", CTLFLAG_RD,
	&sc->chain_free, 0, "number of free chain elements");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "chain_free_lowwater", CTLFLAG_RD,
	&sc->chain_free_lowwater, 0,"lowest number of free chain elements");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_chains", CTLFLAG_RD,
	&sc->max_chains, 0,"maximum chain frames that will be allocated");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "max_io_pages", CTLFLAG_RD,
	&sc->max_io_pages, 0,"maximum pages to allow per I/O (if <1 use "
	"IOCFacts)");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0,
	"enable SSU to SATA SSD/HDD at shutdown");

	SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "chain_alloc_fail", CTLFLAG_RD,
	&sc->chain_alloc_fail, "chain allocation failures");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "spinup_wait_time", CTLFLAG_RD,
	&sc->spinup_wait_time, DEFAULT_SPINUP_WAIT, "seconds to wait for "
	"spinup after SATA ID error");

	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "mapping_table_dump",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	mps_mapping_dump, "A", "Mapping Table Dump");

	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "encl_table_dump",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT, sc, 0,
	mps_mapping_encl_dump, "A", "Enclosure Table Dump");

	SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "dump_reqs",
	CTLTYPE_OPAQUE \| CTLFLAG_RD \| CTLFLAG_SKIP \| CTLFLAG_NEEDGIANT,
	sc, 0, mps_dump_reqs, "I", "Dump Active Requests");

	SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
	OID_AUTO, "use_phy_num", CTLFLAG_RD, &sc->use_phynum, 0,
	"Use the phy number for enumeration");
	}

	static struct mps_debug_string {
	char *name;
	int flag;
	} mps_debug_strings[] = {
	{"info", MPS_INFO},
	{"fault", MPS_FAULT},
	{"event", MPS_EVENT},
	{"log", MPS_LOG},
	{"recovery", MPS_RECOVERY},
	{"error", MPS_ERROR},
	{"init", MPS_INIT},
	{"xinfo", MPS_XINFO},
	{"user", MPS_USER},
	{"mapping", MPS_MAPPING},
	{"trace", MPS_TRACE}
	};

	enum mps_debug_level_combiner {
	COMB_NONE,
	COMB_ADD,
	COMB_SUB
	};

	static int
	mps_debug_sysctl(SYSCTL_HANDLER_ARGS)
	{
	struct mps_softc *sc;
	struct mps_debug_string *string;
	struct sbuf *sbuf;
	char *buffer;
	size_t sz;
	int i, len, debug, error;

	sc = (struct mps_softc *)arg1;

	error = sysctl_wire_old_buffer(req, 0);
	if (error != 0)
	return (error);

	sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
	debug = sc->mps_debug;

	sbuf_printf(sbuf, "%#x", debug);

	sz = sizeof(mps_debug_strings) / sizeof(mps_debug_strings[0]);
	for (i = 0; i < sz; i++) {
	string = &mps_debug_strings[i];
	if (debug & string->flag)
	sbuf_printf(sbuf, ",%s", string->name);
	}

	error = sbuf_finish(sbuf);
	sbuf_delete(sbuf);

	if (error \|\| req->newptr == NULL)
	return (error);

	len = req->newlen - req->newidx;
	if (len == 0)
	return (0);

	buffer = malloc(len, M_MPT2, M_ZERO\|M_WAITOK);
	error = SYSCTL_IN(req, buffer, len);

	mps_parse_debug(sc, buffer);

	free(buffer, M_MPT2);
	return (error);
	}

	static void
	mps_parse_debug(struct mps_softc sc, char list)
	{
	struct mps_debug_string *string;
	enum mps_debug_level_combiner op;
	char token, endtoken;
	size_t sz;
	int flags, i;

	if (list == NULL \|\| *list == '\0')
	return;

	if (*list == '+') {
	op = COMB_ADD;
	list++;
	} else if (*list == '-') {
	op = COMB_SUB;
	list++;
	} else
	op = COMB_NONE;
	if (*list == '\0')
	return;

	flags = 0;
	sz = sizeof(mps_debug_strings) / sizeof(mps_debug_strings[0]);
	while ((token = strsep(&list, ":,")) != NULL) {

	/* Handle integer flags */
	flags \|= strtol(token, &endtoken, 0);
	if (token != endtoken)
	continue;

	/* Handle text flags */
	for (i = 0; i < sz; i++) {
	string = &mps_debug_strings[i];
	if (strcasecmp(token, string->name) == 0) {
	flags \|= string->flag;
	break;
	}
	}
	}

	switch (op) {
	case COMB_NONE:
	sc->mps_debug = flags;
	break;
	case COMB_ADD:
	sc->mps_debug \|= flags;
	break;
	case COMB_SUB:
	sc->mps_debug &= (~flags);
	break;
	}

	return;
	}

	struct mps_dumpreq_hdr {
	uint32_t smid;
	uint32_t state;
	uint32_t numframes;
	uint32_t deschi;
	uint32_t desclo;
	};

	static int
	mps_dump_reqs(SYSCTL_HANDLER_ARGS)
	{
	struct mps_softc *sc;
	struct mps_chain chain, chain1;
	struct mps_command *cm;
	struct mps_dumpreq_hdr hdr;
	struct sbuf *sb;
	uint32_t smid, state;
	int i, numreqs, error = 0;

	sc = (struct mps_softc *)arg1;

	if ((error = priv_check(curthread, PRIV_DRIVER)) != 0) {
	printf("priv check error %d\n", error);
	return (error);
	}

	state = MPS_CM_STATE_INQUEUE;
	smid = 1;
	numreqs = sc->num_reqs;

	if (req->newptr != NULL)
	return (EINVAL);

	if (smid == 0 \|\| smid > sc->num_reqs)
	return (EINVAL);
	if (numreqs <= 0 \|\| (numreqs + smid > sc->num_reqs))
	numreqs = sc->num_reqs;
	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);

	/* Best effort, no locking */
	for (i = smid; i < numreqs; i++) {
	cm = &sc->commands[i];
	if (cm->cm_state != state)
	continue;
	hdr.smid = i;
	hdr.state = cm->cm_state;
	hdr.numframes = 1;
	hdr.deschi = cm->cm_desc.Words.High;
	hdr.desclo = cm->cm_desc.Words.Low;
	TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,
	chain1)
	hdr.numframes++;
	sbuf_bcat(sb, &hdr, sizeof(hdr));
	sbuf_bcat(sb, cm->cm_req, 128);
	TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,
	chain1)
	sbuf_bcat(sb, chain->chain, 128);
	}

	error = sbuf_finish(sb);
	sbuf_delete(sb);
	return (error);
	}

	int
	mps_attach(struct mps_softc *sc)
	{
	int error;

	MPS_FUNCTRACE(sc);
	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	mtx_init(&sc->mps_mtx, "MPT2SAS lock", NULL, MTX_DEF);
	callout_init_mtx(&sc->periodic, &sc->mps_mtx, 0);
	callout_init_mtx(&sc->device_check_callout, &sc->mps_mtx, 0);
	TAILQ_INIT(&sc->event_list);
	timevalclear(&sc->lastfail);

	if ((error = mps_transition_ready(sc)) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "failed to transition "
	"ready\n");
	return (error);
	}

	sc->facts = malloc(sizeof(MPI2_IOC_FACTS_REPLY), M_MPT2,
	M_ZERO\|M_NOWAIT);
	if(!sc->facts) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "Cannot allocate memory, "
	"exit\n");
	return (ENOMEM);
	}

	/*
	* Get IOC Facts and allocate all structures based on this information.
	* A Diag Reset will also call mps_iocfacts_allocate and re-read the IOC
	* Facts. If relevant values have changed in IOC Facts, this function
	* will free all of the memory based on IOC Facts and reallocate that
	* memory. If this fails, any allocated memory should already be freed.
	*/
	if ((error = mps_iocfacts_allocate(sc, TRUE)) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "IOC Facts based allocation "
	"failed with error %d, exit\n", error);
	return (error);
	}

	/* Start the periodic watchdog check on the IOC Doorbell */
	mps_periodic(sc);

	/*
	* The portenable will kick off discovery events that will drive the
	* rest of the initialization process. The CAM/SAS module will
	* hold up the boot sequence until discovery is complete.
	*/
	sc->mps_ich.ich_func = mps_startup;
	sc->mps_ich.ich_arg = sc;
	if (config_intrhook_establish(&sc->mps_ich) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_ERROR,
	"Cannot establish MPS config hook\n");
	error = EINVAL;
	}

	/*
	* Allow IR to shutdown gracefully when shutdown occurs.
	*/
	sc->shutdown_eh = EVENTHANDLER_REGISTER(shutdown_final,
	mpssas_ir_shutdown, sc, SHUTDOWN_PRI_DEFAULT);

	if (sc->shutdown_eh == NULL)
	mps_dprint(sc, MPS_INIT\|MPS_ERROR,
	"shutdown event registration failed\n");

	mps_setup_sysctl(sc);

	sc->mps_flags \|= MPS_FLAGS_ATTACH_DONE;
	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);

	return (error);
	}

	/* Run through any late-start handlers. */
	static void
	mps_startup(void *arg)
	{
	struct mps_softc *sc;

	sc = (struct mps_softc *)arg;
	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	mps_lock(sc);
	mps_unmask_intr(sc);

	/* initialize device mapping tables */
	mps_base_static_config_pages(sc);
	mps_mapping_initialize(sc);
	mpssas_startup(sc);
	mps_unlock(sc);

	mps_dprint(sc, MPS_INIT, "disestablish config intrhook\n");
	config_intrhook_disestablish(&sc->mps_ich);
	sc->mps_ich.ich_arg = NULL;

	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);
	}

	/* Periodic watchdog. Is called with the driver lock already held. */
	static void
	mps_periodic(void *arg)
	{
	struct mps_softc *sc;
	uint32_t db;

	sc = (struct mps_softc *)arg;
	if (sc->mps_flags & MPS_FLAGS_SHUTDOWN)
	return;

	db = mps_regread(sc, MPI2_DOORBELL_OFFSET);
	if ((db & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
	mps_dprint(sc, MPS_FAULT, "IOC Fault 0x%08x, Resetting\n", db);
	mps_reinit(sc);
	}

	callout_reset(&sc->periodic, MPS_PERIODIC_DELAY * hz, mps_periodic, sc);
	}

	static void
	mps_log_evt_handler(struct mps_softc *sc, uintptr_t data,
	MPI2_EVENT_NOTIFICATION_REPLY *event)
	{
	MPI2_EVENT_DATA_LOG_ENTRY_ADDED *entry;

	MPS_DPRINT_EVENT(sc, generic, event);

	switch (event->Event) {
	case MPI2_EVENT_LOG_DATA:
	mps_dprint(sc, MPS_EVENT, "MPI2_EVENT_LOG_DATA:\n");
	if (sc->mps_debug & MPS_EVENT)
	hexdump(event->EventData, event->EventDataLength, NULL, 0);
	break;
	case MPI2_EVENT_LOG_ENTRY_ADDED:
	entry = (MPI2_EVENT_DATA_LOG_ENTRY_ADDED *)event->EventData;
	mps_dprint(sc, MPS_EVENT, "MPI2_EVENT_LOG_ENTRY_ADDED event "
	"0x%x Sequence %d:\n", entry->LogEntryQualifier,
	entry->LogSequence);
	break;
	default:
	break;
	}
	return;
	}

	static int
	mps_attach_log(struct mps_softc *sc)
	{
	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];

	bzero(events, 16);
	setbit(events, MPI2_EVENT_LOG_DATA);
	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);

	mps_register_events(sc, events, mps_log_evt_handler, NULL,
	&sc->mps_log_eh);

	return (0);
	}

	static int
	mps_detach_log(struct mps_softc *sc)
	{

	if (sc->mps_log_eh != NULL)
	mps_deregister_events(sc, sc->mps_log_eh);
	return (0);
	}

	/*
	* Free all of the driver resources and detach submodules. Should be called
	* without the lock held.
	*/
	int
	mps_free(struct mps_softc *sc)
	{
	int error;

	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);
	/* Turn off the watchdog */
	mps_lock(sc);
	sc->mps_flags \|= MPS_FLAGS_SHUTDOWN;
	mps_unlock(sc);
	/* Lock must not be held for this */
	callout_drain(&sc->periodic);
	callout_drain(&sc->device_check_callout);

	if (((error = mps_detach_log(sc)) != 0) \|\|
	((error = mps_detach_sas(sc)) != 0)) {
	mps_dprint(sc, MPS_INIT\|MPS_FAULT, "failed to detach "
	"subsystems, exit\n");
	return (error);
	}

	mps_detach_user(sc);

	/* Put the IOC back in the READY state. */
	mps_lock(sc);
	if ((error = mps_transition_ready(sc)) != 0) {
	mps_unlock(sc);
	return (error);
	}
	mps_unlock(sc);

	if (sc->facts != NULL)
	free(sc->facts, M_MPT2);

	/*
	* Free all buffers that are based on IOC Facts. A Diag Reset may need
	* to free these buffers too.
	*/
	mps_iocfacts_free(sc);

	if (sc->sysctl_tree != NULL)
	sysctl_ctx_free(&sc->sysctl_ctx);

	/* Deregister the shutdown function */
	if (sc->shutdown_eh != NULL)
	EVENTHANDLER_DEREGISTER(shutdown_final, sc->shutdown_eh);

	mtx_destroy(&sc->mps_mtx);
	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);

	return (0);
	}

	static __inline void
	mps_complete_command(struct mps_softc sc, struct mps_command cm)
	{
	MPS_FUNCTRACE(sc);

	if (cm == NULL) {
	mps_dprint(sc, MPS_ERROR, "Completing NULL command\n");
	return;
	}

	if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
	cm->cm_flags \|= MPS_CM_FLAGS_COMPLETE;

	if (cm->cm_complete != NULL) {
	mps_dprint(sc, MPS_TRACE,
	"%s cm %p calling cm_complete %p data %p reply %p\n",
	__func__, cm, cm->cm_complete, cm->cm_complete_data,
	cm->cm_reply);
	cm->cm_complete(sc, cm);
	}

	if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
	mps_dprint(sc, MPS_TRACE, "waking up %p\n", cm);
	wakeup(cm);
	}

	if (cm->cm_sc->io_cmds_active != 0) {
	cm->cm_sc->io_cmds_active--;
	} else {
	mps_dprint(sc, MPS_ERROR, "Warning: io_cmds_active is "
	"out of sync - resynching to 0\n");
	}
	}


	static void
	mps_sas_log_info(struct mps_softc *sc , u32 log_info)
	{
	union loginfo_type {
	u32 loginfo;
	struct {
	u32 subcode:16;
	u32 code:8;
	u32 originator:4;
	u32 bus_type:4;
	} dw;
	};
	union loginfo_type sas_loginfo;
	char *originator_str = NULL;

	sas_loginfo.loginfo = log_info;
	if (sas_loginfo.dw.bus_type != 3 /SAS/)
	return;

	/* each nexus loss loginfo */
	if (log_info == 0x31170000)
	return;

	/* eat the loginfos associated with task aborts */
	if ((log_info == 30050000 \|\| log_info ==
	0x31140000 \|\| log_info == 0x31130000))
	return;

	switch (sas_loginfo.dw.originator) {
	case 0:
	originator_str = "IOP";
	break;
	case 1:
	originator_str = "PL";
	break;
	case 2:
	originator_str = "IR";
	break;
	}

	mps_dprint(sc, MPS_LOG, "log_info(0x%08x): originator(%s), "
	"code(0x%02x), sub_code(0x%04x)\n", log_info,
	originator_str, sas_loginfo.dw.code,
	sas_loginfo.dw.subcode);
	}

	static void
	mps_display_reply_info(struct mps_softc sc, uint8_t reply)
	{
	MPI2DefaultReply_t *mpi_reply;
	u16 sc_status;

	mpi_reply = (MPI2DefaultReply_t*)reply;
	sc_status = le16toh(mpi_reply->IOCStatus);
	if (sc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
	mps_sas_log_info(sc, le32toh(mpi_reply->IOCLogInfo));
	}
	void
	mps_intr(void *data)
	{
	struct mps_softc *sc;
	uint32_t status;

	sc = (struct mps_softc *)data;
	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);

	/*
	* Check interrupt status register to flush the bus. This is
	* needed for both INTx interrupts and driver-driven polling
	*/
	status = mps_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);
	if ((status & MPI2_HIS_REPLY_DESCRIPTOR_INTERRUPT) == 0)
	return;

	mps_lock(sc);
	mps_intr_locked(data);
	mps_unlock(sc);
	return;
	}

	/*
	* In theory, MSI/MSIX interrupts shouldn't need to read any registers on the
	* chip. Hopefully this theory is correct.
	*/
	void
	mps_intr_msi(void *data)
	{
	struct mps_softc *sc;

	sc = (struct mps_softc *)data;
	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
	mps_lock(sc);
	mps_intr_locked(data);
	mps_unlock(sc);
	return;
	}

	/*
	* The locking is overly broad and simplistic, but easy to deal with for now.
	*/
	void
	mps_intr_locked(void *data)
	{
	MPI2_REPLY_DESCRIPTORS_UNION *desc;
	MPI2_DIAG_RELEASE_REPLY *rel_rep;
	mps_fw_diagnostic_buffer_t *pBuffer;
	struct mps_softc *sc;
	struct mps_command *cm = NULL;
	uint64_t tdesc;
	uint8_t flags;
	u_int pq;

	sc = (struct mps_softc *)data;

	pq = sc->replypostindex;
	mps_dprint(sc, MPS_TRACE,
	"%s sc %p starting with replypostindex %u\n",
	__func__, sc, sc->replypostindex);

	for ( ;; ) {
	cm = NULL;
	desc = &sc->post_queue[sc->replypostindex];

	/*
	* Copy and clear out the descriptor so that any reentry will
	* immediately know that this descriptor has already been
	* looked at. There is unfortunate casting magic because the
	* MPI API doesn't have a cardinal 64bit type.
	*/
	tdesc = 0xffffffffffffffff;
	tdesc = atomic_swap_64((uint64_t *)desc, tdesc);
	desc = (MPI2_REPLY_DESCRIPTORS_UNION *)&tdesc;

	flags = desc->Default.ReplyFlags &
	MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
	if ((flags == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
	\|\| (le32toh(desc->Words.High) == 0xffffffff))
	break;

	/* increment the replypostindex now, so that event handlers
	* and cm completion handlers which decide to do a diag
	* reset can zero it without it getting incremented again
	* afterwards, and we break out of this loop on the next
	* iteration since the reply post queue has been cleared to
	* 0xFF and all descriptors look unused (which they are).
	*/
	if (++sc->replypostindex >= sc->pqdepth)
	sc->replypostindex = 0;

	switch (flags) {
	case MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS:
	cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)];
	KASSERT(cm->cm_state == MPS_CM_STATE_INQUEUE,
	("command not inqueue\n"));
	cm->cm_state = MPS_CM_STATE_BUSY;
	cm->cm_reply = NULL;
	break;
	case MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY:
	{
	uint32_t baddr;
	uint8_t *reply;

	/*
	* Re-compose the reply address from the address
	* sent back from the chip. The ReplyFrameAddress
	* is the lower 32 bits of the physical address of
	* particular reply frame. Convert that address to
	* host format, and then use that to provide the
	* offset against the virtual address base
	* (sc->reply_frames).
	*/
	baddr = le32toh(desc->AddressReply.ReplyFrameAddress);
	reply = sc->reply_frames +
	(baddr - ((uint32_t)sc->reply_busaddr));
	/*
	* Make sure the reply we got back is in a valid
	* range. If not, go ahead and panic here, since
	* we'll probably panic as soon as we deference the
	* reply pointer anyway.
	*/
	if ((reply < sc->reply_frames)
	\|\| (reply > (sc->reply_frames +
	(sc->fqdepth * sc->replyframesz)))) {
	printf("%s: WARNING: reply %p out of range!\n",
	__func__, reply);
	printf("%s: reply_frames %p, fqdepth %d, "
	"frame size %d\n", __func__,
	sc->reply_frames, sc->fqdepth,
	sc->replyframesz);
	printf("%s: baddr %#x,\n", __func__, baddr);
	/* LSI-TODO. See Linux Code for Graceful exit */
	panic("Reply address out of range");
	}
	if (le16toh(desc->AddressReply.SMID) == 0) {
	if (((MPI2_DEFAULT_REPLY *)reply)->Function ==
	MPI2_FUNCTION_DIAG_BUFFER_POST) {
	/*
	* If SMID is 0 for Diag Buffer Post,
	* this implies that the reply is due to
	* a release function with a status that
	* the buffer has been released. Set
	* the buffer flags accordingly.
	*/
	rel_rep =
	(MPI2_DIAG_RELEASE_REPLY *)reply;
	if ((le16toh(rel_rep->IOCStatus) &
	MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED)
	{
	pBuffer =
	&sc->fw_diag_buffer_list[
	rel_rep->BufferType];
	pBuffer->valid_data = TRUE;
	pBuffer->owned_by_firmware =
	FALSE;
	pBuffer->immediate = FALSE;
	}
	} else
	mps_dispatch_event(sc, baddr,
	(MPI2_EVENT_NOTIFICATION_REPLY *)
	reply);
	} else {
	/*
	* Ignore commands not in INQUEUE state
	* since they've already been completed
	* via another path.
	*/
	cm = &sc->commands[
	le16toh(desc->AddressReply.SMID)];
	if (cm->cm_state == MPS_CM_STATE_INQUEUE) {
	cm->cm_state = MPS_CM_STATE_BUSY;
	cm->cm_reply = reply;
	cm->cm_reply_data = le32toh(
	desc->AddressReply.ReplyFrameAddress);
	} else {
	mps_dprint(sc, MPS_RECOVERY,
	"Bad state for ADDRESS_REPLY status,"
	" ignoring state %d cm %p\n",
	cm->cm_state, cm);
	}
	}
	break;
	}
	case MPI2_RPY_DESCRIPT_FLAGS_TARGETASSIST_SUCCESS:
	case MPI2_RPY_DESCRIPT_FLAGS_TARGET_COMMAND_BUFFER:
	case MPI2_RPY_DESCRIPT_FLAGS_RAID_ACCELERATOR_SUCCESS:
	default:
	/* Unhandled */
	mps_dprint(sc, MPS_ERROR, "Unhandled reply 0x%x\n",
	desc->Default.ReplyFlags);
	cm = NULL;
	break;
	}


	if (cm != NULL) {
	// Print Error reply frame
	if (cm->cm_reply)
	mps_display_reply_info(sc,cm->cm_reply);
	mps_complete_command(sc, cm);
	}
	}

	if (pq != sc->replypostindex) {
	mps_dprint(sc, MPS_TRACE, "%s sc %p writing postindex %d\n",
	__func__, sc, sc->replypostindex);
	mps_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET,
	sc->replypostindex);
	}

	return;
	}

	static void
	mps_dispatch_event(struct mps_softc *sc, uintptr_t data,
	MPI2_EVENT_NOTIFICATION_REPLY *reply)
	{
	struct mps_event_handle *eh;
	int event, handled = 0;

	event = le16toh(reply->Event);
	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
	if (isset(eh->mask, event)) {
	eh->callback(sc, data, reply);
	handled++;
	}
	}

	if (handled == 0)
	mps_dprint(sc, MPS_EVENT, "Unhandled event 0x%x\n", le16toh(event));

	/*
	* This is the only place that the event/reply should be freed.
	* Anything wanting to hold onto the event data should have
	* already copied it into their own storage.
	*/
	mps_free_reply(sc, data);
	}

	static void
	mps_reregister_events_complete(struct mps_softc sc, struct mps_command cm)
	{
	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);

	if (cm->cm_reply)
	MPS_DPRINT_EVENT(sc, generic,
	(MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply);

	mps_free_command(sc, cm);

	/* next, send a port enable */
	mpssas_startup(sc);
	}

	/*
	* For both register_events and update_events, the caller supplies a bitmap
	* of events that it _wants_. These functions then turn that into a bitmask
	* suitable for the controller.
	*/
	int
	mps_register_events(struct mps_softc sc, u32 mask,
	mps_evt_callback_t cb, void data, struct mps_event_handle **handle)
	{
	struct mps_event_handle *eh;
	int error = 0;

	eh = malloc(sizeof(struct mps_event_handle), M_MPT2, M_WAITOK\|M_ZERO);
	- if(!eh) {
	- mps_dprint(sc, MPS_ERROR, "Cannot allocate event memory\n");
	- return (ENOMEM);
	- }
	eh->callback = cb;
	eh->data = data;
	TAILQ_INSERT_TAIL(&sc->event_list, eh, eh_list);
	if (mask != NULL)
	error = mps_update_events(sc, eh, mask);
	*handle = eh;

	return (error);
	}

	int
	mps_update_events(struct mps_softc sc, struct mps_event_handle handle,
	u32 *mask)
	{
	MPI2_EVENT_NOTIFICATION_REQUEST *evtreq;
	MPI2_EVENT_NOTIFICATION_REPLY *reply = NULL;
	struct mps_command *cm;
	int error, i;

	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);

	if ((mask != NULL) && (handle != NULL))
	bcopy(mask, &handle->mask[0], sizeof(u32) *
	MPI2_EVENT_NOTIFY_EVENTMASK_WORDS);

	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
	sc->event_mask[i] = -1;

	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
	sc->event_mask[i] &= ~handle->mask[i];


	if ((cm = mps_alloc_command(sc)) == NULL)
	return (EBUSY);
	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;
	evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
	evtreq->MsgFlags = 0;
	evtreq->SASBroadcastPrimitiveMasks = 0;
	#ifdef MPS_DEBUG_ALL_EVENTS
	{
	u_char fullmask[16];
	memset(fullmask, 0x00, 16);
	bcopy(fullmask, &evtreq->EventMasks[0], sizeof(u32) *
	MPI2_EVENT_NOTIFY_EVENTMASK_WORDS);
	}
	#else
	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
	evtreq->EventMasks[i] =
	htole32(sc->event_mask[i]);
	#endif
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_data = NULL;

	error = mps_wait_command(sc, &cm, 60, 0);
	if (cm != NULL)
	reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply;
	if ((reply == NULL) \|\|
	(reply->IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
	error = ENXIO;

	if (reply)
	MPS_DPRINT_EVENT(sc, generic, reply);

	mps_dprint(sc, MPS_TRACE, "%s finished error %d\n", __func__, error);

	if (cm != NULL)
	mps_free_command(sc, cm);
	return (error);
	}

	static int
	mps_reregister_events(struct mps_softc *sc)
	{
	MPI2_EVENT_NOTIFICATION_REQUEST *evtreq;
	struct mps_command *cm;
	struct mps_event_handle *eh;
	int error, i;

	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);

	/* first, reregister events */

	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
	sc->event_mask[i] = -1;

	TAILQ_FOREACH(eh, &sc->event_list, eh_list) {
	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
	sc->event_mask[i] &= ~eh->mask[i];
	}

	if ((cm = mps_alloc_command(sc)) == NULL)
	return (EBUSY);
	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;
	evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
	evtreq->MsgFlags = 0;
	evtreq->SASBroadcastPrimitiveMasks = 0;
	#ifdef MPS_DEBUG_ALL_EVENTS
	{
	u_char fullmask[16];
	memset(fullmask, 0x00, 16);
	bcopy(fullmask, &evtreq->EventMasks[0], sizeof(u32) *
	MPI2_EVENT_NOTIFY_EVENTMASK_WORDS);
	}
	#else
	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
	evtreq->EventMasks[i] =
	htole32(sc->event_mask[i]);
	#endif
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_data = NULL;
	cm->cm_complete = mps_reregister_events_complete;

	error = mps_map_command(sc, cm);

	mps_dprint(sc, MPS_TRACE, "%s finished with error %d\n", __func__,
	error);
	return (error);
	}

	void
	mps_deregister_events(struct mps_softc sc, struct mps_event_handle handle)
	{

	TAILQ_REMOVE(&sc->event_list, handle, eh_list);
	free(handle, M_MPT2);
	}

	/*
	* Add a chain element as the next SGE for the specified command.
	* Reset cm_sge and cm_sgesize to indicate all the available space.
	*/
	static int
	mps_add_chain(struct mps_command *cm)
	{
	MPI2_SGE_CHAIN32 *sgc;
	struct mps_chain *chain;
	u_int space;

	if (cm->cm_sglsize < MPS_SGC_SIZE)
	panic("MPS: Need SGE Error Code\n");

	chain = mps_alloc_chain(cm->cm_sc);
	if (chain == NULL)
	return (ENOBUFS);

	space = cm->cm_sc->reqframesz;

	/*
	* Note: a double-linked list is used to make it easier to
	* walk for debugging.
	*/
	TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link);

	sgc = (MPI2_SGE_CHAIN32 *)&cm->cm_sge->MpiChain;
	sgc->Length = htole16(space);
	sgc->NextChainOffset = 0;
	/* TODO Looks like bug in Setting sgc->Flags.
	* sgc->Flags = ( MPI2_SGE_FLAGS_CHAIN_ELEMENT \| MPI2_SGE_FLAGS_64_BIT_ADDRESSING \|
	* MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT
	* This is fine.. because we are not using simple element. In case of
	* MPI2_SGE_CHAIN32, we have separate Length and Flags feild.
	*/
	sgc->Flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT;
	sgc->Address = htole32(chain->chain_busaddr);

	cm->cm_sge = (MPI2_SGE_IO_UNION *)&chain->chain->MpiSimple;
	cm->cm_sglsize = space;
	return (0);
	}

	/*
	* Add one scatter-gather element (chain, simple, transaction context)
	* to the scatter-gather list for a command. Maintain cm_sglsize and
	* cm_sge as the remaining size and pointer to the next SGE to fill
	* in, respectively.
	*/
	int
	mps_push_sge(struct mps_command cm, void sgep, size_t len, int segsleft)
	{
	MPI2_SGE_TRANSACTION_UNION *tc = sgep;
	MPI2_SGE_SIMPLE64 *sge = sgep;
	int error, type;
	uint32_t saved_buf_len, saved_address_low, saved_address_high;

	type = (tc->Flags & MPI2_SGE_FLAGS_ELEMENT_MASK);

	#ifdef INVARIANTS
	switch (type) {
	case MPI2_SGE_FLAGS_TRANSACTION_ELEMENT: {
	if (len != tc->DetailsLength + 4)
	panic("TC %p length %u or %zu?", tc,
	tc->DetailsLength + 4, len);
	}
	break;
	case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
	/* Driver only uses 32-bit chain elements */
	if (len != MPS_SGC_SIZE)
	panic("CHAIN %p length %u or %zu?", sgep,
	MPS_SGC_SIZE, len);
	break;
	case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
	/* Driver only uses 64-bit SGE simple elements */
	if (len != MPS_SGE64_SIZE)
	panic("SGE simple %p length %u or %zu?", sge,
	MPS_SGE64_SIZE, len);
	if (((le32toh(sge->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT) &
	MPI2_SGE_FLAGS_ADDRESS_SIZE) == 0)
	panic("SGE simple %p not marked 64-bit?", sge);

	break;
	default:
	panic("Unexpected SGE %p, flags %02x", tc, tc->Flags);
	}
	#endif

	/*
	* case 1: 1 more segment, enough room for it
	* case 2: 2 more segments, enough room for both
	* case 3: >=2 more segments, only enough room for 1 and a chain
	* case 4: >=1 more segment, enough room for only a chain
	* case 5: >=1 more segment, no room for anything (error)
	*/

	/*
	* There should be room for at least a chain element, or this
	* code is buggy. Case (5).
	*/
	if (cm->cm_sglsize < MPS_SGC_SIZE)
	panic("MPS: Need SGE Error Code\n");

	if (segsleft >= 1 && cm->cm_sglsize < len + MPS_SGC_SIZE) {
	/*
	* 1 or more segment, enough room for only a chain.
	* Hope the previous element wasn't a Simple entry
	* that needed to be marked with
	* MPI2_SGE_FLAGS_LAST_ELEMENT. Case (4).
	*/
	if ((error = mps_add_chain(cm)) != 0)
	return (error);
	}

	if (segsleft >= 2 &&
	cm->cm_sglsize < len + MPS_SGC_SIZE + MPS_SGE64_SIZE) {
	/*
	* There are 2 or more segments left to add, and only
	* enough room for 1 and a chain. Case (3).
	*
	* Mark as last element in this chain if necessary.
	*/
	if (type == MPI2_SGE_FLAGS_SIMPLE_ELEMENT) {
	sge->FlagsLength \|= htole32(
	MPI2_SGE_FLAGS_LAST_ELEMENT << MPI2_SGE_FLAGS_SHIFT);
	}

	/*
	* Add the item then a chain. Do the chain now,
	* rather than on the next iteration, to simplify
	* understanding the code.
	*/
	cm->cm_sglsize -= len;
	bcopy(sgep, cm->cm_sge, len);
	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);
	return (mps_add_chain(cm));
	}

	#ifdef INVARIANTS
	/* Case 1: 1 more segment, enough room for it. */
	if (segsleft == 1 && cm->cm_sglsize < len)
	panic("1 seg left and no room? %u versus %zu",
	cm->cm_sglsize, len);

	/* Case 2: 2 more segments, enough room for both */
	if (segsleft == 2 && cm->cm_sglsize < len + MPS_SGE64_SIZE)
	panic("2 segs left and no room? %u versus %zu",
	cm->cm_sglsize, len);
	#endif

	if (segsleft == 1 && type == MPI2_SGE_FLAGS_SIMPLE_ELEMENT) {
	/*
	* If this is a bi-directional request, need to account for that
	* here. Save the pre-filled sge values. These will be used
	* either for the 2nd SGL or for a single direction SGL. If
	* cm_out_len is non-zero, this is a bi-directional request, so
	* fill in the OUT SGL first, then the IN SGL, otherwise just
	* fill in the IN SGL. Note that at this time, when filling in
	* 2 SGL's for a bi-directional request, they both use the same
	* DMA buffer (same cm command).
	*/
	saved_buf_len = le32toh(sge->FlagsLength) & 0x00FFFFFF;
	saved_address_low = sge->Address.Low;
	saved_address_high = sge->Address.High;
	if (cm->cm_out_len) {
	sge->FlagsLength = htole32(cm->cm_out_len \|
	((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_SGE_FLAGS_END_OF_BUFFER \|
	MPI2_SGE_FLAGS_HOST_TO_IOC \|
	MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
	MPI2_SGE_FLAGS_SHIFT));
	cm->cm_sglsize -= len;
	bcopy(sgep, cm->cm_sge, len);
	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge
	+ len);
	}
	saved_buf_len \|=
	((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_SGE_FLAGS_END_OF_BUFFER \|
	MPI2_SGE_FLAGS_LAST_ELEMENT \|
	MPI2_SGE_FLAGS_END_OF_LIST \|
	MPI2_SGE_FLAGS_64_BIT_ADDRESSING) <<
	MPI2_SGE_FLAGS_SHIFT);
	if (cm->cm_flags & MPS_CM_FLAGS_DATAIN) {
	saved_buf_len \|=
	((uint32_t)(MPI2_SGE_FLAGS_IOC_TO_HOST) <<
	MPI2_SGE_FLAGS_SHIFT);
	} else {
	saved_buf_len \|=
	((uint32_t)(MPI2_SGE_FLAGS_HOST_TO_IOC) <<
	MPI2_SGE_FLAGS_SHIFT);
	}
	sge->FlagsLength = htole32(saved_buf_len);
	sge->Address.Low = saved_address_low;
	sge->Address.High = saved_address_high;
	}

	cm->cm_sglsize -= len;
	bcopy(sgep, cm->cm_sge, len);
	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);
	return (0);
	}

	/*
	* Add one dma segment to the scatter-gather list for a command.
	*/
	int
	mps_add_dmaseg(struct mps_command *cm, vm_paddr_t pa, size_t len, u_int flags,
	int segsleft)
	{
	MPI2_SGE_SIMPLE64 sge;

	/*
	* This driver always uses 64-bit address elements for simplicity.
	*/
	bzero(&sge, sizeof(sge));
	flags \|= MPI2_SGE_FLAGS_SIMPLE_ELEMENT \|
	MPI2_SGE_FLAGS_64_BIT_ADDRESSING;
	sge.FlagsLength = htole32(len \| (flags << MPI2_SGE_FLAGS_SHIFT));
	mps_from_u64(pa, &sge.Address);

	return (mps_push_sge(cm, &sge, sizeof sge, segsleft));
	}

	static void
	mps_data_cb(void arg, bus_dma_segment_t segs, int nsegs, int error)
	{
	struct mps_softc *sc;
	struct mps_command *cm;
	u_int i, dir, sflags;

	cm = (struct mps_command *)arg;
	sc = cm->cm_sc;

	/*
	* In this case, just print out a warning and let the chip tell the
	* user they did the wrong thing.
	*/
	if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) {
	mps_dprint(sc, MPS_ERROR,
	"%s: warning: busdma returned %d segments, "
	"more than the %d allowed\n", __func__, nsegs,
	cm->cm_max_segs);
	}

	/*
	* Set up DMA direction flags. Bi-directional requests are also handled
	* here. In that case, both direction flags will be set.
	*/
	sflags = 0;
	if (cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) {
	/*
	* We have to add a special case for SMP passthrough, there
	* is no easy way to generically handle it. The first
	* S/G element is used for the command (therefore the
	* direction bit needs to be set). The second one is used
	* for the reply. We'll leave it to the caller to make
	* sure we only have two buffers.
	*/
	/*
	* Even though the busdma man page says it doesn't make
	* sense to have both direction flags, it does in this case.
	* We have one s/g element being accessed in each direction.
	*/
	dir = BUS_DMASYNC_PREWRITE \| BUS_DMASYNC_PREREAD;

	/*
	* Set the direction flag on the first buffer in the SMP
	* passthrough request. We'll clear it for the second one.
	*/
	sflags \|= MPI2_SGE_FLAGS_DIRECTION \|
	MPI2_SGE_FLAGS_END_OF_BUFFER;
	} else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT) {
	sflags \|= MPI2_SGE_FLAGS_HOST_TO_IOC;
	dir = BUS_DMASYNC_PREWRITE;
	} else
	dir = BUS_DMASYNC_PREREAD;

	for (i = 0; i < nsegs; i++) {
	if ((cm->cm_flags & MPS_CM_FLAGS_SMP_PASS) && (i != 0)) {
	sflags &= ~MPI2_SGE_FLAGS_DIRECTION;
	}
	error = mps_add_dmaseg(cm, segs[i].ds_addr, segs[i].ds_len,
	sflags, nsegs - i);
	if (error != 0) {
	/* Resource shortage, roll back! */
	if (ratecheck(&sc->lastfail, &mps_chainfail_interval))
	mps_dprint(sc, MPS_INFO, "Out of chain frames, "
	"consider increasing hw.mps.max_chains.\n");
	cm->cm_flags \|= MPS_CM_FLAGS_CHAIN_FAILED;
	mps_complete_command(sc, cm);
	return;
	}
	}

	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
	mps_enqueue_request(sc, cm);

	return;
	}

	static void
	mps_data_cb2(void arg, bus_dma_segment_t segs, int nsegs, bus_size_t mapsize,
	int error)
	{
	mps_data_cb(arg, segs, nsegs, error);
	}

	/*
	* This is the routine to enqueue commands ansynchronously.
	* Note that the only error path here is from bus_dmamap_load(), which can
	* return EINPROGRESS if it is waiting for resources. Other than this, it's
	* assumed that if you have a command in-hand, then you have enough credits
	* to use it.
	*/
	int
	mps_map_command(struct mps_softc sc, struct mps_command cm)
	{
	int error = 0;

	if (cm->cm_flags & MPS_CM_FLAGS_USE_UIO) {
	error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap,
	&cm->cm_uio, mps_data_cb2, cm, 0);
	} else if (cm->cm_flags & MPS_CM_FLAGS_USE_CCB) {
	error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap,
	cm->cm_data, mps_data_cb, cm, 0);
	} else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) {
	error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap,
	cm->cm_data, cm->cm_length, mps_data_cb, cm, 0);
	} else {
	/* Add a zero-length element as needed */
	if (cm->cm_sge != NULL)
	mps_add_dmaseg(cm, 0, 0, 0, 1);
	mps_enqueue_request(sc, cm);
	}

	return (error);
	}

	/*
	* This is the routine to enqueue commands synchronously. An error of
	* EINPROGRESS from mps_map_command() is ignored since the command will
	* be executed and enqueued automatically. Other errors come from msleep().
	*/
	int
	mps_wait_command(struct mps_softc sc, struct mps_command *cmp, int timeout,
	int sleep_flag)
	{
	int error, rc;
	struct timeval cur_time, start_time;
	struct mps_command cm = cmp;

	if (sc->mps_flags & MPS_FLAGS_DIAGRESET)
	return EBUSY;

	cm->cm_complete = NULL;
	cm->cm_flags \|= MPS_CM_FLAGS_POLLED;
	error = mps_map_command(sc, cm);
	if ((error != 0) && (error != EINPROGRESS))
	return (error);

	/*
	* Check for context and wait for 50 mSec at a time until time has
	* expired or the command has finished. If msleep can't be used, need
	* to poll.
	*/
	if (curthread->td_no_sleeping != 0)
	sleep_flag = NO_SLEEP;
	getmicrouptime(&start_time);
	if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) {
	cm->cm_flags \|= MPS_CM_FLAGS_WAKEUP;
	error = msleep(cm, &sc->mps_mtx, 0, "mpswait", timeout*hz);
	if (error == EWOULDBLOCK) {
	/*
	* Record the actual elapsed time in the case of a
	* timeout for the message below.
	*/
	getmicrouptime(&cur_time);
	timevalsub(&cur_time, &start_time);
	}
	} else {
	while ((cm->cm_flags & MPS_CM_FLAGS_COMPLETE) == 0) {
	mps_intr_locked(sc);
	if (sleep_flag == CAN_SLEEP)
	pause("mpswait", hz/20);
	else
	DELAY(50000);

	getmicrouptime(&cur_time);
	timevalsub(&cur_time, &start_time);
	if (cur_time.tv_sec > timeout) {
	error = EWOULDBLOCK;
	break;
	}
	}
	}

	if (error == EWOULDBLOCK) {
	if (cm->cm_timeout_handler == NULL) {
	mps_dprint(sc, MPS_FAULT, "Calling Reinit from %s, timeout=%d,"
	" elapsed=%jd\n", __func__, timeout,
	(intmax_t)cur_time.tv_sec);
	rc = mps_reinit(sc);
	mps_dprint(sc, MPS_FAULT, "Reinit %s\n", (rc == 0) ? "success" :
	"failed");
	} else
	cm->cm_timeout_handler(sc, cm);
	if (sc->mps_flags & MPS_FLAGS_REALLOCATED) {
	/*
	* Tell the caller that we freed the command in a
	* reinit.
	*/
	*cmp = NULL;
	}
	error = ETIMEDOUT;
	}
	return (error);
	}

	/*
	* The MPT driver had a verbose interface for config pages. In this driver,
	* reduce it to much simpler terms, similar to the Linux driver.
	*/
	int
	mps_read_config_page(struct mps_softc sc, struct mps_config_params params)
	{
	MPI2_CONFIG_REQUEST *req;
	struct mps_command *cm;
	int error;

	if (sc->mps_flags & MPS_FLAGS_BUSY) {
	return (EBUSY);
	}

	cm = mps_alloc_command(sc);
	if (cm == NULL) {
	return (EBUSY);
	}

	req = (MPI2_CONFIG_REQUEST *)cm->cm_req;
	req->Function = MPI2_FUNCTION_CONFIG;
	req->Action = params->action;
	req->SGLFlags = 0;
	req->ChainOffset = 0;
	req->PageAddress = params->page_address;
	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
	MPI2_CONFIG_EXTENDED_PAGE_HEADER *hdr;

	hdr = &params->hdr.Ext;
	req->ExtPageType = hdr->ExtPageType;
	req->ExtPageLength = hdr->ExtPageLength;
	req->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
	req->Header.PageLength = 0; /* Must be set to zero */
	req->Header.PageNumber = hdr->PageNumber;
	req->Header.PageVersion = hdr->PageVersion;
	} else {
	MPI2_CONFIG_PAGE_HEADER *hdr;

	hdr = &params->hdr.Struct;
	req->Header.PageType = hdr->PageType;
	req->Header.PageNumber = hdr->PageNumber;
	req->Header.PageLength = hdr->PageLength;
	req->Header.PageVersion = hdr->PageVersion;
	}

	cm->cm_data = params->buffer;
	cm->cm_length = params->length;
	if (cm->cm_data != NULL) {
	cm->cm_sge = &req->PageBufferSGE;
	cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
	cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE \| MPS_CM_FLAGS_DATAIN;
	} else
	cm->cm_sge = NULL;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;

	cm->cm_complete_data = params;
	if (params->callback != NULL) {
	cm->cm_complete = mps_config_complete;
	return (mps_map_command(sc, cm));
	} else {
	error = mps_wait_command(sc, &cm, 0, CAN_SLEEP);
	if (error) {
	mps_dprint(sc, MPS_FAULT,
	"Error %d reading config page\n", error);
	if (cm != NULL)
	mps_free_command(sc, cm);
	return (error);
	}
	mps_config_complete(sc, cm);
	}

	return (0);
	}

	int
	mps_write_config_page(struct mps_softc sc, struct mps_config_params params)
	{
	return (EINVAL);
	}

	static void
	mps_config_complete(struct mps_softc sc, struct mps_command cm)
	{
	MPI2_CONFIG_REPLY *reply;
	struct mps_config_params *params;

	MPS_FUNCTRACE(sc);
	params = cm->cm_complete_data;

	if (cm->cm_data != NULL) {
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
	BUS_DMASYNC_POSTREAD);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
	}

	/*
	* XXX KDM need to do more error recovery? This results in the
	* device in question not getting probed.
	*/
	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	params->status = MPI2_IOCSTATUS_BUSY;
	goto done;
	}

	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;
	if (reply == NULL) {
	params->status = MPI2_IOCSTATUS_BUSY;
	goto done;
	}
	params->status = reply->IOCStatus;
	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
	params->hdr.Ext.ExtPageType = reply->ExtPageType;
	params->hdr.Ext.ExtPageLength = reply->ExtPageLength;
	params->hdr.Ext.PageType = reply->Header.PageType;
	params->hdr.Ext.PageNumber = reply->Header.PageNumber;
	params->hdr.Ext.PageVersion = reply->Header.PageVersion;
	} else {
	params->hdr.Struct.PageType = reply->Header.PageType;
	params->hdr.Struct.PageNumber = reply->Header.PageNumber;
	params->hdr.Struct.PageLength = reply->Header.PageLength;
	params->hdr.Struct.PageVersion = reply->Header.PageVersion;
	}

	done:
	mps_free_command(sc, cm);
	if (params->callback != NULL)
	params->callback(sc, params);

	return;
	}
	diff --git a/sys/dev/mps/mps_sas.c b/sys/dev/mps/mps_sas.c
	index fd3f47c63697..74fced8abfab 100644
	--- a/sys/dev/mps/mps_sas.c
	+++ b/sys/dev/mps/mps_sas.c
	@@ -1,3428 +1,3413 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2009 Yahoo! Inc.
	* Copyright (c) 2011-2015 LSI Corp.
	* Copyright (c) 2013-2015 Avago Technologies
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/* Communications core for Avago Technologies (LSI) MPT2 */

	/* TODO Move headers to mpsvar */
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/selinfo.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/bio.h>
	#include <sys/malloc.h>
	#include <sys/uio.h>
	#include <sys/sysctl.h>
	#include <sys/endian.h>
	#include <sys/queue.h>
	#include <sys/kthread.h>
	#include <sys/taskqueue.h>
	#include <sys/sbuf.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>

	#include <machine/stdarg.h>

	#include <cam/cam.h>
	#include <cam/cam_ccb.h>
	#include <cam/cam_xpt.h>
	#include <cam/cam_debug.h>
	#include <cam/cam_sim.h>
	#include <cam/cam_xpt_sim.h>
	#include <cam/cam_xpt_periph.h>
	#include <cam/cam_periph.h>
	#include <cam/scsi/scsi_all.h>
	#include <cam/scsi/scsi_message.h>
	#include <cam/scsi/smp_all.h>

	#include <dev/mps/mpi/mpi2_type.h>
	#include <dev/mps/mpi/mpi2.h>
	#include <dev/mps/mpi/mpi2_ioc.h>
	#include <dev/mps/mpi/mpi2_sas.h>
	#include <dev/mps/mpi/mpi2_cnfg.h>
	#include <dev/mps/mpi/mpi2_init.h>
	#include <dev/mps/mpi/mpi2_tool.h>
	#include <dev/mps/mps_ioctl.h>
	#include <dev/mps/mpsvar.h>
	#include <dev/mps/mps_table.h>
	#include <dev/mps/mps_sas.h>

	#define MPSSAS_DISCOVERY_TIMEOUT 20
	#define MPSSAS_MAX_DISCOVERY_TIMEOUTS 10 /* 200 seconds */

	/*
	* static array to check SCSI OpCode for EEDP protection bits
	*/
	#define PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
	#define PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
	#define PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
	static uint8_t op_code_prot[256] = {
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	};

	MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");

	static void mpssas_remove_device(struct mps_softc , struct mps_command );
	static void mpssas_remove_complete(struct mps_softc , struct mps_command );
	static void mpssas_action(struct cam_sim sim, union ccb ccb);
	static void mpssas_poll(struct cam_sim *sim);
	static int mpssas_send_abort(struct mps_softc sc, struct mps_command tm,
	struct mps_command *cm);
	static void mpssas_scsiio_timeout(void *data);
	static void mpssas_abort_complete(struct mps_softc sc, struct mps_command cm);
	static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
	struct mps_command cm, union ccb ccb);
	static void mpssas_action_scsiio(struct mpssas_softc , union ccb );
	static void mpssas_scsiio_complete(struct mps_softc , struct mps_command );
	static void mpssas_action_resetdev(struct mpssas_softc , union ccb );
	static void mpssas_smpio_complete(struct mps_softc sc, struct mps_command cm);
	static void mpssas_send_smpcmd(struct mpssas_softc sassc, union ccb ccb,
	uint64_t sasaddr);
	static void mpssas_action_smpio(struct mpssas_softc sassc, union ccb ccb);
	static void mpssas_resetdev_complete(struct mps_softc , struct mps_command );
	static void mpssas_async(void *callback_arg, uint32_t code,
	struct cam_path path, void arg);
	static int mpssas_send_portenable(struct mps_softc *sc);
	static void mpssas_portenable_complete(struct mps_softc *sc,
	struct mps_command *cm);

	struct mpssas_target *
	mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
	{
	struct mpssas_target *target;
	int i;

	for (i = start; i < sassc->maxtargets; i++) {
	target = &sassc->targets[i];
	if (target->handle == handle)
	return (target);
	}

	return (NULL);
	}

	/* we need to freeze the simq during attach and diag reset, to avoid failing
	* commands before device handles have been found by discovery. Since
	* discovery involves reading config pages and possibly sending commands,
	* discovery actions may continue even after we receive the end of discovery
	* event, so refcount discovery actions instead of assuming we can unfreeze
	* the simq when we get the event.
	*/
	void
	mpssas_startup_increment(struct mpssas_softc *sassc)
	{
	MPS_FUNCTRACE(sassc->sc);

	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
	if (sassc->startup_refcount++ == 0) {
	/* just starting, freeze the simq */
	mps_dprint(sassc->sc, MPS_INIT,
	"%s freezing simq\n", __func__);
	xpt_hold_boot();
	xpt_freeze_simq(sassc->sim, 1);
	}
	mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
	sassc->startup_refcount);
	}
	}

	void
	mpssas_release_simq_reinit(struct mpssas_softc *sassc)
	{
	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
	sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
	xpt_release_simq(sassc->sim, 1);
	mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
	}
	}

	void
	mpssas_startup_decrement(struct mpssas_softc *sassc)
	{
	MPS_FUNCTRACE(sassc->sc);

	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
	if (--sassc->startup_refcount == 0) {
	/* finished all discovery-related actions, release
	* the simq and rescan for the latest topology.
	*/
	mps_dprint(sassc->sc, MPS_INIT,
	"%s releasing simq\n", __func__);
	sassc->flags &= ~MPSSAS_IN_STARTUP;
	xpt_release_simq(sassc->sim, 1);
	xpt_release_boot();
	}
	mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
	sassc->startup_refcount);
	}
	}

	/*
	* The firmware requires us to stop sending commands when we're doing task
	* management.
	* XXX The logic for serializing the device has been made lazy and moved to
	* mpssas_prepare_for_tm().
	*/
	struct mps_command *
	mpssas_alloc_tm(struct mps_softc *sc)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mps_command *tm;

	tm = mps_alloc_high_priority_command(sc);
	if (tm == NULL)
	return (NULL);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
	return tm;
	}

	void
	mpssas_free_tm(struct mps_softc sc, struct mps_command tm)
	{
	int target_id = 0xFFFFFFFF;

	if (tm == NULL)
	return;

	/*
	* For TM's the devq is frozen for the device. Unfreeze it here and
	* free the resources used for freezing the devq. Must clear the
	* INRESET flag as well or scsi I/O will not work.
	*/
	if (tm->cm_targ != NULL) {
	tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
	target_id = tm->cm_targ->tid;
	}
	if (tm->cm_ccb) {
	mps_dprint(sc, MPS_INFO, "Unfreezing devq for target ID %d\n",
	target_id);
	xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
	xpt_free_path(tm->cm_ccb->ccb_h.path);
	xpt_free_ccb(tm->cm_ccb);
	}

	mps_free_high_priority_command(sc, tm);
	}

	void
	mpssas_rescan_target(struct mps_softc sc, struct mpssas_target targ)
	{
	struct mpssas_softc *sassc = sc->sassc;
	path_id_t pathid;
	target_id_t targetid;
	union ccb *ccb;

	MPS_FUNCTRACE(sc);
	pathid = cam_sim_path(sassc->sim);
	if (targ == NULL)
	targetid = CAM_TARGET_WILDCARD;
	else
	targetid = targ - sassc->targets;

	/*
	* Allocate a CCB and schedule a rescan.
	*/
	ccb = xpt_alloc_ccb_nowait();
	if (ccb == NULL) {
	mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
	return;
	}

	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
	targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
	mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
	xpt_free_ccb(ccb);
	return;
	}

	if (targetid == CAM_TARGET_WILDCARD)
	ccb->ccb_h.func_code = XPT_SCAN_BUS;
	else
	ccb->ccb_h.func_code = XPT_SCAN_TGT;

	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
	xpt_rescan(ccb);
	}

	static void
	mpssas_log_command(struct mps_command cm, u_int level, const char fmt, ...)
	{
	struct sbuf sb;
	va_list ap;
	char str[224];
	char path_str[64];

	if (cm == NULL)
	return;

	/* No need to be in here if debugging isn't enabled */
	if ((cm->cm_sc->mps_debug & level) == 0)
	return;

	sbuf_new(&sb, str, sizeof(str), 0);

	va_start(ap, fmt);

	if (cm->cm_ccb != NULL) {
	xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
	sizeof(path_str));
	sbuf_cat(&sb, path_str);
	if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
	scsi_command_string(&cm->cm_ccb->csio, &sb);
	sbuf_printf(&sb, "length %d ",
	cm->cm_ccb->csio.dxfer_len);
	}
	}
	else {
	sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
	cam_sim_name(cm->cm_sc->sassc->sim),
	cam_sim_unit(cm->cm_sc->sassc->sim),
	cam_sim_bus(cm->cm_sc->sassc->sim),
	cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
	cm->cm_lun);
	}

	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
	sbuf_vprintf(&sb, fmt, ap);
	sbuf_finish(&sb);
	mps_print_field(cm->cm_sc, "%s", sbuf_data(&sb));

	va_end(ap);
	}


	static void
	mpssas_remove_volume(struct mps_softc sc, struct mps_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	struct mpssas_target *targ;
	uint16_t handle;

	MPS_FUNCTRACE(sc);

	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
	targ = tm->cm_targ;

	if (reply == NULL) {
	/* XXX retry the remove after the diag reset completes? */
	mps_dprint(sc, MPS_FAULT,
	"%s NULL reply resetting device 0x%04x\n", __func__,
	handle);
	mpssas_free_tm(sc, tm);
	return;
	}

	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) {
	mps_dprint(sc, MPS_ERROR,
	"IOCStatus = 0x%x while resetting device 0x%x\n",
	le16toh(reply->IOCStatus), handle);
	}

	mps_dprint(sc, MPS_XINFO,
	"Reset aborted %u commands\n", reply->TerminationCount);
	mps_free_reply(sc, tm->cm_reply_data);
	tm->cm_reply = NULL; /* Ensures the reply won't get re-freed */

	mps_dprint(sc, MPS_XINFO,
	"clearing target %u handle 0x%04x\n", targ->tid, handle);

	/*
	* Don't clear target if remove fails because things will get confusing.
	* Leave the devname and sasaddr intact so that we know to avoid reusing
	* this target id if possible, and so we can assign the same target id
	* to this device if it comes back in the future.
	*/
	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	targ = tm->cm_targ;
	targ->handle = 0x0;
	targ->encl_handle = 0x0;
	targ->encl_slot = 0x0;
	targ->exp_dev_handle = 0x0;
	targ->phy_num = 0x0;
	targ->linkrate = 0x0;
	targ->devinfo = 0x0;
	targ->flags = 0x0;
	}

	mpssas_free_tm(sc, tm);
	}


	/*
	* No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
	* Otherwise Volume Delete is same as Bare Drive Removal.
	*/
	void
	mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mps_softc *sc;
	struct mps_command *tm;
	struct mpssas_target *targ = NULL;

	MPS_FUNCTRACE(sassc->sc);
	sc = sassc->sc;

	#ifdef WD_SUPPORT
	/*
	* If this is a WD controller, determine if the disk should be exposed
	* to the OS or not. If disk should be exposed, return from this
	* function without doing anything.
	*/
	if (sc->WD_available && (sc->WD_hide_expose ==
	MPS_WD_EXPOSE_ALWAYS)) {
	return;
	}
	#endif //WD_SUPPORT

	targ = mpssas_find_target_by_handle(sassc, 0, handle);
	if (targ == NULL) {
	/* FIXME: what is the action? */
	/* We don't know about this device? */
	mps_dprint(sc, MPS_ERROR,
	"%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
	return;
	}

	targ->flags \|= MPSSAS_TARGET_INREMOVAL;

	tm = mpssas_alloc_tm(sc);
	if (tm == NULL) {
	mps_dprint(sc, MPS_ERROR,
	"%s: command alloc failure\n", __func__);
	return;
	}

	mpssas_rescan_target(sc, targ);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = targ->handle;
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;

	/* SAS Hard Link Reset / SATA Link Reset */
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;

	tm->cm_targ = targ;
	tm->cm_data = NULL;
	tm->cm_complete = mpssas_remove_volume;
	tm->cm_complete_data = (void *)(uintptr_t)handle;

	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
	__func__, targ->tid);
	mpssas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);

	mps_map_command(sc, tm);
	}

	/*
	* The MPT2 firmware performs debounce on the link to avoid transient link
	* errors and false removals. When it does decide that link has been lost
	* and a device need to go away, it expects that the host will perform a
	* target reset and then an op remove. The reset has the side-effect of
	* aborting any outstanding requests for the device, which is required for
	* the op-remove to succeed. It's not clear if the host should check for
	* the device coming back alive after the reset.
	*/
	void
	mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mps_softc *sc;
	struct mps_command *cm;
	struct mpssas_target *targ = NULL;

	MPS_FUNCTRACE(sassc->sc);

	sc = sassc->sc;

	targ = mpssas_find_target_by_handle(sassc, 0, handle);
	if (targ == NULL) {
	/* FIXME: what is the action? */
	/* We don't know about this device? */
	mps_dprint(sc, MPS_ERROR,
	"%s : invalid handle 0x%x \n", __func__, handle);
	return;
	}

	targ->flags \|= MPSSAS_TARGET_INREMOVAL;

	cm = mpssas_alloc_tm(sc);
	if (cm == NULL) {
	mps_dprint(sc, MPS_ERROR,
	"%s: command alloc failure\n", __func__);
	return;
	}

	mpssas_rescan_target(sc, targ);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
	memset(req, 0, sizeof(*req));
	req->DevHandle = htole16(targ->handle);
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;

	/* SAS Hard Link Reset / SATA Link Reset */
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;

	cm->cm_targ = targ;
	cm->cm_data = NULL;
	cm->cm_complete = mpssas_remove_device;
	cm->cm_complete_data = (void *)(uintptr_t)handle;

	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
	__func__, targ->tid);
	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);

	mps_map_command(sc, cm);
	}

	static void
	mpssas_remove_device(struct mps_softc sc, struct mps_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
	struct mpssas_target *targ;
	uint16_t handle;

	MPS_FUNCTRACE(sc);

	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_ERROR,
	"%s: cm_flags = %#x for remove of handle %#04x! "
	"This should not happen!\n", __func__, tm->cm_flags,
	handle);
	}

	if (reply == NULL) {
	/* XXX retry the remove after the diag reset completes? */
	mps_dprint(sc, MPS_FAULT,
	"%s NULL reply resetting device 0x%04x\n", __func__,
	handle);
	mpssas_free_tm(sc, tm);
	return;
	}

	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) {
	mps_dprint(sc, MPS_ERROR,
	"IOCStatus = 0x%x while resetting device 0x%x\n",
	le16toh(reply->IOCStatus), handle);
	}

	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
	le32toh(reply->TerminationCount));
	mps_free_reply(sc, tm->cm_reply_data);
	tm->cm_reply = NULL; /* Ensures the reply won't get re-freed */

	/* Reuse the existing command */
	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
	memset(req, 0, sizeof(*req));
	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
	req->DevHandle = htole16(handle);
	tm->cm_data = NULL;
	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	tm->cm_complete = mpssas_remove_complete;
	tm->cm_complete_data = (void *)(uintptr_t)handle;

	/*
	* Wait to send the REMOVE_DEVICE until all the commands have cleared.
	* They should be aborted or time out and we'll kick thus off there
	* if so.
	*/
	if (TAILQ_FIRST(&targ->commands) == NULL) {
	mps_dprint(sc, MPS_INFO, "No pending commands: starting remove_device\n");
	mps_map_command(sc, tm);
	targ->pending_remove_tm = NULL;
	} else {
	targ->pending_remove_tm = tm;
	}


	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
	targ->tid, handle);
	}

	static void
	mpssas_remove_complete(struct mps_softc sc, struct mps_command tm)
	{
	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
	uint16_t handle;
	struct mpssas_target *targ;
	struct mpssas_lun *lun;

	MPS_FUNCTRACE(sc);

	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
	targ = tm->cm_targ;

	/*
	* At this point, we should have no pending commands for the target.
	* The remove target has just completed.
	*/
	KASSERT(TAILQ_FIRST(&targ->commands) == NULL,
	("%s: no commands should be pending\n", __func__));


	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_XINFO,
	"%s: cm_flags = %#x for remove of handle %#04x! "
	"This should not happen!\n", __func__, tm->cm_flags,
	handle);
	mpssas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	/* most likely a chip reset */
	mps_dprint(sc, MPS_FAULT,
	"%s NULL reply removing device 0x%04x\n", __func__, handle);
	mpssas_free_tm(sc, tm);
	return;
	}

	mps_dprint(sc, MPS_XINFO,
	"%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
	handle, le16toh(reply->IOCStatus));

	/*
	* Don't clear target if remove fails because things will get confusing.
	* Leave the devname and sasaddr intact so that we know to avoid reusing
	* this target id if possible, and so we can assign the same target id
	* to this device if it comes back in the future.
	*/
	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	targ->handle = 0x0;
	targ->encl_handle = 0x0;
	targ->encl_slot = 0x0;
	targ->exp_dev_handle = 0x0;
	targ->phy_num = 0x0;
	targ->linkrate = 0x0;
	targ->devinfo = 0x0;
	targ->flags = 0x0;

	while(!SLIST_EMPTY(&targ->luns)) {
	lun = SLIST_FIRST(&targ->luns);
	SLIST_REMOVE_HEAD(&targ->luns, lun_link);
	free(lun, M_MPT2);
	}
	}


	mpssas_free_tm(sc, tm);
	}

	static int
	mpssas_register_events(struct mps_softc *sc)
	{
	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];

	bzero(events, 16);
	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
	setbit(events, MPI2_EVENT_IR_VOLUME);
	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);

	mps_register_events(sc, events, mpssas_evt_handler, NULL,
	&sc->sassc->mpssas_eh);

	return (0);
	}

	int
	mps_attach_sas(struct mps_softc *sc)
	{
	struct mpssas_softc *sassc;
	cam_status status;
	int unit, error = 0, reqs;

	MPS_FUNCTRACE(sc);
	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);

	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK\|M_ZERO);
	- if(!sassc) {
	- mps_dprint(sc, MPS_INIT\|MPS_ERROR,
	- "Cannot allocate SAS controller memory\n");
	- return (ENOMEM);
	- }

	/*
	* XXX MaxTargets could change during a reinit. Since we don't
	* resize the targets[] array during such an event, cache the value
	* of MaxTargets here so that we don't get into trouble later. This
	* should move into the reinit logic.
	*/
	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
	sassc->targets = malloc(sizeof(struct mpssas_target) *
	sassc->maxtargets, M_MPT2, M_WAITOK\|M_ZERO);
	- if(!sassc->targets) {
	- mps_dprint(sc, MPS_INIT\|MPS_ERROR,
	- "Cannot allocate SAS target memory\n");
	- free(sassc, M_MPT2);
	- return (ENOMEM);
	- }
	sc->sassc = sassc;
	sassc->sc = sc;

	reqs = sc->num_reqs - sc->num_prireqs - 1;
	if ((sassc->devq = cam_simq_alloc(reqs)) == NULL) {
	mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
	error = ENOMEM;
	goto out;
	}

	unit = device_get_unit(sc->mps_dev);
	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
	unit, &sc->mps_mtx, reqs, reqs, sassc->devq);
	if (sassc->sim == NULL) {
	mps_dprint(sc, MPS_INIT\|MPS_ERROR, "Cannot allocate SIM\n");
	error = EINVAL;
	goto out;
	}

	TAILQ_INIT(&sassc->ev_queue);

	/* Initialize taskqueue for Event Handling */
	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT \| M_ZERO,
	taskqueue_thread_enqueue, &sassc->ev_tq);
	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
	device_get_nameunit(sc->mps_dev));

	mps_lock(sc);

	/*
	* XXX There should be a bus for every port on the adapter, but since
	* we're just going to fake the topology for now, we'll pretend that
	* everything is just a target on a single bus.
	*/
	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
	mps_dprint(sc, MPS_INIT\|MPS_ERROR,
	"Error %d registering SCSI bus\n", error);
	mps_unlock(sc);
	goto out;
	}

	/*
	* Assume that discovery events will start right away.
	*
	* Hold off boot until discovery is complete.
	*/
	sassc->flags \|= MPSSAS_IN_STARTUP \| MPSSAS_IN_DISCOVERY;
	sc->sassc->startup_refcount = 0;
	mpssas_startup_increment(sassc);

	callout_init(&sassc->discovery_callout, 1 /mpsafe/);

	/*
	* Register for async events so we can determine the EEDP
	* capabilities of devices.
	*/
	status = xpt_create_path(&sassc->path, /periph/NULL,
	cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
	CAM_LUN_WILDCARD);
	if (status != CAM_REQ_CMP) {
	mps_dprint(sc, MPS_ERROR\|MPS_INIT,
	"Error %#x creating sim path\n", status);
	sassc->path = NULL;
	} else {
	int event;

	event = AC_ADVINFO_CHANGED;
	status = xpt_register_async(event, mpssas_async, sc,
	sassc->path);
	if (status != CAM_REQ_CMP) {
	mps_dprint(sc, MPS_ERROR,
	"Error %#x registering async handler for "
	"AC_ADVINFO_CHANGED events\n", status);
	xpt_free_path(sassc->path);
	sassc->path = NULL;
	}
	}
	if (status != CAM_REQ_CMP) {
	/*
	* EEDP use is the exception, not the rule.
	* Warn the user, but do not fail to attach.
	*/
	mps_printf(sc, "EEDP capabilities disabled.\n");
	}

	mps_unlock(sc);

	mpssas_register_events(sc);
	out:
	if (error)
	mps_detach_sas(sc);

	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);
	return (error);
	}

	int
	mps_detach_sas(struct mps_softc *sc)
	{
	struct mpssas_softc *sassc;
	struct mpssas_lun lun, lun_tmp;
	struct mpssas_target *targ;
	int i;

	MPS_FUNCTRACE(sc);

	if (sc->sassc == NULL)
	return (0);

	sassc = sc->sassc;
	mps_deregister_events(sc, sassc->mpssas_eh);

	/*
	* Drain and free the event handling taskqueue with the lock
	* unheld so that any parallel processing tasks drain properly
	* without deadlocking.
	*/
	if (sassc->ev_tq != NULL)
	taskqueue_free(sassc->ev_tq);

	/* Make sure CAM doesn't wedge if we had to bail out early. */
	mps_lock(sc);

	while (sassc->startup_refcount != 0)
	mpssas_startup_decrement(sassc);

	/* Deregister our async handler */
	if (sassc->path != NULL) {
	xpt_register_async(0, mpssas_async, sc, sassc->path);
	xpt_free_path(sassc->path);
	sassc->path = NULL;
	}

	if (sassc->flags & MPSSAS_IN_STARTUP)
	xpt_release_simq(sassc->sim, 1);

	if (sassc->sim != NULL) {
	xpt_bus_deregister(cam_sim_path(sassc->sim));
	cam_sim_free(sassc->sim, FALSE);
	}

	mps_unlock(sc);

	if (sassc->devq != NULL)
	cam_simq_free(sassc->devq);

	for(i=0; i< sassc->maxtargets ;i++) {
	targ = &sassc->targets[i];
	SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
	free(lun, M_MPT2);
	}
	}
	free(sassc->targets, M_MPT2);
	free(sassc, M_MPT2);
	sc->sassc = NULL;

	return (0);
	}

	void
	mpssas_discovery_end(struct mpssas_softc *sassc)
	{
	struct mps_softc *sc = sassc->sc;

	MPS_FUNCTRACE(sc);

	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
	callout_stop(&sassc->discovery_callout);

	/*
	* After discovery has completed, check the mapping table for any
	* missing devices and update their missing counts. Only do this once
	* whenever the driver is initialized so that missing counts aren't
	* updated unnecessarily. Note that just because discovery has
	* completed doesn't mean that events have been processed yet. The
	* check_devices function is a callout timer that checks if ALL devices
	* are missing. If so, it will wait a little longer for events to
	* complete and keep resetting itself until some device in the mapping
	* table is not missing, meaning that event processing has started.
	*/
	if (sc->track_mapping_events) {
	mps_dprint(sc, MPS_XINFO \| MPS_MAPPING, "Discovery has "
	"completed. Check for missing devices in the mapping "
	"table.\n");
	callout_reset(&sc->device_check_callout,
	MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices,
	sc);
	}
	}

	static void
	mpssas_action(struct cam_sim sim, union ccb ccb)
	{
	struct mpssas_softc *sassc;

	sassc = cam_sim_softc(sim);

	MPS_FUNCTRACE(sassc->sc);
	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
	ccb->ccb_h.func_code);
	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);

	switch (ccb->ccb_h.func_code) {
	case XPT_PATH_INQ:
	{
	struct ccb_pathinq *cpi = &ccb->cpi;
	struct mps_softc *sc = sassc->sc;

	cpi->version_num = 1;
	cpi->hba_inquiry = PI_SDTR_ABLE\|PI_TAG_ABLE\|PI_WIDE_16;
	cpi->target_sprt = 0;
	cpi->hba_misc = PIM_NOBUSRESET \| PIM_UNMAPPED \| PIM_NOSCAN;
	cpi->hba_eng_cnt = 0;
	cpi->max_target = sassc->maxtargets - 1;
	cpi->max_lun = 255;

	/*
	* initiator_id is set here to an ID outside the set of valid
	* target IDs (including volumes).
	*/
	cpi->initiator_id = sassc->maxtargets;
	strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
	strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
	strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
	cpi->unit_number = cam_sim_unit(sim);
	cpi->bus_id = cam_sim_bus(sim);
	cpi->base_transfer_speed = 150000;
	cpi->transport = XPORT_SAS;
	cpi->transport_version = 0;
	cpi->protocol = PROTO_SCSI;
	cpi->protocol_version = SCSI_REV_SPC;
	cpi->maxio = sc->maxio;
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	}
	case XPT_GET_TRAN_SETTINGS:
	{
	struct ccb_trans_settings *cts;
	struct ccb_trans_settings_sas *sas;
	struct ccb_trans_settings_scsi *scsi;
	struct mpssas_target *targ;

	cts = &ccb->cts;
	sas = &cts->xport_specific.sas;
	scsi = &cts->proto_specific.scsi;

	KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
	cts->ccb_h.target_id));
	targ = &sassc->targets[cts->ccb_h.target_id];
	if (targ->handle == 0x0) {
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	break;
	}

	cts->protocol_version = SCSI_REV_SPC2;
	cts->transport = XPORT_SAS;
	cts->transport_version = 0;

	sas->valid = CTS_SAS_VALID_SPEED;
	switch (targ->linkrate) {
	case 0x08:
	sas->bitrate = 150000;
	break;
	case 0x09:
	sas->bitrate = 300000;
	break;
	case 0x0a:
	sas->bitrate = 600000;
	break;
	default:
	sas->valid = 0;
	}

	cts->protocol = PROTO_SCSI;
	scsi->valid = CTS_SCSI_VALID_TQ;
	scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;

	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	}
	case XPT_CALC_GEOMETRY:
	cam_calc_geometry(&ccb->ccg, /extended/1);
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	case XPT_RESET_DEV:
	mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
	mpssas_action_resetdev(sassc, ccb);
	return;
	case XPT_RESET_BUS:
	case XPT_ABORT:
	case XPT_TERM_IO:
	mps_dprint(sassc->sc, MPS_XINFO,
	"mpssas_action faking success for abort or reset\n");
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	break;
	case XPT_SCSI_IO:
	mpssas_action_scsiio(sassc, ccb);
	return;
	case XPT_SMP_IO:
	mpssas_action_smpio(sassc, ccb);
	return;
	default:
	mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
	break;
	}
	xpt_done(ccb);

	}

	static void
	mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
	target_id_t target_id, lun_id_t lun_id)
	{
	path_id_t path_id = cam_sim_path(sc->sassc->sim);
	struct cam_path *path;

	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
	ac_code, target_id, (uintmax_t)lun_id);

	if (xpt_create_path(&path, NULL,
	path_id, target_id, lun_id) != CAM_REQ_CMP) {
	mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
	"notification\n");
	return;
	}

	xpt_async(ac_code, path, NULL);
	xpt_free_path(path);
	}

	static void
	mpssas_complete_all_commands(struct mps_softc *sc)
	{
	struct mps_command *cm;
	int i;
	int completed;

	MPS_FUNCTRACE(sc);
	mtx_assert(&sc->mps_mtx, MA_OWNED);

	/* complete all commands with a NULL reply */
	for (i = 1; i < sc->num_reqs; i++) {
	cm = &sc->commands[i];
	if (cm->cm_state == MPS_CM_STATE_FREE)
	continue;

	cm->cm_state = MPS_CM_STATE_BUSY;
	cm->cm_reply = NULL;
	completed = 0;

	if (cm->cm_flags & MPS_CM_FLAGS_SATA_ID_TIMEOUT) {
	MPASS(cm->cm_data);
	free(cm->cm_data, M_MPT2);
	cm->cm_data = NULL;
	}

	if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
	cm->cm_flags \|= MPS_CM_FLAGS_COMPLETE;

	if (cm->cm_complete != NULL) {
	mpssas_log_command(cm, MPS_RECOVERY,
	"completing cm %p state %x ccb %p for diag reset\n",
	cm, cm->cm_state, cm->cm_ccb);

	cm->cm_complete(sc, cm);
	completed = 1;
	} else if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
	mpssas_log_command(cm, MPS_RECOVERY,
	"waking up cm %p state %x ccb %p for diag reset\n",
	cm, cm->cm_state, cm->cm_ccb);
	wakeup(cm);
	completed = 1;
	}

	if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
	/* this should never happen, but if it does, log */
	mpssas_log_command(cm, MPS_RECOVERY,
	"cm %p state %x flags 0x%x ccb %p during diag "
	"reset\n", cm, cm->cm_state, cm->cm_flags,
	cm->cm_ccb);
	}
	}

	sc->io_cmds_active = 0;
	}

	void
	mpssas_handle_reinit(struct mps_softc *sc)
	{
	int i;

	/* Go back into startup mode and freeze the simq, so that CAM
	* doesn't send any commands until after we've rediscovered all
	* targets and found the proper device handles for them.
	*
	* After the reset, portenable will trigger discovery, and after all
	* discovery-related activities have finished, the simq will be
	* released.
	*/
	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
	sc->sassc->flags \|= MPSSAS_IN_STARTUP;
	sc->sassc->flags \|= MPSSAS_IN_DISCOVERY;
	mpssas_startup_increment(sc->sassc);

	/* notify CAM of a bus reset */
	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
	CAM_LUN_WILDCARD);

	/* complete and cleanup after all outstanding commands */
	mpssas_complete_all_commands(sc);

	mps_dprint(sc, MPS_INIT,
	"%s startup %u after command completion\n", __func__,
	sc->sassc->startup_refcount);

	/* zero all the target handles, since they may change after the
	* reset, and we have to rediscover all the targets and use the new
	* handles.
	*/
	for (i = 0; i < sc->sassc->maxtargets; i++) {
	if (sc->sassc->targets[i].outstanding != 0)
	mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
	i, sc->sassc->targets[i].outstanding);
	sc->sassc->targets[i].handle = 0x0;
	sc->sassc->targets[i].exp_dev_handle = 0x0;
	sc->sassc->targets[i].outstanding = 0;
	sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
	}
	}

	static void
	mpssas_tm_timeout(void *data)
	{
	struct mps_command *tm = data;
	struct mps_softc *sc = tm->cm_sc;

	mtx_assert(&sc->mps_mtx, MA_OWNED);

	mpssas_log_command(tm, MPS_INFO\|MPS_RECOVERY,
	"task mgmt %p timed out\n", tm);

	KASSERT(tm->cm_state == MPS_CM_STATE_INQUEUE,
	("command not inqueue\n"));

	tm->cm_state = MPS_CM_STATE_BUSY;
	mps_reinit(sc);
	}

	static void
	mpssas_logical_unit_reset_complete(struct mps_softc sc, struct mps_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	unsigned int cm_count = 0;
	struct mps_command *cm;
	struct mpssas_target *targ;

	callout_stop(&tm->cm_callout);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	* XXXSL So should it be an assertion?
	*/
	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_RECOVERY\|MPS_ERROR,
	"%s: cm_flags = %#x for LUN reset! "
	"This should not happen!\n", __func__, tm->cm_flags);
	mpssas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	mps_dprint(sc, MPS_RECOVERY, "NULL reset reply for tm %p\n",
	tm);
	if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
	/* this completion was due to a reset, just cleanup */
	mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
	"reset, ignoring NULL LUN reset reply\n");
	targ->tm = NULL;
	mpssas_free_tm(sc, tm);
	}
	else {
	/* we should have gotten a reply. */
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY, "NULL reply on "
	"LUN reset attempt, resetting controller\n");
	mps_reinit(sc);
	}
	return;
	}

	mps_dprint(sc, MPS_RECOVERY,
	"logical unit reset status 0x%x code 0x%x count %u\n",
	le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
	le32toh(reply->TerminationCount));

	/*
	* See if there are any outstanding commands for this LUN.
	* This could be made more efficient by using a per-LU data
	* structure of some sort.
	*/
	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
	if (cm->cm_lun == tm->cm_lun)
	cm_count++;
	}

	if (cm_count == 0) {
	mps_dprint(sc, MPS_RECOVERY\|MPS_INFO,
	"Finished recovery after LUN reset for target %u\n",
	targ->tid);

	mpssas_announce_reset(sc, AC_SENT_BDR, targ->tid, tm->cm_lun);

	/*
	* We've finished recovery for this logical unit. check and
	* see if some other logical unit has a timedout command
	* that needs to be processed.
	*/
	cm = TAILQ_FIRST(&targ->timedout_commands);
	if (cm) {
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY,
	"More commands to abort for target %u\n",
	targ->tid);
	mpssas_send_abort(sc, tm, cm);
	} else {
	targ->tm = NULL;
	mpssas_free_tm(sc, tm);
	}
	} else {
	/*
	* If we still have commands for this LUN, the reset
	* effectively failed, regardless of the status reported.
	* Escalate to a target reset.
	*/
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY,
	"logical unit reset complete for target %u, but still "
	"have %u command(s), sending target reset\n", targ->tid,
	cm_count);
	mpssas_send_reset(sc, tm,
	MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
	}
	}

	static void
	mpssas_target_reset_complete(struct mps_softc sc, struct mps_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpssas_target *targ;

	callout_stop(&tm->cm_callout);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
	"This should not happen!\n", __func__, tm->cm_flags);
	mpssas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	mps_dprint(sc, MPS_RECOVERY,
	"NULL target reset reply for tm %pi TaskMID %u\n",
	tm, le16toh(req->TaskMID));
	if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
	/* this completion was due to a reset, just cleanup */
	mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
	"reset, ignoring NULL target reset reply\n");
	targ->tm = NULL;
	mpssas_free_tm(sc, tm);
	} else {
	/* we should have gotten a reply. */
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY, "NULL reply on "
	"target reset attempt, resetting controller\n");
	mps_reinit(sc);
	}
	return;
	}

	mps_dprint(sc, MPS_RECOVERY,
	"target reset status 0x%x code 0x%x count %u\n",
	le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
	le32toh(reply->TerminationCount));

	if (targ->outstanding == 0) {
	/* we've finished recovery for this target and all
	* of its logical units.
	*/
	mps_dprint(sc, MPS_RECOVERY\|MPS_INFO,
	"Finished reset recovery for target %u\n", targ->tid);

	mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
	CAM_LUN_WILDCARD);

	targ->tm = NULL;
	mpssas_free_tm(sc, tm);
	} else {
	/*
	* After a target reset, if this target still has
	* outstanding commands, the reset effectively failed,
	* regardless of the status reported. escalate.
	*/
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY,
	"Target reset complete for target %u, but still have %u "
	"command(s), resetting controller\n", targ->tid,
	targ->outstanding);
	mps_reinit(sc);
	}
	}

	#define MPS_RESET_TIMEOUT 30

	int
	mpssas_send_reset(struct mps_softc sc, struct mps_command tm, uint8_t type)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpssas_target *target;
	int err;

	target = tm->cm_targ;
	if (target->handle == 0) {
	mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
	__func__, target->tid);
	return -1;
	}

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = htole16(target->handle);
	req->TaskType = type;

	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
	/* XXX Need to handle invalid LUNs */
	MPS_SET_LUN(req->LUN, tm->cm_lun);
	tm->cm_targ->logical_unit_resets++;
	mps_dprint(sc, MPS_RECOVERY\|MPS_INFO,
	"Sending logical unit reset to target %u lun %d\n",
	target->tid, tm->cm_lun);
	tm->cm_complete = mpssas_logical_unit_reset_complete;
	mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
	/*
	* Target reset method =
	* SAS Hard Link Reset / SATA Link Reset
	*/
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
	tm->cm_targ->target_resets++;
	mps_dprint(sc, MPS_RECOVERY\|MPS_INFO,
	"Sending target reset to target %u\n", target->tid);
	tm->cm_complete = mpssas_target_reset_complete;
	mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
	} else {
	mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
	return -1;
	}

	tm->cm_data = NULL;
	tm->cm_complete_data = (void *)tm;

	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
	mpssas_tm_timeout, tm);

	err = mps_map_command(sc, tm);
	if (err)
	mps_dprint(sc, MPS_ERROR\|MPS_RECOVERY,
	"error %d sending reset type %u\n",
	err, type);

	return err;
	}


	static void
	mpssas_abort_complete(struct mps_softc sc, struct mps_command tm)
	{
	struct mps_command *cm;
	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpssas_target *targ;

	callout_stop(&tm->cm_callout);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	targ = tm->cm_targ;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_RECOVERY,
	"cm_flags = %#x for abort %p TaskMID %u!\n",
	tm->cm_flags, tm, le16toh(req->TaskMID));
	mpssas_free_tm(sc, tm);
	return;
	}

	if (reply == NULL) {
	mps_dprint(sc, MPS_RECOVERY,
	"NULL abort reply for tm %p TaskMID %u\n",
	tm, le16toh(req->TaskMID));
	if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
	/* this completion was due to a reset, just cleanup */
	mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
	"reset, ignoring NULL abort reply\n");
	targ->tm = NULL;
	mpssas_free_tm(sc, tm);
	} else {
	/* we should have gotten a reply. */
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY, "NULL reply on "
	"abort attempt, resetting controller\n");
	mps_reinit(sc);
	}
	return;
	}

	mps_dprint(sc, MPS_RECOVERY,
	"abort TaskMID %u status 0x%x code 0x%x count %u\n",
	le16toh(req->TaskMID),
	le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
	le32toh(reply->TerminationCount));

	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
	if (cm == NULL) {
	/*
	* If there are no more timedout commands, we're done with
	* error recovery for this target.
	*/
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY,
	"Finished abort recovery for target %u\n", targ->tid);

	targ->tm = NULL;
	mpssas_free_tm(sc, tm);
	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
	/* abort success, but we have more timedout commands to abort */
	mps_dprint(sc, MPS_INFO\|MPS_RECOVERY,
	"Continuing abort recovery for target %u\n", targ->tid);

	mpssas_send_abort(sc, tm, cm);
	} else {
	/* we didn't get a command completion, so the abort
	* failed as far as we're concerned. escalate.
	*/
	mps_dprint(sc, MPS_RECOVERY,
	"Abort failed for target %u, sending logical unit reset\n",
	targ->tid);

	mpssas_send_reset(sc, tm,
	MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
	}
	}

	#define MPS_ABORT_TIMEOUT 5

	static int
	mpssas_send_abort(struct mps_softc sc, struct mps_command tm, struct mps_command *cm)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mpssas_target *targ;
	int err;

	targ = cm->cm_targ;
	if (targ->handle == 0) {
	mps_dprint(sc, MPS_ERROR\|MPS_RECOVERY,
	"%s null devhandle for target_id %d\n",
	__func__, cm->cm_ccb->ccb_h.target_id);
	return -1;
	}

	mpssas_log_command(cm, MPS_RECOVERY\|MPS_INFO,
	"Aborting command %p\n", cm);

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = htole16(targ->handle);
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;

	/* XXX Need to handle invalid LUNs */
	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);

	req->TaskMID = htole16(cm->cm_desc.Default.SMID);

	tm->cm_data = NULL;
	tm->cm_complete = mpssas_abort_complete;
	tm->cm_complete_data = (void *)tm;
	tm->cm_targ = cm->cm_targ;
	tm->cm_lun = cm->cm_lun;

	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
	mpssas_tm_timeout, tm);

	targ->aborts++;

	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);

	err = mps_map_command(sc, tm);
	if (err)
	mps_dprint(sc, MPS_ERROR\|MPS_RECOVERY,
	"error %d sending abort for cm %p SMID %u\n",
	err, cm, req->TaskMID);
	return err;
	}

	static void
	mpssas_scsiio_timeout(void *data)
	{
	sbintime_t elapsed, now;
	union ccb *ccb;
	struct mps_softc *sc;
	struct mps_command *cm;
	struct mpssas_target *targ;

	cm = (struct mps_command *)data;
	sc = cm->cm_sc;
	ccb = cm->cm_ccb;
	now = sbinuptime();

	MPS_FUNCTRACE(sc);
	mtx_assert(&sc->mps_mtx, MA_OWNED);

	mps_dprint(sc, MPS_XINFO\|MPS_RECOVERY, "Timeout checking cm %p\n", sc);

	/*
	* Run the interrupt handler to make sure it's not pending. This
	* isn't perfect because the command could have already completed
	* and been re-used, though this is unlikely.
	*/
	mps_intr_locked(sc);
	if (cm->cm_flags & MPS_CM_FLAGS_ON_RECOVERY) {
	mpssas_log_command(cm, MPS_XINFO,
	"SCSI command %p almost timed out\n", cm);
	return;
	}

	if (cm->cm_ccb == NULL) {
	mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
	return;
	}

	targ = cm->cm_targ;
	targ->timeouts++;

	elapsed = now - ccb->ccb_h.qos.sim_data;
	mpssas_log_command(cm, MPS_INFO\|MPS_RECOVERY,
	"Command timeout on target %u(0x%04x) %d set, %d.%d elapsed\n",
	targ->tid, targ->handle, ccb->ccb_h.timeout,
	sbintime_getsec(elapsed), elapsed & 0xffffffff);

	/* XXX first, check the firmware state, to see if it's still
	* operational. if not, do a diag reset.
	*/
	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
	cm->cm_flags \|= MPS_CM_FLAGS_ON_RECOVERY \| MPS_CM_FLAGS_TIMEDOUT;
	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);

	if (targ->tm != NULL) {
	/* target already in recovery, just queue up another
	* timedout command to be processed later.
	*/
	mps_dprint(sc, MPS_RECOVERY,
	"queued timedout cm %p for processing by tm %p\n",
	cm, targ->tm);
	} else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
	mps_dprint(sc, MPS_RECOVERY\|MPS_INFO,
	"Sending abort to target %u for SMID %d\n", targ->tid,
	cm->cm_desc.Default.SMID);
	mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
	cm, targ->tm);

	/* start recovery by aborting the first timedout command */
	mpssas_send_abort(sc, targ->tm, cm);
	} else {
	/* XXX queue this target up for recovery once a TM becomes
	* available. The firmware only has a limited number of
	* HighPriority credits for the high priority requests used
	* for task management, and we ran out.
	*
	* Isilon: don't worry about this for now, since we have
	* more credits than disks in an enclosure, and limit
	* ourselves to one TM per target for recovery.
	*/
	mps_dprint(sc, MPS_ERROR\|MPS_RECOVERY,
	"timedout cm %p failed to allocate a tm\n", cm);
	}

	}

	static void
	mpssas_action_scsiio(struct mpssas_softc sassc, union ccb ccb)
	{
	MPI2_SCSI_IO_REQUEST *req;
	struct ccb_scsiio *csio;
	struct mps_softc *sc;
	struct mpssas_target *targ;
	struct mpssas_lun *lun;
	struct mps_command *cm;
	uint8_t i, lba_byte, *ref_tag_addr;
	uint16_t eedp_flags;
	uint32_t mpi_control;

	sc = sassc->sc;
	MPS_FUNCTRACE(sc);
	mtx_assert(&sc->mps_mtx, MA_OWNED);

	csio = &ccb->csio;
	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_SCSI_IO\n",
	csio->ccb_h.target_id));
	targ = &sassc->targets[csio->ccb_h.target_id];
	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
	if (targ->handle == 0x0) {
	mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
	__func__, csio->ccb_h.target_id);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}
	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
	mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
	"supported %u\n", __func__, csio->ccb_h.target_id);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}
	/*
	* Sometimes, it is possible to get a command that is not "In
	* Progress" and was actually aborted by the upper layer. Check for
	* this here and complete the command without error.
	*/
	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
	mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
	"target %u\n", __func__, csio->ccb_h.target_id);
	xpt_done(ccb);
	return;
	}
	/*
	* If devinfo is 0 this will be a volume. In that case don't tell CAM
	* that the volume has timed out. We want volumes to be enumerated
	* until they are deleted/removed, not just failed. In either event,
	* we're removing the target due to a firmware event telling us
	* the device is now gone (as opposed to some transient event). Since
	* we're opting to remove failed devices from the OS's view, we need
	* to propagate that status up the stack.
	*/
	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
	if (targ->devinfo == 0)
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}

	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
	mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	xpt_done(ccb);
	return;
	}

	/*
	* If target has a reset in progress, freeze the devq and return. The
	* devq will be released when the TM reset is finished.
	*/
	if (targ->flags & MPSSAS_TARGET_INRESET) {
	ccb->ccb_h.status = CAM_BUSY \| CAM_DEV_QFRZN;
	mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
	__func__, targ->tid);
	xpt_freeze_devq(ccb->ccb_h.path, 1);
	xpt_done(ccb);
	return;
	}

	cm = mps_alloc_command(sc);
	if (cm == NULL \|\| (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
	if (cm != NULL) {
	mps_free_command(sc, cm);
	}
	if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
	xpt_freeze_simq(sassc->sim, 1);
	sassc->flags \|= MPSSAS_QUEUE_FROZEN;
	}
	ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
	ccb->ccb_h.status \|= CAM_REQUEUE_REQ;
	xpt_done(ccb);
	return;
	}

	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
	bzero(req, sizeof(*req));
	req->DevHandle = htole16(targ->handle);
	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
	req->MsgFlags = 0;
	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
	req->SenseBufferLength = MPS_SENSE_LEN;
	req->SGLFlags = 0;
	req->ChainOffset = 0;
	req->SGLOffset0 = 24; /* 32bit word offset to the SGL */
	req->SGLOffset1= 0;
	req->SGLOffset2= 0;
	req->SGLOffset3= 0;
	req->SkipCount = 0;
	req->DataLength = htole32(csio->dxfer_len);
	req->BidirectionalDataLength = 0;
	req->IoFlags = htole16(csio->cdb_len);
	req->EEDPFlags = 0;

	/* Note: BiDirectional transfers are not supported */
	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
	case CAM_DIR_IN:
	mpi_control = MPI2_SCSIIO_CONTROL_READ;
	cm->cm_flags \|= MPS_CM_FLAGS_DATAIN;
	break;
	case CAM_DIR_OUT:
	mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
	cm->cm_flags \|= MPS_CM_FLAGS_DATAOUT;
	break;
	case CAM_DIR_NONE:
	default:
	mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
	break;
	}

	if (csio->cdb_len == 32)
	mpi_control \|= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
	/*
	* It looks like the hardware doesn't require an explicit tag
	* number for each transaction. SAM Task Management not supported
	* at the moment.
	*/
	switch (csio->tag_action) {
	case MSG_HEAD_OF_Q_TAG:
	mpi_control \|= MPI2_SCSIIO_CONTROL_HEADOFQ;
	break;
	case MSG_ORDERED_Q_TAG:
	mpi_control \|= MPI2_SCSIIO_CONTROL_ORDEREDQ;
	break;
	case MSG_ACA_TASK:
	mpi_control \|= MPI2_SCSIIO_CONTROL_ACAQ;
	break;
	case CAM_TAG_ACTION_NONE:
	case MSG_SIMPLE_Q_TAG:
	default:
	mpi_control \|= MPI2_SCSIIO_CONTROL_SIMPLEQ;
	break;
	}
	mpi_control \|= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
	req->Control = htole32(mpi_control);
	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
	mps_free_command(sc, cm);
	mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
	xpt_done(ccb);
	return;
	}

	if (csio->ccb_h.flags & CAM_CDB_POINTER)
	bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
	else
	bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
	req->IoFlags = htole16(csio->cdb_len);

	/*
	* Check if EEDP is supported and enabled. If it is then check if the
	* SCSI opcode could be using EEDP. If so, make sure the LUN exists and
	* is formatted for EEDP support. If all of this is true, set CDB up
	* for EEDP transfer.
	*/
	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
	if (sc->eedp_enabled && eedp_flags) {
	SLIST_FOREACH(lun, &targ->luns, lun_link) {
	if (lun->lun_id == csio->ccb_h.target_lun) {
	break;
	}
	}

	if ((lun != NULL) && (lun->eedp_formatted)) {
	req->EEDPBlockSize = htole16(lun->eedp_block_size);
	eedp_flags \|= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG \|
	MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG \|
	MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
	req->EEDPFlags = htole16(eedp_flags);

	/*
	* If CDB less than 32, fill in Primary Ref Tag with
	* low 4 bytes of LBA. If CDB is 32, tag stuff is
	* already there. Also, set protection bit. FreeBSD
	* currently does not support CDBs bigger than 16, but
	* the code doesn't hurt, and will be here for the
	* future.
	*/
	if (csio->cdb_len != 32) {
	lba_byte = (csio->cdb_len == 16) ? 6 : 2;
	ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
	PrimaryReferenceTag;
	for (i = 0; i < 4; i++) {
	*ref_tag_addr =
	req->CDB.CDB32[lba_byte + i];
	ref_tag_addr++;
	}
	req->CDB.EEDP32.PrimaryReferenceTag =
	htole32(req->CDB.EEDP32.PrimaryReferenceTag);
	req->CDB.EEDP32.PrimaryApplicationTagMask =
	0xFFFF;
	req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) \|
	0x20;
	} else {
	eedp_flags \|=
	MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
	req->EEDPFlags = htole16(eedp_flags);
	req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
	0x1F) \| 0x20;
	}
	}
	}

	cm->cm_length = csio->dxfer_len;
	if (cm->cm_length != 0) {
	cm->cm_data = ccb;
	cm->cm_flags \|= MPS_CM_FLAGS_USE_CCB;
	} else {
	cm->cm_data = NULL;
	}
	cm->cm_sge = &req->SGL;
	cm->cm_sglsize = (32 - 24) * 4;
	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
	cm->cm_complete = mpssas_scsiio_complete;
	cm->cm_complete_data = ccb;
	cm->cm_targ = targ;
	cm->cm_lun = csio->ccb_h.target_lun;
	cm->cm_ccb = ccb;

	/*
	* If HBA is a WD and the command is not for a retry, try to build a
	* direct I/O message. If failed, or the command is for a retry, send
	* the I/O to the IR volume itself.
	*/
	if (sc->WD_valid_config) {
	if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
	mpssas_direct_drive_io(sassc, cm, ccb);
	} else {
	mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
	}
	}

	#if defined(BUF_TRACKING) \|\| defined(FULL_BUF_TRACKING)
	if (csio->bio != NULL)
	biotrack(csio->bio, __func__);
	#endif
	csio->ccb_h.qos.sim_data = sbinuptime();
	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
	mpssas_scsiio_timeout, cm, 0);

	targ->issued++;
	targ->outstanding++;
	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
	ccb->ccb_h.status \|= CAM_SIM_QUEUED;

	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
	__func__, cm, ccb, targ->outstanding);

	mps_map_command(sc, cm);
	return;
	}

	/**
	* mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
	*/
	static void
	mps_sc_failed_io_info(struct mps_softc sc, struct ccb_scsiio csio,
	Mpi2SCSIIOReply_t *mpi_reply)
	{
	u32 response_info;
	u8 *response_bytes;
	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
	MPI2_IOCSTATUS_MASK;
	u8 scsi_state = mpi_reply->SCSIState;
	u8 scsi_status = mpi_reply->SCSIStatus;
	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
	const char desc_ioc_state, desc_scsi_status;

	if (log_info == 0x31170000)
	return;

	desc_ioc_state = mps_describe_table(mps_iocstatus_string,
	ioc_status);
	desc_scsi_status = mps_describe_table(mps_scsi_status_string,
	scsi_status);

	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
	le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);

	/*
	*We can add more detail about underflow data here
	* TO-DO
	*/
	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
	"scsi_state %b\n", desc_scsi_status, scsi_status,
	scsi_state, "\20" "\1AutosenseValid" "\2AutosenseFailed"
	"\3NoScsiStatus" "\4Terminated" "\5Response InfoValid");

	if (sc->mps_debug & MPS_XINFO &&
	scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
	mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
	scsi_sense_print(csio);
	mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
	}

	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
	response_info = le32toh(mpi_reply->ResponseInfo);
	response_bytes = (u8 *)&response_info;
	mps_dprint(sc, MPS_XINFO, "response code(0x%1x): %s\n",
	response_bytes[0],
	mps_describe_table(mps_scsi_taskmgmt_string,
	response_bytes[0]));
	}
	}

	static void
	mpssas_scsiio_complete(struct mps_softc sc, struct mps_command cm)
	{
	MPI2_SCSI_IO_REPLY *rep;
	union ccb *ccb;
	struct ccb_scsiio *csio;
	struct mpssas_softc *sassc;
	struct scsi_vpd_supported_page_list *vpd_list = NULL;
	u8 *TLR_bits, TLR_on;
	int dir = 0, i;
	u16 alloc_len;
	struct mpssas_target *target;
	target_id_t target_id;

	MPS_FUNCTRACE(sc);
	mps_dprint(sc, MPS_TRACE,
	"cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
	cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
	cm->cm_targ->outstanding);

	callout_stop(&cm->cm_callout);
	mtx_assert(&sc->mps_mtx, MA_OWNED);

	sassc = sc->sassc;
	ccb = cm->cm_complete_data;
	csio = &ccb->csio;
	target_id = csio->ccb_h.target_id;
	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
	/*
	* XXX KDM if the chain allocation fails, does it matter if we do
	* the sync and unload here? It is simpler to do it in every case,
	* assuming it doesn't cause problems.
	*/
	if (cm->cm_data != NULL) {
	if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
	dir = BUS_DMASYNC_POSTREAD;
	else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
	dir = BUS_DMASYNC_POSTWRITE;
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
	}

	cm->cm_targ->completed++;
	cm->cm_targ->outstanding--;
	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
	ccb->ccb_h.status &= ~(CAM_STATUS_MASK \| CAM_SIM_QUEUED);

	#if defined(BUF_TRACKING) \|\| defined(FULL_BUF_TRACKING)
	if (ccb->csio.bio != NULL)
	biotrack(ccb->csio.bio, __func__);
	#endif

	if (cm->cm_flags & MPS_CM_FLAGS_ON_RECOVERY) {
	TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
	KASSERT(cm->cm_state == MPS_CM_STATE_BUSY,
	("Not busy for CM_FLAGS_TIMEDOUT: %d\n", cm->cm_state));
	cm->cm_flags &= ~MPS_CM_FLAGS_ON_RECOVERY;
	if (cm->cm_reply != NULL)
	mpssas_log_command(cm, MPS_RECOVERY,
	"completed timedout cm %p ccb %p during recovery "
	"ioc %x scsi %x state %x xfer %u\n",
	cm, cm->cm_ccb, le16toh(rep->IOCStatus),
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	else
	mpssas_log_command(cm, MPS_RECOVERY,
	"completed timedout cm %p ccb %p during recovery\n",
	cm, cm->cm_ccb);
	} else if (cm->cm_targ->tm != NULL) {
	if (cm->cm_reply != NULL)
	mpssas_log_command(cm, MPS_RECOVERY,
	"completed cm %p ccb %p during recovery "
	"ioc %x scsi %x state %x xfer %u\n",
	cm, cm->cm_ccb, le16toh(rep->IOCStatus),
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	else
	mpssas_log_command(cm, MPS_RECOVERY,
	"completed cm %p ccb %p during recovery\n",
	cm, cm->cm_ccb);
	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
	mpssas_log_command(cm, MPS_RECOVERY,
	"reset completed cm %p ccb %p\n",
	cm, cm->cm_ccb);
	}

	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	/*
	* We ran into an error after we tried to map the command,
	* so we're getting a callback without queueing the command
	* to the hardware. So we set the status here, and it will
	* be retained below. We'll go through the "fast path",
	* because there can be no reply when we haven't actually
	* gone out to the hardware.
	*/
	mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);

	/*
	* Currently the only error included in the mask is
	* MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
	* chain frames. We need to freeze the queue until we get
	* a command that completed without this error, which will
	* hopefully have some chain frames attached that we can
	* use. If we wanted to get smarter about it, we would
	* only unfreeze the queue in this condition when we're
	* sure that we're getting some chain frames back. That's
	* probably unnecessary.
	*/
	if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
	xpt_freeze_simq(sassc->sim, 1);
	sassc->flags \|= MPSSAS_QUEUE_FROZEN;
	mps_dprint(sc, MPS_XINFO, "Error sending command, "
	"freezing SIM queue\n");
	}
	}

	/*
	* If this is a Start Stop Unit command and it was issued by the driver
	* during shutdown, decrement the refcount to account for all of the
	* commands that were sent. All SSU commands should be completed before
	* shutdown completes, meaning SSU_refcount will be 0 after SSU_started
	* is TRUE.
	*/
	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
	mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
	sc->SSU_refcount--;
	}

	/* Take the fast path to completion */
	if (cm->cm_reply == NULL) {
	if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
	if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
	mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
	else {
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	ccb->csio.scsi_status = SCSI_STATUS_OK;
	}
	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
	ccb->ccb_h.status \|= CAM_RELEASE_SIMQ;
	sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
	mps_dprint(sc, MPS_XINFO,
	"Unfreezing SIM queue\n");
	}
	}

	/*
	* There are two scenarios where the status won't be
	* CAM_REQ_CMP. The first is if MPS_CM_FLAGS_ERROR_MASK is
	* set, the second is in the MPS_FLAGS_DIAGRESET above.
	*/
	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
	/*
	* Freeze the dev queue so that commands are
	* executed in the correct order after error
	* recovery.
	*/
	ccb->ccb_h.status \|= CAM_DEV_QFRZN;
	xpt_freeze_devq(ccb->ccb_h.path, /count/ 1);
	}
	mps_free_command(sc, cm);
	xpt_done(ccb);
	return;
	}

	mpssas_log_command(cm, MPS_XINFO,
	"ioc %x scsi %x state %x xfer %u\n",
	le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));

	/*
	* If this is a Direct Drive I/O, reissue the I/O to the original IR
	* Volume if an error occurred (normal I/O retry). Use the original
	* CCB, but set a flag that this will be a retry so that it's sent to
	* the original volume. Free the command but reuse the CCB.
	*/
	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
	mps_free_command(sc, cm);
	ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
	mpssas_action_scsiio(sassc, ccb);
	return;
	} else
	ccb->ccb_h.sim_priv.entries[0].field = 0;

	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
	csio->resid = cm->cm_length - le32toh(rep->TransferCount);
	/* FALLTHROUGH */
	case MPI2_IOCSTATUS_SUCCESS:
	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:

	if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
	mpssas_log_command(cm, MPS_XINFO, "recovered error\n");

	/* Completion failed at the transport level. */
	if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS \|
	MPI2_SCSI_STATE_TERMINATED)) {
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	break;
	}

	/* In a modern packetized environment, an autosense failure
	* implies that there's not much else that can be done to
	* recover the command.
	*/
	if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
	mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
	break;
	}

	/*
	* CAM doesn't care about SAS Response Info data, but if this is
	* the state check if TLR should be done. If not, clear the
	* TLR_bits for the target.
	*/
	if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
	((le32toh(rep->ResponseInfo) &
	MPI2_SCSI_RI_MASK_REASONCODE) ==
	MPS_SCSI_RI_INVALID_FRAME)) {
	sc->mapping_table[target_id].TLR_bits =
	(u8)MPI2_SCSIIO_CONTROL_NO_TLR;
	}

	/*
	* Intentionally override the normal SCSI status reporting
	* for these two cases. These are likely to happen in a
	* multi-initiator environment, and we want to make sure that
	* CAM retries these commands rather than fail them.
	*/
	if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) \|\|
	(rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
	mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
	break;
	}

	/* Handle normal status and sense */
	csio->scsi_status = rep->SCSIStatus;
	if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);

	if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
	int sense_len, returned_sense_len;

	returned_sense_len = min(le32toh(rep->SenseCount),
	sizeof(struct scsi_sense_data));
	if (returned_sense_len < ccb->csio.sense_len)
	ccb->csio.sense_resid = ccb->csio.sense_len -
	returned_sense_len;
	else
	ccb->csio.sense_resid = 0;

	sense_len = min(returned_sense_len,
	ccb->csio.sense_len - ccb->csio.sense_resid);
	bzero(&ccb->csio.sense_data,
	sizeof(ccb->csio.sense_data));
	bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
	ccb->ccb_h.status \|= CAM_AUTOSNS_VALID;
	}

	/*
	* Check if this is an INQUIRY command. If it's a VPD inquiry,
	* and it's page code 0 (Supported Page List), and there is
	* inquiry data, and this is for a sequential access device, and
	* the device is an SSP target, and TLR is supported by the
	* controller, turn the TLR_bits value ON if page 0x90 is
	* supported.
	*/
	if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
	(csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
	(csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
	((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
	(csio->data_ptr != NULL) &&
	((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
	(sc->control_TLR) &&
	(sc->mapping_table[target_id].device_info &
	MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
	vpd_list = (struct scsi_vpd_supported_page_list *)
	csio->data_ptr;
	TLR_bits = &sc->mapping_table[target_id].TLR_bits;
	*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
	TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
	alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
	csio->cdb_io.cdb_bytes[4];
	alloc_len -= csio->resid;
	for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
	if (vpd_list->list[i] == 0x90) {
	*TLR_bits = TLR_on;
	break;
	}
	}
	}

	/*
	* If this is a SATA direct-access end device, mark it so that
	* a SCSI StartStopUnit command will be sent to it when the
	* driver is being shutdown.
	*/
	if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
	((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
	(sc->mapping_table[target_id].device_info &
	MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
	((sc->mapping_table[target_id].device_info &
	MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
	MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
	target = &sassc->targets[target_id];
	target->supports_SSU = TRUE;
	mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
	target_id);
	}
	break;
	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
	/*
	* If devinfo is 0 this will be a volume. In that case don't
	* tell CAM that the volume is not there. We want volumes to
	* be enumerated until they are deleted/removed, not just
	* failed.
	*/
	if (cm->cm_targ->devinfo == 0)
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	break;
	case MPI2_IOCSTATUS_INVALID_SGL:
	mps_print_scsiio_cmd(sc, cm);
	mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
	break;
	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
	/*
	* This is one of the responses that comes back when an I/O
	* has been aborted. If it is because of a timeout that we
	* initiated, just set the status to CAM_CMD_TIMEOUT.
	* Otherwise set it to CAM_REQ_ABORTED. The effect on the
	* command is the same (it gets retried, subject to the
	* retry counter), the only difference is what gets printed
	* on the console.
	*/
	if (cm->cm_flags & MPS_CM_FLAGS_TIMEDOUT)
	mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
	else
	mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
	break;
	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
	/* resid is ignored for this condition */
	csio->resid = 0;
	mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
	break;
	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
	/*
	* These can sometimes be transient transport-related
	* errors, and sometimes persistent drive-related errors.
	* We used to retry these without decrementing the retry
	* count by returning CAM_REQUEUE_REQ. Unfortunately, if
	* we hit a persistent drive problem that returns one of
	* these error codes, we would retry indefinitely. So,
	* return CAM_REQ_CMP_ERROR so that we decrement the retry
	* count and avoid infinite retries. We're taking the
	* potential risk of flagging false failures in the event
	* of a topology-related error (e.g. a SAS expander problem
	* causes a command addressed to a drive to fail), but
	* avoiding getting into an infinite retry loop. However,
	* if we get them while were moving a device, we should
	* fail the request as 'not there' because the device
	* is effectively gone.
	*/
	if (cm->cm_targ->flags & MPSSAS_TARGET_INREMOVAL)
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	else
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	mps_dprint(sc, MPS_INFO,
	"Controller reported %s tgt %u SMID %u loginfo %x%s\n",
	mps_describe_table(mps_iocstatus_string,
	le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK),
	target_id, cm->cm_desc.Default.SMID,
	le32toh(rep->IOCLogInfo),
	(cm->cm_targ->flags & MPSSAS_TARGET_INREMOVAL) ? " departing" : "");
	mps_dprint(sc, MPS_XINFO,
	"SCSIStatus %x SCSIState %x xfercount %u\n",
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	break;
	case MPI2_IOCSTATUS_INVALID_FUNCTION:
	case MPI2_IOCSTATUS_INTERNAL_ERROR:
	case MPI2_IOCSTATUS_INVALID_VPID:
	case MPI2_IOCSTATUS_INVALID_FIELD:
	case MPI2_IOCSTATUS_INVALID_STATE:
	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
	default:
	mpssas_log_command(cm, MPS_XINFO,
	"completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
	le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
	rep->SCSIStatus, rep->SCSIState,
	le32toh(rep->TransferCount));
	csio->resid = cm->cm_length;
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	break;
	}

	mps_sc_failed_io_info(sc,csio,rep);

	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
	ccb->ccb_h.status \|= CAM_RELEASE_SIMQ;
	sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
	mps_dprint(sc, MPS_XINFO, "Command completed, "
	"unfreezing SIM queue\n");
	}

	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
	ccb->ccb_h.status \|= CAM_DEV_QFRZN;
	xpt_freeze_devq(ccb->ccb_h.path, /count/ 1);
	}

	/*
	* Check to see if we're removing the device. If so, and this is the
	* last command on the queue, proceed with the deferred removal of the
	* device. Note, for removing a volume, this won't trigger because
	* pending_remove_tm will be NULL.
	*/
	if (cm->cm_targ->flags & MPSSAS_TARGET_INREMOVAL) {
	if (TAILQ_FIRST(&cm->cm_targ->commands) == NULL &&
	cm->cm_targ->pending_remove_tm != NULL) {
	mps_dprint(sc, MPS_INFO, "Last pending command complete: starting remove_device\n");
	mps_map_command(sc, cm->cm_targ->pending_remove_tm);
	cm->cm_targ->pending_remove_tm = NULL;
	}
	}

	mps_free_command(sc, cm);
	xpt_done(ccb);
	}

	/* All Request reached here are Endian safe */
	static void
	mpssas_direct_drive_io(struct mpssas_softc sassc, struct mps_command cm,
	union ccb *ccb) {
	pMpi2SCSIIORequest_t pIO_req;
	struct mps_softc *sc = sassc->sc;
	uint64_t virtLBA;
	uint32_t physLBA, stripe_offset, stripe_unit;
	uint32_t io_size, column;
	uint8_t ptrLBA, lba_idx, physLBA_byte, CDB;

	/*
	* If this is a valid SCSI command (Read6, Read10, Read16, Write6,
	* Write10, or Write16), build a direct I/O message. Otherwise, the I/O
	* will be sent to the IR volume itself. Since Read6 and Write6 are a
	* bit different than the 10/16 CDBs, handle them separately.
	*/
	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
	CDB = pIO_req->CDB.CDB32;

	/*
	* Handle 6 byte CDBs.
	*/
	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) \|\|
	(CDB[0] == WRITE_6))) {
	/*
	* Get the transfer size in blocks.
	*/
	io_size = (cm->cm_length >> sc->DD_block_exponent);

	/*
	* Get virtual LBA given in the CDB.
	*/
	virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) \|
	((uint64_t)CDB[2] << 8) \| (uint64_t)CDB[3];

	/*
	* Check that LBA range for I/O does not exceed volume's
	* MaxLBA.
	*/
	if ((virtLBA + (uint64_t)io_size - 1) <=
	sc->DD_max_lba) {
	/*
	* Check if the I/O crosses a stripe boundary. If not,
	* translate the virtual LBA to a physical LBA and set
	* the DevHandle for the PhysDisk to be used. If it
	* does cross a boundary, do normal I/O. To get the
	* right DevHandle to use, get the map number for the
	* column, then use that map number to look up the
	* DevHandle of the PhysDisk.
	*/
	stripe_offset = (uint32_t)virtLBA &
	(sc->DD_stripe_size - 1);
	if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
	physLBA = (uint32_t)virtLBA >>
	sc->DD_stripe_exponent;
	stripe_unit = physLBA / sc->DD_num_phys_disks;
	column = physLBA % sc->DD_num_phys_disks;
	pIO_req->DevHandle =
	htole16(sc->DD_column_map[column].dev_handle);
	/* ???? Is this endian safe*/
	cm->cm_desc.SCSIIO.DevHandle =
	pIO_req->DevHandle;

	physLBA = (stripe_unit <<
	sc->DD_stripe_exponent) + stripe_offset;
	ptrLBA = &pIO_req->CDB.CDB32[1];
	physLBA_byte = (uint8_t)(physLBA >> 16);
	*ptrLBA = physLBA_byte;
	ptrLBA = &pIO_req->CDB.CDB32[2];
	physLBA_byte = (uint8_t)(physLBA >> 8);
	*ptrLBA = physLBA_byte;
	ptrLBA = &pIO_req->CDB.CDB32[3];
	physLBA_byte = (uint8_t)physLBA;
	*ptrLBA = physLBA_byte;

	/*
	* Set flag that Direct Drive I/O is
	* being done.
	*/
	cm->cm_flags \|= MPS_CM_FLAGS_DD_IO;
	}
	}
	return;
	}

	/*
	* Handle 10, 12 or 16 byte CDBs.
	*/
	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) \|\|
	(CDB[0] == WRITE_10) \|\| (CDB[0] == READ_16) \|\|
	(CDB[0] == WRITE_16) \|\| (CDB[0] == READ_12) \|\|
	(CDB[0] == WRITE_12))) {
	/*
	* For 16-byte CDB's, verify that the upper 4 bytes of the CDB
	* are 0. If not, this is accessing beyond 2TB so handle it in
	* the else section. 10-byte and 12-byte CDB's are OK.
	* FreeBSD sends very rare 12 byte READ/WRITE, but driver is
	* ready to accept 12byte CDB for Direct IOs.
	*/
	if ((CDB[0] == READ_10 \|\| CDB[0] == WRITE_10) \|\|
	(CDB[0] == READ_12 \|\| CDB[0] == WRITE_12) \|\|
	!(CDB[2] \| CDB[3] \| CDB[4] \| CDB[5])) {
	/*
	* Get the transfer size in blocks.
	*/
	io_size = (cm->cm_length >> sc->DD_block_exponent);

	/*
	* Get virtual LBA. Point to correct lower 4 bytes of
	* LBA in the CDB depending on command.
	*/
	lba_idx = ((CDB[0] == READ_12) \|\|
	(CDB[0] == WRITE_12) \|\|
	(CDB[0] == READ_10) \|\|
	(CDB[0] == WRITE_10))? 2 : 6;
	virtLBA = ((uint64_t)CDB[lba_idx] << 24) \|
	((uint64_t)CDB[lba_idx + 1] << 16) \|
	((uint64_t)CDB[lba_idx + 2] << 8) \|
	(uint64_t)CDB[lba_idx + 3];

	/*
	* Check that LBA range for I/O does not exceed volume's
	* MaxLBA.
	*/
	if ((virtLBA + (uint64_t)io_size - 1) <=
	sc->DD_max_lba) {
	/*
	* Check if the I/O crosses a stripe boundary.
	* If not, translate the virtual LBA to a
	* physical LBA and set the DevHandle for the
	* PhysDisk to be used. If it does cross a
	* boundary, do normal I/O. To get the right
	* DevHandle to use, get the map number for the
	* column, then use that map number to look up
	* the DevHandle of the PhysDisk.
	*/
	stripe_offset = (uint32_t)virtLBA &
	(sc->DD_stripe_size - 1);
	if ((stripe_offset + io_size) <=
	sc->DD_stripe_size) {
	physLBA = (uint32_t)virtLBA >>
	sc->DD_stripe_exponent;
	stripe_unit = physLBA /
	sc->DD_num_phys_disks;
	column = physLBA %
	sc->DD_num_phys_disks;
	pIO_req->DevHandle =
	htole16(sc->DD_column_map[column].
	dev_handle);
	cm->cm_desc.SCSIIO.DevHandle =
	pIO_req->DevHandle;

	physLBA = (stripe_unit <<
	sc->DD_stripe_exponent) +
	stripe_offset;
	ptrLBA =
	&pIO_req->CDB.CDB32[lba_idx];
	physLBA_byte = (uint8_t)(physLBA >> 24);
	*ptrLBA = physLBA_byte;
	ptrLBA =
	&pIO_req->CDB.CDB32[lba_idx + 1];
	physLBA_byte = (uint8_t)(physLBA >> 16);
	*ptrLBA = physLBA_byte;
	ptrLBA =
	&pIO_req->CDB.CDB32[lba_idx + 2];
	physLBA_byte = (uint8_t)(physLBA >> 8);
	*ptrLBA = physLBA_byte;
	ptrLBA =
	&pIO_req->CDB.CDB32[lba_idx + 3];
	physLBA_byte = (uint8_t)physLBA;
	*ptrLBA = physLBA_byte;

	/*
	* Set flag that Direct Drive I/O is
	* being done.
	*/
	cm->cm_flags \|= MPS_CM_FLAGS_DD_IO;
	}
	}
	} else {
	/*
	* 16-byte CDB and the upper 4 bytes of the CDB are not
	* 0. Get the transfer size in blocks.
	*/
	io_size = (cm->cm_length >> sc->DD_block_exponent);

	/*
	* Get virtual LBA.
	*/
	virtLBA = ((uint64_t)CDB[2] << 54) \|
	((uint64_t)CDB[3] << 48) \|
	((uint64_t)CDB[4] << 40) \|
	((uint64_t)CDB[5] << 32) \|
	((uint64_t)CDB[6] << 24) \|
	((uint64_t)CDB[7] << 16) \|
	((uint64_t)CDB[8] << 8) \|
	(uint64_t)CDB[9];

	/*
	* Check that LBA range for I/O does not exceed volume's
	* MaxLBA.
	*/
	if ((virtLBA + (uint64_t)io_size - 1) <=
	sc->DD_max_lba) {
	/*
	* Check if the I/O crosses a stripe boundary.
	* If not, translate the virtual LBA to a
	* physical LBA and set the DevHandle for the
	* PhysDisk to be used. If it does cross a
	* boundary, do normal I/O. To get the right
	* DevHandle to use, get the map number for the
	* column, then use that map number to look up
	* the DevHandle of the PhysDisk.
	*/
	stripe_offset = (uint32_t)virtLBA &
	(sc->DD_stripe_size - 1);
	if ((stripe_offset + io_size) <=
	sc->DD_stripe_size) {
	physLBA = (uint32_t)(virtLBA >>
	sc->DD_stripe_exponent);
	stripe_unit = physLBA /
	sc->DD_num_phys_disks;
	column = physLBA %
	sc->DD_num_phys_disks;
	pIO_req->DevHandle =
	htole16(sc->DD_column_map[column].
	dev_handle);
	cm->cm_desc.SCSIIO.DevHandle =
	pIO_req->DevHandle;

	physLBA = (stripe_unit <<
	sc->DD_stripe_exponent) +
	stripe_offset;

	/*
	* Set upper 4 bytes of LBA to 0. We
	* assume that the phys disks are less
	* than 2 TB's in size. Then, set the
	* lower 4 bytes.
	*/
	pIO_req->CDB.CDB32[2] = 0;
	pIO_req->CDB.CDB32[3] = 0;
	pIO_req->CDB.CDB32[4] = 0;
	pIO_req->CDB.CDB32[5] = 0;
	ptrLBA = &pIO_req->CDB.CDB32[6];
	physLBA_byte = (uint8_t)(physLBA >> 24);
	*ptrLBA = physLBA_byte;
	ptrLBA = &pIO_req->CDB.CDB32[7];
	physLBA_byte = (uint8_t)(physLBA >> 16);
	*ptrLBA = physLBA_byte;
	ptrLBA = &pIO_req->CDB.CDB32[8];
	physLBA_byte = (uint8_t)(physLBA >> 8);
	*ptrLBA = physLBA_byte;
	ptrLBA = &pIO_req->CDB.CDB32[9];
	physLBA_byte = (uint8_t)physLBA;
	*ptrLBA = physLBA_byte;

	/*
	* Set flag that Direct Drive I/O is
	* being done.
	*/
	cm->cm_flags \|= MPS_CM_FLAGS_DD_IO;
	}
	}
	}
	}
	}

	static void
	mpssas_smpio_complete(struct mps_softc sc, struct mps_command cm)
	{
	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
	MPI2_SMP_PASSTHROUGH_REQUEST *req;
	uint64_t sasaddr;
	union ccb *ccb;

	ccb = cm->cm_complete_data;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and SMP
	* commands require two S/G elements only. That should be handled
	* in the standard request size.
	*/
	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
	__func__, cm->cm_flags);
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
	if (rpl == NULL) {
	mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
	sasaddr = le32toh(req->SASAddress.Low);
	sasaddr \|= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;

	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS \|\|
	rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
	mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
	__func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
	"%#jx completed successfully\n", __func__,
	(uintmax_t)sasaddr);

	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	else
	mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);

	bailout:
	/*
	* We sync in both directions because we had DMAs in the S/G list
	* in both directions.
	*/
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
	mps_free_command(sc, cm);
	xpt_done(ccb);
	}

	static void
	mpssas_send_smpcmd(struct mpssas_softc sassc, union ccb ccb, uint64_t sasaddr)
	{
	struct mps_command *cm;
	uint8_t request, response;
	MPI2_SMP_PASSTHROUGH_REQUEST *req;
	struct mps_softc *sc;
	int error;

	sc = sassc->sc;
	error = 0;

	/*
	* XXX We don't yet support physical addresses here.
	*/
	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
	case CAM_DATA_PADDR:
	case CAM_DATA_SG_PADDR:
	mps_dprint(sc, MPS_ERROR,
	"%s: physical addresses not supported\n", __func__);
	mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
	xpt_done(ccb);
	return;
	case CAM_DATA_SG:
	/*
	* The chip does not support more than one buffer for the
	* request or response.
	*/
	if ((ccb->smpio.smp_request_sglist_cnt > 1)
	\|\| (ccb->smpio.smp_response_sglist_cnt > 1)) {
	mps_dprint(sc, MPS_ERROR,
	"%s: multiple request or response "
	"buffer segments not supported for SMP\n",
	__func__);
	mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
	xpt_done(ccb);
	return;
	}

	/*
	* The CAM_SCATTER_VALID flag was originally implemented
	* for the XPT_SCSI_IO CCB, which only has one data pointer.
	* We have two. So, just take that flag to mean that we
	* might have S/G lists, and look at the S/G segment count
	* to figure out whether that is the case for each individual
	* buffer.
	*/
	if (ccb->smpio.smp_request_sglist_cnt != 0) {
	bus_dma_segment_t *req_sg;

	req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
	request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
	} else
	request = ccb->smpio.smp_request;

	if (ccb->smpio.smp_response_sglist_cnt != 0) {
	bus_dma_segment_t *rsp_sg;

	rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
	response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
	} else
	response = ccb->smpio.smp_response;
	break;
	case CAM_DATA_VADDR:
	request = ccb->smpio.smp_request;
	response = ccb->smpio.smp_response;
	break;
	default:
	mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
	xpt_done(ccb);
	return;
	}

	cm = mps_alloc_command(sc);
	if (cm == NULL) {
	mps_dprint(sc, MPS_ERROR,
	"%s: cannot allocate command\n", __func__);
	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
	xpt_done(ccb);
	return;
	}

	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
	bzero(req, sizeof(*req));
	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;

	/* Allow the chip to use any route to this SAS address. */
	req->PhysicalPort = 0xff;

	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
	req->SGLFlags =
	MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE \| MPI2_SGLFLAGS_SGL_TYPE_MPI;

	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
	"address %#jx\n", __func__, (uintmax_t)sasaddr);

	mpi_init_sge(cm, req, &req->SGL);

	/*
	* Set up a uio to pass into mps_map_command(). This allows us to
	* do one map command, and one busdma call in there.
	*/
	cm->cm_uio.uio_iov = cm->cm_iovec;
	cm->cm_uio.uio_iovcnt = 2;
	cm->cm_uio.uio_segflg = UIO_SYSSPACE;

	/*
	* The read/write flag isn't used by busdma, but set it just in
	* case. This isn't exactly accurate, either, since we're going in
	* both directions.
	*/
	cm->cm_uio.uio_rw = UIO_WRITE;

	cm->cm_iovec[0].iov_base = request;
	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
	cm->cm_iovec[1].iov_base = response;
	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;

	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
	cm->cm_iovec[1].iov_len;

	/*
	* Trigger a warning message in mps_data_cb() for the user if we
	* wind up exceeding two S/G segments. The chip expects one
	* segment for the request and another for the response.
	*/
	cm->cm_max_segs = 2;

	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete = mpssas_smpio_complete;
	cm->cm_complete_data = ccb;

	/*
	* Tell the mapping code that we're using a uio, and that this is
	* an SMP passthrough request. There is a little special-case
	* logic there (in mps_data_cb()) to handle the bidirectional
	* transfer.
	*/
	cm->cm_flags \|= MPS_CM_FLAGS_USE_UIO \| MPS_CM_FLAGS_SMP_PASS \|
	MPS_CM_FLAGS_DATAIN \| MPS_CM_FLAGS_DATAOUT;

	/* The chip data format is little endian. */
	req->SASAddress.High = htole32(sasaddr >> 32);
	req->SASAddress.Low = htole32(sasaddr);

	/*
	* XXX Note that we don't have a timeout/abort mechanism here.
	* From the manual, it looks like task management requests only
	* work for SCSI IO and SATA passthrough requests. We may need to
	* have a mechanism to retry requests in the event of a chip reset
	* at least. Hopefully the chip will insure that any errors short
	* of that are relayed back to the driver.
	*/
	error = mps_map_command(sc, cm);
	if ((error != 0) && (error != EINPROGRESS)) {
	mps_dprint(sc, MPS_ERROR,
	"%s: error %d returned from mps_map_command()\n",
	__func__, error);
	goto bailout_error;
	}

	return;

	bailout_error:
	mps_free_command(sc, cm);
	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
	xpt_done(ccb);
	return;

	}

	static void
	mpssas_action_smpio(struct mpssas_softc sassc, union ccb ccb)
	{
	struct mps_softc *sc;
	struct mpssas_target *targ;
	uint64_t sasaddr = 0;

	sc = sassc->sc;

	/*
	* Make sure the target exists.
	*/
	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
	targ = &sassc->targets[ccb->ccb_h.target_id];
	if (targ->handle == 0x0) {
	mps_dprint(sc, MPS_ERROR,
	"%s: target %d does not exist!\n", __func__,
	ccb->ccb_h.target_id);
	mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
	xpt_done(ccb);
	return;
	}

	/*
	* If this device has an embedded SMP target, we'll talk to it
	* directly.
	* figure out what the expander's address is.
	*/
	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
	sasaddr = targ->sasaddr;

	/*
	* If we don't have a SAS address for the expander yet, try
	* grabbing it from the page 0x83 information cached in the
	* transport layer for this target. LSI expanders report the
	* expander SAS address as the port-associated SAS address in
	* Inquiry VPD page 0x83. Maxim expanders don't report it in page
	* 0x83.
	*
	* XXX KDM disable this for now, but leave it commented out so that
	* it is obvious that this is another possible way to get the SAS
	* address.
	*
	* The parent handle method below is a little more reliable, and
	* the other benefit is that it works for devices other than SES
	* devices. So you can send a SMP request to a da(4) device and it
	* will get routed to the expander that device is attached to.
	* (Assuming the da(4) device doesn't contain an SMP target...)
	*/
	#if 0
	if (sasaddr == 0)
	sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
	#endif

	/*
	* If we still don't have a SAS address for the expander, look for
	* the parent device of this device, which is probably the expander.
	*/
	if (sasaddr == 0) {
	#ifdef OLD_MPS_PROBE
	struct mpssas_target *parent_target;
	#endif

	if (targ->parent_handle == 0x0) {
	mps_dprint(sc, MPS_ERROR,
	"%s: handle %d does not have a valid "
	"parent handle!\n", __func__, targ->handle);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}
	#ifdef OLD_MPS_PROBE
	parent_target = mpssas_find_target_by_handle(sassc, 0,
	targ->parent_handle);

	if (parent_target == NULL) {
	mps_dprint(sc, MPS_ERROR,
	"%s: handle %d does not have a valid "
	"parent target!\n", __func__, targ->handle);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}

	if ((parent_target->devinfo &
	MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
	mps_dprint(sc, MPS_ERROR,
	"%s: handle %d parent %d does not "
	"have an SMP target!\n", __func__,
	targ->handle, parent_target->handle);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;

	}

	sasaddr = parent_target->sasaddr;
	#else /* OLD_MPS_PROBE */
	if ((targ->parent_devinfo &
	MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
	mps_dprint(sc, MPS_ERROR,
	"%s: handle %d parent %d does not "
	"have an SMP target!\n", __func__,
	targ->handle, targ->parent_handle);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;

	}
	if (targ->parent_sasaddr == 0x0) {
	mps_dprint(sc, MPS_ERROR,
	"%s: handle %d parent handle %d does "
	"not have a valid SAS address!\n",
	__func__, targ->handle, targ->parent_handle);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}

	sasaddr = targ->parent_sasaddr;
	#endif /* OLD_MPS_PROBE */

	}

	if (sasaddr == 0) {
	mps_dprint(sc, MPS_INFO,
	"%s: unable to find SAS address for handle %d\n",
	__func__, targ->handle);
	mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
	goto bailout;
	}
	mpssas_send_smpcmd(sassc, ccb, sasaddr);

	return;

	bailout:
	xpt_done(ccb);

	}

	static void
	mpssas_action_resetdev(struct mpssas_softc sassc, union ccb ccb)
	{
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
	struct mps_softc *sc;
	struct mps_command *tm;
	struct mpssas_target *targ;

	MPS_FUNCTRACE(sassc->sc);
	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);

	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
	("Target %d out of bounds in XPT_RESET_DEV\n",
	ccb->ccb_h.target_id));
	sc = sassc->sc;
	tm = mpssas_alloc_tm(sc);
	if (tm == NULL) {
	mps_dprint(sc, MPS_ERROR,
	"command alloc failure in mpssas_action_resetdev\n");
	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
	xpt_done(ccb);
	return;
	}

	targ = &sassc->targets[ccb->ccb_h.target_id];
	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
	req->DevHandle = htole16(targ->handle);
	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;

	/* SAS Hard Link Reset / SATA Link Reset */
	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;

	tm->cm_data = NULL;
	tm->cm_complete = mpssas_resetdev_complete;
	tm->cm_complete_data = ccb;
	tm->cm_targ = targ;

	mpssas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
	mps_map_command(sc, tm);
	}

	static void
	mpssas_resetdev_complete(struct mps_softc sc, struct mps_command tm)
	{
	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
	union ccb *ccb;

	MPS_FUNCTRACE(sc);
	mtx_assert(&sc->mps_mtx, MA_OWNED);

	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
	ccb = tm->cm_complete_data;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* task management commands don't have S/G lists.
	*/
	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	MPI2_SCSI_TASK_MANAGE_REQUEST *req;

	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;

	mps_dprint(sc, MPS_ERROR,
	"%s: cm_flags = %#x for reset of handle %#04x! "
	"This should not happen!\n", __func__, tm->cm_flags,
	req->DevHandle);
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
	goto bailout;
	}

	mps_dprint(sc, MPS_XINFO,
	"%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
	le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));

	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
	mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
	CAM_LUN_WILDCARD);
	}
	else
	mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);

	bailout:

	mpssas_free_tm(sc, tm);
	xpt_done(ccb);
	}

	static void
	mpssas_poll(struct cam_sim *sim)
	{
	struct mpssas_softc *sassc;

	sassc = cam_sim_softc(sim);

	if (sassc->sc->mps_debug & MPS_TRACE) {
	/* frequent debug messages during a panic just slow
	* everything down too much.
	*/
	mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
	sassc->sc->mps_debug &= ~MPS_TRACE;
	}

	mps_intr_locked(sassc->sc);
	}

	static void
	mpssas_async(void callback_arg, uint32_t code, struct cam_path path,
	void *arg)
	{
	struct mps_softc *sc;

	sc = (struct mps_softc *)callback_arg;

	switch (code) {
	case AC_ADVINFO_CHANGED: {
	struct mpssas_target *target;
	struct mpssas_softc *sassc;
	struct scsi_read_capacity_data_long rcap_buf;
	struct ccb_dev_advinfo cdai;
	struct mpssas_lun *lun;
	lun_id_t lunid;
	int found_lun;
	uintptr_t buftype;

	buftype = (uintptr_t)arg;

	found_lun = 0;
	sassc = sc->sassc;

	/*
	* We're only interested in read capacity data changes.
	*/
	if (buftype != CDAI_TYPE_RCAPLONG)
	break;

	/*
	* We should have a handle for this, but check to make sure.
	*/
	KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
	("Target %d out of bounds in mpssas_async\n",
	xpt_path_target_id(path)));
	target = &sassc->targets[xpt_path_target_id(path)];
	if (target->handle == 0)
	break;

	lunid = xpt_path_lun_id(path);

	SLIST_FOREACH(lun, &target->luns, lun_link) {
	if (lun->lun_id == lunid) {
	found_lun = 1;
	break;
	}
	}

	if (found_lun == 0) {
	lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
	M_NOWAIT \| M_ZERO);
	if (lun == NULL) {
	mps_dprint(sc, MPS_ERROR, "Unable to alloc "
	"LUN for EEDP support.\n");
	break;
	}
	lun->lun_id = lunid;
	SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
	}

	bzero(&rcap_buf, sizeof(rcap_buf));
	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
	cdai.ccb_h.flags = CAM_DIR_IN;
	cdai.buftype = CDAI_TYPE_RCAPLONG;
	cdai.flags = CDAI_FLAG_NONE;
	cdai.bufsiz = sizeof(rcap_buf);
	cdai.buf = (uint8_t *)&rcap_buf;
	xpt_action((union ccb *)&cdai);
	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
	cam_release_devq(cdai.ccb_h.path,
	0, 0, 0, FALSE);

	if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
	&& (rcap_buf.prot & SRC16_PROT_EN)) {
	switch (rcap_buf.prot & SRC16_P_TYPE) {
	case SRC16_PTYPE_1:
	case SRC16_PTYPE_3:
	lun->eedp_formatted = TRUE;
	lun->eedp_block_size =
	scsi_4btoul(rcap_buf.length);
	break;
	case SRC16_PTYPE_2:
	default:
	lun->eedp_formatted = FALSE;
	lun->eedp_block_size = 0;
	break;
	}
	} else {
	lun->eedp_formatted = FALSE;
	lun->eedp_block_size = 0;
	}
	break;
	}
	default:
	break;
	}
	}

	/*
	* Set the INRESET flag for this target so that no I/O will be sent to
	* the target until the reset has completed. If an I/O request does
	* happen, the devq will be frozen. The CCB holds the path which is
	* used to release the devq. The devq is released and the CCB is freed
	* when the TM completes.
	*/
	void
	mpssas_prepare_for_tm(struct mps_softc sc, struct mps_command tm,
	struct mpssas_target *target, lun_id_t lun_id)
	{
	union ccb *ccb;
	path_id_t path_id;

	ccb = xpt_alloc_ccb_nowait();
	if (ccb) {
	path_id = cam_sim_path(sc->sassc->sim);
	if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
	target->tid, lun_id) != CAM_REQ_CMP) {
	xpt_free_ccb(ccb);
	} else {
	tm->cm_ccb = ccb;
	tm->cm_targ = target;
	target->flags \|= MPSSAS_TARGET_INRESET;
	}
	}
	}

	int
	mpssas_startup(struct mps_softc *sc)
	{

	/*
	* Send the port enable message and set the wait_for_port_enable flag.
	* This flag helps to keep the simq frozen until all discovery events
	* are processed.
	*/
	sc->wait_for_port_enable = 1;
	mpssas_send_portenable(sc);
	return (0);
	}

	static int
	mpssas_send_portenable(struct mps_softc *sc)
	{
	MPI2_PORT_ENABLE_REQUEST *request;
	struct mps_command *cm;

	MPS_FUNCTRACE(sc);

	if ((cm = mps_alloc_command(sc)) == NULL)
	return (EBUSY);
	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
	request->Function = MPI2_FUNCTION_PORT_ENABLE;
	request->MsgFlags = 0;
	request->VP_ID = 0;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete = mpssas_portenable_complete;
	cm->cm_data = NULL;
	cm->cm_sge = NULL;

	mps_map_command(sc, cm);
	mps_dprint(sc, MPS_XINFO,
	"mps_send_portenable finished cm %p req %p complete %p\n",
	cm, cm->cm_req, cm->cm_complete);
	return (0);
	}

	static void
	mpssas_portenable_complete(struct mps_softc sc, struct mps_command cm)
	{
	MPI2_PORT_ENABLE_REPLY *reply;
	struct mpssas_softc *sassc;

	MPS_FUNCTRACE(sc);
	sassc = sc->sassc;

	/*
	* Currently there should be no way we can hit this case. It only
	* happens when we have a failure to allocate chain frames, and
	* port enable commands don't have S/G lists.
	*/
	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
	mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
	"This should not happen!\n", __func__, cm->cm_flags);
	}

	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
	if (reply == NULL)
	mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS)
	mps_dprint(sc, MPS_FAULT, "Portenable failed\n");

	mps_free_command(sc, cm);

	/*
	* Get WarpDrive info after discovery is complete but before the scan
	* starts. At this point, all devices are ready to be exposed to the
	* OS. If devices should be hidden instead, take them out of the
	* 'targets' array before the scan. The devinfo for a disk will have
	* some info and a volume's will be 0. Use that to remove disks.
	*/
	mps_wd_config_pages(sc);

	/*
	* Done waiting for port enable to complete. Decrement the refcount.
	* If refcount is 0, discovery is complete and a rescan of the bus can
	* take place. Since the simq was explicitly frozen before port
	* enable, it must be explicitly released here to keep the
	* freeze/release count in sync.
	*/
	sc->wait_for_port_enable = 0;
	sc->port_enable_complete = 1;
	wakeup(&sc->port_enable_complete);
	mpssas_startup_decrement(sassc);
	}

	int
	mpssas_check_id(struct mpssas_softc *sassc, int id)
	{
	struct mps_softc *sc = sassc->sc;
	char *ids;
	char *name;

	ids = &sc->exclude_ids[0];
	while((name = strsep(&ids, ",")) != NULL) {
	if (name[0] == '\0')
	continue;
	if (strtol(name, NULL, 0) == (long)id)
	return (1);
	}

	return (0);
	}

	void
	mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
	{
	struct mpssas_softc *sassc;
	struct mpssas_lun lun, lun_tmp;
	struct mpssas_target *targ;
	int i;

	sassc = sc->sassc;
	/*
	* The number of targets is based on IOC Facts, so free all of
	* the allocated LUNs for each target and then the target buffer
	* itself.
	*/
	for (i=0; i< maxtargets; i++) {
	targ = &sassc->targets[i];
	SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
	free(lun, M_MPT2);
	}
	}
	free(sassc->targets, M_MPT2);

	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
	M_MPT2, M_WAITOK\|M_ZERO);
	- if (!sassc->targets) {
	- panic("%s failed to alloc targets with error %d\n",
	- __func__, ENOMEM);
	- }
	}
	diff --git a/sys/dev/mps/mps_user.c b/sys/dev/mps/mps_user.c
	index 0a2506070849..15ae463a1ad1 100644
	--- a/sys/dev/mps/mps_user.c
	+++ b/sys/dev/mps/mps_user.c
	@@ -1,2520 +1,2513 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 2008 Yahoo!, Inc.
	* All rights reserved.
	* Written by: John Baldwin <jhb@FreeBSD.org>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the author nor the names of any co-contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD userland interface
	*/
	/*-
	* Copyright (c) 2011-2015 LSI Corp.
	* Copyright (c) 2013-2015 Avago Technologies
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/* TODO Move headers to mpsvar */
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/selinfo.h>
	#include <sys/module.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/bio.h>
	#include <sys/abi_compat.h>
	#include <sys/malloc.h>
	#include <sys/uio.h>
	#include <sys/sysctl.h>
	#include <sys/ioccom.h>
	#include <sys/endian.h>
	#include <sys/queue.h>
	#include <sys/kthread.h>
	#include <sys/taskqueue.h>
	#include <sys/proc.h>
	#include <sys/sysent.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>

	#include <cam/cam.h>
	#include <cam/cam_ccb.h>
	#include <cam/scsi/scsi_all.h>

	#include <dev/mps/mpi/mpi2_type.h>
	#include <dev/mps/mpi/mpi2.h>
	#include <dev/mps/mpi/mpi2_ioc.h>
	#include <dev/mps/mpi/mpi2_cnfg.h>
	#include <dev/mps/mpi/mpi2_init.h>
	#include <dev/mps/mpi/mpi2_tool.h>
	#include <dev/mps/mps_ioctl.h>
	#include <dev/mps/mpsvar.h>
	#include <dev/mps/mps_table.h>
	#include <dev/mps/mps_sas.h>
	#include <dev/pci/pcivar.h>
	#include <dev/pci/pcireg.h>

	static d_open_t mps_open;
	static d_close_t mps_close;
	static d_ioctl_t mps_ioctl_devsw;

	static struct cdevsw mps_cdevsw = {
	.d_version = D_VERSION,
	.d_flags = 0,
	.d_open = mps_open,
	.d_close = mps_close,
	.d_ioctl = mps_ioctl_devsw,
	.d_name = "mps",
	};

	typedef int (mps_user_f)(struct mps_command , struct mps_usr_command );
	static mps_user_f mpi_pre_ioc_facts;
	static mps_user_f mpi_pre_port_facts;
	static mps_user_f mpi_pre_fw_download;
	static mps_user_f mpi_pre_fw_upload;
	static mps_user_f mpi_pre_sata_passthrough;
	static mps_user_f mpi_pre_smp_passthrough;
	static mps_user_f mpi_pre_config;
	static mps_user_f mpi_pre_sas_io_unit_control;

	static int mps_user_read_cfg_header(struct mps_softc *,
	struct mps_cfg_page_req *);
	static int mps_user_read_cfg_page(struct mps_softc *,
	struct mps_cfg_page_req , void );
	static int mps_user_read_extcfg_header(struct mps_softc *,
	struct mps_ext_cfg_page_req *);
	static int mps_user_read_extcfg_page(struct mps_softc *,
	struct mps_ext_cfg_page_req , void );
	static int mps_user_write_cfg_page(struct mps_softc *,
	struct mps_cfg_page_req , void );
	static int mps_user_setup_request(struct mps_command *,
	struct mps_usr_command *);
	static int mps_user_command(struct mps_softc , struct mps_usr_command );

	static int mps_user_pass_thru(struct mps_softc sc, mps_pass_thru_t data);
	static void mps_user_get_adapter_data(struct mps_softc *sc,
	mps_adapter_data_t *data);
	static void mps_user_read_pci_info(struct mps_softc *sc,
	mps_pci_info_t *data);
	static uint8_t mps_get_fw_diag_buffer_number(struct mps_softc *sc,
	uint32_t unique_id);
	static int mps_post_fw_diag_buffer(struct mps_softc *sc,
	mps_fw_diagnostic_buffer_t pBuffer, uint32_t return_code);
	static int mps_release_fw_diag_buffer(struct mps_softc *sc,
	mps_fw_diagnostic_buffer_t pBuffer, uint32_t return_code,
	uint32_t diag_type);
	static int mps_diag_register(struct mps_softc *sc,
	mps_fw_diag_register_t diag_register, uint32_t return_code);
	static int mps_diag_unregister(struct mps_softc *sc,
	mps_fw_diag_unregister_t diag_unregister, uint32_t return_code);
	static int mps_diag_query(struct mps_softc sc, mps_fw_diag_query_t diag_query,
	uint32_t *return_code);
	static int mps_diag_read_buffer(struct mps_softc *sc,
	mps_diag_read_buffer_t diag_read_buffer, uint8_t ioctl_buf,
	uint32_t *return_code);
	static int mps_diag_release(struct mps_softc *sc,
	mps_fw_diag_release_t diag_release, uint32_t return_code);
	static int mps_do_diag_action(struct mps_softc *sc, uint32_t action,
	uint8_t diag_action, uint32_t length, uint32_t return_code);
	static int mps_user_diag_action(struct mps_softc sc, mps_diag_action_t data);
	static void mps_user_event_query(struct mps_softc sc, mps_event_query_t data);
	static void mps_user_event_enable(struct mps_softc *sc,
	mps_event_enable_t *data);
	static int mps_user_event_report(struct mps_softc *sc,
	mps_event_report_t *data);
	static int mps_user_reg_access(struct mps_softc sc, mps_reg_access_t data);
	static int mps_user_btdh(struct mps_softc sc, mps_btdh_mapping_t data);

	MALLOC_DEFINE(M_MPSUSER, "mps_user", "Buffers for mps(4) ioctls");

	int
	mps_attach_user(struct mps_softc *sc)
	{
	int unit;

	unit = device_get_unit(sc->mps_dev);
	sc->mps_cdev = make_dev(&mps_cdevsw, unit, UID_ROOT, GID_OPERATOR, 0640,
	"mps%d", unit);
	if (sc->mps_cdev == NULL) {
	return (ENOMEM);
	}
	sc->mps_cdev->si_drv1 = sc;
	return (0);
	}

	void
	mps_detach_user(struct mps_softc *sc)
	{

	/* XXX: do a purge of pending requests? */
	if (sc->mps_cdev != NULL)
	destroy_dev(sc->mps_cdev);
	}

	static int
	mps_open(struct cdev dev, int flags, int fmt, struct thread td)
	{

	return (0);
	}

	static int
	mps_close(struct cdev dev, int flags, int fmt, struct thread td)
	{

	return (0);
	}

	static int
	mps_user_read_cfg_header(struct mps_softc *sc,
	struct mps_cfg_page_req *page_req)
	{
	MPI2_CONFIG_PAGE_HEADER *hdr;
	struct mps_config_params params;
	int error;

	hdr = &params.hdr.Struct;
	params.action = MPI2_CONFIG_ACTION_PAGE_HEADER;
	params.page_address = le32toh(page_req->page_address);
	hdr->PageVersion = 0;
	hdr->PageLength = 0;
	hdr->PageNumber = page_req->header.PageNumber;
	hdr->PageType = page_req->header.PageType;
	params.buffer = NULL;
	params.length = 0;
	params.callback = NULL;

	if ((error = mps_read_config_page(sc, &params)) != 0) {
	/*
	* Leave the request. Without resetting the chip, it's
	* still owned by it and we'll just get into trouble
	* freeing it now. Mark it as abandoned so that if it
	* shows up later it can be freed.
	*/
	mps_printf(sc, "read_cfg_header timed out\n");
	return (ETIMEDOUT);
	}

	page_req->ioc_status = htole16(params.status);
	if ((page_req->ioc_status & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	bcopy(hdr, &page_req->header, sizeof(page_req->header));
	}

	return (0);
	}

	static int
	mps_user_read_cfg_page(struct mps_softc sc, struct mps_cfg_page_req page_req,
	void *buf)
	{
	MPI2_CONFIG_PAGE_HEADER reqhdr, hdr;
	struct mps_config_params params;
	int error;

	reqhdr = buf;
	hdr = &params.hdr.Struct;
	hdr->PageVersion = reqhdr->PageVersion;
	hdr->PageLength = reqhdr->PageLength;
	hdr->PageNumber = reqhdr->PageNumber;
	hdr->PageType = reqhdr->PageType & MPI2_CONFIG_PAGETYPE_MASK;
	params.action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
	params.page_address = le32toh(page_req->page_address);
	params.buffer = buf;
	params.length = le32toh(page_req->len);
	params.callback = NULL;

	if ((error = mps_read_config_page(sc, &params)) != 0) {
	mps_printf(sc, "mps_user_read_cfg_page timed out\n");
	return (ETIMEDOUT);
	}

	page_req->ioc_status = htole16(params.status);
	return (0);
	}

	static int
	mps_user_read_extcfg_header(struct mps_softc *sc,
	struct mps_ext_cfg_page_req *ext_page_req)
	{
	MPI2_CONFIG_EXTENDED_PAGE_HEADER *hdr;
	struct mps_config_params params;
	int error;

	hdr = &params.hdr.Ext;
	params.action = MPI2_CONFIG_ACTION_PAGE_HEADER;
	hdr->PageVersion = ext_page_req->header.PageVersion;
	hdr->PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
	hdr->ExtPageLength = 0;
	hdr->PageNumber = ext_page_req->header.PageNumber;
	hdr->ExtPageType = ext_page_req->header.ExtPageType;
	params.page_address = le32toh(ext_page_req->page_address);
	params.buffer = NULL;
	params.length = 0;
	params.callback = NULL;

	if ((error = mps_read_config_page(sc, &params)) != 0) {
	/*
	* Leave the request. Without resetting the chip, it's
	* still owned by it and we'll just get into trouble
	* freeing it now. Mark it as abandoned so that if it
	* shows up later it can be freed.
	*/
	mps_printf(sc, "mps_user_read_extcfg_header timed out\n");
	return (ETIMEDOUT);
	}

	ext_page_req->ioc_status = htole16(params.status);
	if ((ext_page_req->ioc_status & MPI2_IOCSTATUS_MASK) ==
	MPI2_IOCSTATUS_SUCCESS) {
	ext_page_req->header.PageVersion = hdr->PageVersion;
	ext_page_req->header.PageNumber = hdr->PageNumber;
	ext_page_req->header.PageType = hdr->PageType;
	ext_page_req->header.ExtPageLength = hdr->ExtPageLength;
	ext_page_req->header.ExtPageType = hdr->ExtPageType;
	}

	return (0);
	}

	static int
	mps_user_read_extcfg_page(struct mps_softc *sc,
	struct mps_ext_cfg_page_req ext_page_req, void buf)
	{
	MPI2_CONFIG_EXTENDED_PAGE_HEADER reqhdr, hdr;
	struct mps_config_params params;
	int error;

	reqhdr = buf;
	hdr = &params.hdr.Ext;
	params.action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
	params.page_address = le32toh(ext_page_req->page_address);
	hdr->PageVersion = reqhdr->PageVersion;
	hdr->PageType = MPI2_CONFIG_PAGETYPE_EXTENDED;
	hdr->PageNumber = reqhdr->PageNumber;
	hdr->ExtPageType = reqhdr->ExtPageType;
	hdr->ExtPageLength = reqhdr->ExtPageLength;
	params.buffer = buf;
	params.length = le32toh(ext_page_req->len);
	params.callback = NULL;

	if ((error = mps_read_config_page(sc, &params)) != 0) {
	mps_printf(sc, "mps_user_read_extcfg_page timed out\n");
	return (ETIMEDOUT);
	}

	ext_page_req->ioc_status = htole16(params.status);
	return (0);
	}

	static int
	mps_user_write_cfg_page(struct mps_softc *sc,
	struct mps_cfg_page_req page_req, void buf)
	{
	MPI2_CONFIG_PAGE_HEADER reqhdr, hdr;
	struct mps_config_params params;
	u_int hdr_attr;
	int error;

	reqhdr = buf;
	hdr = &params.hdr.Struct;
	hdr_attr = reqhdr->PageType & MPI2_CONFIG_PAGEATTR_MASK;
	if (hdr_attr != MPI2_CONFIG_PAGEATTR_CHANGEABLE &&
	hdr_attr != MPI2_CONFIG_PAGEATTR_PERSISTENT) {
	mps_printf(sc, "page type 0x%x not changeable\n",
	reqhdr->PageType & MPI2_CONFIG_PAGETYPE_MASK);
	return (EINVAL);
	}

	/*
	* There isn't any point in restoring stripped out attributes
	* if you then mask them going down to issue the request.
	*/

	hdr->PageVersion = reqhdr->PageVersion;
	hdr->PageLength = reqhdr->PageLength;
	hdr->PageNumber = reqhdr->PageNumber;
	hdr->PageType = reqhdr->PageType;
	params.action = MPI2_CONFIG_ACTION_PAGE_WRITE_CURRENT;
	params.page_address = le32toh(page_req->page_address);
	params.buffer = buf;
	params.length = le32toh(page_req->len);
	params.callback = NULL;

	if ((error = mps_write_config_page(sc, &params)) != 0) {
	mps_printf(sc, "mps_write_cfg_page timed out\n");
	return (ETIMEDOUT);
	}

	page_req->ioc_status = htole16(params.status);
	return (0);
	}

	void
	mpi_init_sge(struct mps_command cm, void req, void *sge)
	{
	int off, space;

	space = (int)cm->cm_sc->reqframesz;
	off = (uintptr_t)sge - (uintptr_t)req;

	KASSERT(off < space, ("bad pointers %p %p, off %d, space %d",
	req, sge, off, space));

	cm->cm_sge = sge;
	cm->cm_sglsize = space - off;
	}

	/*
	* Prepare the mps_command for an IOC_FACTS request.
	*/
	static int
	mpi_pre_ioc_facts(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_IOC_FACTS_REQUEST req = (void )cm->cm_req;
	MPI2_IOC_FACTS_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	cm->cm_sge = NULL;
	cm->cm_sglsize = 0;
	return (0);
	}

	/*
	* Prepare the mps_command for a PORT_FACTS request.
	*/
	static int
	mpi_pre_port_facts(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_PORT_FACTS_REQUEST req = (void )cm->cm_req;
	MPI2_PORT_FACTS_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	cm->cm_sge = NULL;
	cm->cm_sglsize = 0;
	return (0);
	}

	/*
	* Prepare the mps_command for a FW_DOWNLOAD request.
	*/
	static int
	mpi_pre_fw_download(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_FW_DOWNLOAD_REQUEST req = (void )cm->cm_req;
	MPI2_FW_DOWNLOAD_REPLY *rpl;
	MPI2_FW_DOWNLOAD_TCSGE tc;
	int error;

	/*
	* This code assumes there is room in the request's SGL for
	* the TransactionContext plus at least a SGL chain element.
	*/
	CTASSERT(sizeof req->SGL >= sizeof tc + MPS_SGC_SIZE);

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	if (cmd->len == 0)
	return (EINVAL);

	error = copyin(cmd->buf, cm->cm_data, cmd->len);
	if (error != 0)
	return (error);

	mpi_init_sge(cm, req, &req->SGL);
	bzero(&tc, sizeof tc);

	/*
	* For now, the F/W image must be provided in a single request.
	*/
	if ((req->MsgFlags & MPI2_FW_DOWNLOAD_MSGFLGS_LAST_SEGMENT) == 0)
	return (EINVAL);
	if (req->TotalImageSize != cmd->len)
	return (EINVAL);

	/*
	* The value of the first two elements is specified in the
	* Fusion-MPT Message Passing Interface document.
	*/
	tc.ContextSize = 0;
	tc.DetailsLength = 12;
	tc.ImageOffset = 0;
	tc.ImageSize = cmd->len;

	cm->cm_flags \|= MPS_CM_FLAGS_DATAOUT;

	return (mps_push_sge(cm, &tc, sizeof tc, 0));
	}

	/*
	* Prepare the mps_command for a FW_UPLOAD request.
	*/
	static int
	mpi_pre_fw_upload(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_FW_UPLOAD_REQUEST req = (void )cm->cm_req;
	MPI2_FW_UPLOAD_REPLY *rpl;
	MPI2_FW_UPLOAD_TCSGE tc;

	/*
	* This code assumes there is room in the request's SGL for
	* the TransactionContext plus at least a SGL chain element.
	*/
	CTASSERT(sizeof req->SGL >= sizeof tc + MPS_SGC_SIZE);

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpi_init_sge(cm, req, &req->SGL);
	bzero(&tc, sizeof tc);

	/*
	* The value of the first two elements is specified in the
	* Fusion-MPT Message Passing Interface document.
	*/
	tc.ContextSize = 0;
	tc.DetailsLength = 12;
	/*
	* XXX Is there any reason to fetch a partial image? I.e. to
	* set ImageOffset to something other than 0?
	*/
	tc.ImageOffset = 0;
	tc.ImageSize = cmd->len;

	cm->cm_flags \|= MPS_CM_FLAGS_DATAIN;

	return (mps_push_sge(cm, &tc, sizeof tc, 0));
	}

	/*
	* Prepare the mps_command for a SATA_PASSTHROUGH request.
	*/
	static int
	mpi_pre_sata_passthrough(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_SATA_PASSTHROUGH_REQUEST req = (void )cm->cm_req;
	MPI2_SATA_PASSTHROUGH_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpi_init_sge(cm, req, &req->SGL);
	return (0);
	}

	/*
	* Prepare the mps_command for a SMP_PASSTHROUGH request.
	*/
	static int
	mpi_pre_smp_passthrough(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_SMP_PASSTHROUGH_REQUEST req = (void )cm->cm_req;
	MPI2_SMP_PASSTHROUGH_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpi_init_sge(cm, req, &req->SGL);
	return (0);
	}

	/*
	* Prepare the mps_command for a CONFIG request.
	*/
	static int
	mpi_pre_config(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_CONFIG_REQUEST req = (void )cm->cm_req;
	MPI2_CONFIG_REPLY *rpl;

	if (cmd->req_len != sizeof *req)
	return (EINVAL);
	if (cmd->rpl_len != sizeof *rpl)
	return (EINVAL);

	mpi_init_sge(cm, req, &req->PageBufferSGE);
	return (0);
	}

	/*
	* Prepare the mps_command for a SAS_IO_UNIT_CONTROL request.
	*/
	static int
	mpi_pre_sas_io_unit_control(struct mps_command *cm,
	struct mps_usr_command *cmd)
	{

	cm->cm_sge = NULL;
	cm->cm_sglsize = 0;
	return (0);
	}

	/*
	* A set of functions to prepare an mps_command for the various
	* supported requests.
	*/
	struct mps_user_func {
	U8 Function;
	mps_user_f *f_pre;
	} mps_user_func_list[] = {
	{ MPI2_FUNCTION_IOC_FACTS, mpi_pre_ioc_facts },
	{ MPI2_FUNCTION_PORT_FACTS, mpi_pre_port_facts },
	{ MPI2_FUNCTION_FW_DOWNLOAD, mpi_pre_fw_download },
	{ MPI2_FUNCTION_FW_UPLOAD, mpi_pre_fw_upload },
	{ MPI2_FUNCTION_SATA_PASSTHROUGH, mpi_pre_sata_passthrough },
	{ MPI2_FUNCTION_SMP_PASSTHROUGH, mpi_pre_smp_passthrough},
	{ MPI2_FUNCTION_CONFIG, mpi_pre_config},
	{ MPI2_FUNCTION_SAS_IO_UNIT_CONTROL, mpi_pre_sas_io_unit_control },
	{ 0xFF, NULL } /* list end */
	};

	static int
	mps_user_setup_request(struct mps_command cm, struct mps_usr_command cmd)
	{
	MPI2_REQUEST_HEADER hdr = (MPI2_REQUEST_HEADER )cm->cm_req;
	struct mps_user_func *f;

	for (f = mps_user_func_list; f->f_pre != NULL; f++) {
	if (hdr->Function == f->Function)
	return (f->f_pre(cm, cmd));
	}
	return (EINVAL);
	}

	static int
	mps_user_command(struct mps_softc sc, struct mps_usr_command cmd)
	{
	MPI2_REQUEST_HEADER *hdr;
	MPI2_DEFAULT_REPLY *rpl;
	void *buf = NULL;
	struct mps_command *cm = NULL;
	int err = 0;
	int sz;

	mps_lock(sc);
	cm = mps_alloc_command(sc);

	if (cm == NULL) {
	mps_printf(sc, "%s: no mps requests\n", __func__);
	err = ENOMEM;
	goto RetFree;
	}
	mps_unlock(sc);

	hdr = (MPI2_REQUEST_HEADER *)cm->cm_req;

	mps_dprint(sc, MPS_USER, "%s: req %p %d rpl %p %d\n", __func__,
	cmd->req, cmd->req_len, cmd->rpl, cmd->rpl_len);

	if (cmd->req_len > (int)sc->reqframesz) {
	err = EINVAL;
	goto RetFreeUnlocked;
	}
	err = copyin(cmd->req, hdr, cmd->req_len);
	if (err != 0)
	goto RetFreeUnlocked;

	mps_dprint(sc, MPS_USER, "%s: Function %02X MsgFlags %02X\n", __func__,
	hdr->Function, hdr->MsgFlags);

	if (cmd->len > 0) {
	buf = malloc(cmd->len, M_MPSUSER, M_WAITOK\|M_ZERO);
	cm->cm_data = buf;
	cm->cm_length = cmd->len;
	} else {
	cm->cm_data = NULL;
	cm->cm_length = 0;
	}

	cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;

	err = mps_user_setup_request(cm, cmd);
	if (err == EINVAL) {
	mps_printf(sc, "%s: unsupported parameter or unsupported "
	"function in request (function = 0x%X)\n", __func__,
	hdr->Function);
	}
	if (err != 0)
	goto RetFreeUnlocked;

	mps_lock(sc);
	err = mps_wait_command(sc, &cm, 60, CAN_SLEEP);

	if (err \|\| (cm == NULL)) {
	mps_printf(sc, "%s: invalid request: error %d\n",
	__func__, err);
	goto RetFree;
	}

	rpl = (MPI2_DEFAULT_REPLY *)cm->cm_reply;
	if (rpl != NULL)
	sz = rpl->MsgLength * 4;
	else
	sz = 0;

	if (sz > cmd->rpl_len) {
	mps_printf(sc, "%s: user reply buffer (%d) smaller than "
	"returned buffer (%d)\n", __func__, cmd->rpl_len, sz);
	sz = cmd->rpl_len;
	}

	mps_unlock(sc);
	copyout(rpl, cmd->rpl, sz);
	if (buf != NULL)
	copyout(buf, cmd->buf, cmd->len);
	mps_dprint(sc, MPS_USER, "%s: reply size %d\n", __func__, sz);

	RetFreeUnlocked:
	mps_lock(sc);
	RetFree:
	if (cm != NULL)
	mps_free_command(sc, cm);
	mps_unlock(sc);
	if (buf != NULL)
	free(buf, M_MPSUSER);
	return (err);
	}

	static int
	mps_user_pass_thru(struct mps_softc sc, mps_pass_thru_t data)
	{
	MPI2_REQUEST_HEADER *hdr, tmphdr;
	MPI2_DEFAULT_REPLY *rpl = NULL;
	struct mps_command *cm = NULL;
	int err = 0, dir = 0, sz;
	uint8_t function = 0;
	u_int sense_len;
	struct mpssas_target *targ = NULL;

	/*
	* Only allow one passthru command at a time. Use the MPS_FLAGS_BUSY
	* bit to denote that a passthru is being processed.
	*/
	mps_lock(sc);
	if (sc->mps_flags & MPS_FLAGS_BUSY) {
	mps_dprint(sc, MPS_USER, "%s: Only one passthru command "
	"allowed at a single time.", __func__);
	mps_unlock(sc);
	return (EBUSY);
	}
	sc->mps_flags \|= MPS_FLAGS_BUSY;
	mps_unlock(sc);

	/*
	* Do some validation on data direction. Valid cases are:
	* 1) DataSize is 0 and direction is NONE
	* 2) DataSize is non-zero and one of:
	* a) direction is READ or
	* b) direction is WRITE or
	* c) direction is BOTH and DataOutSize is non-zero
	* If valid and the direction is BOTH, change the direction to READ.
	* if valid and the direction is not BOTH, make sure DataOutSize is 0.
	*/
	if (((data->DataSize == 0) &&
	(data->DataDirection == MPS_PASS_THRU_DIRECTION_NONE)) \|\|
	((data->DataSize != 0) &&
	((data->DataDirection == MPS_PASS_THRU_DIRECTION_READ) \|\|
	(data->DataDirection == MPS_PASS_THRU_DIRECTION_WRITE) \|\|
	((data->DataDirection == MPS_PASS_THRU_DIRECTION_BOTH) &&
	(data->DataOutSize != 0))))) {
	if (data->DataDirection == MPS_PASS_THRU_DIRECTION_BOTH)
	data->DataDirection = MPS_PASS_THRU_DIRECTION_READ;
	else
	data->DataOutSize = 0;
	} else {
	err = EINVAL;
	goto RetFreeUnlocked;
	}

	mps_dprint(sc, MPS_USER, "%s: req 0x%jx %d rpl 0x%jx %d "
	"data in 0x%jx %d data out 0x%jx %d data dir %d\n", __func__,
	data->PtrRequest, data->RequestSize, data->PtrReply,
	data->ReplySize, data->PtrData, data->DataSize,
	data->PtrDataOut, data->DataOutSize, data->DataDirection);

	/*
	* copy in the header so we know what we're dealing with before we
	* commit to allocating a command for it.
	*/
	err = copyin(PTRIN(data->PtrRequest), &tmphdr, data->RequestSize);
	if (err != 0)
	goto RetFreeUnlocked;

	if (data->RequestSize > (int)sc->reqframesz) {
	err = EINVAL;
	goto RetFreeUnlocked;
	}

	function = tmphdr.Function;
	mps_dprint(sc, MPS_USER, "%s: Function %02X MsgFlags %02X\n", __func__,
	function, tmphdr.MsgFlags);

	/*
	* Handle a passthru TM request.
	*/
	if (function == MPI2_FUNCTION_SCSI_TASK_MGMT) {
	MPI2_SCSI_TASK_MANAGE_REQUEST *task;

	mps_lock(sc);
	cm = mpssas_alloc_tm(sc);
	if (cm == NULL) {
	err = EINVAL;
	goto Ret;
	}

	/* Copy the header in. Only a small fixup is needed. */
	task = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
	bcopy(&tmphdr, task, data->RequestSize);
	task->TaskMID = cm->cm_desc.Default.SMID;

	cm->cm_data = NULL;
	cm->cm_complete = NULL;
	cm->cm_complete_data = NULL;

	targ = mpssas_find_target_by_handle(sc->sassc, 0,
	task->DevHandle);
	if (targ == NULL) {
	mps_dprint(sc, MPS_INFO,
	"%s %d : invalid handle for requested TM 0x%x \n",
	__func__, __LINE__, task->DevHandle);
	err = 1;
	} else {
	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
	err = mps_wait_command(sc, &cm, 30, CAN_SLEEP);
	}

	if (err != 0) {
	err = EIO;
	mps_dprint(sc, MPS_FAULT, "%s: task management failed",
	__func__);
	}
	/*
	* Copy the reply data and sense data to user space.
	*/
	if ((cm != NULL) && (cm->cm_reply != NULL)) {
	rpl = (MPI2_DEFAULT_REPLY *)cm->cm_reply;
	sz = rpl->MsgLength * 4;

	if (sz > data->ReplySize) {
	mps_printf(sc, "%s: user reply buffer (%d) "
	"smaller than returned buffer (%d)\n",
	__func__, data->ReplySize, sz);
	}
	mps_unlock(sc);
	copyout(cm->cm_reply, PTRIN(data->PtrReply),
	data->ReplySize);
	mps_lock(sc);
	}
	mpssas_free_tm(sc, cm);
	goto Ret;
	}

	mps_lock(sc);
	cm = mps_alloc_command(sc);

	if (cm == NULL) {
	mps_printf(sc, "%s: no mps requests\n", __func__);
	err = ENOMEM;
	goto Ret;
	}
	mps_unlock(sc);

	hdr = (MPI2_REQUEST_HEADER *)cm->cm_req;
	bcopy(&tmphdr, hdr, data->RequestSize);

	/*
	* Do some checking to make sure the IOCTL request contains a valid
	* request. Then set the SGL info.
	*/
	mpi_init_sge(cm, hdr, (void )((uint8_t )hdr + data->RequestSize));

	/*
	* Set up for read, write or both. From check above, DataOutSize will
	* be 0 if direction is READ or WRITE, but it will have some non-zero
	* value if the direction is BOTH. So, just use the biggest size to get
	* the cm_data buffer size. If direction is BOTH, 2 SGLs need to be set
	* up; the first is for the request and the second will contain the
	* response data. cm_out_len needs to be set here and this will be used
	* when the SGLs are set up.
	*/
	cm->cm_data = NULL;
	cm->cm_length = MAX(data->DataSize, data->DataOutSize);
	cm->cm_out_len = data->DataOutSize;
	cm->cm_flags = 0;
	if (cm->cm_length != 0) {
	cm->cm_data = malloc(cm->cm_length, M_MPSUSER, M_WAITOK \|
	M_ZERO);
	cm->cm_flags = MPS_CM_FLAGS_DATAIN;
	if (data->DataOutSize) {
	cm->cm_flags \|= MPS_CM_FLAGS_DATAOUT;
	err = copyin(PTRIN(data->PtrDataOut),
	cm->cm_data, data->DataOutSize);
	} else if (data->DataDirection ==
	MPS_PASS_THRU_DIRECTION_WRITE) {
	cm->cm_flags = MPS_CM_FLAGS_DATAOUT;
	err = copyin(PTRIN(data->PtrData),
	cm->cm_data, data->DataSize);
	}
	if (err != 0)
	mps_dprint(sc, MPS_FAULT, "%s: failed to copy "
	"IOCTL data from user space\n", __func__);
	}
	cm->cm_flags \|= MPS_CM_FLAGS_SGE_SIMPLE;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;

	/*
	* Set up Sense buffer and SGL offset for IO passthru. SCSI IO request
	* uses SCSI IO descriptor.
	*/
	if ((function == MPI2_FUNCTION_SCSI_IO_REQUEST) \|\|
	(function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH)) {
	MPI2_SCSI_IO_REQUEST *scsi_io_req;

	scsi_io_req = (MPI2_SCSI_IO_REQUEST *)hdr;
	/*
	* Put SGE for data and data_out buffer at the end of
	* scsi_io_request message header (64 bytes in total).
	* Following above SGEs, the residual space will be used by
	* sense data.
	*/
	scsi_io_req->SenseBufferLength = (uint8_t)(data->RequestSize -
	64);
	scsi_io_req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);

	/*
	* Set SGLOffset0 value. This is the number of dwords that SGL
	* is offset from the beginning of MPI2_SCSI_IO_REQUEST struct.
	*/
	scsi_io_req->SGLOffset0 = 24;

	/*
	* Setup descriptor info. RAID passthrough must use the
	* default request descriptor which is already set, so if this
	* is a SCSI IO request, change the descriptor to SCSI IO.
	* Also, if this is a SCSI IO request, handle the reply in the
	* mpssas_scsio_complete function.
	*/
	if (function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
	cm->cm_desc.SCSIIO.RequestFlags =
	MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
	cm->cm_desc.SCSIIO.DevHandle = scsi_io_req->DevHandle;

	/*
	* Make sure the DevHandle is not 0 because this is a
	* likely error.
	*/
	if (scsi_io_req->DevHandle == 0) {
	err = EINVAL;
	goto RetFreeUnlocked;
	}
	}
	}

	mps_lock(sc);

	err = mps_wait_command(sc, &cm, 30, CAN_SLEEP);

	if (err \|\| (cm == NULL)) {
	mps_printf(sc, "%s: invalid request: error %d\n", __func__,
	err);
	mps_unlock(sc);
	goto RetFreeUnlocked;
	}

	/*
	* Sync the DMA data, if any. Then copy the data to user space.
	*/
	if (cm->cm_data != NULL) {
	if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
	dir = BUS_DMASYNC_POSTREAD;
	else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
	dir = BUS_DMASYNC_POSTWRITE;
	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);

	if (cm->cm_flags & MPS_CM_FLAGS_DATAIN) {
	mps_unlock(sc);
	err = copyout(cm->cm_data,
	PTRIN(data->PtrData), data->DataSize);
	mps_lock(sc);
	if (err != 0)
	mps_dprint(sc, MPS_FAULT, "%s: failed to copy "
	"IOCTL data to user space\n", __func__);
	}
	}

	/*
	* Copy the reply data and sense data to user space.
	*/
	if (cm->cm_reply != NULL) {
	rpl = (MPI2_DEFAULT_REPLY *)cm->cm_reply;
	sz = rpl->MsgLength * 4;

	if (sz > data->ReplySize) {
	mps_printf(sc, "%s: user reply buffer (%d) smaller "
	"than returned buffer (%d)\n", __func__,
	data->ReplySize, sz);
	}
	mps_unlock(sc);
	copyout(cm->cm_reply, PTRIN(data->PtrReply), data->ReplySize);
	mps_lock(sc);

	if ((function == MPI2_FUNCTION_SCSI_IO_REQUEST) \|\|
	(function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH)) {
	if (((MPI2_SCSI_IO_REPLY *)rpl)->SCSIState &
	MPI2_SCSI_STATE_AUTOSENSE_VALID) {
	sense_len =
	MIN((le32toh(((MPI2_SCSI_IO_REPLY *)rpl)->
	SenseCount)), sizeof(struct
	scsi_sense_data));
	mps_unlock(sc);
	copyout(cm->cm_sense, (PTRIN(data->PtrReply +
	sizeof(MPI2_SCSI_IO_REPLY))), sense_len);
	mps_lock(sc);
	}
	}
	}
	mps_unlock(sc);

	RetFreeUnlocked:
	mps_lock(sc);

	if (cm != NULL) {
	if (cm->cm_data)
	free(cm->cm_data, M_MPSUSER);
	mps_free_command(sc, cm);
	}
	Ret:
	sc->mps_flags &= ~MPS_FLAGS_BUSY;
	mps_unlock(sc);

	return (err);
	}

	static void
	mps_user_get_adapter_data(struct mps_softc sc, mps_adapter_data_t data)
	{
	Mpi2ConfigReply_t mpi_reply;
	Mpi2BiosPage3_t config_page;

	/*
	* Use the PCI interface functions to get the Bus, Device, and Function
	* information.
	*/
	data->PciInformation.u.bits.BusNumber = pci_get_bus(sc->mps_dev);
	data->PciInformation.u.bits.DeviceNumber = pci_get_slot(sc->mps_dev);
	data->PciInformation.u.bits.FunctionNumber =
	pci_get_function(sc->mps_dev);

	/*
	* Get the FW version that should already be saved in IOC Facts.
	*/
	data->MpiFirmwareVersion = sc->facts->FWVersion.Word;

	/*
	* General device info.
	*/
	data->AdapterType = MPSIOCTL_ADAPTER_TYPE_SAS2;
	if (sc->mps_flags & MPS_FLAGS_WD_AVAILABLE)
	data->AdapterType = MPSIOCTL_ADAPTER_TYPE_SAS2_SSS6200;
	data->PCIDeviceHwId = pci_get_device(sc->mps_dev);
	data->PCIDeviceHwRev = pci_read_config(sc->mps_dev, PCIR_REVID, 1);
	data->SubSystemId = pci_get_subdevice(sc->mps_dev);
	data->SubsystemVendorId = pci_get_subvendor(sc->mps_dev);

	/*
	* Get the driver version.
	*/
	strcpy((char *)&data->DriverVersion[0], MPS_DRIVER_VERSION);

	/*
	* Need to get BIOS Config Page 3 for the BIOS Version.
	*/
	data->BiosVersion = 0;
	mps_lock(sc);
	if (mps_config_get_bios_pg3(sc, &mpi_reply, &config_page))
	printf("%s: Error while retrieving BIOS Version\n", __func__);
	else
	data->BiosVersion = config_page.BiosVersion;
	mps_unlock(sc);
	}

	static void
	mps_user_read_pci_info(struct mps_softc sc, mps_pci_info_t data)
	{
	int i;

	/*
	* Use the PCI interface functions to get the Bus, Device, and Function
	* information.
	*/
	data->BusNumber = pci_get_bus(sc->mps_dev);
	data->DeviceNumber = pci_get_slot(sc->mps_dev);
	data->FunctionNumber = pci_get_function(sc->mps_dev);

	/*
	* Now get the interrupt vector and the pci header. The vector can
	* only be 0 right now. The header is the first 256 bytes of config
	* space.
	*/
	data->InterruptVector = 0;
	for (i = 0; i < sizeof (data->PciHeader); i++) {
	data->PciHeader[i] = pci_read_config(sc->mps_dev, i, 1);
	}
	}

	static uint8_t
	mps_get_fw_diag_buffer_number(struct mps_softc *sc, uint32_t unique_id)
	{
	uint8_t index;

	for (index = 0; index < MPI2_DIAG_BUF_TYPE_COUNT; index++) {
	if (sc->fw_diag_buffer_list[index].unique_id == unique_id) {
	return (index);
	}
	}

	return (MPS_FW_DIAGNOSTIC_UID_NOT_FOUND);
	}

	static int
	mps_post_fw_diag_buffer(struct mps_softc *sc,
	mps_fw_diagnostic_buffer_t pBuffer, uint32_t return_code)
	{
	MPI2_DIAG_BUFFER_POST_REQUEST *req;
	MPI2_DIAG_BUFFER_POST_REPLY *reply = NULL;
	struct mps_command *cm = NULL;
	int i, status;

	/*
	* If buffer is not enabled, just leave.
	*/
	*return_code = MPS_FW_DIAG_ERROR_POST_FAILED;
	if (!pBuffer->enabled) {
	return (MPS_DIAG_FAILURE);
	}

	/*
	* Clear some flags initially.
	*/
	pBuffer->force_release = FALSE;
	pBuffer->valid_data = FALSE;
	pBuffer->owned_by_firmware = FALSE;

	/*
	* Get a command.
	*/
	cm = mps_alloc_command(sc);
	if (cm == NULL) {
	mps_printf(sc, "%s: no mps requests\n", __func__);
	return (MPS_DIAG_FAILURE);
	}

	/*
	* Build the request for releasing the FW Diag Buffer and send it.
	*/
	req = (MPI2_DIAG_BUFFER_POST_REQUEST *)cm->cm_req;
	req->Function = MPI2_FUNCTION_DIAG_BUFFER_POST;
	req->BufferType = pBuffer->buffer_type;
	req->ExtendedType = pBuffer->extended_type;
	req->BufferLength = pBuffer->size;
	for (i = 0; i < (sizeof(req->ProductSpecific) / 4); i++)
	req->ProductSpecific[i] = pBuffer->product_specific[i];
	mps_from_u64(sc->fw_diag_busaddr, &req->BufferAddress);
	cm->cm_data = NULL;
	cm->cm_length = 0;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete_data = NULL;

	/*
	* Send command synchronously.
	*/
	status = mps_wait_command(sc, &cm, 30, CAN_SLEEP);
	if (status \|\| (cm == NULL)) {
	mps_printf(sc, "%s: invalid request: error %d\n", __func__,
	status);
	status = MPS_DIAG_FAILURE;
	goto done;
	}

	/*
	* Process POST reply.
	*/
	reply = (MPI2_DIAG_BUFFER_POST_REPLY *)cm->cm_reply;
	if (reply == NULL) {
	mps_printf(sc, "%s: reply is NULL, probably due to "
	"reinitialization\n", __func__);
	status = MPS_DIAG_FAILURE;
	goto done;
	}
	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) {
	status = MPS_DIAG_FAILURE;
	mps_dprint(sc, MPS_FAULT, "%s: post of FW Diag Buffer failed "
	"with IOCStatus = 0x%x, IOCLogInfo = 0x%x and "
	"TransferLength = 0x%x\n", __func__,
	le16toh(reply->IOCStatus), le32toh(reply->IOCLogInfo),
	le32toh(reply->TransferLength));
	goto done;
	}

	/*
	* Post was successful.
	*/
	pBuffer->valid_data = TRUE;
	pBuffer->owned_by_firmware = TRUE;
	*return_code = MPS_FW_DIAG_ERROR_SUCCESS;
	status = MPS_DIAG_SUCCESS;

	done:
	if (cm != NULL)
	mps_free_command(sc, cm);
	return (status);
	}

	static int
	mps_release_fw_diag_buffer(struct mps_softc *sc,
	mps_fw_diagnostic_buffer_t pBuffer, uint32_t return_code,
	uint32_t diag_type)
	{
	MPI2_DIAG_RELEASE_REQUEST *req;
	MPI2_DIAG_RELEASE_REPLY *reply = NULL;
	struct mps_command *cm = NULL;
	int status;

	/*
	* If buffer is not enabled, just leave.
	*/
	*return_code = MPS_FW_DIAG_ERROR_RELEASE_FAILED;
	if (!pBuffer->enabled) {
	mps_dprint(sc, MPS_USER, "%s: This buffer type is not "
	"supported by the IOC", __func__);
	return (MPS_DIAG_FAILURE);
	}

	/*
	* Clear some flags initially.
	*/
	pBuffer->force_release = FALSE;
	pBuffer->valid_data = FALSE;
	pBuffer->owned_by_firmware = FALSE;

	/*
	* Get a command.
	*/
	cm = mps_alloc_command(sc);
	if (cm == NULL) {
	mps_printf(sc, "%s: no mps requests\n", __func__);
	return (MPS_DIAG_FAILURE);
	}

	/*
	* Build the request for releasing the FW Diag Buffer and send it.
	*/
	req = (MPI2_DIAG_RELEASE_REQUEST *)cm->cm_req;
	req->Function = MPI2_FUNCTION_DIAG_RELEASE;
	req->BufferType = pBuffer->buffer_type;
	cm->cm_data = NULL;
	cm->cm_length = 0;
	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
	cm->cm_complete_data = NULL;

	/*
	* Send command synchronously.
	*/
	status = mps_wait_command(sc, &cm, 30, CAN_SLEEP);
	if (status \|\| (cm == NULL)) {
	mps_printf(sc, "%s: invalid request: error %d\n", __func__,
	status);
	status = MPS_DIAG_FAILURE;
	goto done;
	}

	/*
	* Process RELEASE reply.
	*/
	reply = (MPI2_DIAG_RELEASE_REPLY *)cm->cm_reply;
	if (reply == NULL) {
	mps_printf(sc, "%s: reply is NULL, probably due to "
	"reinitialization\n", __func__);
	status = MPS_DIAG_FAILURE;
	goto done;
	}
	if (((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
	MPI2_IOCSTATUS_SUCCESS) \|\| pBuffer->owned_by_firmware) {
	status = MPS_DIAG_FAILURE;
	mps_dprint(sc, MPS_FAULT, "%s: release of FW Diag Buffer "
	"failed with IOCStatus = 0x%x and IOCLogInfo = 0x%x\n",
	__func__, le16toh(reply->IOCStatus),
	le32toh(reply->IOCLogInfo));
	goto done;
	}

	/*
	* Release was successful.
	*/
	*return_code = MPS_FW_DIAG_ERROR_SUCCESS;
	status = MPS_DIAG_SUCCESS;

	/*
	* If this was for an UNREGISTER diag type command, clear the unique ID.
	*/
	if (diag_type == MPS_FW_DIAG_TYPE_UNREGISTER) {
	pBuffer->unique_id = MPS_FW_DIAG_INVALID_UID;
	}

	done:
	if (cm != NULL)
	mps_free_command(sc, cm);

	return (status);
	}

	static int
	mps_diag_register(struct mps_softc sc, mps_fw_diag_register_t diag_register,
	uint32_t *return_code)
	{
	mps_fw_diagnostic_buffer_t *pBuffer;
	struct mps_busdma_context *ctx;
	uint8_t extended_type, buffer_type, i;
	uint32_t buffer_size;
	uint32_t unique_id;
	int status;
	int error;

	extended_type = diag_register->ExtendedType;
	buffer_type = diag_register->BufferType;
	buffer_size = diag_register->RequestedBufferSize;
	unique_id = diag_register->UniqueId;
	ctx = NULL;
	error = 0;

	/*
	* Check for valid buffer type
	*/
	if (buffer_type >= MPI2_DIAG_BUF_TYPE_COUNT) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	return (MPS_DIAG_FAILURE);
	}

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should not be found. If it is, the ID is already in use.
	*/
	i = mps_get_fw_diag_buffer_number(sc, unique_id);
	pBuffer = &sc->fw_diag_buffer_list[buffer_type];
	if (i != MPS_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}

	/*
	* The buffer's unique ID should not be registered yet, and the given
	* unique ID cannot be 0.
	*/
	if ((pBuffer->unique_id != MPS_FW_DIAG_INVALID_UID) \|\|
	(unique_id == MPS_FW_DIAG_INVALID_UID)) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}

	/*
	* If this buffer is already posted as immediate, just change owner.
	*/
	if (pBuffer->immediate && pBuffer->owned_by_firmware &&
	(pBuffer->unique_id == MPS_FW_DIAG_INVALID_UID)) {
	pBuffer->immediate = FALSE;
	pBuffer->unique_id = unique_id;
	return (MPS_DIAG_SUCCESS);
	}

	/*
	* Post a new buffer after checking if it's enabled. The DMA buffer
	* that is allocated will be contiguous (nsegments = 1).
	*/
	if (!pBuffer->enabled) {
	*return_code = MPS_FW_DIAG_ERROR_NO_BUFFER;
	return (MPS_DIAG_FAILURE);
	}
	if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */
	1, 0, /* algnmnt, boundary */
	BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	buffer_size, /* maxsize */
	1, /* nsegments */
	buffer_size, /* maxsegsize */
	0, /* flags */
	NULL, NULL, /* lockfunc, lockarg */
	&sc->fw_diag_dmat)) {
	mps_dprint(sc, MPS_ERROR,
	"Cannot allocate FW diag buffer DMA tag\n");
	*return_code = MPS_FW_DIAG_ERROR_NO_BUFFER;
	status = MPS_DIAG_FAILURE;
	goto bailout;
	}
	if (bus_dmamem_alloc(sc->fw_diag_dmat, (void **)&sc->fw_diag_buffer,
	BUS_DMA_NOWAIT, &sc->fw_diag_map)) {
	mps_dprint(sc, MPS_ERROR,
	"Cannot allocate FW diag buffer memory\n");
	*return_code = MPS_FW_DIAG_ERROR_NO_BUFFER;
	status = MPS_DIAG_FAILURE;
	goto bailout;
	}
	bzero(sc->fw_diag_buffer, buffer_size);

	ctx = malloc(sizeof(*ctx), M_MPSUSER, M_WAITOK \| M_ZERO);
	- if (ctx == NULL) {
	- device_printf(sc->mps_dev, "%s: context malloc failed\n",
	- __func__);
	- *return_code = MPS_FW_DIAG_ERROR_NO_BUFFER;
	- status = MPS_DIAG_FAILURE;
	- goto bailout;
	- }
	ctx->addr = &sc->fw_diag_busaddr;
	ctx->buffer_dmat = sc->fw_diag_dmat;
	ctx->buffer_dmamap = sc->fw_diag_map;
	ctx->softc = sc;
	error = bus_dmamap_load(sc->fw_diag_dmat, sc->fw_diag_map,
	sc->fw_diag_buffer, buffer_size, mps_memaddr_wait_cb,
	ctx, 0);

	if (error == EINPROGRESS) {

	/* XXX KDM */
	device_printf(sc->mps_dev, "%s: Deferred bus_dmamap_load\n",
	__func__);
	/*
	* Wait for the load to complete. If we're interrupted,
	* bail out.
	*/
	mps_lock(sc);
	if (ctx->completed == 0) {
	error = msleep(ctx, &sc->mps_mtx, PCATCH, "mpswait", 0);
	if (error != 0) {
	/*
	* We got an error from msleep(9). This is
	* most likely due to a signal. Tell
	* mpr_memaddr_wait_cb() that we've abandoned
	* the context, so it needs to clean up when
	* it is called.
	*/
	ctx->abandoned = 1;

	/* The callback will free this memory */
	ctx = NULL;
	mps_unlock(sc);

	device_printf(sc->mps_dev, "Cannot "
	"bus_dmamap_load FW diag buffer, error = "
	"%d returned from msleep\n", error);
	*return_code = MPS_FW_DIAG_ERROR_NO_BUFFER;
	status = MPS_DIAG_FAILURE;
	goto bailout;
	}
	}
	mps_unlock(sc);
	}

	if ((error != 0) \|\| (ctx->error != 0)) {
	device_printf(sc->mps_dev, "Cannot bus_dmamap_load FW diag "
	"buffer, %serror = %d\n", error ? "" : "callback ",
	error ? error : ctx->error);
	*return_code = MPS_FW_DIAG_ERROR_NO_BUFFER;
	status = MPS_DIAG_FAILURE;
	goto bailout;
	}

	bus_dmamap_sync(sc->fw_diag_dmat, sc->fw_diag_map, BUS_DMASYNC_PREREAD);

	pBuffer->size = buffer_size;

	/*
	* Copy the given info to the diag buffer and post the buffer.
	*/
	pBuffer->buffer_type = buffer_type;
	pBuffer->immediate = FALSE;
	if (buffer_type == MPI2_DIAG_BUF_TYPE_TRACE) {
	for (i = 0; i < (sizeof (pBuffer->product_specific) / 4);
	i++) {
	pBuffer->product_specific[i] =
	diag_register->ProductSpecific[i];
	}
	}
	pBuffer->extended_type = extended_type;
	pBuffer->unique_id = unique_id;
	status = mps_post_fw_diag_buffer(sc, pBuffer, return_code);

	bailout:
	/*
	* In case there was a failure, free the DMA buffer.
	*/
	if (status == MPS_DIAG_FAILURE) {
	if (sc->fw_diag_busaddr != 0) {
	bus_dmamap_unload(sc->fw_diag_dmat, sc->fw_diag_map);
	sc->fw_diag_busaddr = 0;
	}
	if (sc->fw_diag_buffer != NULL) {
	bus_dmamem_free(sc->fw_diag_dmat, sc->fw_diag_buffer,
	sc->fw_diag_map);
	sc->fw_diag_buffer = NULL;
	}
	if (sc->fw_diag_dmat != NULL) {
	bus_dma_tag_destroy(sc->fw_diag_dmat);
	sc->fw_diag_dmat = NULL;
	}
	}

	if (ctx != NULL)
	free(ctx, M_MPSUSER);

	return (status);
	}

	static int
	mps_diag_unregister(struct mps_softc *sc,
	mps_fw_diag_unregister_t diag_unregister, uint32_t return_code)
	{
	mps_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i;
	uint32_t unique_id;
	int status;

	unique_id = diag_unregister->UniqueId;

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should be there.
	*/
	i = mps_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPS_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}

	pBuffer = &sc->fw_diag_buffer_list[i];

	/*
	* Try to release the buffer from FW before freeing it. If release
	* fails, don't free the DMA buffer in case FW tries to access it
	* later. If buffer is not owned by firmware, can't release it.
	*/
	if (!pBuffer->owned_by_firmware) {
	status = MPS_DIAG_SUCCESS;
	} else {
	status = mps_release_fw_diag_buffer(sc, pBuffer, return_code,
	MPS_FW_DIAG_TYPE_UNREGISTER);
	}

	/*
	* At this point, return the current status no matter what happens with
	* the DMA buffer.
	*/
	pBuffer->unique_id = MPS_FW_DIAG_INVALID_UID;
	if (status == MPS_DIAG_SUCCESS) {
	if (sc->fw_diag_busaddr != 0) {
	bus_dmamap_unload(sc->fw_diag_dmat, sc->fw_diag_map);
	sc->fw_diag_busaddr = 0;
	}
	if (sc->fw_diag_buffer != NULL) {
	bus_dmamem_free(sc->fw_diag_dmat, sc->fw_diag_buffer,
	sc->fw_diag_map);
	sc->fw_diag_buffer = NULL;
	}
	if (sc->fw_diag_dmat != NULL) {
	bus_dma_tag_destroy(sc->fw_diag_dmat);
	sc->fw_diag_dmat = NULL;
	}
	}

	return (status);
	}

	static int
	mps_diag_query(struct mps_softc sc, mps_fw_diag_query_t diag_query,
	uint32_t *return_code)
	{
	mps_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i;
	uint32_t unique_id;

	unique_id = diag_query->UniqueId;

	/*
	* If ID is valid, query on ID.
	* If ID is invalid, query on buffer type.
	*/
	if (unique_id == MPS_FW_DIAG_INVALID_UID) {
	i = diag_query->BufferType;
	if (i >= MPI2_DIAG_BUF_TYPE_COUNT) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}
	} else {
	i = mps_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPS_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}
	}

	/*
	* Fill query structure with the diag buffer info.
	*/
	pBuffer = &sc->fw_diag_buffer_list[i];
	diag_query->BufferType = pBuffer->buffer_type;
	diag_query->ExtendedType = pBuffer->extended_type;
	if (diag_query->BufferType == MPI2_DIAG_BUF_TYPE_TRACE) {
	for (i = 0; i < (sizeof(diag_query->ProductSpecific) / 4);
	i++) {
	diag_query->ProductSpecific[i] =
	pBuffer->product_specific[i];
	}
	}
	diag_query->TotalBufferSize = pBuffer->size;
	diag_query->DriverAddedBufferSize = 0;
	diag_query->UniqueId = pBuffer->unique_id;
	diag_query->ApplicationFlags = 0;
	diag_query->DiagnosticFlags = 0;

	/*
	* Set/Clear application flags
	*/
	if (pBuffer->immediate) {
	diag_query->ApplicationFlags &= ~MPS_FW_DIAG_FLAG_APP_OWNED;
	} else {
	diag_query->ApplicationFlags \|= MPS_FW_DIAG_FLAG_APP_OWNED;
	}
	if (pBuffer->valid_data \|\| pBuffer->owned_by_firmware) {
	diag_query->ApplicationFlags \|= MPS_FW_DIAG_FLAG_BUFFER_VALID;
	} else {
	diag_query->ApplicationFlags &= ~MPS_FW_DIAG_FLAG_BUFFER_VALID;
	}
	if (pBuffer->owned_by_firmware) {
	diag_query->ApplicationFlags \|=
	MPS_FW_DIAG_FLAG_FW_BUFFER_ACCESS;
	} else {
	diag_query->ApplicationFlags &=
	~MPS_FW_DIAG_FLAG_FW_BUFFER_ACCESS;
	}

	return (MPS_DIAG_SUCCESS);
	}

	static int
	mps_diag_read_buffer(struct mps_softc *sc,
	mps_diag_read_buffer_t diag_read_buffer, uint8_t ioctl_buf,
	uint32_t *return_code)
	{
	mps_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i, *pData;
	uint32_t unique_id;
	int status;

	unique_id = diag_read_buffer->UniqueId;

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should be there.
	*/
	i = mps_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPS_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}

	pBuffer = &sc->fw_diag_buffer_list[i];

	/*
	* Make sure requested read is within limits
	*/
	if (diag_read_buffer->StartingOffset + diag_read_buffer->BytesToRead >
	pBuffer->size) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	return (MPS_DIAG_FAILURE);
	}

	/* Sync the DMA map before we copy to userland. */
	bus_dmamap_sync(sc->fw_diag_dmat, sc->fw_diag_map,
	BUS_DMASYNC_POSTREAD);

	/*
	* Copy the requested data from DMA to the diag_read_buffer. The DMA
	* buffer that was allocated is one contiguous buffer.
	*/
	pData = (uint8_t *)(sc->fw_diag_buffer +
	diag_read_buffer->StartingOffset);
	if (copyout(pData, ioctl_buf, diag_read_buffer->BytesToRead) != 0)
	return (MPS_DIAG_FAILURE);
	diag_read_buffer->Status = 0;

	/*
	* Set or clear the Force Release flag.
	*/
	if (pBuffer->force_release) {
	diag_read_buffer->Flags \|= MPS_FW_DIAG_FLAG_FORCE_RELEASE;
	} else {
	diag_read_buffer->Flags &= ~MPS_FW_DIAG_FLAG_FORCE_RELEASE;
	}

	/*
	* If buffer is to be reregistered, make sure it's not already owned by
	* firmware first.
	*/
	status = MPS_DIAG_SUCCESS;
	if (!pBuffer->owned_by_firmware) {
	if (diag_read_buffer->Flags & MPS_FW_DIAG_FLAG_REREGISTER) {
	status = mps_post_fw_diag_buffer(sc, pBuffer,
	return_code);
	}
	}

	return (status);
	}

	static int
	mps_diag_release(struct mps_softc sc, mps_fw_diag_release_t diag_release,
	uint32_t *return_code)
	{
	mps_fw_diagnostic_buffer_t *pBuffer;
	uint8_t i;
	uint32_t unique_id;
	int status;

	unique_id = diag_release->UniqueId;

	/*
	* Get the current buffer and look up the unique ID. The unique ID
	* should be there.
	*/
	i = mps_get_fw_diag_buffer_number(sc, unique_id);
	if (i == MPS_FW_DIAGNOSTIC_UID_NOT_FOUND) {
	*return_code = MPS_FW_DIAG_ERROR_INVALID_UID;
	return (MPS_DIAG_FAILURE);
	}

	pBuffer = &sc->fw_diag_buffer_list[i];

	/*
	* If buffer is not owned by firmware, it's already been released.
	*/
	if (!pBuffer->owned_by_firmware) {
	*return_code = MPS_FW_DIAG_ERROR_ALREADY_RELEASED;
	return (MPS_DIAG_FAILURE);
	}

	/*
	* Release the buffer.
	*/
	status = mps_release_fw_diag_buffer(sc, pBuffer, return_code,
	MPS_FW_DIAG_TYPE_RELEASE);
	return (status);
	}

	static int
	mps_do_diag_action(struct mps_softc sc, uint32_t action, uint8_t diag_action,
	uint32_t length, uint32_t *return_code)
	{
	mps_fw_diag_register_t diag_register;
	mps_fw_diag_unregister_t diag_unregister;
	mps_fw_diag_query_t diag_query;
	mps_diag_read_buffer_t diag_read_buffer;
	mps_fw_diag_release_t diag_release;
	int status = MPS_DIAG_SUCCESS;
	uint32_t original_return_code;

	original_return_code = *return_code;
	*return_code = MPS_FW_DIAG_ERROR_SUCCESS;

	switch (action) {
	case MPS_FW_DIAG_TYPE_REGISTER:
	if (!length) {
	*return_code =
	MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPS_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_register,
	sizeof(diag_register)) != 0)
	return (MPS_DIAG_FAILURE);
	status = mps_diag_register(sc, &diag_register,
	return_code);
	break;

	case MPS_FW_DIAG_TYPE_UNREGISTER:
	if (length < sizeof(diag_unregister)) {
	*return_code =
	MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPS_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_unregister,
	sizeof(diag_unregister)) != 0)
	return (MPS_DIAG_FAILURE);
	status = mps_diag_unregister(sc, &diag_unregister,
	return_code);
	break;

	case MPS_FW_DIAG_TYPE_QUERY:
	if (length < sizeof (diag_query)) {
	*return_code =
	MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPS_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_query, sizeof(diag_query))
	!= 0)
	return (MPS_DIAG_FAILURE);
	status = mps_diag_query(sc, &diag_query, return_code);
	if (status == MPS_DIAG_SUCCESS)
	if (copyout(&diag_query, diag_action,
	sizeof (diag_query)) != 0)
	return (MPS_DIAG_FAILURE);
	break;

	case MPS_FW_DIAG_TYPE_READ_BUFFER:
	if (copyin(diag_action, &diag_read_buffer,
	sizeof(diag_read_buffer)) != 0)
	return (MPS_DIAG_FAILURE);
	if (length < diag_read_buffer.BytesToRead) {
	*return_code =
	MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPS_DIAG_FAILURE;
	break;
	}
	status = mps_diag_read_buffer(sc, &diag_read_buffer,
	PTRIN(diag_read_buffer.PtrDataBuffer),
	return_code);
	if (status == MPS_DIAG_SUCCESS) {
	if (copyout(&diag_read_buffer, diag_action,
	sizeof(diag_read_buffer) -
	sizeof(diag_read_buffer.PtrDataBuffer)) !=
	0)
	return (MPS_DIAG_FAILURE);
	}
	break;

	case MPS_FW_DIAG_TYPE_RELEASE:
	if (length < sizeof(diag_release)) {
	*return_code =
	MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPS_DIAG_FAILURE;
	break;
	}
	if (copyin(diag_action, &diag_release,
	sizeof(diag_release)) != 0)
	return (MPS_DIAG_FAILURE);
	status = mps_diag_release(sc, &diag_release,
	return_code);
	break;

	default:
	*return_code = MPS_FW_DIAG_ERROR_INVALID_PARAMETER;
	status = MPS_DIAG_FAILURE;
	break;
	}

	if ((status == MPS_DIAG_FAILURE) &&
	(original_return_code == MPS_FW_DIAG_NEW) &&
	(*return_code != MPS_FW_DIAG_ERROR_SUCCESS))
	status = MPS_DIAG_SUCCESS;

	return (status);
	}

	static int
	mps_user_diag_action(struct mps_softc sc, mps_diag_action_t data)
	{
	int status;

	/*
	* Only allow one diag action at one time.
	*/
	if (sc->mps_flags & MPS_FLAGS_BUSY) {
	mps_dprint(sc, MPS_USER, "%s: Only one FW diag command "
	"allowed at a single time.", __func__);
	return (EBUSY);
	}
	sc->mps_flags \|= MPS_FLAGS_BUSY;

	/*
	* Send diag action request
	*/
	if (data->Action == MPS_FW_DIAG_TYPE_REGISTER \|\|
	data->Action == MPS_FW_DIAG_TYPE_UNREGISTER \|\|
	data->Action == MPS_FW_DIAG_TYPE_QUERY \|\|
	data->Action == MPS_FW_DIAG_TYPE_READ_BUFFER \|\|
	data->Action == MPS_FW_DIAG_TYPE_RELEASE) {
	status = mps_do_diag_action(sc, data->Action,
	PTRIN(data->PtrDiagAction), data->Length,
	&data->ReturnCode);
	} else
	status = EINVAL;

	sc->mps_flags &= ~MPS_FLAGS_BUSY;
	return (status);
	}

	/*
	* Copy the event recording mask and the event queue size out. For
	* clarification, the event recording mask (events_to_record) is not the same
	* thing as the event mask (event_mask). events_to_record has a bit set for
	* every event type that is to be recorded by the driver, and event_mask has a
	* bit cleared for every event that is allowed into the driver from the IOC.
	* They really have nothing to do with each other.
	*/
	static void
	mps_user_event_query(struct mps_softc sc, mps_event_query_t data)
	{
	uint8_t i;

	mps_lock(sc);
	data->Entries = MPS_EVENT_QUEUE_SIZE;

	for (i = 0; i < 4; i++) {
	data->Types[i] = sc->events_to_record[i];
	}
	mps_unlock(sc);
	}

	/*
	* Set the driver's event mask according to what's been given. See
	* mps_user_event_query for explanation of the event recording mask and the IOC
	* event mask. It's the app's responsibility to enable event logging by setting
	* the bits in events_to_record. Initially, no events will be logged.
	*/
	static void
	mps_user_event_enable(struct mps_softc sc, mps_event_enable_t data)
	{
	uint8_t i;

	mps_lock(sc);
	for (i = 0; i < 4; i++) {
	sc->events_to_record[i] = data->Types[i];
	}
	mps_unlock(sc);
	}

	/*
	* Copy out the events that have been recorded, up to the max events allowed.
	*/
	static int
	mps_user_event_report(struct mps_softc sc, mps_event_report_t data)
	{
	int status = 0;
	uint32_t size;

	mps_lock(sc);
	size = data->Size;
	if ((size >= sizeof(sc->recorded_events)) && (status == 0)) {
	mps_unlock(sc);
	if (copyout((void *)sc->recorded_events,
	PTRIN(data->PtrEvents), size) != 0)
	status = EFAULT;
	mps_lock(sc);
	} else {
	/*
	* data->Size value is not large enough to copy event data.
	*/
	status = EFAULT;
	}

	/*
	* Change size value to match the number of bytes that were copied.
	*/
	if (status == 0)
	data->Size = sizeof(sc->recorded_events);
	mps_unlock(sc);

	return (status);
	}

	/*
	* Record events into the driver from the IOC if they are not masked.
	*/
	void
	mpssas_record_event(struct mps_softc *sc,
	MPI2_EVENT_NOTIFICATION_REPLY *event_reply)
	{
	uint32_t event;
	int i, j;
	uint16_t event_data_len;
	boolean_t sendAEN = FALSE;

	event = event_reply->Event;

	/*
	* Generate a system event to let anyone who cares know that a
	* LOG_ENTRY_ADDED event has occurred. This is sent no matter what the
	* event mask is set to.
	*/
	if (event == MPI2_EVENT_LOG_ENTRY_ADDED) {
	sendAEN = TRUE;
	}

	/*
	* Record the event only if its corresponding bit is set in
	* events_to_record. event_index is the index into recorded_events and
	* event_number is the overall number of an event being recorded since
	* start-of-day. event_index will roll over; event_number will never
	* roll over.
	*/
	i = (uint8_t)(event / 32);
	j = (uint8_t)(event % 32);
	if ((i < 4) && ((1 << j) & sc->events_to_record[i])) {
	i = sc->event_index;
	sc->recorded_events[i].Type = event;
	sc->recorded_events[i].Number = ++sc->event_number;
	bzero(sc->recorded_events[i].Data, MPS_MAX_EVENT_DATA_LENGTH *
	4);
	event_data_len = event_reply->EventDataLength;

	if (event_data_len > 0) {
	/*
	* Limit data to size in m_event entry
	*/
	if (event_data_len > MPS_MAX_EVENT_DATA_LENGTH) {
	event_data_len = MPS_MAX_EVENT_DATA_LENGTH;
	}
	for (j = 0; j < event_data_len; j++) {
	sc->recorded_events[i].Data[j] =
	event_reply->EventData[j];
	}

	/*
	* check for index wrap-around
	*/
	if (++i == MPS_EVENT_QUEUE_SIZE) {
	i = 0;
	}
	sc->event_index = (uint8_t)i;

	/*
	* Set flag to send the event.
	*/
	sendAEN = TRUE;
	}
	}

	/*
	* Generate a system event if flag is set to let anyone who cares know
	* that an event has occurred.
	*/
	if (sendAEN) {
	//SLM-how to send a system event (see kqueue, kevent)
	// (void) ddi_log_sysevent(mpt->m_dip, DDI_VENDOR_LSI, "MPT_SAS",
	// "SAS", NULL, NULL, DDI_NOSLEEP);
	}
	}

	static int
	mps_user_reg_access(struct mps_softc sc, mps_reg_access_t data)
	{
	int status = 0;

	switch (data->Command) {
	/*
	* IO access is not supported.
	*/
	case REG_IO_READ:
	case REG_IO_WRITE:
	mps_dprint(sc, MPS_USER, "IO access is not supported. "
	"Use memory access.");
	status = EINVAL;
	break;

	case REG_MEM_READ:
	data->RegData = mps_regread(sc, data->RegOffset);
	break;

	case REG_MEM_WRITE:
	mps_regwrite(sc, data->RegOffset, data->RegData);
	break;

	default:
	status = EINVAL;
	break;
	}

	return (status);
	}

	static int
	mps_user_btdh(struct mps_softc sc, mps_btdh_mapping_t data)
	{
	uint8_t bt2dh = FALSE;
	uint8_t dh2bt = FALSE;
	uint16_t dev_handle, bus, target;

	bus = data->Bus;
	target = data->TargetID;
	dev_handle = data->DevHandle;

	/*
	* When DevHandle is 0xFFFF and Bus/Target are not 0xFFFF, use Bus/
	* Target to get DevHandle. When Bus/Target are 0xFFFF and DevHandle is
	* not 0xFFFF, use DevHandle to get Bus/Target. Anything else is
	* invalid.
	*/
	if ((bus == 0xFFFF) && (target == 0xFFFF) && (dev_handle != 0xFFFF))
	dh2bt = TRUE;
	if ((dev_handle == 0xFFFF) && (bus != 0xFFFF) && (target != 0xFFFF))
	bt2dh = TRUE;
	if (!dh2bt && !bt2dh)
	return (EINVAL);

	/*
	* Only handle bus of 0. Make sure target is within range.
	*/
	if (bt2dh) {
	if (bus != 0)
	return (EINVAL);

	if (target > sc->max_devices) {
	mps_dprint(sc, MPS_FAULT, "Target ID is out of range "
	"for Bus/Target to DevHandle mapping.");
	return (EINVAL);
	}
	dev_handle = sc->mapping_table[target].dev_handle;
	if (dev_handle)
	data->DevHandle = dev_handle;
	} else {
	bus = 0;
	target = mps_mapping_get_tid_from_handle(sc, dev_handle);
	data->Bus = bus;
	data->TargetID = target;
	}

	return (0);
	}

	static int
	mps_ioctl(struct cdev dev, u_long cmd, void arg, int flag,
	struct thread *td)
	{
	struct mps_softc *sc;
	struct mps_cfg_page_req *page_req;
	struct mps_ext_cfg_page_req *ext_page_req;
	void *mps_page;
	int error, msleep_ret;

	mps_page = NULL;
	sc = dev->si_drv1;
	page_req = (void *)arg;
	ext_page_req = (void *)arg;

	switch (cmd) {
	case MPSIO_READ_CFG_HEADER:
	mps_lock(sc);
	error = mps_user_read_cfg_header(sc, page_req);
	mps_unlock(sc);
	break;
	case MPSIO_READ_CFG_PAGE:
	mps_page = malloc(page_req->len, M_MPSUSER, M_WAITOK \| M_ZERO);
	error = copyin(page_req->buf, mps_page,
	sizeof(MPI2_CONFIG_PAGE_HEADER));
	if (error)
	break;
	mps_lock(sc);
	error = mps_user_read_cfg_page(sc, page_req, mps_page);
	mps_unlock(sc);
	if (error)
	break;
	error = copyout(mps_page, page_req->buf, page_req->len);
	break;
	case MPSIO_READ_EXT_CFG_HEADER:
	mps_lock(sc);
	error = mps_user_read_extcfg_header(sc, ext_page_req);
	mps_unlock(sc);
	break;
	case MPSIO_READ_EXT_CFG_PAGE:
	mps_page = malloc(ext_page_req->len, M_MPSUSER, M_WAITOK\|M_ZERO);
	error = copyin(ext_page_req->buf, mps_page,
	sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER));
	if (error)
	break;
	mps_lock(sc);
	error = mps_user_read_extcfg_page(sc, ext_page_req, mps_page);
	mps_unlock(sc);
	if (error)
	break;
	error = copyout(mps_page, ext_page_req->buf, ext_page_req->len);
	break;
	case MPSIO_WRITE_CFG_PAGE:
	mps_page = malloc(page_req->len, M_MPSUSER, M_WAITOK\|M_ZERO);
	error = copyin(page_req->buf, mps_page, page_req->len);
	if (error)
	break;
	mps_lock(sc);
	error = mps_user_write_cfg_page(sc, page_req, mps_page);
	mps_unlock(sc);
	break;
	case MPSIO_MPS_COMMAND:
	error = mps_user_command(sc, (struct mps_usr_command *)arg);
	break;
	case MPTIOCTL_PASS_THRU:
	/*
	* The user has requested to pass through a command to be
	* executed by the MPT firmware. Call our routine which does
	* this. Only allow one passthru IOCTL at one time.
	*/
	error = mps_user_pass_thru(sc, (mps_pass_thru_t *)arg);
	break;
	case MPTIOCTL_GET_ADAPTER_DATA:
	/*
	* The user has requested to read adapter data. Call our
	* routine which does this.
	*/
	error = 0;
	mps_user_get_adapter_data(sc, (mps_adapter_data_t *)arg);
	break;
	case MPTIOCTL_GET_PCI_INFO:
	/*
	* The user has requested to read pci info. Call
	* our routine which does this.
	*/
	mps_lock(sc);
	error = 0;
	mps_user_read_pci_info(sc, (mps_pci_info_t *)arg);
	mps_unlock(sc);
	break;
	case MPTIOCTL_RESET_ADAPTER:
	mps_lock(sc);
	sc->port_enable_complete = 0;
	uint32_t reinit_start = time_uptime;
	error = mps_reinit(sc);
	/* Sleep for 300 second. */
	msleep_ret = msleep(&sc->port_enable_complete, &sc->mps_mtx, PRIBIO,
	"mps_porten", 300 * hz);
	mps_unlock(sc);
	if (msleep_ret)
	printf("Port Enable did not complete after Diag "
	"Reset msleep error %d.\n", msleep_ret);
	else
	mps_dprint(sc, MPS_USER,
	"Hard Reset with Port Enable completed in %d seconds.\n",
	(uint32_t) (time_uptime - reinit_start));
	break;
	case MPTIOCTL_DIAG_ACTION:
	/*
	* The user has done a diag buffer action. Call our routine
	* which does this. Only allow one diag action at one time.
	*/
	mps_lock(sc);
	error = mps_user_diag_action(sc, (mps_diag_action_t *)arg);
	mps_unlock(sc);
	break;
	case MPTIOCTL_EVENT_QUERY:
	/*
	* The user has done an event query. Call our routine which does
	* this.
	*/
	error = 0;
	mps_user_event_query(sc, (mps_event_query_t *)arg);
	break;
	case MPTIOCTL_EVENT_ENABLE:
	/*
	* The user has done an event enable. Call our routine which
	* does this.
	*/
	error = 0;
	mps_user_event_enable(sc, (mps_event_enable_t *)arg);
	break;
	case MPTIOCTL_EVENT_REPORT:
	/*
	* The user has done an event report. Call our routine which
	* does this.
	*/
	error = mps_user_event_report(sc, (mps_event_report_t *)arg);
	break;
	case MPTIOCTL_REG_ACCESS:
	/*
	* The user has requested register access. Call our routine
	* which does this.
	*/
	mps_lock(sc);
	error = mps_user_reg_access(sc, (mps_reg_access_t *)arg);
	mps_unlock(sc);
	break;
	case MPTIOCTL_BTDH_MAPPING:
	/*
	* The user has requested to translate a bus/target to a
	* DevHandle or a DevHandle to a bus/target. Call our routine
	* which does this.
	*/
	error = mps_user_btdh(sc, (mps_btdh_mapping_t *)arg);
	break;
	default:
	error = ENOIOCTL;
	break;
	}

	if (mps_page != NULL)
	free(mps_page, M_MPSUSER);

	return (error);
	}

	#ifdef COMPAT_FREEBSD32

	struct mps_cfg_page_req32 {
	MPI2_CONFIG_PAGE_HEADER header;
	uint32_t page_address;
	uint32_t buf;
	int len;
	uint16_t ioc_status;
	};

	struct mps_ext_cfg_page_req32 {
	MPI2_CONFIG_EXTENDED_PAGE_HEADER header;
	uint32_t page_address;
	uint32_t buf;
	int len;
	uint16_t ioc_status;
	};

	struct mps_raid_action32 {
	uint8_t action;
	uint8_t volume_bus;
	uint8_t volume_id;
	uint8_t phys_disk_num;
	uint32_t action_data_word;
	uint32_t buf;
	int len;
	uint32_t volume_status;
	uint32_t action_data[4];
	uint16_t action_status;
	uint16_t ioc_status;
	uint8_t write;
	};

	struct mps_usr_command32 {
	uint32_t req;
	uint32_t req_len;
	uint32_t rpl;
	uint32_t rpl_len;
	uint32_t buf;
	int len;
	uint32_t flags;
	};

	#define MPSIO_READ_CFG_HEADER32 _IOWR('M', 200, struct mps_cfg_page_req32)
	#define MPSIO_READ_CFG_PAGE32 _IOWR('M', 201, struct mps_cfg_page_req32)
	#define MPSIO_READ_EXT_CFG_HEADER32 _IOWR('M', 202, struct mps_ext_cfg_page_req32)
	#define MPSIO_READ_EXT_CFG_PAGE32 _IOWR('M', 203, struct mps_ext_cfg_page_req32)
	#define MPSIO_WRITE_CFG_PAGE32 _IOWR('M', 204, struct mps_cfg_page_req32)
	#define MPSIO_RAID_ACTION32 _IOWR('M', 205, struct mps_raid_action32)
	#define MPSIO_MPS_COMMAND32 _IOWR('M', 210, struct mps_usr_command32)

	static int
	mps_ioctl32(struct cdev dev, u_long cmd32, void _arg, int flag,
	struct thread *td)
	{
	struct mps_cfg_page_req32 *page32 = _arg;
	struct mps_ext_cfg_page_req32 *ext32 = _arg;
	struct mps_raid_action32 *raid32 = _arg;
	struct mps_usr_command32 *user32 = _arg;
	union {
	struct mps_cfg_page_req page;
	struct mps_ext_cfg_page_req ext;
	struct mps_raid_action raid;
	struct mps_usr_command user;
	} arg;
	u_long cmd;
	int error;

	switch (cmd32) {
	case MPSIO_READ_CFG_HEADER32:
	case MPSIO_READ_CFG_PAGE32:
	case MPSIO_WRITE_CFG_PAGE32:
	if (cmd32 == MPSIO_READ_CFG_HEADER32)
	cmd = MPSIO_READ_CFG_HEADER;
	else if (cmd32 == MPSIO_READ_CFG_PAGE32)
	cmd = MPSIO_READ_CFG_PAGE;
	else
	cmd = MPSIO_WRITE_CFG_PAGE;
	CP(*page32, arg.page, header);
	CP(*page32, arg.page, page_address);
	PTRIN_CP(*page32, arg.page, buf);
	CP(*page32, arg.page, len);
	CP(*page32, arg.page, ioc_status);
	break;

	case MPSIO_READ_EXT_CFG_HEADER32:
	case MPSIO_READ_EXT_CFG_PAGE32:
	if (cmd32 == MPSIO_READ_EXT_CFG_HEADER32)
	cmd = MPSIO_READ_EXT_CFG_HEADER;
	else
	cmd = MPSIO_READ_EXT_CFG_PAGE;
	CP(*ext32, arg.ext, header);
	CP(*ext32, arg.ext, page_address);
	PTRIN_CP(*ext32, arg.ext, buf);
	CP(*ext32, arg.ext, len);
	CP(*ext32, arg.ext, ioc_status);
	break;

	case MPSIO_RAID_ACTION32:
	cmd = MPSIO_RAID_ACTION;
	CP(*raid32, arg.raid, action);
	CP(*raid32, arg.raid, volume_bus);
	CP(*raid32, arg.raid, volume_id);
	CP(*raid32, arg.raid, phys_disk_num);
	CP(*raid32, arg.raid, action_data_word);
	PTRIN_CP(*raid32, arg.raid, buf);
	CP(*raid32, arg.raid, len);
	CP(*raid32, arg.raid, volume_status);
	bcopy(raid32->action_data, arg.raid.action_data,
	sizeof arg.raid.action_data);
	CP(*raid32, arg.raid, ioc_status);
	CP(*raid32, arg.raid, write);
	break;

	case MPSIO_MPS_COMMAND32:
	cmd = MPSIO_MPS_COMMAND;
	PTRIN_CP(*user32, arg.user, req);
	CP(*user32, arg.user, req_len);
	PTRIN_CP(*user32, arg.user, rpl);
	CP(*user32, arg.user, rpl_len);
	PTRIN_CP(*user32, arg.user, buf);
	CP(*user32, arg.user, len);
	CP(*user32, arg.user, flags);
	break;
	default:
	return (ENOIOCTL);
	}

	error = mps_ioctl(dev, cmd, &arg, flag, td);
	if (error == 0 && (cmd32 & IOC_OUT) != 0) {
	switch (cmd32) {
	case MPSIO_READ_CFG_HEADER32:
	case MPSIO_READ_CFG_PAGE32:
	case MPSIO_WRITE_CFG_PAGE32:
	CP(arg.page, *page32, header);
	CP(arg.page, *page32, page_address);
	PTROUT_CP(arg.page, *page32, buf);
	CP(arg.page, *page32, len);
	CP(arg.page, *page32, ioc_status);
	break;

	case MPSIO_READ_EXT_CFG_HEADER32:
	case MPSIO_READ_EXT_CFG_PAGE32:
	CP(arg.ext, *ext32, header);
	CP(arg.ext, *ext32, page_address);
	PTROUT_CP(arg.ext, *ext32, buf);
	CP(arg.ext, *ext32, len);
	CP(arg.ext, *ext32, ioc_status);
	break;

	case MPSIO_RAID_ACTION32:
	CP(arg.raid, *raid32, action);
	CP(arg.raid, *raid32, volume_bus);
	CP(arg.raid, *raid32, volume_id);
	CP(arg.raid, *raid32, phys_disk_num);
	CP(arg.raid, *raid32, action_data_word);
	PTROUT_CP(arg.raid, *raid32, buf);
	CP(arg.raid, *raid32, len);
	CP(arg.raid, *raid32, volume_status);
	bcopy(arg.raid.action_data, raid32->action_data,
	sizeof arg.raid.action_data);
	CP(arg.raid, *raid32, ioc_status);
	CP(arg.raid, *raid32, write);
	break;

	case MPSIO_MPS_COMMAND32:
	PTROUT_CP(arg.user, *user32, req);
	CP(arg.user, *user32, req_len);
	PTROUT_CP(arg.user, *user32, rpl);
	CP(arg.user, *user32, rpl_len);
	PTROUT_CP(arg.user, *user32, buf);
	CP(arg.user, *user32, len);
	CP(arg.user, *user32, flags);
	break;
	}
	}

	return (error);
	}
	#endif /* COMPAT_FREEBSD32 */

	static int
	mps_ioctl_devsw(struct cdev *dev, u_long com, caddr_t arg, int flag,
	struct thread *td)
	{
	#ifdef COMPAT_FREEBSD32
	if (SV_CURPROC_FLAG(SV_ILP32))
	return (mps_ioctl32(dev, com, arg, flag, td));
	#endif
	return (mps_ioctl(dev, com, arg, flag, td));
	}
	diff --git a/sys/dev/ntb/ntb_hw/ntb_hw_intel.c b/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
	index 9182461fe9a5..2dfe49cc1fd9 100644
	--- a/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
	+++ b/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
	@@ -1,3215 +1,3218 @@
	/*-
	* Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org>
	* Copyright (C) 2013 Intel Corporation
	* Copyright (C) 2015 EMC Corporation
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/*
	* The Non-Transparent Bridge (NTB) is a device that allows you to connect
	* two or more systems using a PCI-e links, providing remote memory access.
	*
	* This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs.
	*
	* NOTE: Much of the code in this module is shared with Linux. Any patches may
	* be picked up and redistributed in Linux with a dual GPL/BSD license.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/kernel.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/endian.h>
	#include <sys/interrupt.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/pciio.h>
	+#include <sys/taskqueue.h>
	+#include <sys/tree.h>
	#include <sys/queue.h>
	#include <sys/rman.h>
	#include <sys/sbuf.h>
	#include <sys/sysctl.h>
	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <machine/bus.h>
	#include <machine/intr_machdep.h>
	#include <machine/resource.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	+#include <dev/iommu/iommu.h>

	#include "ntb_hw_intel.h"
	#include "../ntb.h"

	#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)

	#define NTB_HB_TIMEOUT 1 /* second */
	#define ATOM_LINK_RECOVERY_TIME 500 /* ms */
	#define BAR_HIGH_MASK (~((1ull << 12) - 1))

	#define NTB_MSIX_VER_GUARD 0xaabbccdd
	#define NTB_MSIX_RECEIVED 0xe0f0e0f0

	/*
	* PCI constants could be somewhere more generic, but aren't defined/used in
	* pci.c.
	*/
	#define PCI_MSIX_ENTRY_SIZE 16
	#define PCI_MSIX_ENTRY_LOWER_ADDR 0
	#define PCI_MSIX_ENTRY_UPPER_ADDR 4
	#define PCI_MSIX_ENTRY_DATA 8

	enum ntb_device_type {
	NTB_XEON,
	NTB_ATOM
	};

	/* ntb_conn_type are hardware numbers, cannot change. */
	enum ntb_conn_type {
	NTB_CONN_TRANSPARENT = 0,
	NTB_CONN_B2B = 1,
	NTB_CONN_RP = 2,
	};

	enum ntb_b2b_direction {
	NTB_DEV_USD = 0,
	NTB_DEV_DSD = 1,
	};

	enum ntb_bar {
	NTB_CONFIG_BAR = 0,
	NTB_B2B_BAR_1,
	NTB_B2B_BAR_2,
	NTB_B2B_BAR_3,
	NTB_MAX_BARS
	};

	enum {
	NTB_MSIX_GUARD = 0,
	NTB_MSIX_DATA0,
	NTB_MSIX_DATA1,
	NTB_MSIX_DATA2,
	NTB_MSIX_OFS0,
	NTB_MSIX_OFS1,
	NTB_MSIX_OFS2,
	NTB_MSIX_DONE,
	NTB_MAX_MSIX_SPAD
	};

	/* Device features and workarounds */
	#define HAS_FEATURE(ntb, feature) \
	(((ntb)->features & (feature)) != 0)

	struct ntb_hw_info {
	uint32_t device_id;
	const char *desc;
	enum ntb_device_type type;
	uint32_t features;
	};

	struct ntb_pci_bar_info {
	bus_space_tag_t pci_bus_tag;
	bus_space_handle_t pci_bus_handle;
	int pci_resource_id;
	struct resource *pci_resource;
	vm_paddr_t pbase;
	caddr_t vbase;
	vm_size_t size;
	vm_memattr_t map_mode;

	/* Configuration register offsets */
	uint32_t psz_off;
	uint32_t ssz_off;
	uint32_t pbarxlat_off;
	};

	struct ntb_int_info {
	struct resource *res;
	int rid;
	void *tag;
	};

	struct ntb_vec {
	struct ntb_softc *ntb;
	uint32_t num;
	unsigned masked;
	};

	struct ntb_reg {
	uint32_t ntb_ctl;
	uint32_t lnk_sta;
	uint8_t db_size;
	unsigned mw_bar[NTB_MAX_BARS];
	};

	struct ntb_alt_reg {
	uint32_t db_bell;
	uint32_t db_mask;
	uint32_t spad;
	};

	struct ntb_xlat_reg {
	uint32_t bar0_base;
	uint32_t bar2_base;
	uint32_t bar4_base;
	uint32_t bar5_base;

	uint32_t bar2_xlat;
	uint32_t bar4_xlat;
	uint32_t bar5_xlat;

	uint32_t bar2_limit;
	uint32_t bar4_limit;
	uint32_t bar5_limit;
	};

	struct ntb_b2b_addr {
	uint64_t bar0_addr;
	uint64_t bar2_addr64;
	uint64_t bar4_addr64;
	uint64_t bar4_addr32;
	uint64_t bar5_addr32;
	};

	struct ntb_msix_data {
	uint32_t nmd_ofs;
	uint32_t nmd_data;
	};

	struct ntb_softc {
	/* ntb.c context. Do not move! Must go first! */
	void *ntb_store;

	device_t device;
	enum ntb_device_type type;
	uint32_t features;

	struct ntb_pci_bar_info bar_info[NTB_MAX_BARS];
	struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS];
	uint32_t allocated_interrupts;

	struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS];
	struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS];
	bool peer_msix_good;
	bool peer_msix_done;
	struct ntb_pci_bar_info *peer_lapic_bar;
	struct callout peer_msix_work;

	bus_dma_tag_t bar0_dma_tag;
	bus_dmamap_t bar0_dma_map;

	struct callout heartbeat_timer;
	struct callout lr_timer;

	struct ntb_vec *msix_vec;

	uint32_t ppd;
	enum ntb_conn_type conn_type;
	enum ntb_b2b_direction dev_type;

	/* Offset of peer bar0 in B2B BAR */
	uint64_t b2b_off;
	/* Memory window used to access peer bar0 */
	#define B2B_MW_DISABLED UINT8_MAX
	uint8_t b2b_mw_idx;
	uint32_t msix_xlat;
	uint8_t msix_mw_idx;

	uint8_t mw_count;
	uint8_t spad_count;
	uint8_t db_count;
	uint8_t db_vec_count;
	uint8_t db_vec_shift;

	/* Protects local db_mask. */
	#define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock)
	#define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock)
	#define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f))
	struct mtx db_mask_lock;

	volatile uint32_t ntb_ctl;
	volatile uint32_t lnk_sta;

	uint64_t db_valid_mask;
	uint64_t db_link_mask;
	uint64_t db_mask;
	uint64_t fake_db; /* NTB_SB01BASE_LOCKUP*/
	uint64_t force_db; /* NTB_SB01BASE_LOCKUP*/

	int last_ts; /* ticks @ last irq */

	const struct ntb_reg *reg;
	const struct ntb_alt_reg *self_reg;
	const struct ntb_alt_reg *peer_reg;
	const struct ntb_xlat_reg *xlat_reg;
	};

	#ifdef __i386__
	static __inline uint64_t
	bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
	bus_size_t offset)
	{

	return (bus_space_read_4(tag, handle, offset) \|
	((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32);
	}

	static __inline void
	bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
	bus_size_t offset, uint64_t val)
	{

	bus_space_write_4(tag, handle, offset, val);
	bus_space_write_4(tag, handle, offset + 4, val >> 32);
	}
	#endif

	#define intel_ntb_bar_read(SIZE, bar, offset) \
	bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
	ntb->bar_info[(bar)].pci_bus_handle, (offset))
	#define intel_ntb_bar_write(SIZE, bar, offset, val) \
	bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
	ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
	#define intel_ntb_reg_read(SIZE, offset) \
	intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
	#define intel_ntb_reg_write(SIZE, offset, val) \
	intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
	#define intel_ntb_mw_read(SIZE, offset) \
	intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
	offset)
	#define intel_ntb_mw_write(SIZE, offset, val) \
	intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
	offset, val)

	static int intel_ntb_probe(device_t device);
	static int intel_ntb_attach(device_t device);
	static int intel_ntb_detach(device_t device);
	static uint64_t intel_ntb_db_valid_mask(device_t dev);
	static void intel_ntb_spad_clear(device_t dev);
	static uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector);
	static bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed,
	enum ntb_width *width);
	static int intel_ntb_link_enable(device_t dev, enum ntb_speed speed,
	enum ntb_width width);
	static int intel_ntb_link_disable(device_t dev);
	static int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val);
	static int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val);

	static unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx);
	static inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
	static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar);
	static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar,
	uint32_t base, uint32_t xlat, uint32_t *lmt);
	static int intel_ntb_map_pci_bars(struct ntb_softc *ntb);
	static int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx,
	vm_memattr_t);
	static void print_map_success(struct ntb_softc , struct ntb_pci_bar_info ,
	const char *);
	static int map_mmr_bar(struct ntb_softc ntb, struct ntb_pci_bar_info bar);
	static int map_memory_window_bar(struct ntb_softc *ntb,
	struct ntb_pci_bar_info *bar);
	static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);
	static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
	static int intel_ntb_init_isr(struct ntb_softc *ntb);
	static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
	static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
	static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);
	static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
	static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec);
	static void ndev_vec_isr(void *arg);
	static void ndev_irq_isr(void *arg);
	static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
	static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t);
	static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t);
	static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
	static void intel_ntb_free_msix_vec(struct ntb_softc *ntb);
	static void intel_ntb_get_msix_info(struct ntb_softc *ntb);
	static void intel_ntb_exchange_msix(void *);
	static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
	static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
	static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
	static int intel_ntb_detect_atom(struct ntb_softc *ntb);
	static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
	static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
	static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
	static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
	static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
	enum ntb_bar regbar);
	static void xeon_set_sbar_base_and_limit(struct ntb_softc *,
	uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
	static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
	enum ntb_bar idx);
	static int xeon_setup_b2b_mw(struct ntb_softc *,
	const struct ntb_b2b_addr addr, const struct ntb_b2b_addr peer_addr);
	static inline bool link_is_up(struct ntb_softc *ntb);
	static inline bool _xeon_link_is_up(struct ntb_softc *ntb);
	static inline bool atom_link_is_err(struct ntb_softc *ntb);
	static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *);
	static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *);
	static void atom_link_hb(void *arg);
	static void recover_atom_link(void *arg);
	static bool intel_ntb_poll_link(struct ntb_softc *ntb);
	static void save_bar_parameters(struct ntb_pci_bar_info *bar);
	static void intel_ntb_sysctl_init(struct ntb_softc *);
	static int sysctl_handle_features(SYSCTL_HANDLER_ARGS);
	static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS);
	static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS);
	static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS);
	static int sysctl_handle_register(SYSCTL_HANDLER_ARGS);

	static unsigned g_ntb_hw_debug_level;
	SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN,
	&g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose");
	#define intel_ntb_printf(lvl, ...) do { \
	if ((lvl) <= g_ntb_hw_debug_level) { \
	device_printf(ntb->device, __VA_ARGS__); \
	} \
	} while (0)

	#define _NTB_PAT_UC 0
	#define _NTB_PAT_WC 1
	#define _NTB_PAT_WT 4
	#define _NTB_PAT_WP 5
	#define _NTB_PAT_WB 6
	#define _NTB_PAT_UCM 7
	static unsigned g_ntb_mw_pat = _NTB_PAT_UC;
	SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN,
	&g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): "
	"UC: " __XSTRING(_NTB_PAT_UC) ", "
	"WC: " __XSTRING(_NTB_PAT_WC) ", "
	"WT: " __XSTRING(_NTB_PAT_WT) ", "
	"WP: " __XSTRING(_NTB_PAT_WP) ", "
	"WB: " __XSTRING(_NTB_PAT_WB) ", "
	"UC-: " __XSTRING(_NTB_PAT_UCM));

	static inline vm_memattr_t
	intel_ntb_pat_flags(void)
	{

	switch (g_ntb_mw_pat) {
	case _NTB_PAT_WC:
	return (VM_MEMATTR_WRITE_COMBINING);
	case _NTB_PAT_WT:
	return (VM_MEMATTR_WRITE_THROUGH);
	case _NTB_PAT_WP:
	return (VM_MEMATTR_WRITE_PROTECTED);
	case _NTB_PAT_WB:
	return (VM_MEMATTR_WRITE_BACK);
	case _NTB_PAT_UCM:
	return (VM_MEMATTR_WEAK_UNCACHEABLE);
	case _NTB_PAT_UC:
	/* FALLTHROUGH */
	default:
	return (VM_MEMATTR_UNCACHEABLE);
	}
	}

	/*
	* Well, this obviously doesn't belong here, but it doesn't seem to exist
	* anywhere better yet.
	*/
	static inline const char *
	intel_ntb_vm_memattr_to_str(vm_memattr_t pat)
	{

	switch (pat) {
	case VM_MEMATTR_WRITE_COMBINING:
	return ("WRITE_COMBINING");
	case VM_MEMATTR_WRITE_THROUGH:
	return ("WRITE_THROUGH");
	case VM_MEMATTR_WRITE_PROTECTED:
	return ("WRITE_PROTECTED");
	case VM_MEMATTR_WRITE_BACK:
	return ("WRITE_BACK");
	case VM_MEMATTR_WEAK_UNCACHEABLE:
	return ("UNCACHED");
	case VM_MEMATTR_UNCACHEABLE:
	return ("UNCACHEABLE");
	default:
	return ("UNKNOWN");
	}
	}

	static int g_ntb_msix_idx = 1;
	SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx,
	0, "Use this memory window to access the peer MSIX message complex on "
	"certain Xeon-based NTB systems, as a workaround for a hardware errata. "
	"Like b2b_mw_idx, negative values index from the last available memory "
	"window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)");

	static int g_ntb_mw_idx = -1;
	SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx,
	0, "Use this memory window to access the peer NTB registers. A "
	"non-negative value starts from the first MW index; a negative value "
	"starts from the last MW index. The default is -1, i.e., the last "
	"available memory window. Both sides of the NTB MUST set the same "
	"value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)");

	/* Hardware owns the low 16 bits of features. */
	#define NTB_BAR_SIZE_4K (1 << 0)
	#define NTB_SDOORBELL_LOCKUP (1 << 1)
	#define NTB_SB01BASE_LOCKUP (1 << 2)
	#define NTB_B2BDOORBELL_BIT14 (1 << 3)
	/* Software/configuration owns the top 16 bits. */
	#define NTB_SPLIT_BAR (1ull << 16)

	#define NTB_FEATURES_STR \
	"\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
	"\02SDOORBELL_LOCKUP\01BAR_SIZE_4K"

	static struct ntb_hw_info pci_ids[] = {
	/* XXX: PS/SS IDs left out until they are supported. */
	{ 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B",
	NTB_ATOM, 0 },

	{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
	NTB_XEON, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 },
	{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
	NTB_XEON, NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 },
	{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
	NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|
	NTB_SB01BASE_LOCKUP \| NTB_BAR_SIZE_4K },
	{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
	NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|
	NTB_SB01BASE_LOCKUP },
	{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
	NTB_SDOORBELL_LOCKUP \| NTB_B2BDOORBELL_BIT14 \|
	NTB_SB01BASE_LOCKUP },
	};

	static const struct ntb_reg atom_reg = {
	.ntb_ctl = ATOM_NTBCNTL_OFFSET,
	.lnk_sta = ATOM_LINK_STATUS_OFFSET,
	.db_size = sizeof(uint64_t),
	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
	};

	static const struct ntb_alt_reg atom_pri_reg = {
	.db_bell = ATOM_PDOORBELL_OFFSET,
	.db_mask = ATOM_PDBMSK_OFFSET,
	.spad = ATOM_SPAD_OFFSET,
	};

	static const struct ntb_alt_reg atom_b2b_reg = {
	.db_bell = ATOM_B2B_DOORBELL_OFFSET,
	.spad = ATOM_B2B_SPAD_OFFSET,
	};

	static const struct ntb_xlat_reg atom_sec_xlat = {
	#if 0
	/* "FIXME" says the Linux driver. */
	.bar0_base = ATOM_SBAR0BASE_OFFSET,
	.bar2_base = ATOM_SBAR2BASE_OFFSET,
	.bar4_base = ATOM_SBAR4BASE_OFFSET,

	.bar2_limit = ATOM_SBAR2LMT_OFFSET,
	.bar4_limit = ATOM_SBAR4LMT_OFFSET,
	#endif

	.bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
	.bar4_xlat = ATOM_SBAR4XLAT_OFFSET,
	};

	static const struct ntb_reg xeon_reg = {
	.ntb_ctl = XEON_NTBCNTL_OFFSET,
	.lnk_sta = XEON_LINK_STATUS_OFFSET,
	.db_size = sizeof(uint16_t),
	.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 },
	};

	static const struct ntb_alt_reg xeon_pri_reg = {
	.db_bell = XEON_PDOORBELL_OFFSET,
	.db_mask = XEON_PDBMSK_OFFSET,
	.spad = XEON_SPAD_OFFSET,
	};

	static const struct ntb_alt_reg xeon_b2b_reg = {
	.db_bell = XEON_B2B_DOORBELL_OFFSET,
	.spad = XEON_B2B_SPAD_OFFSET,
	};

	static const struct ntb_xlat_reg xeon_sec_xlat = {
	.bar0_base = XEON_SBAR0BASE_OFFSET,
	.bar2_base = XEON_SBAR2BASE_OFFSET,
	.bar4_base = XEON_SBAR4BASE_OFFSET,
	.bar5_base = XEON_SBAR5BASE_OFFSET,

	.bar2_limit = XEON_SBAR2LMT_OFFSET,
	.bar4_limit = XEON_SBAR4LMT_OFFSET,
	.bar5_limit = XEON_SBAR5LMT_OFFSET,

	.bar2_xlat = XEON_SBAR2XLAT_OFFSET,
	.bar4_xlat = XEON_SBAR4XLAT_OFFSET,
	.bar5_xlat = XEON_SBAR5XLAT_OFFSET,
	};

	static struct ntb_b2b_addr xeon_b2b_usd_addr = {
	.bar0_addr = XEON_B2B_BAR0_ADDR,
	.bar2_addr64 = XEON_B2B_BAR2_ADDR64,
	.bar4_addr64 = XEON_B2B_BAR4_ADDR64,
	.bar4_addr32 = XEON_B2B_BAR4_ADDR32,
	.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
	};

	static struct ntb_b2b_addr xeon_b2b_dsd_addr = {
	.bar0_addr = XEON_B2B_BAR0_ADDR,
	.bar2_addr64 = XEON_B2B_BAR2_ADDR64,
	.bar4_addr64 = XEON_B2B_BAR4_ADDR64,
	.bar4_addr32 = XEON_B2B_BAR4_ADDR32,
	.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
	};

	SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"B2B MW segment overrides -- MUST be the same on both sides");

	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,
	&xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
	"hardware, use this 64-bit address on the bus between the NTB devices for "
	"the window at BAR2, on the upstream side of the link. MUST be the same "
	"address on both sides.");
	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN,
	&xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4.");
	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN,
	&xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 "
	"(split-BAR mode).");
	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN,
	&xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 "
	"(split-BAR mode).");

	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN,
	&xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
	"hardware, use this 64-bit address on the bus between the NTB devices for "
	"the window at BAR2, on the downstream side of the link. MUST be the same"
	" address on both sides.");
	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN,
	&xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4.");
	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN,
	&xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 "
	"(split-BAR mode).");
	SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN,
	&xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 "
	"(split-BAR mode).");

	/*
	* OS <-> Driver interface structures
	*/
	MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");

	/*
	* OS <-> Driver linkage functions
	*/
	static int
	intel_ntb_probe(device_t device)
	{
	struct ntb_hw_info *p;

	p = intel_ntb_get_device_info(pci_get_devid(device));
	if (p == NULL)
	return (ENXIO);

	device_set_desc(device, p->desc);
	return (0);
	}

	static int
	intel_ntb_attach(device_t device)
	{
	struct ntb_softc *ntb;
	struct ntb_hw_info *p;
	int error;

	ntb = device_get_softc(device);
	p = intel_ntb_get_device_info(pci_get_devid(device));

	ntb->device = device;
	ntb->type = p->type;
	ntb->features = p->features;
	ntb->b2b_mw_idx = B2B_MW_DISABLED;
	ntb->msix_mw_idx = B2B_MW_DISABLED;

	/* Heartbeat timer for NTB_ATOM since there is no link interrupt */
	callout_init(&ntb->heartbeat_timer, 1);
	callout_init(&ntb->lr_timer, 1);
	callout_init(&ntb->peer_msix_work, 1);
	mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);

	if (ntb->type == NTB_ATOM)
	error = intel_ntb_detect_atom(ntb);
	else
	error = intel_ntb_detect_xeon(ntb);
	if (error != 0)
	goto out;

	intel_ntb_detect_max_mw(ntb);

	pci_enable_busmaster(ntb->device);

	error = intel_ntb_map_pci_bars(ntb);
	if (error != 0)
	goto out;
	if (ntb->type == NTB_ATOM)
	error = intel_ntb_atom_init_dev(ntb);
	else
	error = intel_ntb_xeon_init_dev(ntb);
	if (error != 0)
	goto out;

	intel_ntb_spad_clear(device);

	intel_ntb_poll_link(ntb);

	intel_ntb_sysctl_init(ntb);

	/* Attach children to this controller */
	error = ntb_register_device(device);

	out:
	if (error != 0)
	intel_ntb_detach(device);
	return (error);
	}

	static int
	intel_ntb_detach(device_t device)
	{
	struct ntb_softc *ntb;

	ntb = device_get_softc(device);

	/* Detach & delete all children */
	ntb_unregister_device(device);

	if (ntb->self_reg != NULL) {
	DB_MASK_LOCK(ntb);
	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask);
	DB_MASK_UNLOCK(ntb);
	}
	callout_drain(&ntb->heartbeat_timer);
	callout_drain(&ntb->lr_timer);
	callout_drain(&ntb->peer_msix_work);
	pci_disable_busmaster(ntb->device);
	if (ntb->type == NTB_XEON)
	intel_ntb_teardown_xeon(ntb);
	intel_ntb_teardown_interrupts(ntb);

	mtx_destroy(&ntb->db_mask_lock);

	intel_ntb_unmap_pci_bar(ntb);

	return (0);
	}

	/*
	* Driver internal routines
	*/
	static inline enum ntb_bar
	intel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
	{

	KASSERT(mw < ntb->mw_count,
	("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
	KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw"));

	return (ntb->reg->mw_bar[mw]);
	}

	static inline bool
	bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar)
	{
	/* XXX This assertion could be stronger. */
	KASSERT(bar < NTB_MAX_BARS, ("bogus bar"));
	return (bar < NTB_B2B_BAR_2 \|\| !HAS_FEATURE(ntb, NTB_SPLIT_BAR));
	}

	static inline void
	bar_get_xlat_params(struct ntb_softc ntb, enum ntb_bar bar, uint32_t base,
	uint32_t xlat, uint32_t lmt)
	{
	uint32_t basev, lmtv, xlatv;

	switch (bar) {
	case NTB_B2B_BAR_1:
	basev = ntb->xlat_reg->bar2_base;
	lmtv = ntb->xlat_reg->bar2_limit;
	xlatv = ntb->xlat_reg->bar2_xlat;
	break;
	case NTB_B2B_BAR_2:
	basev = ntb->xlat_reg->bar4_base;
	lmtv = ntb->xlat_reg->bar4_limit;
	xlatv = ntb->xlat_reg->bar4_xlat;
	break;
	case NTB_B2B_BAR_3:
	basev = ntb->xlat_reg->bar5_base;
	lmtv = ntb->xlat_reg->bar5_limit;
	xlatv = ntb->xlat_reg->bar5_xlat;
	break;
	default:
	KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS,
	("bad bar"));
	basev = lmtv = xlatv = 0;
	break;
	}

	if (base != NULL)
	*base = basev;
	if (xlat != NULL)
	*xlat = xlatv;
	if (lmt != NULL)
	*lmt = lmtv;
	}

	static int
	intel_ntb_map_pci_bars(struct ntb_softc *ntb)
	{
	struct ntb_pci_bar_info *bar;
	int rc;

	bar = &ntb->bar_info[NTB_CONFIG_BAR];
	bar->pci_resource_id = PCIR_BAR(0);
	rc = map_mmr_bar(ntb, bar);
	if (rc != 0)
	goto out;

	/*
	* At least on Xeon v4 NTB device leaks to host some remote side
	* BAR0 writes supposed to update scratchpad registers. I am not
	* sure why it happens, but it may be related to the fact that
	* on a link side BAR0 is 32KB, while on a host side it is 64KB.
	* Without this hack DMAR blocks those accesses as not allowed.
	*/
	if (bus_dma_tag_create(bus_get_dma_tag(ntb->device), 1, 0,
	BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
	bar->size, 1, bar->size, 0, NULL, NULL, &ntb->bar0_dma_tag)) {
	device_printf(ntb->device, "Unable to create BAR0 tag\n");
	return (ENOMEM);
	}
	if (bus_dmamap_create(ntb->bar0_dma_tag, 0, &ntb->bar0_dma_map)) {
	device_printf(ntb->device, "Unable to create BAR0 map\n");
	return (ENOMEM);
	}
	- if (bus_dma_dmar_load_ident(ntb->bar0_dma_tag, ntb->bar0_dma_map,
	+ if (bus_dma_iommu_load_ident(ntb->bar0_dma_tag, ntb->bar0_dma_map,
	bar->pbase, bar->size, 0)) {
	device_printf(ntb->device, "Unable to load BAR0 map\n");
	return (ENOMEM);
	}

	bar = &ntb->bar_info[NTB_B2B_BAR_1];
	bar->pci_resource_id = PCIR_BAR(2);
	rc = map_memory_window_bar(ntb, bar);
	if (rc != 0)
	goto out;
	bar->psz_off = XEON_PBAR23SZ_OFFSET;
	bar->ssz_off = XEON_SBAR23SZ_OFFSET;
	bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;

	bar = &ntb->bar_info[NTB_B2B_BAR_2];
	bar->pci_resource_id = PCIR_BAR(4);
	rc = map_memory_window_bar(ntb, bar);
	if (rc != 0)
	goto out;
	bar->psz_off = XEON_PBAR4SZ_OFFSET;
	bar->ssz_off = XEON_SBAR4SZ_OFFSET;
	bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;

	if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
	goto out;

	bar = &ntb->bar_info[NTB_B2B_BAR_3];
	bar->pci_resource_id = PCIR_BAR(5);
	rc = map_memory_window_bar(ntb, bar);
	bar->psz_off = XEON_PBAR5SZ_OFFSET;
	bar->ssz_off = XEON_SBAR5SZ_OFFSET;
	bar->pbarxlat_off = XEON_PBAR5XLAT_OFFSET;

	out:
	if (rc != 0)
	device_printf(ntb->device,
	"unable to allocate pci resource\n");
	return (rc);
	}

	static void
	print_map_success(struct ntb_softc ntb, struct ntb_pci_bar_info bar,
	const char *kind)
	{

	device_printf(ntb->device,
	"Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n",
	PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
	(char *)bar->vbase + bar->size - 1,
	(void )bar->pbase, (void )(bar->pbase + bar->size - 1),
	(uintmax_t)bar->size, kind);
	}

	static int
	map_mmr_bar(struct ntb_softc ntb, struct ntb_pci_bar_info bar)
	{

	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
	&bar->pci_resource_id, RF_ACTIVE);
	if (bar->pci_resource == NULL)
	return (ENXIO);

	save_bar_parameters(bar);
	bar->map_mode = VM_MEMATTR_UNCACHEABLE;
	print_map_success(ntb, bar, "mmr");
	return (0);
	}

	static int
	map_memory_window_bar(struct ntb_softc ntb, struct ntb_pci_bar_info bar)
	{
	int rc;
	vm_memattr_t mapmode;
	uint8_t bar_size_bits = 0;

	bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
	&bar->pci_resource_id, RF_ACTIVE);

	if (bar->pci_resource == NULL)
	return (ENXIO);

	save_bar_parameters(bar);
	/*
	* Ivytown NTB BAR sizes are misreported by the hardware due to a
	* hardware issue. To work around this, query the size it should be
	* configured to by the device and modify the resource to correspond to
	* this new size. The BIOS on systems with this problem is required to
	* provide enough address space to allow the driver to make this change
	* safely.
	*
	* Ideally I could have just specified the size when I allocated the
	* resource like:
	* bus_alloc_resource(ntb->device,
	* SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
	* 1ul << bar_size_bits, RF_ACTIVE);
	* but the PCI driver does not honor the size in this call, so we have
	* to modify it after the fact.
	*/
	if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) {
	if (bar->pci_resource_id == PCIR_BAR(2))
	bar_size_bits = pci_read_config(ntb->device,
	XEON_PBAR23SZ_OFFSET, 1);
	else
	bar_size_bits = pci_read_config(ntb->device,
	XEON_PBAR45SZ_OFFSET, 1);

	rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
	bar->pci_resource, bar->pbase,
	bar->pbase + (1ul << bar_size_bits) - 1);
	if (rc != 0) {
	device_printf(ntb->device,
	"unable to resize bar\n");
	return (rc);
	}

	save_bar_parameters(bar);
	}

	bar->map_mode = VM_MEMATTR_UNCACHEABLE;
	print_map_success(ntb, bar, "mw");

	/*
	* Optionally, mark MW BARs as anything other than UC to improve
	* performance.
	*/
	mapmode = intel_ntb_pat_flags();
	if (mapmode == bar->map_mode)
	return (0);

	rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode);
	if (rc == 0) {
	bar->map_mode = mapmode;
	device_printf(ntb->device,
	"Marked BAR%d v:[%p-%p] p:[%p-%p] as "
	"%s.\n",
	PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
	(char *)bar->vbase + bar->size - 1,
	(void )bar->pbase, (void )(bar->pbase + bar->size - 1),
	intel_ntb_vm_memattr_to_str(mapmode));
	} else
	device_printf(ntb->device,
	"Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as "
	"%s: %d\n",
	PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
	(char *)bar->vbase + bar->size - 1,
	(void )bar->pbase, (void )(bar->pbase + bar->size - 1),
	intel_ntb_vm_memattr_to_str(mapmode), rc);
	/* Proceed anyway */
	return (0);
	}

	static void
	intel_ntb_unmap_pci_bar(struct ntb_softc *ntb)
	{
	struct ntb_pci_bar_info *bar;
	int i;

	if (ntb->bar0_dma_map != NULL) {
	bus_dmamap_unload(ntb->bar0_dma_tag, ntb->bar0_dma_map);
	bus_dmamap_destroy(ntb->bar0_dma_tag, ntb->bar0_dma_map);
	}
	if (ntb->bar0_dma_tag != NULL)
	bus_dma_tag_destroy(ntb->bar0_dma_tag);
	for (i = 0; i < NTB_MAX_BARS; i++) {
	bar = &ntb->bar_info[i];
	if (bar->pci_resource != NULL)
	bus_release_resource(ntb->device, SYS_RES_MEMORY,
	bar->pci_resource_id, bar->pci_resource);
	}
	}

	static int
	intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
	{
	uint32_t i;
	int rc;

	for (i = 0; i < num_vectors; i++) {
	ntb->int_info[i].rid = i + 1;
	ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
	SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
	if (ntb->int_info[i].res == NULL) {
	device_printf(ntb->device,
	"bus_alloc_resource failed\n");
	return (ENOMEM);
	}
	ntb->int_info[i].tag = NULL;
	ntb->allocated_interrupts++;
	rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
	INTR_MPSAFE \| INTR_TYPE_MISC, NULL, ndev_vec_isr,
	&ntb->msix_vec[i], &ntb->int_info[i].tag);
	if (rc != 0) {
	device_printf(ntb->device, "bus_setup_intr failed\n");
	return (ENXIO);
	}
	}
	return (0);
	}

	/*
	* The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector
	* cannot be allocated for each MSI-X message. JHB seems to think remapping
	* should be okay. This tunable should enable us to test that hypothesis
	* when someone gets their hands on some Xeon hardware.
	*/
	static int ntb_force_remap_mode;
	SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN,
	&ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped"
	" to a smaller number of ithreads, even if the desired number are "
	"available");

	/*
	* In case it is NOT ok, give consumers an abort button.
	*/
	static int ntb_prefer_intx;
	SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN,
	&ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather "
	"than remapping MSI-X messages over available slots (match Linux driver "
	"behavior)");

	/*
	* Remap the desired number of MSI-X messages to available ithreads in a simple
	* round-robin fashion.
	*/
	static int
	intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
	{
	u_int *vectors;
	uint32_t i;
	int rc;

	if (ntb_prefer_intx != 0)
	return (ENXIO);

	vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO \| M_WAITOK);

	for (i = 0; i < desired; i++)
	vectors[i] = (i % avail) + 1;

	rc = pci_remap_msix(dev, desired, vectors);
	free(vectors, M_NTB);
	return (rc);
	}

	static int
	intel_ntb_init_isr(struct ntb_softc *ntb)
	{
	uint32_t desired_vectors, num_vectors;
	int rc;

	ntb->allocated_interrupts = 0;
	ntb->last_ts = ticks;

	/*
	* Mask all doorbell interrupts. (Except link events!)
	*/
	DB_MASK_LOCK(ntb);
	ntb->db_mask = ntb->db_valid_mask;
	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
	DB_MASK_UNLOCK(ntb);

	num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
	ntb->db_count);
	if (desired_vectors >= 1) {
	rc = pci_alloc_msix(ntb->device, &num_vectors);

	if (ntb_force_remap_mode != 0 && rc == 0 &&
	num_vectors == desired_vectors)
	num_vectors--;

	if (rc == 0 && num_vectors < desired_vectors) {
	rc = intel_ntb_remap_msix(ntb->device, desired_vectors,
	num_vectors);
	if (rc == 0)
	num_vectors = desired_vectors;
	else
	pci_release_msi(ntb->device);
	}
	if (rc != 0)
	num_vectors = 1;
	} else
	num_vectors = 1;

	if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	device_printf(ntb->device,
	"Errata workaround does not support MSI or INTX\n");
	return (EINVAL);
	}

	ntb->db_vec_count = 1;
	ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT;
	rc = intel_ntb_setup_legacy_interrupt(ntb);
	} else {
	if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS &&
	HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	device_printf(ntb->device,
	"Errata workaround expects %d doorbell bits\n",
	XEON_NONLINK_DB_MSIX_BITS);
	return (EINVAL);
	}

	intel_ntb_create_msix_vec(ntb, num_vectors);
	rc = intel_ntb_setup_msix(ntb, num_vectors);
	}
	if (rc != 0) {
	device_printf(ntb->device,
	"Error allocating interrupts: %d\n", rc);
	intel_ntb_free_msix_vec(ntb);
	}

	return (rc);
	}

	static int
	intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb)
	{
	int rc;

	ntb->int_info[0].rid = 0;
	ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
	&ntb->int_info[0].rid, RF_SHAREABLE\|RF_ACTIVE);
	if (ntb->int_info[0].res == NULL) {
	device_printf(ntb->device, "bus_alloc_resource failed\n");
	return (ENOMEM);
	}

	ntb->int_info[0].tag = NULL;
	ntb->allocated_interrupts = 1;

	rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
	INTR_MPSAFE \| INTR_TYPE_MISC, NULL, ndev_irq_isr,
	ntb, &ntb->int_info[0].tag);
	if (rc != 0) {
	device_printf(ntb->device, "bus_setup_intr failed\n");
	return (ENXIO);
	}

	return (0);
	}

	static void
	intel_ntb_teardown_interrupts(struct ntb_softc *ntb)
	{
	struct ntb_int_info *current_int;
	int i;

	for (i = 0; i < ntb->allocated_interrupts; i++) {
	current_int = &ntb->int_info[i];
	if (current_int->tag != NULL)
	bus_teardown_intr(ntb->device, current_int->res,
	current_int->tag);

	if (current_int->res != NULL)
	bus_release_resource(ntb->device, SYS_RES_IRQ,
	rman_get_rid(current_int->res), current_int->res);
	}

	intel_ntb_free_msix_vec(ntb);
	pci_release_msi(ntb->device);
	}

	/*
	* Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it
	* out to make code clearer.
	*/
	static inline uint64_t
	db_ioread(struct ntb_softc *ntb, uint64_t regoff)
	{

	if (ntb->type == NTB_ATOM)
	return (intel_ntb_reg_read(8, regoff));

	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));

	return (intel_ntb_reg_read(2, regoff));
	}

	static inline void
	db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
	{

	KASSERT((val & ~ntb->db_valid_mask) == 0,
	("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
	(uintmax_t)(val & ~ntb->db_valid_mask),
	(uintmax_t)ntb->db_valid_mask));

	if (regoff == ntb->self_reg->db_mask)
	DB_MASK_ASSERT(ntb, MA_OWNED);
	db_iowrite_raw(ntb, regoff, val);
	}

	static inline void
	db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
	{

	if (ntb->type == NTB_ATOM) {
	intel_ntb_reg_write(8, regoff, val);
	return;
	}

	KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
	intel_ntb_reg_write(2, regoff, (uint16_t)val);
	}

	static void
	intel_ntb_db_set_mask(device_t dev, uint64_t bits)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	DB_MASK_LOCK(ntb);
	ntb->db_mask \|= bits;
	if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
	DB_MASK_UNLOCK(ntb);
	}

	static void
	intel_ntb_db_clear_mask(device_t dev, uint64_t bits)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	uint64_t ibits;
	int i;

	KASSERT((bits & ~ntb->db_valid_mask) == 0,
	("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
	(uintmax_t)(bits & ~ntb->db_valid_mask),
	(uintmax_t)ntb->db_valid_mask));

	DB_MASK_LOCK(ntb);
	ibits = ntb->fake_db & ntb->db_mask & bits;
	ntb->db_mask &= ~bits;
	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	/* Simulate fake interrupts if unmasked DB bits are set. */
	ntb->force_db \|= ibits;
	for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
	if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0)
	swi_sched(ntb->int_info[i].tag, 0);
	}
	} else {
	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
	}
	DB_MASK_UNLOCK(ntb);
	}

	static uint64_t
	intel_ntb_db_read(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
	return (ntb->fake_db);

	return (db_ioread(ntb, ntb->self_reg->db_bell));
	}

	static void
	intel_ntb_db_clear(device_t dev, uint64_t bits)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	KASSERT((bits & ~ntb->db_valid_mask) == 0,
	("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
	(uintmax_t)(bits & ~ntb->db_valid_mask),
	(uintmax_t)ntb->db_valid_mask));

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	DB_MASK_LOCK(ntb);
	ntb->fake_db &= ~bits;
	DB_MASK_UNLOCK(ntb);
	return;
	}

	db_iowrite(ntb, ntb->self_reg->db_bell, bits);
	}

	static inline uint64_t
	intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
	{
	uint64_t shift, mask;

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	/*
	* Remap vectors in custom way to make at least first
	* three doorbells to not generate stray events.
	* This breaks Linux compatibility (if one existed)
	* when more then one DB is used (not by if_ntb).
	*/
	if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1)
	return (1 << db_vector);
	if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1)
	return (0x7ffc);
	}

	shift = ntb->db_vec_shift;
	mask = (1ull << shift) - 1;
	return (mask << (shift * db_vector));
	}

	static void
	intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
	{
	uint64_t vec_mask;

	ntb->last_ts = ticks;
	vec_mask = intel_ntb_vec_mask(ntb, vec);

	if ((vec_mask & ntb->db_link_mask) != 0) {
	if (intel_ntb_poll_link(ntb))
	ntb_link_event(ntb->device);
	}

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
	(vec_mask & ntb->db_link_mask) == 0) {
	DB_MASK_LOCK(ntb);

	/*
	* Do not report same DB events again if not cleared yet,
	* unless the mask was just cleared for them and this
	* interrupt handler call can be the consequence of it.
	*/
	vec_mask &= ~ntb->fake_db \| ntb->force_db;
	ntb->force_db &= ~vec_mask;

	/* Update our internal doorbell register. */
	ntb->fake_db \|= vec_mask;

	/* Do not report masked DB events. */
	vec_mask &= ~ntb->db_mask;

	DB_MASK_UNLOCK(ntb);
	}

	if ((vec_mask & ntb->db_valid_mask) != 0)
	ntb_db_event(ntb->device, vec);
	}

	static void
	ndev_vec_isr(void *arg)
	{
	struct ntb_vec *nvec = arg;

	intel_ntb_interrupt(nvec->ntb, nvec->num);
	}

	static void
	ndev_irq_isr(void *arg)
	{
	/* If we couldn't set up MSI-X, we only have the one vector. */
	intel_ntb_interrupt(arg, 0);
	}

	static int
	intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors)
	{
	uint32_t i;

	ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB,
	M_ZERO \| M_WAITOK);
	for (i = 0; i < num_vectors; i++) {
	ntb->msix_vec[i].num = i;
	ntb->msix_vec[i].ntb = ntb;
	}

	return (0);
	}

	static void
	intel_ntb_free_msix_vec(struct ntb_softc *ntb)
	{

	if (ntb->msix_vec == NULL)
	return;

	free(ntb->msix_vec, M_NTB);
	ntb->msix_vec = NULL;
	}

	static void
	intel_ntb_get_msix_info(struct ntb_softc *ntb)
	{
	struct pci_devinfo *dinfo;
	struct pcicfg_msix *msix;
	uint32_t laddr, data, i, offset;

	dinfo = device_get_ivars(ntb->device);
	msix = &dinfo->cfg.msix;

	CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data));

	for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
	offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE;

	laddr = bus_read_4(msix->msix_table_res, offset +
	PCI_MSIX_ENTRY_LOWER_ADDR);
	intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr);

	KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE,
	("local MSIX addr 0x%x not in MSI base 0x%x", laddr,
	MSI_INTEL_ADDR_BASE));
	ntb->msix_data[i].nmd_ofs = laddr;

	data = bus_read_4(msix->msix_table_res, offset +
	PCI_MSIX_ENTRY_DATA);
	intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data);

	ntb->msix_data[i].nmd_data = data;
	}
	}

	static struct ntb_hw_info *
	intel_ntb_get_device_info(uint32_t device_id)
	{
	struct ntb_hw_info *ep;

	for (ep = pci_ids; ep < &pci_ids[nitems(pci_ids)]; ep++) {
	if (ep->device_id == device_id)
	return (ep);
	}
	return (NULL);
	}

	static void
	intel_ntb_teardown_xeon(struct ntb_softc *ntb)
	{

	if (ntb->reg != NULL)
	intel_ntb_link_disable(ntb->device);
	}

	static void
	intel_ntb_detect_max_mw(struct ntb_softc *ntb)
	{

	if (ntb->type == NTB_ATOM) {
	ntb->mw_count = ATOM_MW_COUNT;
	return;
	}

	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
	ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
	else
	ntb->mw_count = XEON_SNB_MW_COUNT;
	}

	static int
	intel_ntb_detect_xeon(struct ntb_softc *ntb)
	{
	uint8_t ppd, conn_type;

	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
	ntb->ppd = ppd;

	if ((ppd & XEON_PPD_DEV_TYPE) != 0)
	ntb->dev_type = NTB_DEV_DSD;
	else
	ntb->dev_type = NTB_DEV_USD;

	if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
	ntb->features \|= NTB_SPLIT_BAR;

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
	!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	device_printf(ntb->device,
	"Can not apply SB01BASE_LOCKUP workaround "
	"with split BARs disabled!\n");
	device_printf(ntb->device,
	"Expect system hangs under heavy NTB traffic!\n");
	ntb->features &= ~NTB_SB01BASE_LOCKUP;
	}

	/*
	* SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP
	* errata workaround; only do one at a time.
	*/
	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
	ntb->features &= ~NTB_SDOORBELL_LOCKUP;

	conn_type = ppd & XEON_PPD_CONN_TYPE;
	switch (conn_type) {
	case NTB_CONN_B2B:
	ntb->conn_type = conn_type;
	break;
	case NTB_CONN_RP:
	case NTB_CONN_TRANSPARENT:
	default:
	device_printf(ntb->device, "Unsupported connection type: %u\n",
	(unsigned)conn_type);
	return (ENXIO);
	}
	return (0);
	}

	static int
	intel_ntb_detect_atom(struct ntb_softc *ntb)
	{
	uint32_t ppd, conn_type;

	ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
	ntb->ppd = ppd;

	if ((ppd & ATOM_PPD_DEV_TYPE) != 0)
	ntb->dev_type = NTB_DEV_DSD;
	else
	ntb->dev_type = NTB_DEV_USD;

	conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8;
	switch (conn_type) {
	case NTB_CONN_B2B:
	ntb->conn_type = conn_type;
	break;
	default:
	device_printf(ntb->device, "Unsupported NTB configuration\n");
	return (ENXIO);
	}
	return (0);
	}

	static int
	intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
	{
	int rc;

	ntb->spad_count = XEON_SPAD_COUNT;
	ntb->db_count = XEON_DB_COUNT;
	ntb->db_link_mask = XEON_DB_LINK_BIT;
	ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT;
	ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT;

	if (ntb->conn_type != NTB_CONN_B2B) {
	device_printf(ntb->device, "Connection type %d not supported\n",
	ntb->conn_type);
	return (ENXIO);
	}

	ntb->reg = &xeon_reg;
	ntb->self_reg = &xeon_pri_reg;
	ntb->peer_reg = &xeon_b2b_reg;
	ntb->xlat_reg = &xeon_sec_xlat;

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	ntb->force_db = ntb->fake_db = 0;
	ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) %
	ntb->mw_count;
	intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n",
	g_ntb_msix_idx, ntb->msix_mw_idx);
	rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx,
	VM_MEMATTR_UNCACHEABLE);
	KASSERT(rc == 0, ("shouldn't fail"));
	} else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
	/*
	* There is a Xeon hardware errata related to writes to SDOORBELL or
	* B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
	* which may hang the system. To workaround this, use a memory
	* window to access the interrupt and scratch pad registers on the
	* remote system.
	*/
	ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) %
	ntb->mw_count;
	intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n",
	g_ntb_mw_idx, ntb->b2b_mw_idx);
	rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx,
	VM_MEMATTR_UNCACHEABLE);
	KASSERT(rc == 0, ("shouldn't fail"));
	} else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14))
	/*
	* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
	* mirrored to the remote system. Shrink the number of bits by one,
	* since bit 14 is the last bit.
	*
	* On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
	* anyway. Nor for non-B2B connection types.
	*/
	ntb->db_count = XEON_DB_COUNT - 1;

	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;

	if (ntb->dev_type == NTB_DEV_USD)
	rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr,
	&xeon_b2b_usd_addr);
	else
	rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr,
	&xeon_b2b_dsd_addr);
	if (rc != 0)
	return (rc);

	/* Enable Bus Master and Memory Space on the secondary side */
	intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET,
	PCIM_CMD_MEMEN \| PCIM_CMD_BUSMASTEREN);

	/*
	* Mask all doorbell interrupts.
	*/
	DB_MASK_LOCK(ntb);
	ntb->db_mask = ntb->db_valid_mask;
	db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
	DB_MASK_UNLOCK(ntb);

	rc = intel_ntb_init_isr(ntb);
	return (rc);
	}

	static int
	intel_ntb_atom_init_dev(struct ntb_softc *ntb)
	{
	int error;

	KASSERT(ntb->conn_type == NTB_CONN_B2B,
	("Unsupported NTB configuration (%d)\n", ntb->conn_type));

	ntb->spad_count = ATOM_SPAD_COUNT;
	ntb->db_count = ATOM_DB_COUNT;
	ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT;
	ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT;
	ntb->db_valid_mask = (1ull << ntb->db_count) - 1;

	ntb->reg = &atom_reg;
	ntb->self_reg = &atom_pri_reg;
	ntb->peer_reg = &atom_b2b_reg;
	ntb->xlat_reg = &atom_sec_xlat;

	/*
	* FIXME - MSI-X bug on early Atom HW, remove once internal issue is
	* resolved. Mask transaction layer internal parity errors.
	*/
	pci_write_config(ntb->device, 0xFC, 0x4, 4);

	configure_atom_secondary_side_bars(ntb);

	/* Enable Bus Master and Memory Space on the secondary side */
	intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET,
	PCIM_CMD_MEMEN \| PCIM_CMD_BUSMASTEREN);

	error = intel_ntb_init_isr(ntb);
	if (error != 0)
	return (error);

	/* Initiate PCI-E link training */
	intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);

	callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb);

	return (0);
	}

	/* XXX: Linux driver doesn't seem to do any of this for Atom. */
	static void
	configure_atom_secondary_side_bars(struct ntb_softc *ntb)
	{

	if (ntb->dev_type == NTB_DEV_USD) {
	intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
	XEON_B2B_BAR2_ADDR64);
	intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
	XEON_B2B_BAR4_ADDR64);
	intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64);
	intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64);
	} else {
	intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
	XEON_B2B_BAR2_ADDR64);
	intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
	XEON_B2B_BAR4_ADDR64);
	intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64);
	intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64);
	}
	}


	/*
	* When working around Xeon SDOORBELL errata by remapping remote registers in a
	* MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW
	* remains for use by a higher layer.
	*
	* Will only be used if working around SDOORBELL errata and the BIOS-configured
	* MW size is sufficiently large.
	*/
	static unsigned int ntb_b2b_mw_share;
	SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share,
	0, "If enabled (non-zero), prefer to share half of the B2B peer register "
	"MW with higher level consumers. Both sides of the NTB MUST set the same "
	"value here.");

	static void
	xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
	enum ntb_bar regbar)
	{
	struct ntb_pci_bar_info *bar;
	uint8_t bar_sz;

	if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3)
	return;

	bar = &ntb->bar_info[idx];
	bar_sz = pci_read_config(ntb->device, bar->psz_off, 1);
	if (idx == regbar) {
	if (ntb->b2b_off != 0)
	bar_sz--;
	else
	bar_sz = 0;
	}
	pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1);
	bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1);
	(void)bar_sz;
	}

	static void
	xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr,
	enum ntb_bar idx, enum ntb_bar regbar)
	{
	uint64_t reg_val;
	uint32_t base_reg, lmt_reg;

	bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg);
	if (idx == regbar) {
	if (ntb->b2b_off)
	bar_addr += ntb->b2b_off;
	else
	bar_addr = 0;
	}

	if (!bar_is_64bit(ntb, idx)) {
	intel_ntb_reg_write(4, base_reg, bar_addr);
	reg_val = intel_ntb_reg_read(4, base_reg);
	(void)reg_val;

	intel_ntb_reg_write(4, lmt_reg, bar_addr);
	reg_val = intel_ntb_reg_read(4, lmt_reg);
	(void)reg_val;
	} else {
	intel_ntb_reg_write(8, base_reg, bar_addr);
	reg_val = intel_ntb_reg_read(8, base_reg);
	(void)reg_val;

	intel_ntb_reg_write(8, lmt_reg, bar_addr);
	reg_val = intel_ntb_reg_read(8, lmt_reg);
	(void)reg_val;
	}
	}

	static void
	xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx)
	{
	struct ntb_pci_bar_info *bar;

	bar = &ntb->bar_info[idx];
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
	intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr);
	base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off);
	} else {
	intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr);
	base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off);
	}
	(void)base_addr;
	}

	static int
	xeon_setup_b2b_mw(struct ntb_softc ntb, const struct ntb_b2b_addr addr,
	const struct ntb_b2b_addr *peer_addr)
	{
	struct ntb_pci_bar_info *b2b_bar;
	vm_size_t bar_size;
	uint64_t bar_addr;
	enum ntb_bar b2b_bar_num, i;

	if (ntb->b2b_mw_idx == B2B_MW_DISABLED) {
	b2b_bar = NULL;
	b2b_bar_num = NTB_CONFIG_BAR;
	ntb->b2b_off = 0;
	} else {
	b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx);
	KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS,
	("invalid b2b mw bar"));

	b2b_bar = &ntb->bar_info[b2b_bar_num];
	bar_size = b2b_bar->size;

	if (ntb_b2b_mw_share != 0 &&
	(bar_size >> 1) >= XEON_B2B_MIN_SIZE)
	ntb->b2b_off = bar_size >> 1;
	else if (bar_size >= XEON_B2B_MIN_SIZE) {
	ntb->b2b_off = 0;
	} else {
	device_printf(ntb->device,
	"B2B bar size is too small!\n");
	return (EIO);
	}
	}

	/*
	* Reset the secondary bar sizes to match the primary bar sizes.
	* (Except, disable or halve the size of the B2B secondary bar.)
	*/
	for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++)
	xeon_reset_sbar_size(ntb, i, b2b_bar_num);

	bar_addr = 0;
	if (b2b_bar_num == NTB_CONFIG_BAR)
	bar_addr = addr->bar0_addr;
	else if (b2b_bar_num == NTB_B2B_BAR_1)
	bar_addr = addr->bar2_addr64;
	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR))
	bar_addr = addr->bar4_addr64;
	else if (b2b_bar_num == NTB_B2B_BAR_2)
	bar_addr = addr->bar4_addr32;
	else if (b2b_bar_num == NTB_B2B_BAR_3)
	bar_addr = addr->bar5_addr32;
	else
	KASSERT(false, ("invalid bar"));

	intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr);

	/*
	* Other SBARs are normally hit by the PBAR xlat, except for the b2b
	* register BAR. The B2B BAR is either disabled above or configured
	* half-size. It starts at PBAR xlat + offset.
	*
	* Also set up incoming BAR limits == base (zero length window).
	*/
	xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1,
	b2b_bar_num);
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32,
	NTB_B2B_BAR_2, b2b_bar_num);
	xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32,
	NTB_B2B_BAR_3, b2b_bar_num);
	} else
	xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64,
	NTB_B2B_BAR_2, b2b_bar_num);

	/* Zero incoming translation addrs */
	intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0);
	intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0);

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	uint32_t xlat_reg, lmt_reg;
	enum ntb_bar bar_num;

	/*
	* We point the chosen MSIX MW BAR xlat to remote LAPIC for
	* workaround
	*/
	bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx);
	bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg);
	if (bar_is_64bit(ntb, bar_num)) {
	intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE);
	ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg);
	intel_ntb_reg_write(8, lmt_reg, 0);
	} else {
	intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE);
	ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg);
	intel_ntb_reg_write(4, lmt_reg, 0);
	}

	ntb->peer_lapic_bar = &ntb->bar_info[bar_num];
	}
	(void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET);
	(void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET);

	/* Zero outgoing translation limits (whole bar size windows) */
	intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0);
	intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);

	/* Set outgoing translation offsets */
	xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1);
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2);
	xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3);
	} else
	xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2);

	/* Set the translation offset for B2B registers */
	bar_addr = 0;
	if (b2b_bar_num == NTB_CONFIG_BAR)
	bar_addr = peer_addr->bar0_addr;
	else if (b2b_bar_num == NTB_B2B_BAR_1)
	bar_addr = peer_addr->bar2_addr64;
	else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR))
	bar_addr = peer_addr->bar4_addr64;
	else if (b2b_bar_num == NTB_B2B_BAR_2)
	bar_addr = peer_addr->bar4_addr32;
	else if (b2b_bar_num == NTB_B2B_BAR_3)
	bar_addr = peer_addr->bar5_addr32;
	else
	KASSERT(false, ("invalid bar"));

	/*
	* B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
	* at a time.
	*/
	intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
	intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
	return (0);
	}

	static inline bool
	_xeon_link_is_up(struct ntb_softc *ntb)
	{

	if (ntb->conn_type == NTB_CONN_TRANSPARENT)
	return (true);
	return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
	}

	static inline bool
	link_is_up(struct ntb_softc *ntb)
	{

	if (ntb->type == NTB_XEON)
	return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good \|\|
	!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));

	KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
	return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0);
	}

	static inline bool
	atom_link_is_err(struct ntb_softc *ntb)
	{
	uint32_t status;

	KASSERT(ntb->type == NTB_ATOM, ("ntb type"));

	status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
	if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0)
	return (true);

	status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
	return ((status & ATOM_IBIST_ERR_OFLOW) != 0);
	}

	/* Atom does not have link status interrupt, poll on that platform */
	static void
	atom_link_hb(void *arg)
	{
	struct ntb_softc *ntb = arg;
	sbintime_t timo, poll_ts;

	timo = NTB_HB_TIMEOUT * hz;
	poll_ts = ntb->last_ts + timo;

	/*
	* Delay polling the link status if an interrupt was received, unless
	* the cached link status says the link is down.
	*/
	if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) {
	timo = poll_ts - ticks;
	goto out;
	}

	if (intel_ntb_poll_link(ntb))
	ntb_link_event(ntb->device);

	if (!link_is_up(ntb) && atom_link_is_err(ntb)) {
	/* Link is down with error, proceed with recovery */
	callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb);
	return;
	}

	out:
	callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb);
	}

	static void
	atom_perform_link_restart(struct ntb_softc *ntb)
	{
	uint32_t status;

	/* Driver resets the NTB ModPhy lanes - magic! */
	intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0);
	intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40);
	intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60);
	intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60);

	/* Driver waits 100ms to allow the NTB ModPhy to settle */
	pause("ModPhy", hz / 10);

	/* Clear AER Errors, write to clear */
	status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET);
	status &= PCIM_AER_COR_REPLAY_ROLLOVER;
	intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status);

	/* Clear unexpected electrical idle event in LTSSM, write to clear */
	status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET);
	status \|= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
	intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status);

	/* Clear DeSkew Buffer error, write to clear */
	status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET);
	status \|= ATOM_DESKEWSTS_DBERR;
	intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status);

	status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
	status &= ATOM_IBIST_ERR_OFLOW;
	intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status);

	/* Releases the NTB state machine to allow the link to retrain */
	status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
	status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
	intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status);
	}

	static int
	intel_ntb_port_number(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	return (ntb->dev_type == NTB_DEV_USD ? 0 : 1);
	}

	static int
	intel_ntb_peer_port_count(device_t dev)
	{

	return (1);
	}

	static int
	intel_ntb_peer_port_number(device_t dev, int pidx)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (pidx != 0)
	return (-EINVAL);

	return (ntb->dev_type == NTB_DEV_USD ? 1 : 0);
	}

	static int
	intel_ntb_peer_port_idx(device_t dev, int port)
	{
	int peer_port;

	peer_port = intel_ntb_peer_port_number(dev, 0);
	if (peer_port == -EINVAL \|\| port != peer_port)
	return (-EINVAL);

	return (0);
	}

	static int
	intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused,
	enum ntb_width width __unused)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	uint32_t cntl;

	intel_ntb_printf(2, "%s\n", __func__);

	if (ntb->type == NTB_ATOM) {
	pci_write_config(ntb->device, NTB_PPD_OFFSET,
	ntb->ppd \| ATOM_PPD_INIT_LINK, 4);
	return (0);
	}

	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
	ntb_link_event(dev);
	return (0);
	}

	cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
	cntl &= ~(NTB_CNTL_LINK_DISABLE \| NTB_CNTL_CFG_LOCK);
	cntl \|= NTB_CNTL_P2S_BAR23_SNOOP \| NTB_CNTL_S2P_BAR23_SNOOP;
	cntl \|= NTB_CNTL_P2S_BAR4_SNOOP \| NTB_CNTL_S2P_BAR4_SNOOP;
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
	cntl \|= NTB_CNTL_P2S_BAR5_SNOOP \| NTB_CNTL_S2P_BAR5_SNOOP;
	intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
	return (0);
	}

	static int
	intel_ntb_link_disable(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	uint32_t cntl;

	intel_ntb_printf(2, "%s\n", __func__);

	if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
	ntb_link_event(dev);
	return (0);
	}

	cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
	cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP \| NTB_CNTL_S2P_BAR23_SNOOP);
	cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP \| NTB_CNTL_S2P_BAR4_SNOOP);
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
	cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP \| NTB_CNTL_S2P_BAR5_SNOOP);
	cntl \|= NTB_CNTL_LINK_DISABLE \| NTB_CNTL_CFG_LOCK;
	intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
	return (0);
	}

	static bool
	intel_ntb_link_enabled(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	uint32_t cntl;

	if (ntb->type == NTB_ATOM) {
	cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
	return ((cntl & ATOM_PPD_INIT_LINK) != 0);
	}

	if (ntb->conn_type == NTB_CONN_TRANSPARENT)
	return (true);

	cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
	return ((cntl & NTB_CNTL_LINK_DISABLE) == 0);
	}

	static void
	recover_atom_link(void *arg)
	{
	struct ntb_softc *ntb = arg;
	unsigned speed, width, oldspeed, oldwidth;
	uint32_t status32;

	atom_perform_link_restart(ntb);

	/*
	* There is a potential race between the 2 NTB devices recovering at
	* the same time. If the times are the same, the link will not recover
	* and the driver will be stuck in this loop forever. Add a random
	* interval to the recovery time to prevent this race.
	*/
	status32 = arc4random() % ATOM_LINK_RECOVERY_TIME;
	pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000);

	if (atom_link_is_err(ntb))
	goto retry;

	status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
	if ((status32 & ATOM_CNTL_LINK_DOWN) != 0)
	goto out;

	status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
	width = NTB_LNK_STA_WIDTH(status32);
	speed = status32 & NTB_LINK_SPEED_MASK;

	oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta);
	oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK;
	if (oldwidth != width \|\| oldspeed != speed)
	goto retry;

	out:
	callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb,
	ntb);
	return;

	retry:
	callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link,
	ntb);
	}

	/*
	* Polls the HW link status register(s); returns true if something has changed.
	*/
	static bool
	intel_ntb_poll_link(struct ntb_softc *ntb)
	{
	uint32_t ntb_cntl;
	uint16_t reg_val;

	if (ntb->type == NTB_ATOM) {
	ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
	if (ntb_cntl == ntb->ntb_ctl)
	return (false);

	ntb->ntb_ctl = ntb_cntl;
	ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
	} else {
	db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);

	reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
	if (reg_val == ntb->lnk_sta)
	return (false);

	ntb->lnk_sta = reg_val;

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	if (_xeon_link_is_up(ntb)) {
	if (!ntb->peer_msix_good) {
	callout_reset(&ntb->peer_msix_work, 0,
	intel_ntb_exchange_msix, ntb);
	return (false);
	}
	} else {
	ntb->peer_msix_good = false;
	ntb->peer_msix_done = false;
	}
	}
	}
	return (true);
	}

	static inline enum ntb_speed
	intel_ntb_link_sta_speed(struct ntb_softc *ntb)
	{

	if (!link_is_up(ntb))
	return (NTB_SPEED_NONE);
	return (ntb->lnk_sta & NTB_LINK_SPEED_MASK);
	}

	static inline enum ntb_width
	intel_ntb_link_sta_width(struct ntb_softc *ntb)
	{

	if (!link_is_up(ntb))
	return (NTB_WIDTH_NONE);
	return (NTB_LNK_STA_WIDTH(ntb->lnk_sta));
	}

	SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Driver state, statistics, and HW registers");

	#define NTB_REGSZ_MASK (3ul << 30)
	#define NTB_REG_64 (1ul << 30)
	#define NTB_REG_32 (2ul << 30)
	#define NTB_REG_16 (3ul << 30)
	#define NTB_REG_8 (0ul << 30)

	#define NTB_DB_READ (1ul << 29)
	#define NTB_PCI_REG (1ul << 28)
	#define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK \| NTB_DB_READ \| NTB_PCI_REG)

	static void
	intel_ntb_sysctl_init(struct ntb_softc *ntb)
	{
	struct sysctl_oid_list globals, tree_par, regpar, statpar, *errpar;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid tree, tmptree;

	ctx = device_get_sysctl_ctx(ntb->device);
	globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device));

	SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status",
	CTLFLAG_RD \| CTLTYPE_STRING \| CTLFLAG_NEEDGIANT, ntb, 0,
	sysctl_handle_link_status_human, "A",
	"Link status (human readable)");
	SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active",
	CTLFLAG_RD \| CTLTYPE_UINT \| CTLFLAG_NEEDGIANT, ntb, 0,
	sysctl_handle_link_status, "IU",
	"Link status (1=active, 0=inactive)");
	SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up",
	CTLFLAG_RW \| CTLTYPE_UINT \| CTLFLAG_NEEDGIANT, ntb, 0,
	sysctl_handle_link_admin, "IU",
	"Set/get interface status (1=UP, 0=DOWN)");

	tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"Driver state, statistics, and HW registers");
	tree_par = SYSCTL_CHILDREN(tree);

	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD,
	&ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port");
	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD,
	&ntb->dev_type, 0, "0 - USD; 1 - DSD");
	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD,
	&ntb->ppd, 0, "Raw PPD register (cached)");

	if (ntb->b2b_mw_idx != B2B_MW_DISABLED) {
	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD,
	&ntb->b2b_mw_idx, 0,
	"Index of the MW used for B2B remote register access");
	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off",
	CTLFLAG_RD, &ntb->b2b_off,
	"If non-zero, offset of B2B register region in shared MW");
	}

	SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features",
	CTLFLAG_RD \| CTLTYPE_STRING \| CTLFLAG_NEEDGIANT, ntb, 0,
	sysctl_handle_features, "A", "Features/errata of this NTB device");

	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD,
	__DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0,
	"NTB CTL register (cached)");
	SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD,
	__DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0,
	"LNK STA register (cached)");

	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD,
	&ntb->mw_count, 0, "MW count");
	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD,
	&ntb->spad_count, 0, "Scratchpad count");
	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD,
	&ntb->db_count, 0, "Doorbell count");
	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD,
	&ntb->db_vec_count, 0, "Doorbell vector count");
	SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD,
	&ntb->db_vec_shift, 0, "Doorbell vector shift");

	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD,
	&ntb->db_valid_mask, "Doorbell valid mask");
	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD,
	&ntb->db_link_mask, "Doorbell link mask");
	SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD,
	&ntb->db_mask, "Doorbell mask (cached)");

	tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL,
	"Raw HW registers (big-endian)");
	regpar = SYSCTL_CHILDREN(tmptree);

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->reg->ntb_ctl, sysctl_handle_register, "IU",
	"NTB Control register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| 0x19c, sysctl_handle_register, "IU",
	"NTB Link Capabilities");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| 0x1a0, sysctl_handle_register, "IU",
	"NTB Link Control register");

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| NTB_DB_READ \| ntb->self_reg->db_mask,
	sysctl_handle_register, "QU", "Doorbell mask register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| NTB_DB_READ \| ntb->self_reg->db_bell,
	sysctl_handle_register, "QU", "Doorbell register");

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar2_xlat,
	sysctl_handle_register, "QU", "Incoming XLAT23 register");
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->xlat_reg->bar4_xlat,
	sysctl_handle_register, "IU", "Incoming XLAT4 register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->xlat_reg->bar5_xlat,
	sysctl_handle_register, "IU", "Incoming XLAT5 register");
	} else {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar4_xlat,
	sysctl_handle_register, "QU", "Incoming XLAT45 register");
	}

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar2_limit,
	sysctl_handle_register, "QU", "Incoming LMT23 register");
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->xlat_reg->bar4_limit,
	sysctl_handle_register, "IU", "Incoming LMT4 register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->xlat_reg->bar5_limit,
	sysctl_handle_register, "IU", "Incoming LMT5 register");
	} else {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar4_limit,
	sysctl_handle_register, "QU", "Incoming LMT45 register");
	}

	if (ntb->type == NTB_ATOM)
	return;

	tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "Xeon HW statistics");
	statpar = SYSCTL_CHILDREN(tmptree);
	SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_16 \| XEON_USMEMMISS_OFFSET,
	sysctl_handle_register, "SU", "Upstream Memory Miss");

	tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "Xeon HW errors");
	errpar = SYSCTL_CHILDREN(tmptree);

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| NTB_PPD_OFFSET,
	sysctl_handle_register, "CU", "PPD");

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| XEON_PBAR23SZ_OFFSET,
	sysctl_handle_register, "CU", "PBAR23 SZ (log2)");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| XEON_PBAR4SZ_OFFSET,
	sysctl_handle_register, "CU", "PBAR4 SZ (log2)");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| XEON_PBAR5SZ_OFFSET,
	sysctl_handle_register, "CU", "PBAR5 SZ (log2)");

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| XEON_SBAR23SZ_OFFSET,
	sysctl_handle_register, "CU", "SBAR23 SZ (log2)");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| XEON_SBAR4SZ_OFFSET,
	sysctl_handle_register, "CU", "SBAR4 SZ (log2)");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_8 \| NTB_PCI_REG \| XEON_SBAR5SZ_OFFSET,
	sysctl_handle_register, "CU", "SBAR5 SZ (log2)");

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_16 \| NTB_PCI_REG \| XEON_DEVSTS_OFFSET,
	sysctl_handle_register, "SU", "DEVSTS");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_16 \| NTB_PCI_REG \| XEON_LINK_STATUS_OFFSET,
	sysctl_handle_register, "SU", "LNKSTS");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_16 \| NTB_PCI_REG \| XEON_SLINK_STATUS_OFFSET,
	sysctl_handle_register, "SU", "SLNKSTS");

	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| NTB_PCI_REG \| XEON_UNCERRSTS_OFFSET,
	sysctl_handle_register, "IU", "UNCERRSTS");
	SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| NTB_PCI_REG \| XEON_CORERRSTS_OFFSET,
	sysctl_handle_register, "IU", "CORERRSTS");

	if (ntb->conn_type != NTB_CONN_B2B)
	return;

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01l",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| XEON_B2B_XLAT_OFFSETL,
	sysctl_handle_register, "IU", "Outgoing XLAT0L register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01u",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| XEON_B2B_XLAT_OFFSETU,
	sysctl_handle_register, "IU", "Outgoing XLAT0U register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off,
	sysctl_handle_register, "QU", "Outgoing XLAT23 register");
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
	sysctl_handle_register, "IU", "Outgoing XLAT4 register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off,
	sysctl_handle_register, "IU", "Outgoing XLAT5 register");
	} else {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
	sysctl_handle_register, "QU", "Outgoing XLAT45 register");
	}

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| XEON_PBAR2LMT_OFFSET,
	sysctl_handle_register, "QU", "Outgoing LMT23 register");
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| XEON_PBAR4LMT_OFFSET,
	sysctl_handle_register, "IU", "Outgoing LMT4 register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| XEON_PBAR5LMT_OFFSET,
	sysctl_handle_register, "IU", "Outgoing LMT5 register");
	} else {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| XEON_PBAR4LMT_OFFSET,
	sysctl_handle_register, "QU", "Outgoing LMT45 register");
	}

	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar0_base,
	sysctl_handle_register, "QU", "Secondary BAR01 base register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar2_base,
	sysctl_handle_register, "QU", "Secondary BAR23 base register");
	if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->xlat_reg->bar4_base,
	sysctl_handle_register, "IU",
	"Secondary BAR4 base register");
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_32 \| ntb->xlat_reg->bar5_base,
	sysctl_handle_register, "IU",
	"Secondary BAR5 base register");
	} else {
	SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base",
	CTLFLAG_RD \| CTLTYPE_OPAQUE \| CTLFLAG_NEEDGIANT, ntb,
	NTB_REG_64 \| ntb->xlat_reg->bar4_base,
	sysctl_handle_register, "QU",
	"Secondary BAR45 base register");
	}
	}

	static int
	sysctl_handle_features(SYSCTL_HANDLER_ARGS)
	{
	struct ntb_softc *ntb = arg1;
	struct sbuf sb;
	int error;

	sbuf_new_for_sysctl(&sb, NULL, 256, req);

	sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR);
	error = sbuf_finish(&sb);
	sbuf_delete(&sb);

	if (error \|\| !req->newptr)
	return (error);
	return (EINVAL);
	}

	static int
	sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS)
	{
	struct ntb_softc *ntb = arg1;
	unsigned old, new;
	int error;

	old = intel_ntb_link_enabled(ntb->device);

	error = SYSCTL_OUT(req, &old, sizeof(old));
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	error = SYSCTL_IN(req, &new, sizeof(new));
	if (error != 0)
	return (error);

	intel_ntb_printf(0, "Admin set interface state to '%sabled'\n",
	(new != 0)? "en" : "dis");

	if (new != 0)
	error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
	else
	error = intel_ntb_link_disable(ntb->device);
	return (error);
	}

	static int
	sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS)
	{
	struct ntb_softc *ntb = arg1;
	struct sbuf sb;
	enum ntb_speed speed;
	enum ntb_width width;
	int error;

	sbuf_new_for_sysctl(&sb, NULL, 32, req);

	if (intel_ntb_link_is_up(ntb->device, &speed, &width))
	sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u",
	(unsigned)speed, (unsigned)width);
	else
	sbuf_printf(&sb, "down");

	error = sbuf_finish(&sb);
	sbuf_delete(&sb);

	if (error \|\| !req->newptr)
	return (error);
	return (EINVAL);
	}

	static int
	sysctl_handle_link_status(SYSCTL_HANDLER_ARGS)
	{
	struct ntb_softc *ntb = arg1;
	unsigned res;
	int error;

	res = intel_ntb_link_is_up(ntb->device, NULL, NULL);

	error = SYSCTL_OUT(req, &res, sizeof(res));
	if (error \|\| !req->newptr)
	return (error);
	return (EINVAL);
	}

	static int
	sysctl_handle_register(SYSCTL_HANDLER_ARGS)
	{
	struct ntb_softc *ntb;
	const void *outp;
	uintptr_t sz;
	uint64_t umv;
	char be[sizeof(umv)];
	size_t outsz;
	uint32_t reg;
	bool db, pci;
	int error;

	ntb = arg1;
	reg = arg2 & ~NTB_REGFLAGS_MASK;
	sz = arg2 & NTB_REGSZ_MASK;
	db = (arg2 & NTB_DB_READ) != 0;
	pci = (arg2 & NTB_PCI_REG) != 0;

	KASSERT(!(db && pci), ("bogus"));

	if (db) {
	KASSERT(sz == NTB_REG_64, ("bogus"));
	umv = db_ioread(ntb, reg);
	outsz = sizeof(uint64_t);
	} else {
	switch (sz) {
	case NTB_REG_64:
	if (pci)
	umv = pci_read_config(ntb->device, reg, 8);
	else
	umv = intel_ntb_reg_read(8, reg);
	outsz = sizeof(uint64_t);
	break;
	case NTB_REG_32:
	if (pci)
	umv = pci_read_config(ntb->device, reg, 4);
	else
	umv = intel_ntb_reg_read(4, reg);
	outsz = sizeof(uint32_t);
	break;
	case NTB_REG_16:
	if (pci)
	umv = pci_read_config(ntb->device, reg, 2);
	else
	umv = intel_ntb_reg_read(2, reg);
	outsz = sizeof(uint16_t);
	break;
	case NTB_REG_8:
	if (pci)
	umv = pci_read_config(ntb->device, reg, 1);
	else
	umv = intel_ntb_reg_read(1, reg);
	outsz = sizeof(uint8_t);
	break;
	default:
	panic("bogus");
	break;
	}
	}

	/* Encode bigendian so that sysctl -x is legible. */
	be64enc(be, umv);
	outp = ((char *)be) + sizeof(umv) - outsz;

	error = SYSCTL_OUT(req, outp, outsz);
	if (error \|\| !req->newptr)
	return (error);
	return (EINVAL);
	}

	static unsigned
	intel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx)
	{

	if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 &&
	uidx >= ntb->b2b_mw_idx) \|\|
	(ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx))
	uidx++;
	if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 &&
	uidx >= ntb->b2b_mw_idx) &&
	(ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx))
	uidx++;
	return (uidx);
	}

	#ifndef EARLY_AP_STARTUP
	static int msix_ready;

	static void
	intel_ntb_msix_ready(void *arg __unused)
	{

	msix_ready = 1;
	}
	SYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY,
	intel_ntb_msix_ready, NULL);
	#endif

	static void
	intel_ntb_exchange_msix(void *ctx)
	{
	struct ntb_softc *ntb;
	uint32_t val;
	unsigned i;

	ntb = ctx;

	if (ntb->peer_msix_good)
	goto msix_good;
	if (ntb->peer_msix_done)
	goto msix_done;

	#ifndef EARLY_AP_STARTUP
	/* Block MSIX negotiation until SMP started and IRQ reshuffled. */
	if (!msix_ready)
	goto reschedule;
	#endif

	intel_ntb_get_msix_info(ntb);
	for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
	intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i,
	ntb->msix_data[i].nmd_data);
	intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i,
	ntb->msix_data[i].nmd_ofs - ntb->msix_xlat);
	}
	intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD);

	intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val);
	if (val != NTB_MSIX_VER_GUARD)
	goto reschedule;

	for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
	intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val);
	intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val);
	ntb->peer_msix_data[i].nmd_data = val;
	intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val);
	intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val);
	ntb->peer_msix_data[i].nmd_ofs = val;
	}

	ntb->peer_msix_done = true;

	msix_done:
	intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED);
	intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val);
	if (val != NTB_MSIX_RECEIVED)
	goto reschedule;

	intel_ntb_spad_clear(ntb->device);
	ntb->peer_msix_good = true;
	/* Give peer time to see our NTB_MSIX_RECEIVED. */
	goto reschedule;

	msix_good:
	intel_ntb_poll_link(ntb);
	ntb_link_event(ntb->device);
	return;

	reschedule:
	ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
	if (_xeon_link_is_up(ntb)) {
	callout_reset(&ntb->peer_msix_work,
	hz * (ntb->peer_msix_good ? 2 : 1) / 10,
	intel_ntb_exchange_msix, ntb);
	} else
	intel_ntb_spad_clear(ntb->device);
	}

	/*
	* Public API to the rest of the OS
	*/

	static uint8_t
	intel_ntb_spad_count(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	return (ntb->spad_count);
	}

	static uint8_t
	intel_ntb_mw_count(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	uint8_t res;

	res = ntb->mw_count;
	if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0)
	res--;
	if (ntb->msix_mw_idx != B2B_MW_DISABLED)
	res--;
	return (res);
	}

	static int
	intel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (idx >= ntb->spad_count)
	return (EINVAL);

	intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val);

	return (0);
	}

	/*
	* Zeros the local scratchpad.
	*/
	static void
	intel_ntb_spad_clear(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	unsigned i;

	for (i = 0; i < ntb->spad_count; i++)
	intel_ntb_spad_write(dev, i, 0);
	}

	static int
	intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (idx >= ntb->spad_count)
	return (EINVAL);

	val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx 4);

	return (0);
	}

	static int
	intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (idx >= ntb->spad_count)
	return (EINVAL);

	if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP))
	intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val);
	else
	intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val);

	return (0);
	}

	static int
	intel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (idx >= ntb->spad_count)
	return (EINVAL);

	if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP))
	val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx 4);
	else
	val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx 4);

	return (0);
	}

	static int
	intel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base,
	caddr_t vbase, size_t size, size_t align, size_t align_size,
	bus_addr_t *plimit)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	struct ntb_pci_bar_info *bar;
	bus_addr_t limit;
	size_t bar_b2b_off;
	enum ntb_bar bar_num;

	if (mw_idx >= intel_ntb_mw_count(dev))
	return (EINVAL);
	mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx);

	bar_num = intel_ntb_mw_to_bar(ntb, mw_idx);
	bar = &ntb->bar_info[bar_num];
	bar_b2b_off = 0;
	if (mw_idx == ntb->b2b_mw_idx) {
	KASSERT(ntb->b2b_off != 0,
	("user shouldn't get non-shared b2b mw"));
	bar_b2b_off = ntb->b2b_off;
	}

	if (bar_is_64bit(ntb, bar_num))
	limit = BUS_SPACE_MAXADDR;
	else
	limit = BUS_SPACE_MAXADDR_32BIT;

	if (base != NULL)
	*base = bar->pbase + bar_b2b_off;
	if (vbase != NULL)
	*vbase = bar->vbase + bar_b2b_off;
	if (size != NULL)
	*size = bar->size - bar_b2b_off;
	if (align != NULL)
	*align = bar->size;
	if (align_size != NULL)
	*align_size = 1;
	if (plimit != NULL)
	*plimit = limit;
	return (0);
	}

	static int
	intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	struct ntb_pci_bar_info *bar;
	uint64_t base, limit, reg_val;
	size_t bar_size, mw_size;
	uint32_t base_reg, xlat_reg, limit_reg;
	enum ntb_bar bar_num;

	if (idx >= intel_ntb_mw_count(dev))
	return (EINVAL);
	idx = intel_ntb_user_mw_to_idx(ntb, idx);

	bar_num = intel_ntb_mw_to_bar(ntb, idx);
	bar = &ntb->bar_info[bar_num];

	bar_size = bar->size;
	if (idx == ntb->b2b_mw_idx)
	mw_size = bar_size - ntb->b2b_off;
	else
	mw_size = bar_size;

	/* Hardware requires that addr is aligned to bar size */
	if ((addr & (bar_size - 1)) != 0)
	return (EINVAL);

	if (size > mw_size)
	return (EINVAL);

	bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);

	limit = 0;
	if (bar_is_64bit(ntb, bar_num)) {
	base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK;

	if (limit_reg != 0 && size != mw_size)
	limit = base + size;

	/* Set and verify translation address */
	intel_ntb_reg_write(8, xlat_reg, addr);
	reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK;
	if (reg_val != addr) {
	intel_ntb_reg_write(8, xlat_reg, 0);
	return (EIO);
	}

	/* Set and verify the limit */
	intel_ntb_reg_write(8, limit_reg, limit);
	reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK;
	if (reg_val != limit) {
	intel_ntb_reg_write(8, limit_reg, base);
	intel_ntb_reg_write(8, xlat_reg, 0);
	return (EIO);
	}
	} else {
	/* Configure 32-bit (split) BAR MW */

	if ((addr & UINT32_MAX) != addr)
	return (ERANGE);
	if (((addr + size) & UINT32_MAX) != (addr + size))
	return (ERANGE);

	base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK;

	if (limit_reg != 0 && size != mw_size)
	limit = base + size;

	/* Set and verify translation address */
	intel_ntb_reg_write(4, xlat_reg, addr);
	reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK;
	if (reg_val != addr) {
	intel_ntb_reg_write(4, xlat_reg, 0);
	return (EIO);
	}

	/* Set and verify the limit */
	intel_ntb_reg_write(4, limit_reg, limit);
	reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK;
	if (reg_val != limit) {
	intel_ntb_reg_write(4, limit_reg, base);
	intel_ntb_reg_write(4, xlat_reg, 0);
	return (EIO);
	}
	}
	return (0);
	}

	static int
	intel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx)
	{

	return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0));
	}

	static int
	intel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	struct ntb_pci_bar_info *bar;

	if (idx >= intel_ntb_mw_count(dev))
	return (EINVAL);
	idx = intel_ntb_user_mw_to_idx(ntb, idx);

	bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)];
	*mode = bar->map_mode;
	return (0);
	}

	static int
	intel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (idx >= intel_ntb_mw_count(dev))
	return (EINVAL);

	idx = intel_ntb_user_mw_to_idx(ntb, idx);
	return (intel_ntb_mw_set_wc_internal(ntb, idx, mode));
	}

	static int
	intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode)
	{
	struct ntb_pci_bar_info *bar;
	int rc;

	bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)];
	if (bar->map_mode == mode)
	return (0);

	rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode);
	if (rc == 0)
	bar->map_mode = mode;

	return (rc);
	}

	static void
	intel_ntb_peer_db_set(device_t dev, uint64_t bit)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
	struct ntb_pci_bar_info *lapic;
	unsigned i;

	lapic = ntb->peer_lapic_bar;

	for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
	if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0)
	bus_space_write_4(lapic->pci_bus_tag,
	lapic->pci_bus_handle,
	ntb->peer_msix_data[i].nmd_ofs,
	ntb->peer_msix_data[i].nmd_data);
	}
	return;
	}

	if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
	intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);
	return;
	}

	db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
	}

	static int
	intel_ntb_peer_db_addr(device_t dev, bus_addr_t db_addr, vm_size_t db_size)
	{
	struct ntb_softc *ntb = device_get_softc(dev);
	struct ntb_pci_bar_info *bar;
	uint64_t regoff;

	KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL"));

	if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
	bar = &ntb->bar_info[NTB_CONFIG_BAR];
	regoff = ntb->peer_reg->db_bell;
	} else {
	KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED,
	("invalid b2b idx"));

	bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
	regoff = XEON_PDOORBELL_OFFSET;
	}
	KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh"));

	/* HACK: Specific to current x86 bus implementation. */
	*db_addr = ((uint64_t)bar->pci_bus_handle + regoff);
	*db_size = ntb->reg->db_size;
	return (0);
	}

	static uint64_t
	intel_ntb_db_valid_mask(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	return (ntb->db_valid_mask);
	}

	static int
	intel_ntb_db_vector_count(device_t dev)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	return (ntb->db_vec_count);
	}

	static uint64_t
	intel_ntb_db_vector_mask(device_t dev, uint32_t vector)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (vector > ntb->db_vec_count)
	return (0);
	return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector));
	}

	static bool
	intel_ntb_link_is_up(device_t dev, enum ntb_speed speed, enum ntb_width width)
	{
	struct ntb_softc *ntb = device_get_softc(dev);

	if (speed != NULL)
	*speed = intel_ntb_link_sta_speed(ntb);
	if (width != NULL)
	*width = intel_ntb_link_sta_width(ntb);
	return (link_is_up(ntb));
	}

	static void
	save_bar_parameters(struct ntb_pci_bar_info *bar)
	{

	bar->pci_bus_tag = rman_get_bustag(bar->pci_resource);
	bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource);
	bar->pbase = rman_get_start(bar->pci_resource);
	bar->size = rman_get_size(bar->pci_resource);
	bar->vbase = rman_get_virtual(bar->pci_resource);
	}

	static device_method_t ntb_intel_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, intel_ntb_probe),
	DEVMETHOD(device_attach, intel_ntb_attach),
	DEVMETHOD(device_detach, intel_ntb_detach),
	/* Bus interface */
	DEVMETHOD(bus_child_location_str, ntb_child_location_str),
	DEVMETHOD(bus_print_child, ntb_print_child),
	DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag),
	/* NTB interface */
	DEVMETHOD(ntb_port_number, intel_ntb_port_number),
	DEVMETHOD(ntb_peer_port_count, intel_ntb_peer_port_count),
	DEVMETHOD(ntb_peer_port_number, intel_ntb_peer_port_number),
	DEVMETHOD(ntb_peer_port_idx, intel_ntb_peer_port_idx),
	DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up),
	DEVMETHOD(ntb_link_enable, intel_ntb_link_enable),
	DEVMETHOD(ntb_link_disable, intel_ntb_link_disable),
	DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled),
	DEVMETHOD(ntb_mw_count, intel_ntb_mw_count),
	DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range),
	DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans),
	DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans),
	DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc),
	DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc),
	DEVMETHOD(ntb_spad_count, intel_ntb_spad_count),
	DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear),
	DEVMETHOD(ntb_spad_write, intel_ntb_spad_write),
	DEVMETHOD(ntb_spad_read, intel_ntb_spad_read),
	DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write),
	DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read),
	DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask),
	DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count),
	DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask),
	DEVMETHOD(ntb_db_clear, intel_ntb_db_clear),
	DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask),
	DEVMETHOD(ntb_db_read, intel_ntb_db_read),
	DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask),
	DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr),
	DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set),
	DEVMETHOD_END
	};

	static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods,
	sizeof(struct ntb_softc));
	DRIVER_MODULE(ntb_hw_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL);
	MODULE_DEPEND(ntb_hw_intel, ntb, 1, 1, 1);
	MODULE_VERSION(ntb_hw_intel, 1);
	MODULE_PNP_INFO("W32:vendor/device;D:#", pci, ntb_hw_intel, pci_ids,
	nitems(pci_ids));
	diff --git a/sys/dev/ntb/ntb_hw/ntb_hw_plx.c b/sys/dev/ntb/ntb_hw/ntb_hw_plx.c
	index 97df9ce3a4cc..f3d8af4971a4 100644
	--- a/sys/dev/ntb/ntb_hw/ntb_hw_plx.c
	+++ b/sys/dev/ntb/ntb_hw/ntb_hw_plx.c
	@@ -1,1093 +1,1096 @@
	/*-
	* Copyright (c) 2017-2019 Alexander Motin <mav@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/*
	* The Non-Transparent Bridge (NTB) is a device that allows you to connect
	* two or more systems using a PCI-e links, providing remote memory access.
	*
	* This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/kernel.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/interrupt.h>
	#include <sys/module.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	+#include <sys/taskqueue.h>
	+#include <sys/tree.h>
	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <machine/bus.h>
	#include <machine/intr_machdep.h>
	#include <machine/resource.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	+#include <dev/iommu/iommu.h>

	#include "../ntb.h"

	#define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */
	#define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */
	#define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */
	#define PLX_NUM_DB 16 /* There are 16 doorbells. */
	#define PLX_MAX_SPLIT 128 /* Allow are at most 128 splits. */

	struct ntb_plx_mw_info {
	int mw_bar;
	int mw_64bit;
	int mw_rid;
	struct resource *mw_res;
	vm_paddr_t mw_pbase;
	caddr_t mw_vbase;
	vm_size_t mw_size;
	struct {
	vm_memattr_t mw_map_mode;
	bus_addr_t mw_xlat_addr;
	bus_size_t mw_xlat_size;
	} splits[PLX_MAX_SPLIT];
	};

	struct ntb_plx_softc {
	/* ntb.c context. Do not move! Must go first! */
	void *ntb_store;

	device_t dev;
	struct resource *conf_res;
	int conf_rid;
	u_int ntx; /* NTx number within chip. */
	u_int link; /* Link v/s Virtual side. */
	u_int port; /* Port number within chip. */
	u_int alut; /* A-LUT is enabled for NTx */
	u_int split; /* split BAR2 into 2^x parts */

	int int_rid;
	struct resource *int_res;
	void *int_tag;

	struct ntb_plx_mw_info mw_info[PLX_MAX_BARS];
	int mw_count; /* Number of memory windows. */

	int spad_count1; /* Number of standard spads. */
	int spad_count2; /* Number of extra spads. */
	uint32_t spad_off1; /* Offset of our spads. */
	uint32_t spad_off2; /* Offset of our extra spads. */
	uint32_t spad_offp1; /* Offset of peer spads. */
	uint32_t spad_offp2; /* Offset of peer extra spads. */

	/* Parameters of window shared with peer config access in B2B mode. */
	int b2b_mw; /* Shared window number. */
	uint64_t b2b_off; /* Offset in shared window. */
	};

	#define PLX_NT0_BASE 0x3E000
	#define PLX_NT1_BASE 0x3C000
	#define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE)
	#define PLX_NTX_LINK_OFFSET 0x01000

	/* Bases of NTx our/peer interface registers */
	#define PLX_NTX_OUR_BASE(sc) \
	(PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0))
	#define PLX_NTX_PEER_BASE(sc) \
	(PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET))

	/* Read/write NTx our interface registers */
	#define NTX_READ(sc, reg) \
	bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg))
	#define NTX_WRITE(sc, reg, val) \
	bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val))

	/* Read/write NTx peer interface registers */
	#define PNTX_READ(sc, reg) \
	bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg))
	#define PNTX_WRITE(sc, reg, val) \
	bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val))

	/* Read/write B2B NTx registers */
	#define BNTX_READ(sc, reg) \
	bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \
	PLX_NTX_BASE(sc) + (reg))
	#define BNTX_WRITE(sc, reg, val) \
	bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \
	PLX_NTX_BASE(sc) + (reg), (val))

	#define PLX_PORT_BASE(p) ((p) << 12)
	#define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7)

	#define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208)

	static int ntb_plx_init(device_t dev);
	static int ntb_plx_detach(device_t dev);
	static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx);

	static int
	ntb_plx_probe(device_t dev)
	{

	switch (pci_get_devid(dev)) {
	case 0x87a010b5:
	device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link");
	return (BUS_PROBE_DEFAULT);
	case 0x87a110b5:
	device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link");
	return (BUS_PROBE_DEFAULT);
	case 0x87b010b5:
	device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual");
	return (BUS_PROBE_DEFAULT);
	case 0x87b110b5:
	device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual");
	return (BUS_PROBE_DEFAULT);
	}
	return (ENXIO);
	}

	static int
	ntb_plx_init(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	uint64_t val64;
	int i;
	uint32_t val;

	if (sc->b2b_mw >= 0) {
	/* Set peer BAR0/1 size and address for B2B NTx access. */
	mw = &sc->mw_info[sc->b2b_mw];
	if (mw->mw_64bit) {
	PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */
	val64 = 0x2000000000000000 * mw->mw_bar \| 0x4;
	PNTX_WRITE(sc, PCIR_BAR(0), val64);
	PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32);
	} else {
	PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */
	val = 0x20000000 * mw->mw_bar;
	PNTX_WRITE(sc, PCIR_BAR(0), val);
	}

	/* Set Virtual to Link address translation for B2B. */
	for (i = 0; i < sc->mw_count; i++) {
	mw = &sc->mw_info[i];
	if (mw->mw_64bit) {
	val64 = 0x2000000000000000 * mw->mw_bar;
	NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64);
	NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32);
	} else {
	val = 0x20000000 * mw->mw_bar;
	NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val);
	}
	}

	/* Make sure Virtual to Link A-LUT is disabled. */
	if (sc->alut)
	PNTX_WRITE(sc, 0xc94, 0);

	/* Enable all Link Interface LUT entries for peer. */
	for (i = 0; i < 32; i += 2) {
	PNTX_WRITE(sc, 0xdb4 + i * 2,
	0x00010001 \| ((i + 1) << 19) \| (i << 3));
	}
	}

	/*
	* Enable Virtual Interface LUT entry 0 for 0:0.*.
	* entry 1 for our Requester ID reported by the chip,
	* entries 2-5 for 0/64/128/192:4.* of I/OAT DMA engines.
	* XXX: Its a hack, we can't know all DMA engines, but this covers all
	* I/OAT of Xeon E5/E7 at least from Sandy Bridge till Skylake I saw.
	*/
	val = (NTX_READ(sc, 0xc90) << 16) \| 0x00010001;
	NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val);
	NTX_WRITE(sc, sc->link ? 0xdb8 : 0xd98, 0x40210021);
	NTX_WRITE(sc, sc->link ? 0xdbc : 0xd9c, 0xc0218021);

	/* Set Link to Virtual address translation. */
	for (i = 0; i < sc->mw_count; i++)
	ntb_plx_mw_set_trans_internal(dev, i);

	pci_enable_busmaster(dev);
	if (sc->b2b_mw >= 0)
	PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN \| PCIM_CMD_BUSMASTEREN);

	return (0);
	}

	static void
	ntb_plx_isr(void *arg)
	{
	device_t dev = arg;
	struct ntb_plx_softc *sc = device_get_softc(dev);
	uint32_t val;

	ntb_db_event((device_t)arg, 0);

	if (sc->link) /* Link Interface has no Link Error registers. */
	return;

	val = NTX_READ(sc, 0xfe0);
	if (val == 0)
	return;
	NTX_WRITE(sc, 0xfe0, val);
	if (val & 1)
	device_printf(dev, "Correctable Error\n");
	if (val & 2)
	device_printf(dev, "Uncorrectable Error\n");
	if (val & 4) {
	/* DL_Down resets link side registers, have to reinit. */
	ntb_plx_init(dev);
	ntb_link_event(dev);
	}
	if (val & 8)
	device_printf(dev, "Uncorrectable Error Message Drop\n");
	}

	static int
	ntb_plx_setup_intr(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	int error;

	/*
	* XXX: This hardware supports MSI, but I found it unusable.
	* It generates new MSI only when doorbell register goes from
	* zero, but does not generate it when another bit is set or on
	* partial clear. It makes operation very racy and unreliable.
	* The data book mentions some mask juggling magic to workaround
	* that, but I failed to make it work.
	*/
	sc->int_rid = 0;
	sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
	&sc->int_rid, RF_SHAREABLE\|RF_ACTIVE);
	if (sc->int_res == NULL) {
	device_printf(dev, "bus_alloc_resource failed\n");
	return (ENOMEM);
	}
	error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE \| INTR_TYPE_MISC,
	NULL, ntb_plx_isr, dev, &sc->int_tag);
	if (error != 0) {
	device_printf(dev, "bus_setup_intr failed: %d\n", error);
	return (error);
	}

	if (!sc->link) { /* Link Interface has no Link Error registers. */
	NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */
	NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */
	}
	return (0);
	}

	static void
	ntb_plx_teardown_intr(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	if (!sc->link) /* Link Interface has no Link Error registers. */
	NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */

	if (sc->int_res) {
	bus_teardown_intr(dev, sc->int_res, sc->int_tag);
	bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid,
	sc->int_res);
	}
	}

	static int
	ntb_plx_attach(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	int error = 0, i, j;
	uint32_t val;
	char buf[32];

	/* Identify what we are (what side of what NTx). */
	sc->dev = dev;
	val = pci_read_config(dev, 0xc8c, 4);
	sc->ntx = (val & 1) != 0;
	sc->link = (val & 0x80000000) != 0;

	/* Get access to whole 256KB of chip configuration space via BAR0/1. */
	sc->conf_rid = PCIR_BAR(0);
	sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&sc->conf_rid, RF_ACTIVE);
	if (sc->conf_res == NULL) {
	device_printf(dev, "Can't allocate configuration BAR.\n");
	return (ENXIO);
	}

	/*
	* The device occupies whole bus. In translated TLP slot field
	* keeps LUT index (original bus/slot), function is passed through.
	*/
	- bus_dma_dmar_set_buswide(dev);
	+ bus_dma_iommu_set_buswide(dev);

	/* Identify chip port we are connected to. */
	val = bus_read_4(sc->conf_res, 0x360);
	sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f;

	/* Detect A-LUT enable and size. */
	val >>= 30;
	sc->alut = (val == 0x3) ? 1 : ((val & (1 << sc->ntx)) ? 2 : 0);
	if (sc->alut)
	device_printf(dev, "%u A-LUT entries\n", 128 * sc->alut);

	/* Find configured memory windows at BAR2-5. */
	sc->mw_count = 0;
	for (i = 2; i <= 5; i++) {
	mw = &sc->mw_info[sc->mw_count];
	mw->mw_bar = i;
	mw->mw_rid = PCIR_BAR(mw->mw_bar);
	mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&mw->mw_rid, RF_ACTIVE);
	if (mw->mw_res == NULL)
	continue;
	mw->mw_pbase = rman_get_start(mw->mw_res);
	mw->mw_size = rman_get_size(mw->mw_res);
	mw->mw_vbase = rman_get_virtual(mw->mw_res);
	for (j = 0; j < PLX_MAX_SPLIT; j++)
	mw->splits[j].mw_map_mode = VM_MEMATTR_UNCACHEABLE;
	sc->mw_count++;

	/* Skip over adjacent BAR for 64-bit BARs. */
	val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4);
	if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) {
	mw->mw_64bit = 1;
	i++;
	}
	}

	/* Try to identify B2B mode. */
	i = 1;
	snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev),
	device_get_unit(dev));
	TUNABLE_INT_FETCH(buf, &i);
	if (sc->link) {
	device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n");
	sc->b2b_mw = -1;
	} else if (i == 0) {
	device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n");
	sc->b2b_mw = -1;
	} else {
	device_printf(dev, "NTB-to-NTB (back-to-back) mode\n");

	/* We need at least one memory window for B2B peer access. */
	if (sc->mw_count == 0) {
	device_printf(dev, "No memory window BARs enabled.\n");
	error = ENXIO;
	goto out;
	}
	sc->b2b_mw = sc->mw_count - 1;

	/* Use half of the window for B2B, but no less then 1MB. */
	mw = &sc->mw_info[sc->b2b_mw];
	if (mw->mw_size >= 2 * 1024 * 1024)
	sc->b2b_off = mw->mw_size / 2;
	else
	sc->b2b_off = 0;
	}

	snprintf(buf, sizeof(buf), "hint.%s.%d.split", device_get_name(dev),
	device_get_unit(dev));
	TUNABLE_INT_FETCH(buf, &sc->split);
	if (sc->split > 7) {
	device_printf(dev, "Split value is too high (%u)\n", sc->split);
	sc->split = 0;
	} else if (sc->split > 0 && sc->alut == 0) {
	device_printf(dev, "Can't split with disabled A-LUT\n");
	sc->split = 0;
	} else if (sc->split > 0 && (sc->mw_count == 0 \|\| sc->mw_info[0].mw_bar != 2)) {
	device_printf(dev, "Can't split disabled BAR2\n");
	sc->split = 0;
	} else if (sc->split > 0 && (sc->b2b_mw == 0 && sc->b2b_off == 0)) {
	device_printf(dev, "Can't split BAR2 consumed by B2B\n");
	sc->split = 0;
	} else if (sc->split > 0) {
	device_printf(dev, "Splitting BAR2 into %d memory windows\n",
	1 << sc->split);
	}

	/*
	* Use Physical Layer User Test Pattern as additional scratchpad.
	* Make sure they are present and enabled by writing to them.
	* XXX: Its a hack, but standard 8 registers are not enough.
	*/
	sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c;
	sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c;
	if (sc->b2b_mw >= 0) {
	/* In NTB-to-NTB mode each side has own scratchpads. */
	sc->spad_count1 = PLX_NUM_SPAD;
	bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678);
	if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678)
	sc->spad_count2 = PLX_NUM_SPAD_PATT;
	} else {
	/* Otherwise we have share scratchpads with the peer. */
	if (sc->link) {
	sc->spad_off1 += PLX_NUM_SPAD / 2 * 4;
	sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4;
	} else {
	sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4;
	sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4;
	}
	sc->spad_count1 = PLX_NUM_SPAD / 2;
	bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678);
	if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678)
	sc->spad_count2 = PLX_NUM_SPAD_PATT / 2;
	}

	/* Apply static part of NTB configuration. */
	ntb_plx_init(dev);

	/* Allocate and setup interrupts. */
	error = ntb_plx_setup_intr(dev);
	if (error)
	goto out;

	/* Attach children to this controller */
	error = ntb_register_device(dev);

	out:
	if (error != 0)
	ntb_plx_detach(dev);
	return (error);
	}

	static int
	ntb_plx_detach(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	int i;

	/* Detach & delete all children */
	ntb_unregister_device(dev);

	/* Disable and free interrupts. */
	ntb_plx_teardown_intr(dev);

	/* Free memory resources. */
	for (i = 0; i < sc->mw_count; i++) {
	mw = &sc->mw_info[i];
	bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid,
	mw->mw_res);
	}
	bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res);
	return (0);
	}

	static int
	ntb_plx_port_number(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	return (sc->link ? 1 : 0);
	}

	static int
	ntb_plx_peer_port_count(device_t dev)
	{

	return (1);
	}

	static int
	ntb_plx_peer_port_number(device_t dev, int pidx)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	if (pidx != 0)
	return (-EINVAL);

	return (sc->link ? 0 : 1);
	}

	static int
	ntb_plx_peer_port_idx(device_t dev, int port)
	{
	int peer_port;

	peer_port = ntb_plx_peer_port_number(dev, 0);
	if (peer_port == -EINVAL \|\| port != peer_port)
	return (-EINVAL);

	return (0);
	}

	static bool
	ntb_plx_link_is_up(device_t dev, enum ntb_speed speed, enum ntb_width width)
	{
	uint16_t link;

	link = pcie_read_config(dev, PCIER_LINK_STA, 2);
	if (speed != NULL)
	*speed = (link & PCIEM_LINK_STA_SPEED);
	if (width != NULL)
	*width = (link & PCIEM_LINK_STA_WIDTH) >> 4;
	return ((link & PCIEM_LINK_STA_WIDTH) != 0);
	}

	static int
	ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused,
	enum ntb_width width __unused)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	uint32_t reg, val;

	/* The fact that we see the Link Interface means link is enabled. */
	if (sc->link) {
	ntb_link_event(dev);
	return (0);
	}

	reg = PLX_PORT_CONTROL(sc);
	val = bus_read_4(sc->conf_res, reg);
	if ((val & (1 << (sc->port & 7))) == 0) {
	/* If already enabled, generate fake link event and exit. */
	ntb_link_event(dev);
	return (0);
	}
	val &= ~(1 << (sc->port & 7));
	bus_write_4(sc->conf_res, reg, val);
	return (0);
	}

	static int
	ntb_plx_link_disable(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	uint32_t reg, val;

	/* Link disable for Link Interface would be suicidal. */
	if (sc->link)
	return (0);

	reg = PLX_PORT_CONTROL(sc);
	val = bus_read_4(sc->conf_res, reg);
	val \|= (1 << (sc->port & 7));
	bus_write_4(sc->conf_res, reg, val);
	return (0);
	}

	static bool
	ntb_plx_link_enabled(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	uint32_t reg, val;

	/* The fact that we see the Link Interface means link is enabled. */
	if (sc->link)
	return (TRUE);

	reg = PLX_PORT_CONTROL(sc);
	val = bus_read_4(sc->conf_res, reg);
	return ((val & (1 << (sc->port & 7))) == 0);
	}

	static uint8_t
	ntb_plx_mw_count(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	uint8_t res;

	res = sc->mw_count;
	res += (1 << sc->split) - 1;
	if (sc->b2b_mw >= 0 && sc->b2b_off == 0)
	res--; /* B2B consumed whole window. */
	return (res);
	}

	static unsigned
	ntb_plx_user_mw_to_idx(struct ntb_plx_softc sc, unsigned uidx, unsigned sp)
	{
	unsigned t;

	t = 1 << sc->split;
	if (uidx < t) {
	*sp = uidx;
	return (0);
	}
	*sp = 0;
	return (uidx - (t - 1));
	}

	static int
	ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base,
	caddr_t vbase, size_t size, size_t align, size_t align_size,
	bus_addr_t *plimit)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	size_t off, ss;
	unsigned sp, split;

	mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp);
	if (mw_idx >= sc->mw_count)
	return (EINVAL);
	off = 0;
	if (mw_idx == sc->b2b_mw) {
	KASSERT(sc->b2b_off != 0,
	("user shouldn't get non-shared b2b mw"));
	off = sc->b2b_off;
	}
	mw = &sc->mw_info[mw_idx];
	split = (mw->mw_bar == 2) ? sc->split : 0;
	ss = (mw->mw_size - off) >> split;

	/* Local to remote memory window parameters. */
	if (base != NULL)
	base = mw->mw_pbase + off + ss sp;
	if (vbase != NULL)
	vbase = mw->mw_vbase + off + ss sp;
	if (size != NULL)
	*size = ss;

	/*
	* Remote to local memory window translation address alignment.
	* Translation address has to be aligned to the BAR size, but A-LUT
	* entries re-map addresses can be aligned to 1/128 or 1/256 of it.
	* XXX: In B2B mode we can change BAR size (and so alignmet) live,
	* but there is no way to report it here, so report safe value.
	*/
	if (align != NULL) {
	if (sc->alut && mw->mw_bar == 2)
	*align = (mw->mw_size - off) / 128 / sc->alut;
	else
	*align = mw->mw_size - off;
	}

	/*
	* Remote to local memory window size alignment.
	* The chip has no limit registers, but A-LUT, when available, allows
	* access control with granularity of 1/128 or 1/256 of the BAR size.
	* XXX: In B2B case we can change BAR size live, but there is no way
	* to report it, so report half of the BAR size, that should be safe.
	* In non-B2B case there is no control at all, so report the BAR size.
	*/
	if (align_size != NULL) {
	if (sc->alut && mw->mw_bar == 2)
	*align_size = (mw->mw_size - off) / 128 / sc->alut;
	else if (sc->b2b_mw >= 0)
	*align_size = (mw->mw_size - off) / 2;
	else
	*align_size = mw->mw_size - off;
	}

	/* Remote to local memory window translation address upper limit. */
	if (plimit != NULL)
	*plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR :
	BUS_SPACE_MAXADDR_32BIT;
	return (0);
	}

	static int
	ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	uint64_t addr, eaddr, off, size, bsize, esize, val64;
	uint32_t val;
	unsigned i, sp, split;

	mw = &sc->mw_info[mw_idx];
	off = (mw_idx == sc->b2b_mw) ? sc->b2b_off : 0;
	split = (mw->mw_bar == 2) ? sc->split : 0;

	/* Get BAR size. In case of split or B2RP we can't change it. */
	if (split \|\| sc->b2b_mw < 0) {
	bsize = mw->mw_size - off;
	} else {
	bsize = mw->splits[0].mw_xlat_size;
	if (!powerof2(bsize))
	bsize = 1LL << flsll(bsize);
	if (bsize > 0 && bsize < 1024 * 1024)
	bsize = 1024 * 1024;
	}

	/*
	* While for B2B we can set any BAR size on a link side, for shared
	* window we can't go above preconfigured size due to BAR address
	* alignment requirements.
	*/
	if ((off & (bsize - 1)) != 0)
	return (EINVAL);

	/* In B2B mode set Link Interface BAR size/address. */
	if (sc->b2b_mw >= 0 && mw->mw_64bit) {
	val64 = 0;
	if (bsize > 0)
	val64 = (~(bsize - 1) & ~0xfffff);
	val64 \|= 0xc;
	PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64);
	PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32);

	val64 = 0x2000000000000000 * mw->mw_bar + off;
	PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64);
	PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32);
	} else if (sc->b2b_mw >= 0) {
	val = 0;
	if (bsize > 0)
	val = (~(bsize - 1) & ~0xfffff);
	PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val);

	val64 = 0x20000000 * mw->mw_bar + off;
	PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64);
	}

	/* Set BARs address translation */
	addr = split ? UINT64_MAX : mw->splits[0].mw_xlat_addr;
	if (mw->mw_64bit) {
	PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr);
	PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32);
	} else {
	PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr);
	}

	/* Configure and enable A-LUT if we need it. */
	size = split ? 0 : mw->splits[0].mw_xlat_size;
	if (sc->alut && mw->mw_bar == 2 && (sc->split > 0 \|\|
	((addr & (bsize - 1)) != 0 \|\| size != bsize))) {
	esize = bsize / (128 * sc->alut);
	for (i = sp = 0; i < 128 * sc->alut; i++) {
	if (i % (128 * sc->alut >> sc->split) == 0) {
	eaddr = addr = mw->splits[sp].mw_xlat_addr;
	size = mw->splits[sp++].mw_xlat_size;
	}
	val = sc->link ? 0 : 1;
	if (sc->alut == 1)
	val += 2 * sc->ntx;
	val = 0x1000 sc->alut;
	val += 0x38000 + i * 4 + (i >= 128 ? 0x0e00 : 0);
	bus_write_4(sc->conf_res, val, eaddr);
	bus_write_4(sc->conf_res, val + 0x400, eaddr >> 32);
	bus_write_4(sc->conf_res, val + 0x800,
	(eaddr < addr + size) ? 0x3 : 0);
	eaddr += esize;
	}
	NTX_WRITE(sc, 0xc94, 0x10000000);
	} else if (sc->alut && mw->mw_bar == 2)
	NTX_WRITE(sc, 0xc94, 0);

	return (0);
	}

	static int
	ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	unsigned sp;

	mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp);
	if (mw_idx >= sc->mw_count)
	return (EINVAL);
	mw = &sc->mw_info[mw_idx];
	if (!mw->mw_64bit &&
	((addr & UINT32_MAX) != addr \|\|
	((addr + size) & UINT32_MAX) != (addr + size)))
	return (ERANGE);
	mw->splits[sp].mw_xlat_addr = addr;
	mw->splits[sp].mw_xlat_size = size;
	return (ntb_plx_mw_set_trans_internal(dev, mw_idx));
	}

	static int
	ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx)
	{

	return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0));
	}

	static int
	ntb_plx_mw_get_wc(device_t dev, unsigned mw_idx, vm_memattr_t *mode)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	unsigned sp;

	mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp);
	if (mw_idx >= sc->mw_count)
	return (EINVAL);
	mw = &sc->mw_info[mw_idx];
	*mode = mw->splits[sp].mw_map_mode;
	return (0);
	}

	static int
	ntb_plx_mw_set_wc(device_t dev, unsigned mw_idx, vm_memattr_t mode)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;
	uint64_t off, ss;
	int rc;
	unsigned sp, split;

	mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp);
	if (mw_idx >= sc->mw_count)
	return (EINVAL);
	mw = &sc->mw_info[mw_idx];
	if (mw->splits[sp].mw_map_mode == mode)
	return (0);

	off = 0;
	if (mw_idx == sc->b2b_mw) {
	KASSERT(sc->b2b_off != 0,
	("user shouldn't get non-shared b2b mw"));
	off = sc->b2b_off;
	}

	split = (mw->mw_bar == 2) ? sc->split : 0;
	ss = (mw->mw_size - off) >> split;
	rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off + ss * sp,
	ss, mode);
	if (rc == 0)
	mw->splits[sp].mw_map_mode = mode;
	return (rc);
	}

	static uint8_t
	ntb_plx_spad_count(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	return (sc->spad_count1 + sc->spad_count2);
	}

	static int
	ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	u_int off;

	if (idx >= sc->spad_count1 + sc->spad_count2)
	return (EINVAL);

	if (idx < sc->spad_count1)
	off = sc->spad_off1 + idx * 4;
	else
	off = sc->spad_off2 + (idx - sc->spad_count1) * 4;
	bus_write_4(sc->conf_res, off, val);
	return (0);
	}

	static void
	ntb_plx_spad_clear(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	int i;

	for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++)
	ntb_plx_spad_write(dev, i, 0);
	}

	static int
	ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	u_int off;

	if (idx >= sc->spad_count1 + sc->spad_count2)
	return (EINVAL);

	if (idx < sc->spad_count1)
	off = sc->spad_off1 + idx * 4;
	else
	off = sc->spad_off2 + (idx - sc->spad_count1) * 4;
	*val = bus_read_4(sc->conf_res, off);
	return (0);
	}

	static int
	ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	u_int off;

	if (idx >= sc->spad_count1 + sc->spad_count2)
	return (EINVAL);

	if (idx < sc->spad_count1)
	off = sc->spad_offp1 + idx * 4;
	else
	off = sc->spad_offp2 + (idx - sc->spad_count1) * 4;
	if (sc->b2b_mw >= 0)
	bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val);
	else
	bus_write_4(sc->conf_res, off, val);
	return (0);
	}

	static int
	ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	u_int off;

	if (idx >= sc->spad_count1 + sc->spad_count2)
	return (EINVAL);

	if (idx < sc->spad_count1)
	off = sc->spad_offp1 + idx * 4;
	else
	off = sc->spad_offp2 + (idx - sc->spad_count1) * 4;
	if (sc->b2b_mw >= 0)
	*val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off);
	else
	*val = bus_read_4(sc->conf_res, off);
	return (0);
	}

	static uint64_t
	ntb_plx_db_valid_mask(device_t dev)
	{

	return ((1LL << PLX_NUM_DB) - 1);
	}

	static int
	ntb_plx_db_vector_count(device_t dev)
	{

	return (1);
	}

	static uint64_t
	ntb_plx_db_vector_mask(device_t dev, uint32_t vector)
	{

	if (vector > 0)
	return (0);
	return ((1LL << PLX_NUM_DB) - 1);
	}

	static void
	ntb_plx_db_clear(device_t dev, uint64_t bits)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits);
	}

	static void
	ntb_plx_db_clear_mask(device_t dev, uint64_t bits)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits);
	}

	static uint64_t
	ntb_plx_db_read(device_t dev)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c));
	}

	static void
	ntb_plx_db_set_mask(device_t dev, uint64_t bits)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits);
	}

	static int
	ntb_plx_peer_db_addr(device_t dev, bus_addr_t db_addr, vm_size_t db_size)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);
	struct ntb_plx_mw_info *mw;

	KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL"));

	if (sc->b2b_mw >= 0) {
	mw = &sc->mw_info[sc->b2b_mw];
	*db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c;
	} else {
	*db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc);
	*db_addr += sc->link ? 0xc4c : 0xc5c;
	}
	*db_size = 4;
	return (0);
	}

	static void
	ntb_plx_peer_db_set(device_t dev, uint64_t bit)
	{
	struct ntb_plx_softc *sc = device_get_softc(dev);

	if (sc->b2b_mw >= 0)
	BNTX_WRITE(sc, 0xc4c, bit);
	else
	NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit);
	}

	static device_method_t ntb_plx_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, ntb_plx_probe),
	DEVMETHOD(device_attach, ntb_plx_attach),
	DEVMETHOD(device_detach, ntb_plx_detach),
	/* Bus interface */
	DEVMETHOD(bus_child_location_str, ntb_child_location_str),
	DEVMETHOD(bus_print_child, ntb_print_child),
	DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag),
	/* NTB interface */
	DEVMETHOD(ntb_port_number, ntb_plx_port_number),
	DEVMETHOD(ntb_peer_port_count, ntb_plx_peer_port_count),
	DEVMETHOD(ntb_peer_port_number, ntb_plx_peer_port_number),
	DEVMETHOD(ntb_peer_port_idx, ntb_plx_peer_port_idx),
	DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up),
	DEVMETHOD(ntb_link_enable, ntb_plx_link_enable),
	DEVMETHOD(ntb_link_disable, ntb_plx_link_disable),
	DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled),
	DEVMETHOD(ntb_mw_count, ntb_plx_mw_count),
	DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range),
	DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans),
	DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans),
	DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc),
	DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc),
	DEVMETHOD(ntb_spad_count, ntb_plx_spad_count),
	DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear),
	DEVMETHOD(ntb_spad_write, ntb_plx_spad_write),
	DEVMETHOD(ntb_spad_read, ntb_plx_spad_read),
	DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write),
	DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read),
	DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask),
	DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count),
	DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask),
	DEVMETHOD(ntb_db_clear, ntb_plx_db_clear),
	DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask),
	DEVMETHOD(ntb_db_read, ntb_plx_db_read),
	DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask),
	DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr),
	DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set),
	DEVMETHOD_END
	};

	static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods,
	sizeof(struct ntb_plx_softc));
	DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, ntb_hw_devclass, NULL, NULL);
	MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1);
	MODULE_VERSION(ntb_hw_plx, 1);
	diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
	index e740caa7fc8a..379ad427b6b6 100644
	--- a/sys/dev/pci/pci.c
	+++ b/sys/dev/pci/pci.c
	@@ -1,6630 +1,6691 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 1997, Stefan Esser <se@freebsd.org>
	* Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
	* Copyright (c) 2000, BSDi
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice unmodified, this list of conditions, and the following
	* disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_acpi.h"
	#include "opt_bus.h"

	#include <sys/param.h>
	#include <sys/conf.h>
	#include <sys/endian.h>
	#include <sys/eventhandler.h>
	#include <sys/fcntl.h>
	#include <sys/kernel.h>
	#include <sys/limits.h>
	#include <sys/linker.h>
	#include <sys/malloc.h>
	#include <sys/module.h>
	#include <sys/queue.h>
	#include <sys/sysctl.h>
	#include <sys/systm.h>

	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <vm/vm_extern.h>

	#include <sys/bus.h>
	#include <machine/bus.h>
	#include <sys/rman.h>
	#include <machine/resource.h>
	#include <machine/stdarg.h>

	#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__powerpc__)
	#include <machine/intr_machdep.h>
	#endif

	#include <sys/pciio.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	#include <dev/pci/pci_private.h>

	#ifdef PCI_IOV
	#include <sys/nv.h>
	#include <dev/pci/pci_iov_private.h>
	#endif

	#include <dev/usb/controller/xhcireg.h>
	#include <dev/usb/controller/ehcireg.h>
	#include <dev/usb/controller/ohcireg.h>
	#include <dev/usb/controller/uhcireg.h>

	#include "pcib_if.h"
	#include "pci_if.h"

	#define PCIR_IS_BIOS(cfg, reg) \
	(((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) \|\| \
	((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))

	static int pci_has_quirk(uint32_t devid, int quirk);
	static pci_addr_t pci_mapbase(uint64_t mapreg);
	static const char *pci_maptype(uint64_t mapreg);
	static int pci_maprange(uint64_t mapreg);
	static pci_addr_t pci_rombase(uint64_t mapreg);
	static int pci_romsize(uint64_t testval);
	static void pci_fixancient(pcicfgregs *cfg);
	static int pci_printf(pcicfgregs cfg, const char fmt, ...);

	static int pci_porten(device_t dev);
	static int pci_memen(device_t dev);
	static void pci_assign_interrupt(device_t bus, device_t dev,
	int force_route);
	static int pci_add_map(device_t bus, device_t dev, int reg,
	struct resource_list *rl, int force, int prefetch);
	static int pci_probe(device_t dev);
	static void pci_load_vendor_data(void);
	static int pci_describe_parse_line(char *ptr, int vendor,
	int device, char *desc);
	static char *pci_describe_device(device_t dev);
	static int pci_modevent(module_t mod, int what, void *arg);
	static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
	pcicfgregs *cfg);
	static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
	static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
	int reg, uint32_t *data);
	#if 0
	static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
	int reg, uint32_t data);
	#endif
	static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
	static void pci_mask_msix(device_t dev, u_int index);
	static void pci_unmask_msix(device_t dev, u_int index);
	static int pci_msi_blacklisted(void);
	static int pci_msix_blacklisted(void);
	static void pci_resume_msi(device_t dev);
	static void pci_resume_msix(device_t dev);
	static int pci_remap_intr_method(device_t bus, device_t dev,
	u_int irq);
	static void pci_hint_device_unit(device_t acdev, device_t child,
	const char name, int unitp);
	static int pci_reset_post(device_t dev, device_t child);
	static int pci_reset_prepare(device_t dev, device_t child);
	static int pci_reset_child(device_t dev, device_t child,
	int flags);

	static int pci_get_id_method(device_t dev, device_t child,
	enum pci_id_type type, uintptr_t *rid);

	static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
	int b, int s, int f, uint16_t vid, uint16_t did);

	static device_method_t pci_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, pci_probe),
	DEVMETHOD(device_attach, pci_attach),
	DEVMETHOD(device_detach, pci_detach),
	DEVMETHOD(device_shutdown, bus_generic_shutdown),
	DEVMETHOD(device_suspend, bus_generic_suspend),
	DEVMETHOD(device_resume, pci_resume),

	/* Bus interface */
	DEVMETHOD(bus_print_child, pci_print_child),
	DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
	DEVMETHOD(bus_read_ivar, pci_read_ivar),
	DEVMETHOD(bus_write_ivar, pci_write_ivar),
	DEVMETHOD(bus_driver_added, pci_driver_added),
	DEVMETHOD(bus_setup_intr, pci_setup_intr),
	DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
	DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
	DEVMETHOD(bus_reset_post, pci_reset_post),
	DEVMETHOD(bus_reset_child, pci_reset_child),

	DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
	DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
	DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
	DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
	DEVMETHOD(bus_delete_resource, pci_delete_resource),
	DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
	DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
	DEVMETHOD(bus_release_resource, pci_release_resource),
	DEVMETHOD(bus_activate_resource, pci_activate_resource),
	DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
	DEVMETHOD(bus_child_deleted, pci_child_deleted),
	DEVMETHOD(bus_child_detached, pci_child_detached),
	DEVMETHOD(bus_child_pnpinfo_str, pci_child_pnpinfo_str_method),
	DEVMETHOD(bus_child_location_str, pci_child_location_str_method),
	DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
	DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
	DEVMETHOD(bus_suspend_child, pci_suspend_child),
	DEVMETHOD(bus_resume_child, pci_resume_child),
	DEVMETHOD(bus_rescan, pci_rescan_method),

	/* PCI interface */
	DEVMETHOD(pci_read_config, pci_read_config_method),
	DEVMETHOD(pci_write_config, pci_write_config_method),
	DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
	DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
	DEVMETHOD(pci_enable_io, pci_enable_io_method),
	DEVMETHOD(pci_disable_io, pci_disable_io_method),
	DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
	DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
	DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
	DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
	DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
	DEVMETHOD(pci_find_cap, pci_find_cap_method),
	DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
	DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
	DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
	DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
	DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
	DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
	DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
	DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
	DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
	DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
	DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
	DEVMETHOD(pci_release_msi, pci_release_msi_method),
	DEVMETHOD(pci_msi_count, pci_msi_count_method),
	DEVMETHOD(pci_msix_count, pci_msix_count_method),
	DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
	DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
	DEVMETHOD(pci_get_id, pci_get_id_method),
	DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
	DEVMETHOD(pci_child_added, pci_child_added_method),
	#ifdef PCI_IOV
	DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
	DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
	DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
	#endif

	DEVMETHOD_END
	};

	DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));

	static devclass_t pci_devclass;
	EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, NULL,
	BUS_PASS_BUS);
	MODULE_VERSION(pci, 1);

	static char *pci_vendordata;
	static size_t pci_vendordata_size;

	struct pci_quirk {
	uint32_t devid; /* Vendor/device of the card */
	int type;
	#define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
	#define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
	#define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
	#define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
	#define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
	#define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
	#define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
	int arg1;
	int arg2;
	};

	static const struct pci_quirk pci_quirks[] = {
	/* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
	{ 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
	{ 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
	/* As does the Serverworks OSB4 (the SMBus mapping register) */
	{ 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },

	/*
	* MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
	* or the CMIC-SL (AKA ServerWorks GC_LE).
	*/
	{ 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },

	/*
	* MSI doesn't work on earlier Intel chipsets including
	* E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
	*/
	{ 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
	{ 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },

	/*
	* MSI doesn't work with devices behind the AMD 8131 HT-PCIX
	* bridge.
	*/
	{ 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },

	/*
	* Some virtualization environments emulate an older chipset
	* but support MSI just fine. QEMU uses the Intel 82440.
	*/
	{ 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },

	/*
	* HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
	* controller depending on SoftPciRst register (PM_IO 0x55 [7]).
	* It prevents us from attaching hpet(4) when the bit is unset.
	* Note this quirk only affects SB600 revision A13 and earlier.
	* For SB600 A21 and later, firmware must set the bit to hide it.
	* For SB700 and later, it is unused and hardcoded to zero.
	*/
	{ 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },

	/*
	* Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
	* a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
	* of the command register is set.
	*/
	{ 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
	{ 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
	{ 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
	{ 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
	{ 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },

	/*
	* Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
	* issue MSI interrupts with PCIM_CMD_INTxDIS set either.
	*/
	{ 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
	{ 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
	{ 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
	{ 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
	{ 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
	{ 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */

	/*
	* HPE Gen 10 VGA has a memory range that can't be allocated in the
	* expected place.
	*/
	{ 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },

	{ 0 }
	};

	/* map register information */
	#define PCI_MAPMEM 0x01 /* memory map */
	#define PCI_MAPMEMP 0x02 /* prefetchable memory map */
	#define PCI_MAPPORT 0x04 /* port map */

	struct devlist pci_devq;
	uint32_t pci_generation;
	uint32_t pci_numdevs = 0;
	static int pcie_chipset, pcix_chipset;

	/* sysctl vars */
	SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"PCI bus tuning parameters");

	static int pci_enable_io_modes = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
	&pci_enable_io_modes, 1,
	"Enable I/O and memory bits in the config register. Some BIOSes do not"
	" enable these bits correctly. We'd like to do this all the time, but"
	" there are some peripherals that this causes problems with.");

	static int pci_do_realloc_bars = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
	&pci_do_realloc_bars, 0,
	"Attempt to allocate a new range for any BARs whose original "
	"firmware-assigned ranges fail to allocate during the initial device scan.");

	static int pci_do_power_nodriver = 0;
	SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
	&pci_do_power_nodriver, 0,
	"Place a function into D3 state when no driver attaches to it. 0 means"
	" disable. 1 means conservatively place devices into D3 state. 2 means"
	" aggressively place devices into D3 state. 3 means put absolutely"
	" everything in D3 state.");

	int pci_do_power_resume = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
	&pci_do_power_resume, 1,
	"Transition from D3 -> D0 on resume.");

	int pci_do_power_suspend = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
	&pci_do_power_suspend, 1,
	"Transition from D0 -> D3 on suspend.");

	static int pci_do_msi = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
	"Enable support for MSI interrupts");

	static int pci_do_msix = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
	"Enable support for MSI-X interrupts");

	static int pci_msix_rewrite_table = 0;
	SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
	&pci_msix_rewrite_table, 0,
	"Rewrite entire MSI-X table when updating MSI-X entries");

	static int pci_honor_msi_blacklist = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
	&pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");

	#if defined(__i386__) \|\| defined(__amd64__)
	static int pci_usb_takeover = 1;
	#else
	static int pci_usb_takeover = 0;
	#endif
	SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
	&pci_usb_takeover, 1,
	"Enable early takeover of USB controllers. Disable this if you depend on"
	" BIOS emulation of USB devices, that is you use USB devices (like"
	" keyboard or mouse) but do not load USB drivers");

	static int pci_clear_bars;
	SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
	"Ignore firmware-assigned resources for BARs.");

	#if defined(NEW_PCIB) && defined(PCI_RES_BUS)
	static int pci_clear_buses;
	SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
	"Ignore firmware-assigned bus numbers.");
	#endif

	static int pci_enable_ari = 1;
	SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
	0, "Enable support for PCIe Alternative RID Interpretation");

	int pci_enable_aspm;
	SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
	0, "Enable support for PCIe Active State Power Management");

	static int pci_clear_aer_on_attach = 0;
	SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
	&pci_clear_aer_on_attach, 0,
	"Clear port and device AER state on driver attach");

	static int
	pci_has_quirk(uint32_t devid, int quirk)
	{
	const struct pci_quirk *q;

	for (q = &pci_quirks[0]; q->devid; q++) {
	if (q->devid == devid && q->type == quirk)
	return (1);
	}
	return (0);
	}

	/* Find a device_t by bus/slot/function in domain 0 */

	device_t
	pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
	{

	return (pci_find_dbsf(0, bus, slot, func));
	}

	/* Find a device_t by domain/bus/slot/function */

	device_t
	pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
	{
	struct pci_devinfo *dinfo = NULL;

	STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
	if ((dinfo->cfg.domain == domain) &&
	(dinfo->cfg.bus == bus) &&
	(dinfo->cfg.slot == slot) &&
	(dinfo->cfg.func == func)) {
	break;
	}
	}

	return (dinfo != NULL ? dinfo->cfg.dev : NULL);
	}

	/* Find a device_t by vendor/device ID */

	device_t
	pci_find_device(uint16_t vendor, uint16_t device)
	{
	struct pci_devinfo *dinfo;

	STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
	if ((dinfo->cfg.vendor == vendor) &&
	(dinfo->cfg.device == device)) {
	return (dinfo->cfg.dev);
	}
	}

	return (NULL);
	}

	device_t
	pci_find_class(uint8_t class, uint8_t subclass)
	{
	struct pci_devinfo *dinfo;

	STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
	if (dinfo->cfg.baseclass == class &&
	dinfo->cfg.subclass == subclass) {
	return (dinfo->cfg.dev);
	}
	}

	return (NULL);
	}

	static int
	pci_printf(pcicfgregs cfg, const char fmt, ...)
	{
	va_list ap;
	int retval;

	retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
	cfg->func);
	va_start(ap, fmt);
	retval += vprintf(fmt, ap);
	va_end(ap);
	return (retval);
	}

	/* return base address of memory or port map */

	static pci_addr_t
	pci_mapbase(uint64_t mapreg)
	{

	if (PCI_BAR_MEM(mapreg))
	return (mapreg & PCIM_BAR_MEM_BASE);
	else
	return (mapreg & PCIM_BAR_IO_BASE);
	}

	/* return map type of memory or port map */

	static const char *
	pci_maptype(uint64_t mapreg)
	{

	if (PCI_BAR_IO(mapreg))
	return ("I/O Port");
	if (mapreg & PCIM_BAR_MEM_PREFETCH)
	return ("Prefetchable Memory");
	return ("Memory");
	}

	/* return log2 of map size decoded for memory or port map */

	int
	pci_mapsize(uint64_t testval)
	{
	int ln2size;

	testval = pci_mapbase(testval);
	ln2size = 0;
	if (testval != 0) {
	while ((testval & 1) == 0)
	{
	ln2size++;
	testval >>= 1;
	}
	}
	return (ln2size);
	}

	/* return base address of device ROM */

	static pci_addr_t
	pci_rombase(uint64_t mapreg)
	{

	return (mapreg & PCIM_BIOS_ADDR_MASK);
	}

	/* return log2 of map size decided for device ROM */

	static int
	pci_romsize(uint64_t testval)
	{
	int ln2size;

	testval = pci_rombase(testval);
	ln2size = 0;
	if (testval != 0) {
	while ((testval & 1) == 0)
	{
	ln2size++;
	testval >>= 1;
	}
	}
	return (ln2size);
	}

	/* return log2 of address range supported by map register */

	static int
	pci_maprange(uint64_t mapreg)
	{
	int ln2range = 0;

	if (PCI_BAR_IO(mapreg))
	ln2range = 32;
	else
	switch (mapreg & PCIM_BAR_MEM_TYPE) {
	case PCIM_BAR_MEM_32:
	ln2range = 32;
	break;
	case PCIM_BAR_MEM_1MB:
	ln2range = 20;
	break;
	case PCIM_BAR_MEM_64:
	ln2range = 64;
	break;
	}
	return (ln2range);
	}

	/* adjust some values from PCI 1.0 devices to match 2.0 standards ... */

	static void
	pci_fixancient(pcicfgregs *cfg)
	{
	if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
	return;

	/* PCI to PCI bridges use header type 1 */
	if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
	cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
	}

	/* extract header type specific config data */

	static void
	pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
	{
	#define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
	switch (cfg->hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_NORMAL:
	cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
	cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
	cfg->mingnt = REG(PCIR_MINGNT, 1);
	cfg->maxlat = REG(PCIR_MAXLAT, 1);
	cfg->nummaps = PCI_MAXMAPS_0;
	break;
	case PCIM_HDRTYPE_BRIDGE:
	cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
	cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
	cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
	cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
	cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
	cfg->nummaps = PCI_MAXMAPS_1;
	break;
	case PCIM_HDRTYPE_CARDBUS:
	cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
	cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
	cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
	cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
	cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
	cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
	cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
	cfg->nummaps = PCI_MAXMAPS_2;
	break;
	}
	#undef REG
	}

	/* read configuration header into pcicfgregs structure */
	struct pci_devinfo *
	pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
	{
	#define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
	uint16_t vid, did;

	vid = REG(PCIR_VENDOR, 2);
	did = REG(PCIR_DEVICE, 2);
	if (vid != 0xffff)
	return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));

	return (NULL);
	}

	struct pci_devinfo *
	pci_alloc_devinfo_method(device_t dev)
	{

	return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
	M_WAITOK \| M_ZERO));
	}

	static struct pci_devinfo *
	pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
	uint16_t vid, uint16_t did)
	{
	struct pci_devinfo *devlist_entry;
	pcicfgregs *cfg;

	devlist_entry = PCI_ALLOC_DEVINFO(bus);

	cfg = &devlist_entry->cfg;

	cfg->domain = d;
	cfg->bus = b;
	cfg->slot = s;
	cfg->func = f;
	cfg->vendor = vid;
	cfg->device = did;
	cfg->cmdreg = REG(PCIR_COMMAND, 2);
	cfg->statreg = REG(PCIR_STATUS, 2);
	cfg->baseclass = REG(PCIR_CLASS, 1);
	cfg->subclass = REG(PCIR_SUBCLASS, 1);
	cfg->progif = REG(PCIR_PROGIF, 1);
	cfg->revid = REG(PCIR_REVID, 1);
	cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
	cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
	cfg->lattimer = REG(PCIR_LATTIMER, 1);
	cfg->intpin = REG(PCIR_INTPIN, 1);
	cfg->intline = REG(PCIR_INTLINE, 1);

	cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
	cfg->hdrtype &= ~PCIM_MFDEV;
	STAILQ_INIT(&cfg->maps);

	cfg->iov = NULL;

	pci_fixancient(cfg);
	pci_hdrtypedata(pcib, b, s, f, cfg);

	if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
	pci_read_cap(pcib, cfg);

	STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);

	devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
	devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
	devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
	devlist_entry->conf.pc_sel.pc_func = cfg->func;
	devlist_entry->conf.pc_hdr = cfg->hdrtype;

	devlist_entry->conf.pc_subvendor = cfg->subvendor;
	devlist_entry->conf.pc_subdevice = cfg->subdevice;
	devlist_entry->conf.pc_vendor = cfg->vendor;
	devlist_entry->conf.pc_device = cfg->device;

	devlist_entry->conf.pc_class = cfg->baseclass;
	devlist_entry->conf.pc_subclass = cfg->subclass;
	devlist_entry->conf.pc_progif = cfg->progif;
	devlist_entry->conf.pc_revid = cfg->revid;

	pci_numdevs++;
	pci_generation++;

	return (devlist_entry);
	}
	#undef REG

	static void
	pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
	{
	#define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
	cfg->ea.ea_location + (n), w)
	int num_ent;
	int ptr;
	int a, b;
	uint32_t val;
	int ent_size;
	uint32_t dw[4];
	uint64_t base, max_offset;
	struct pci_ea_entry *eae;

	if (cfg->ea.ea_location == 0)
	return;

	STAILQ_INIT(&cfg->ea.ea_entries);

	/* Determine the number of entries */
	num_ent = REG(PCIR_EA_NUM_ENT, 2);
	num_ent &= PCIM_EA_NUM_ENT_MASK;

	/* Find the first entry to care of */
	ptr = PCIR_EA_FIRST_ENT;

	/* Skip DWORD 2 for type 1 functions */
	if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
	ptr += 4;

	for (a = 0; a < num_ent; a++) {

	eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK \| M_ZERO);
	eae->eae_cfg_offset = cfg->ea.ea_location + ptr;

	/* Read a number of dwords in the entry */
	val = REG(ptr, 4);
	ptr += 4;
	ent_size = (val & PCIM_EA_ES);

	for (b = 0; b < ent_size; b++) {
	dw[b] = REG(ptr, 4);
	ptr += 4;
	}

	eae->eae_flags = val;
	eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;

	base = dw[0] & PCIM_EA_FIELD_MASK;
	max_offset = dw[1] \| ~PCIM_EA_FIELD_MASK;
	b = 2;
	if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
	base \|= (uint64_t)dw[b] << 32UL;
	b++;
	}
	if (((dw[1] & PCIM_EA_IS_64) != 0)
	&& (b < ent_size)) {
	max_offset \|= (uint64_t)dw[b] << 32UL;
	b++;
	}

	eae->eae_base = base;
	eae->eae_max_offset = max_offset;

	STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);

	if (bootverbose) {
	printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
	cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
	(uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
	}
	}
	}
	#undef REG

	static void
	pci_read_cap(device_t pcib, pcicfgregs *cfg)
	{
	#define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
	#define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
	#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__powerpc__)
	uint64_t addr;
	#endif
	uint32_t val;
	int ptr, nextptr, ptrptr;

	switch (cfg->hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_NORMAL:
	case PCIM_HDRTYPE_BRIDGE:
	ptrptr = PCIR_CAP_PTR;
	break;
	case PCIM_HDRTYPE_CARDBUS:
	ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
	break;
	default:
	return; /* no extended capabilities support */
	}
	nextptr = REG(ptrptr, 1); /* sanity check? */

	/*
	* Read capability entries.
	*/
	while (nextptr != 0) {
	/* Sanity check */
	if (nextptr > 255) {
	printf("illegal PCI extended capability offset %d\n",
	nextptr);
	return;
	}
	/* Find the next entry */
	ptr = nextptr;
	nextptr = REG(ptr + PCICAP_NEXTPTR, 1);

	/* Process this entry */
	switch (REG(ptr + PCICAP_ID, 1)) {
	case PCIY_PMG: /* PCI power management */
	if (cfg->pp.pp_cap == 0) {
	cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
	cfg->pp.pp_status = ptr + PCIR_POWER_STATUS;
	cfg->pp.pp_bse = ptr + PCIR_POWER_BSE;
	if ((nextptr - ptr) > PCIR_POWER_DATA)
	cfg->pp.pp_data = ptr + PCIR_POWER_DATA;
	}
	break;
	case PCIY_HT: /* HyperTransport */
	/* Determine HT-specific capability type. */
	val = REG(ptr + PCIR_HT_COMMAND, 2);

	if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
	cfg->ht.ht_slave = ptr;

	#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__powerpc__)
	switch (val & PCIM_HTCMD_CAP_MASK) {
	case PCIM_HTCAP_MSI_MAPPING:
	if (!(val & PCIM_HTCMD_MSI_FIXED)) {
	/* Sanity check the mapping window. */
	addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
	4);
	addr <<= 32;
	addr \|= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
	4);
	if (addr != MSI_INTEL_ADDR_BASE)
	device_printf(pcib,
	"HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
	cfg->domain, cfg->bus,
	cfg->slot, cfg->func,
	(long long)addr);
	} else
	addr = MSI_INTEL_ADDR_BASE;

	cfg->ht.ht_msimap = ptr;
	cfg->ht.ht_msictrl = val;
	cfg->ht.ht_msiaddr = addr;
	break;
	}
	#endif
	break;
	case PCIY_MSI: /* PCI MSI */
	cfg->msi.msi_location = ptr;
	cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
	cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl &
	PCIM_MSICTRL_MMC_MASK)>>1);
	break;
	case PCIY_MSIX: /* PCI MSI-X */
	cfg->msix.msix_location = ptr;
	cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
	cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl &
	PCIM_MSIXCTRL_TABLE_SIZE) + 1;
	val = REG(ptr + PCIR_MSIX_TABLE, 4);
	cfg->msix.msix_table_bar = PCIR_BAR(val &
	PCIM_MSIX_BIR_MASK);
	cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
	val = REG(ptr + PCIR_MSIX_PBA, 4);
	cfg->msix.msix_pba_bar = PCIR_BAR(val &
	PCIM_MSIX_BIR_MASK);
	cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
	break;
	case PCIY_VPD: /* PCI Vital Product Data */
	cfg->vpd.vpd_reg = ptr;
	break;
	case PCIY_SUBVENDOR:
	/* Should always be true. */
	if ((cfg->hdrtype & PCIM_HDRTYPE) ==
	PCIM_HDRTYPE_BRIDGE) {
	val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
	cfg->subvendor = val & 0xffff;
	cfg->subdevice = val >> 16;
	}
	break;
	case PCIY_PCIX: /* PCI-X */
	/*
	* Assume we have a PCI-X chipset if we have
	* at least one PCI-PCI bridge with a PCI-X
	* capability. Note that some systems with
	* PCI-express or HT chipsets might match on
	* this check as well.
	*/
	if ((cfg->hdrtype & PCIM_HDRTYPE) ==
	PCIM_HDRTYPE_BRIDGE)
	pcix_chipset = 1;
	cfg->pcix.pcix_location = ptr;
	break;
	case PCIY_EXPRESS: /* PCI-express */
	/*
	* Assume we have a PCI-express chipset if we have
	* at least one PCI-express device.
	*/
	pcie_chipset = 1;
	cfg->pcie.pcie_location = ptr;
	val = REG(ptr + PCIER_FLAGS, 2);
	cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
	break;
	case PCIY_EA: /* Enhanced Allocation */
	cfg->ea.ea_location = ptr;
	pci_ea_fill_info(pcib, cfg);
	break;
	default:
	break;
	}
	}

	#if defined(__powerpc__)
	/*
	* Enable the MSI mapping window for all HyperTransport
	* slaves. PCI-PCI bridges have their windows enabled via
	* PCIB_MAP_MSI().
	*/
	if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
	!(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
	device_printf(pcib,
	"Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
	cfg->domain, cfg->bus, cfg->slot, cfg->func);
	cfg->ht.ht_msictrl \|= PCIM_HTCMD_MSI_ENABLE;
	WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
	2);
	}
	#endif
	/* REG and WREG use carry through to next functions */
	}

	/*
	* PCI Vital Product Data
	*/

	#define PCI_VPD_TIMEOUT 1000000

	static int
	pci_read_vpd_reg(device_t pcib, pcicfgregs cfg, int reg, uint32_t data)
	{
	int count = PCI_VPD_TIMEOUT;

	KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));

	WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);

	while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
	if (--count < 0)
	return (ENXIO);
	DELAY(1); /* limit looping */
	}
	*data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));

	return (0);
	}

	#if 0
	static int
	pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
	{
	int count = PCI_VPD_TIMEOUT;

	KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));

	WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
	WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg \| 0x8000, 2);
	while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
	if (--count < 0)
	return (ENXIO);
	DELAY(1); /* limit looping */
	}

	return (0);
	}
	#endif

	#undef PCI_VPD_TIMEOUT

	struct vpd_readstate {
	device_t pcib;
	pcicfgregs *cfg;
	uint32_t val;
	int bytesinval;
	int off;
	uint8_t cksum;
	};

	static int
	vpd_nextbyte(struct vpd_readstate vrs, uint8_t data)
	{
	uint32_t reg;
	uint8_t byte;

	if (vrs->bytesinval == 0) {
	if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, &reg))
	return (ENXIO);
	vrs->val = le32toh(reg);
	vrs->off += 4;
	byte = vrs->val & 0xff;
	vrs->bytesinval = 3;
	} else {
	vrs->val = vrs->val >> 8;
	byte = vrs->val & 0xff;
	vrs->bytesinval--;
	}

	vrs->cksum += byte;
	*data = byte;
	return (0);
	}

	static void
	pci_read_vpd(device_t pcib, pcicfgregs *cfg)
	{
	struct vpd_readstate vrs;
	int state;
	int name;
	int remain;
	int i;
	int alloc, off; /* alloc/off for RO/W arrays */
	int cksumvalid;
	int dflen;
	uint8_t byte;
	uint8_t byte2;

	/* init vpd reader */
	vrs.bytesinval = 0;
	vrs.off = 0;
	vrs.pcib = pcib;
	vrs.cfg = cfg;
	vrs.cksum = 0;

	state = 0;
	name = remain = i = 0; /* shut up stupid gcc */
	alloc = off = 0; /* shut up stupid gcc */
	dflen = 0; /* shut up stupid gcc */
	cksumvalid = -1;
	while (state >= 0) {
	if (vpd_nextbyte(&vrs, &byte)) {
	state = -2;
	break;
	}
	#if 0
	printf("vpd: val: %#x, off: %d, bytesinval: %d, byte: %#hhx, " \
	"state: %d, remain: %d, name: %#x, i: %d\n", vrs.val,
	vrs.off, vrs.bytesinval, byte, state, remain, name, i);
	#endif
	switch (state) {
	case 0: /* item name */
	if (byte & 0x80) {
	if (vpd_nextbyte(&vrs, &byte2)) {
	state = -2;
	break;
	}
	remain = byte2;
	if (vpd_nextbyte(&vrs, &byte2)) {
	state = -2;
	break;
	}
	remain \|= byte2 << 8;
	name = byte & 0x7f;
	} else {
	remain = byte & 0x7;
	name = (byte >> 3) & 0xf;
	}
	if (vrs.off + remain - vrs.bytesinval > 0x8000) {
	pci_printf(cfg,
	"VPD data overflow, remain %#x\n", remain);
	state = -1;
	break;
	}
	switch (name) {
	case 0x2: /* String */
	cfg->vpd.vpd_ident = malloc(remain + 1,
	M_DEVBUF, M_WAITOK);
	i = 0;
	state = 1;
	break;
	case 0xf: /* End */
	state = -1;
	break;
	case 0x10: /* VPD-R */
	alloc = 8;
	off = 0;
	cfg->vpd.vpd_ros = malloc(alloc *
	sizeof(*cfg->vpd.vpd_ros), M_DEVBUF,
	M_WAITOK \| M_ZERO);
	state = 2;
	break;
	case 0x11: /* VPD-W */
	alloc = 8;
	off = 0;
	cfg->vpd.vpd_w = malloc(alloc *
	sizeof(*cfg->vpd.vpd_w), M_DEVBUF,
	M_WAITOK \| M_ZERO);
	state = 5;
	break;
	default: /* Invalid data, abort */
	state = -1;
	break;
	}
	break;

	case 1: /* Identifier String */
	cfg->vpd.vpd_ident[i++] = byte;
	remain--;
	if (remain == 0) {
	cfg->vpd.vpd_ident[i] = '\0';
	state = 0;
	}
	break;

	case 2: /* VPD-R Keyword Header */
	if (off == alloc) {
	cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros,
	(alloc = 2) sizeof(*cfg->vpd.vpd_ros),
	M_DEVBUF, M_WAITOK \| M_ZERO);
	}
	cfg->vpd.vpd_ros[off].keyword[0] = byte;
	if (vpd_nextbyte(&vrs, &byte2)) {
	state = -2;
	break;
	}
	cfg->vpd.vpd_ros[off].keyword[1] = byte2;
	if (vpd_nextbyte(&vrs, &byte2)) {
	state = -2;
	break;
	}
	cfg->vpd.vpd_ros[off].len = dflen = byte2;
	if (dflen == 0 &&
	strncmp(cfg->vpd.vpd_ros[off].keyword, "RV",
	2) == 0) {
	/*
	* if this happens, we can't trust the rest
	* of the VPD.
	*/
	pci_printf(cfg, "bad keyword length: %d\n",
	dflen);
	cksumvalid = 0;
	state = -1;
	break;
	} else if (dflen == 0) {
	cfg->vpd.vpd_ros[off].value = malloc(1 *
	sizeof(*cfg->vpd.vpd_ros[off].value),
	M_DEVBUF, M_WAITOK);
	cfg->vpd.vpd_ros[off].value[0] = '\x00';
	} else
	cfg->vpd.vpd_ros[off].value = malloc(
	(dflen + 1) *
	sizeof(*cfg->vpd.vpd_ros[off].value),
	M_DEVBUF, M_WAITOK);
	remain -= 3;
	i = 0;
	/* keep in sync w/ state 3's transistions */
	if (dflen == 0 && remain == 0)
	state = 0;
	else if (dflen == 0)
	state = 2;
	else
	state = 3;
	break;

	case 3: /* VPD-R Keyword Value */
	cfg->vpd.vpd_ros[off].value[i++] = byte;
	if (strncmp(cfg->vpd.vpd_ros[off].keyword,
	"RV", 2) == 0 && cksumvalid == -1) {
	if (vrs.cksum == 0)
	cksumvalid = 1;
	else {
	if (bootverbose)
	pci_printf(cfg,
	"bad VPD cksum, remain %hhu\n",
	vrs.cksum);
	cksumvalid = 0;
	state = -1;
	break;
	}
	}
	dflen--;
	remain--;
	/* keep in sync w/ state 2's transistions */
	if (dflen == 0)
	cfg->vpd.vpd_ros[off++].value[i++] = '\0';
	if (dflen == 0 && remain == 0) {
	cfg->vpd.vpd_rocnt = off;
	cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros,
	off * sizeof(*cfg->vpd.vpd_ros),
	M_DEVBUF, M_WAITOK \| M_ZERO);
	state = 0;
	} else if (dflen == 0)
	state = 2;
	break;

	case 4:
	remain--;
	if (remain == 0)
	state = 0;
	break;

	case 5: /* VPD-W Keyword Header */
	if (off == alloc) {
	cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w,
	(alloc = 2) sizeof(*cfg->vpd.vpd_w),
	M_DEVBUF, M_WAITOK \| M_ZERO);
	}
	cfg->vpd.vpd_w[off].keyword[0] = byte;
	if (vpd_nextbyte(&vrs, &byte2)) {
	state = -2;
	break;
	}
	cfg->vpd.vpd_w[off].keyword[1] = byte2;
	if (vpd_nextbyte(&vrs, &byte2)) {
	state = -2;
	break;
	}
	cfg->vpd.vpd_w[off].len = dflen = byte2;
	cfg->vpd.vpd_w[off].start = vrs.off - vrs.bytesinval;
	cfg->vpd.vpd_w[off].value = malloc((dflen + 1) *
	sizeof(*cfg->vpd.vpd_w[off].value),
	M_DEVBUF, M_WAITOK);
	remain -= 3;
	i = 0;
	/* keep in sync w/ state 6's transistions */
	if (dflen == 0 && remain == 0)
	state = 0;
	else if (dflen == 0)
	state = 5;
	else
	state = 6;
	break;

	case 6: /* VPD-W Keyword Value */
	cfg->vpd.vpd_w[off].value[i++] = byte;
	dflen--;
	remain--;
	/* keep in sync w/ state 5's transistions */
	if (dflen == 0)
	cfg->vpd.vpd_w[off++].value[i++] = '\0';
	if (dflen == 0 && remain == 0) {
	cfg->vpd.vpd_wcnt = off;
	cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w,
	off * sizeof(*cfg->vpd.vpd_w),
	M_DEVBUF, M_WAITOK \| M_ZERO);
	state = 0;
	} else if (dflen == 0)
	state = 5;
	break;

	default:
	pci_printf(cfg, "invalid state: %d\n", state);
	state = -1;
	break;
	}
	}

	if (cksumvalid == 0 \|\| state < -1) {
	/* read-only data bad, clean up */
	if (cfg->vpd.vpd_ros != NULL) {
	for (off = 0; cfg->vpd.vpd_ros[off].value; off++)
	free(cfg->vpd.vpd_ros[off].value, M_DEVBUF);
	free(cfg->vpd.vpd_ros, M_DEVBUF);
	cfg->vpd.vpd_ros = NULL;
	}
	}
	if (state < -1) {
	/* I/O error, clean up */
	pci_printf(cfg, "failed to read VPD data.\n");
	if (cfg->vpd.vpd_ident != NULL) {
	free(cfg->vpd.vpd_ident, M_DEVBUF);
	cfg->vpd.vpd_ident = NULL;
	}
	if (cfg->vpd.vpd_w != NULL) {
	for (off = 0; cfg->vpd.vpd_w[off].value; off++)
	free(cfg->vpd.vpd_w[off].value, M_DEVBUF);
	free(cfg->vpd.vpd_w, M_DEVBUF);
	cfg->vpd.vpd_w = NULL;
	}
	}
	cfg->vpd.vpd_cached = 1;
	#undef REG
	#undef WREG
	}

	int
	pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;

	if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
	pci_read_vpd(device_get_parent(dev), cfg);

	*identptr = cfg->vpd.vpd_ident;

	if (*identptr == NULL)
	return (ENXIO);

	return (0);
	}

	int
	pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
	const char **vptr)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	int i;

	if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
	pci_read_vpd(device_get_parent(dev), cfg);

	for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
	if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
	sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
	*vptr = cfg->vpd.vpd_ros[i].value;
	return (0);
	}

	*vptr = NULL;
	return (ENXIO);
	}

	struct pcicfg_vpd *
	pci_fetch_vpd_list(device_t dev)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	pcicfgregs *cfg = &dinfo->cfg;

	if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
	pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
	return (&cfg->vpd);
	}

	/*
	* Find the requested HyperTransport capability and return the offset
	* in configuration space via the pointer provided. The function
	* returns 0 on success and an error code otherwise.
	*/
	int
	pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
	{
	int ptr, error;
	uint16_t val;

	error = pci_find_cap(child, PCIY_HT, &ptr);
	if (error)
	return (error);

	/*
	* Traverse the capabilities list checking each HT capability
	* to see if it matches the requested HT capability.
	*/
	for (;;) {
	val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
	if (capability == PCIM_HTCAP_SLAVE \|\|
	capability == PCIM_HTCAP_HOST)
	val &= 0xe000;
	else
	val &= PCIM_HTCMD_CAP_MASK;
	if (val == capability) {
	if (capreg != NULL)
	*capreg = ptr;
	return (0);
	}

	/* Skip to the next HT capability. */
	if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
	break;
	}

	return (ENOENT);
	}

	/*
	* Find the next requested HyperTransport capability after start and return
	* the offset in configuration space via the pointer provided. The function
	* returns 0 on success and an error code otherwise.
	*/
	int
	pci_find_next_htcap_method(device_t dev, device_t child, int capability,
	int start, int *capreg)
	{
	int ptr;
	uint16_t val;

	KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
	("start capability is not HyperTransport capability"));
	ptr = start;

	/*
	* Traverse the capabilities list checking each HT capability
	* to see if it matches the requested HT capability.
	*/
	for (;;) {
	/* Skip to the next HT capability. */
	if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
	break;

	val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
	if (capability == PCIM_HTCAP_SLAVE \|\|
	capability == PCIM_HTCAP_HOST)
	val &= 0xe000;
	else
	val &= PCIM_HTCMD_CAP_MASK;
	if (val == capability) {
	if (capreg != NULL)
	*capreg = ptr;
	return (0);
	}
	}

	return (ENOENT);
	}

	/*
	* Find the requested capability and return the offset in
	* configuration space via the pointer provided. The function returns
	* 0 on success and an error code otherwise.
	*/
	int
	pci_find_cap_method(device_t dev, device_t child, int capability,
	int *capreg)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	uint32_t status;
	uint8_t ptr;

	/*
	* Check the CAP_LIST bit of the PCI status register first.
	*/
	status = pci_read_config(child, PCIR_STATUS, 2);
	if (!(status & PCIM_STATUS_CAPPRESENT))
	return (ENXIO);

	/*
	* Determine the start pointer of the capabilities list.
	*/
	switch (cfg->hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_NORMAL:
	case PCIM_HDRTYPE_BRIDGE:
	ptr = PCIR_CAP_PTR;
	break;
	case PCIM_HDRTYPE_CARDBUS:
	ptr = PCIR_CAP_PTR_2;
	break;
	default:
	/* XXX: panic? */
	return (ENXIO); /* no extended capabilities support */
	}
	ptr = pci_read_config(child, ptr, 1);

	/*
	* Traverse the capabilities list.
	*/
	while (ptr != 0) {
	if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
	if (capreg != NULL)
	*capreg = ptr;
	return (0);
	}
	ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
	}

	return (ENOENT);
	}

	/*
	* Find the next requested capability after start and return the offset in
	* configuration space via the pointer provided. The function returns
	* 0 on success and an error code otherwise.
	*/
	int
	pci_find_next_cap_method(device_t dev, device_t child, int capability,
	int start, int *capreg)
	{
	uint8_t ptr;

	KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
	("start capability is not expected capability"));

	ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
	while (ptr != 0) {
	if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
	if (capreg != NULL)
	*capreg = ptr;
	return (0);
	}
	ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
	}

	return (ENOENT);
	}

	/*
	* Find the requested extended capability and return the offset in
	* configuration space via the pointer provided. The function returns
	* 0 on success and an error code otherwise.
	*/
	int
	pci_find_extcap_method(device_t dev, device_t child, int capability,
	int *capreg)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	uint32_t ecap;
	uint16_t ptr;

	/* Only supported for PCI-express devices. */
	if (cfg->pcie.pcie_location == 0)
	return (ENXIO);

	ptr = PCIR_EXTCAP;
	ecap = pci_read_config(child, ptr, 4);
	if (ecap == 0xffffffff \|\| ecap == 0)
	return (ENOENT);
	for (;;) {
	if (PCI_EXTCAP_ID(ecap) == capability) {
	if (capreg != NULL)
	*capreg = ptr;
	return (0);
	}
	ptr = PCI_EXTCAP_NEXTPTR(ecap);
	if (ptr == 0)
	break;
	ecap = pci_read_config(child, ptr, 4);
	}

	return (ENOENT);
	}

	/*
	* Find the next requested extended capability after start and return the
	* offset in configuration space via the pointer provided. The function
	* returns 0 on success and an error code otherwise.
	*/
	int
	pci_find_next_extcap_method(device_t dev, device_t child, int capability,
	int start, int *capreg)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	uint32_t ecap;
	uint16_t ptr;

	/* Only supported for PCI-express devices. */
	if (cfg->pcie.pcie_location == 0)
	return (ENXIO);

	ecap = pci_read_config(child, start, 4);
	KASSERT(PCI_EXTCAP_ID(ecap) == capability,
	("start extended capability is not expected capability"));
	ptr = PCI_EXTCAP_NEXTPTR(ecap);
	while (ptr != 0) {
	ecap = pci_read_config(child, ptr, 4);
	if (PCI_EXTCAP_ID(ecap) == capability) {
	if (capreg != NULL)
	*capreg = ptr;
	return (0);
	}
	ptr = PCI_EXTCAP_NEXTPTR(ecap);
	}

	return (ENOENT);
	}

	/*
	* Support for MSI-X message interrupts.
	*/
	static void
	pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	uint32_t offset;

	KASSERT(msix->msix_table_len > index, ("bogus index"));
	offset = msix->msix_table_offset + index * 16;
	bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
	bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
	bus_write_4(msix->msix_table_res, offset + 8, data);
	}

	void
	pci_enable_msix_method(device_t dev, device_t child, u_int index,
	uint64_t address, uint32_t data)
	{

	if (pci_msix_rewrite_table) {
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;

	/*
	* Some VM hosts require MSIX to be disabled in the
	* control register before updating the MSIX table
	* entries are allowed. It is not enough to only
	* disable MSIX while updating a single entry. MSIX
	* must be disabled while updating all entries in the
	* table.
	*/
	pci_write_config(child,
	msix->msix_location + PCIR_MSIX_CTRL,
	msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
	pci_resume_msix(child);
	} else
	pci_write_msix_entry(child, index, address, data);

	/* Enable MSI -> HT mapping. */
	pci_ht_map_msi(child, address);
	}

	void
	pci_mask_msix(device_t dev, u_int index)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	uint32_t offset, val;

	KASSERT(msix->msix_msgnum > index, ("bogus index"));
	offset = msix->msix_table_offset + index * 16 + 12;
	val = bus_read_4(msix->msix_table_res, offset);
	val \|= PCIM_MSIX_VCTRL_MASK;

	/*
	* Some devices (e.g. Samsung PM961) do not support reads of this
	* register, so always write the new value.
	*/
	bus_write_4(msix->msix_table_res, offset, val);
	}

	void
	pci_unmask_msix(device_t dev, u_int index)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	uint32_t offset, val;

	KASSERT(msix->msix_table_len > index, ("bogus index"));
	offset = msix->msix_table_offset + index * 16 + 12;
	val = bus_read_4(msix->msix_table_res, offset);
	val &= ~PCIM_MSIX_VCTRL_MASK;

	/*
	* Some devices (e.g. Samsung PM961) do not support reads of this
	* register, so always write the new value.
	*/
	bus_write_4(msix->msix_table_res, offset, val);
	}

	int
	pci_pending_msix(device_t dev, u_int index)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	uint32_t offset, bit;

	KASSERT(msix->msix_table_len > index, ("bogus index"));
	offset = msix->msix_pba_offset + (index / 32) * 4;
	bit = 1 << index % 32;
	return (bus_read_4(msix->msix_pba_res, offset) & bit);
	}

	/*
	* Restore MSI-X registers and table during resume. If MSI-X is
	* enabled then walk the virtual table to restore the actual MSI-X
	* table.
	*/
	static void
	pci_resume_msix(device_t dev)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	struct msix_table_entry *mte;
	struct msix_vector *mv;
	int i;

	if (msix->msix_alloc > 0) {
	/* First, mask all vectors. */
	for (i = 0; i < msix->msix_msgnum; i++)
	pci_mask_msix(dev, i);

	/* Second, program any messages with at least one handler. */
	for (i = 0; i < msix->msix_table_len; i++) {
	mte = &msix->msix_table[i];
	if (mte->mte_vector == 0 \|\| mte->mte_handlers == 0)
	continue;
	mv = &msix->msix_vectors[mte->mte_vector - 1];
	pci_write_msix_entry(dev, i, mv->mv_address,
	mv->mv_data);
	pci_unmask_msix(dev, i);
	}
	}
	pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
	msix->msix_ctrl, 2);
	}

	/*
	* Attempt to allocate *count MSI-X messages. The actual number allocated is
	* returned in *count. After this function returns, each message will be
	* available to the driver as SYS_RES_IRQ resources starting at rid 1.
	*/
	int
	pci_alloc_msix_method(device_t dev, device_t child, int *count)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	struct resource_list_entry *rle;
	int actual, error, i, irq, max;

	/* Don't let count == 0 get us into trouble. */
	if (*count == 0)
	return (EINVAL);

	/* If rid 0 is allocated, then fail. */
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
	if (rle != NULL && rle->res != NULL)
	return (ENXIO);

	/* Already have allocated messages? */
	if (cfg->msi.msi_alloc != 0 \|\| cfg->msix.msix_alloc != 0)
	return (ENXIO);

	/* If MSI-X is blacklisted for this system, fail. */
	if (pci_msix_blacklisted())
	return (ENXIO);

	/* MSI-X capability present? */
	if (cfg->msix.msix_location == 0 \|\| !pci_do_msix)
	return (ENODEV);

	/* Make sure the appropriate BARs are mapped. */
	rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
	cfg->msix.msix_table_bar);
	if (rle == NULL \|\| rle->res == NULL \|\|
	!(rman_get_flags(rle->res) & RF_ACTIVE))
	return (ENXIO);
	cfg->msix.msix_table_res = rle->res;
	if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
	rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
	cfg->msix.msix_pba_bar);
	if (rle == NULL \|\| rle->res == NULL \|\|
	!(rman_get_flags(rle->res) & RF_ACTIVE))
	return (ENXIO);
	}
	cfg->msix.msix_pba_res = rle->res;

	if (bootverbose)
	device_printf(child,
	"attempting to allocate %d MSI-X vectors (%d supported)\n",
	*count, cfg->msix.msix_msgnum);
	max = min(*count, cfg->msix.msix_msgnum);
	for (i = 0; i < max; i++) {
	/* Allocate a message. */
	error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
	if (error) {
	if (i == 0)
	return (error);
	break;
	}
	resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
	irq, 1);
	}
	actual = i;

	if (bootverbose) {
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
	if (actual == 1)
	device_printf(child, "using IRQ %ju for MSI-X\n",
	rle->start);
	else {
	int run;

	/*
	* Be fancy and try to print contiguous runs of
	* IRQ values as ranges. 'irq' is the previous IRQ.
	* 'run' is true if we are in a range.
	*/
	device_printf(child, "using IRQs %ju", rle->start);
	irq = rle->start;
	run = 0;
	for (i = 1; i < actual; i++) {
	rle = resource_list_find(&dinfo->resources,
	SYS_RES_IRQ, i + 1);

	/* Still in a run? */
	if (rle->start == irq + 1) {
	run = 1;
	irq++;
	continue;
	}

	/* Finish previous range. */
	if (run) {
	printf("-%d", irq);
	run = 0;
	}

	/* Start new range. */
	printf(",%ju", rle->start);
	irq = rle->start;
	}

	/* Unfinished range? */
	if (run)
	printf("-%d", irq);
	printf(" for MSI-X\n");
	}
	}

	/* Mask all vectors. */
	for (i = 0; i < cfg->msix.msix_msgnum; i++)
	pci_mask_msix(child, i);

	/* Allocate and initialize vector data and virtual table. */
	cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual,
	M_DEVBUF, M_WAITOK \| M_ZERO);
	cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual,
	M_DEVBUF, M_WAITOK \| M_ZERO);
	for (i = 0; i < actual; i++) {
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
	cfg->msix.msix_vectors[i].mv_irq = rle->start;
	cfg->msix.msix_table[i].mte_vector = i + 1;
	}

	/* Update control register to enable MSI-X. */
	cfg->msix.msix_ctrl \|= PCIM_MSIXCTRL_MSIX_ENABLE;
	pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
	cfg->msix.msix_ctrl, 2);

	/* Update counts of alloc'd messages. */
	cfg->msix.msix_alloc = actual;
	cfg->msix.msix_table_len = actual;
	*count = actual;
	return (0);
	}

	/*
	* By default, pci_alloc_msix() will assign the allocated IRQ
	* resources consecutively to the first N messages in the MSI-X table.
	* However, device drivers may want to use different layouts if they
	* either receive fewer messages than they asked for, or they wish to
	* populate the MSI-X table sparsely. This method allows the driver
	* to specify what layout it wants. It must be called after a
	* successful pci_alloc_msix() but before any of the associated
	* SYS_RES_IRQ resources are allocated via bus_alloc_resource().
	*
	* The 'vectors' array contains 'count' message vectors. The array
	* maps directly to the MSI-X table in that index 0 in the array
	* specifies the vector for the first message in the MSI-X table, etc.
	* The vector value in each array index can either be 0 to indicate
	* that no vector should be assigned to a message slot, or it can be a
	* number from 1 to N (where N is the count returned from a
	* succcessful call to pci_alloc_msix()) to indicate which message
	* vector (IRQ) to be used for the corresponding message.
	*
	* On successful return, each message with a non-zero vector will have
	* an associated SYS_RES_IRQ whose rid is equal to the array index +
	* 1. Additionally, if any of the IRQs allocated via the previous
	* call to pci_alloc_msix() are not used in the mapping, those IRQs
	* will be freed back to the system automatically.
	*
	* For example, suppose a driver has a MSI-X table with 6 messages and
	* asks for 6 messages, but pci_alloc_msix() only returns a count of
	* 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
	* C. After the call to pci_alloc_msix(), the device will be setup to
	* have an MSI-X table of ABC--- (where - means no vector assigned).
	* If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
	* then the MSI-X table will look like A-AB-B, and the 'C' vector will
	* be freed back to the system. This device will also have valid
	* SYS_RES_IRQ rids of 1, 3, 4, and 6.
	*
	* In any case, the SYS_RES_IRQ rid X will always map to the message
	* at MSI-X table index X - 1 and will only be valid if a vector is
	* assigned to that table entry.
	*/
	int
	pci_remap_msix_method(device_t dev, device_t child, int count,
	const u_int *vectors)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	struct resource_list_entry *rle;
	int i, irq, j, *used;

	/*
	* Have to have at least one message in the table but the
	* table can't be bigger than the actual MSI-X table in the
	* device.
	*/
	if (count == 0 \|\| count > msix->msix_msgnum)
	return (EINVAL);

	/* Sanity check the vectors. */
	for (i = 0; i < count; i++)
	if (vectors[i] > msix->msix_alloc)
	return (EINVAL);

	/*
	* Make sure there aren't any holes in the vectors to be used.
	* It's a big pain to support it, and it doesn't really make
	* sense anyway. Also, at least one vector must be used.
	*/
	used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK \|
	M_ZERO);
	for (i = 0; i < count; i++)
	if (vectors[i] != 0)
	used[vectors[i] - 1] = 1;
	for (i = 0; i < msix->msix_alloc - 1; i++)
	if (used[i] == 0 && used[i + 1] == 1) {
	free(used, M_DEVBUF);
	return (EINVAL);
	}
	if (used[0] != 1) {
	free(used, M_DEVBUF);
	return (EINVAL);
	}

	/* Make sure none of the resources are allocated. */
	for (i = 0; i < msix->msix_table_len; i++) {
	if (msix->msix_table[i].mte_vector == 0)
	continue;
	if (msix->msix_table[i].mte_handlers > 0) {
	free(used, M_DEVBUF);
	return (EBUSY);
	}
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
	KASSERT(rle != NULL, ("missing resource"));
	if (rle->res != NULL) {
	free(used, M_DEVBUF);
	return (EBUSY);
	}
	}

	/* Free the existing resource list entries. */
	for (i = 0; i < msix->msix_table_len; i++) {
	if (msix->msix_table[i].mte_vector == 0)
	continue;
	resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
	}

	/*
	* Build the new virtual table keeping track of which vectors are
	* used.
	*/
	free(msix->msix_table, M_DEVBUF);
	msix->msix_table = malloc(sizeof(struct msix_table_entry) * count,
	M_DEVBUF, M_WAITOK \| M_ZERO);
	for (i = 0; i < count; i++)
	msix->msix_table[i].mte_vector = vectors[i];
	msix->msix_table_len = count;

	/* Free any unused IRQs and resize the vectors array if necessary. */
	j = msix->msix_alloc - 1;
	if (used[j] == 0) {
	struct msix_vector *vec;

	while (used[j] == 0) {
	PCIB_RELEASE_MSIX(device_get_parent(dev), child,
	msix->msix_vectors[j].mv_irq);
	j--;
	}
	vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF,
	M_WAITOK);
	bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
	(j + 1));
	free(msix->msix_vectors, M_DEVBUF);
	msix->msix_vectors = vec;
	msix->msix_alloc = j + 1;
	}
	free(used, M_DEVBUF);

	/* Map the IRQs onto the rids. */
	for (i = 0; i < count; i++) {
	if (vectors[i] == 0)
	continue;
	irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
	resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
	irq, 1);
	}

	if (bootverbose) {
	device_printf(child, "Remapped MSI-X IRQs as: ");
	for (i = 0; i < count; i++) {
	if (i != 0)
	printf(", ");
	if (vectors[i] == 0)
	printf("---");
	else
	printf("%d",
	msix->msix_vectors[vectors[i] - 1].mv_irq);
	}
	printf("\n");
	}

	return (0);
	}

	static int
	pci_release_msix(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;
	struct resource_list_entry *rle;
	int i;

	/* Do we have any messages to release? */
	if (msix->msix_alloc == 0)
	return (ENODEV);

	/* Make sure none of the resources are allocated. */
	for (i = 0; i < msix->msix_table_len; i++) {
	if (msix->msix_table[i].mte_vector == 0)
	continue;
	if (msix->msix_table[i].mte_handlers > 0)
	return (EBUSY);
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
	KASSERT(rle != NULL, ("missing resource"));
	if (rle->res != NULL)
	return (EBUSY);
	}

	/* Update control register to disable MSI-X. */
	msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
	pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
	msix->msix_ctrl, 2);

	/* Free the resource list entries. */
	for (i = 0; i < msix->msix_table_len; i++) {
	if (msix->msix_table[i].mte_vector == 0)
	continue;
	resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
	}
	free(msix->msix_table, M_DEVBUF);
	msix->msix_table_len = 0;

	/* Release the IRQs. */
	for (i = 0; i < msix->msix_alloc; i++)
	PCIB_RELEASE_MSIX(device_get_parent(dev), child,
	msix->msix_vectors[i].mv_irq);
	free(msix->msix_vectors, M_DEVBUF);
	msix->msix_alloc = 0;
	return (0);
	}

	/*
	* Return the max supported MSI-X messages this device supports.
	* Basically, assuming the MD code can alloc messages, this function
	* should return the maximum value that pci_alloc_msix() can return.
	* Thus, it is subject to the tunables, etc.
	*/
	int
	pci_msix_count_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;

	if (pci_do_msix && msix->msix_location != 0)
	return (msix->msix_msgnum);
	return (0);
	}

	int
	pci_msix_pba_bar_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;

	if (pci_do_msix && msix->msix_location != 0)
	return (msix->msix_pba_bar);
	return (-1);
	}

	int
	pci_msix_table_bar_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msix *msix = &dinfo->cfg.msix;

	if (pci_do_msix && msix->msix_location != 0)
	return (msix->msix_table_bar);
	return (-1);
	}

	/*
	* HyperTransport MSI mapping control
	*/
	void
	pci_ht_map_msi(device_t dev, uint64_t addr)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_ht *ht = &dinfo->cfg.ht;

	if (!ht->ht_msimap)
	return;

	if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
	ht->ht_msiaddr >> 20 == addr >> 20) {
	/* Enable MSI -> HT mapping. */
	ht->ht_msictrl \|= PCIM_HTCMD_MSI_ENABLE;
	pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
	ht->ht_msictrl, 2);
	}

	if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
	/* Disable MSI -> HT mapping. */
	ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
	pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
	ht->ht_msictrl, 2);
	}
	}

	int
	pci_get_max_payload(device_t dev)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	int cap;
	uint16_t val;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return (0);
	val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
	val &= PCIEM_CTL_MAX_PAYLOAD;
	val >>= 5;
	return (1 << (val + 7));
	}

	int
	pci_get_max_read_req(device_t dev)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	int cap;
	uint16_t val;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return (0);
	val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
	val &= PCIEM_CTL_MAX_READ_REQUEST;
	val >>= 12;
	return (1 << (val + 7));
	}

	int
	pci_set_max_read_req(device_t dev, int size)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	int cap;
	uint16_t val;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return (0);
	if (size < 128)
	size = 128;
	if (size > 4096)
	size = 4096;
	size = (1 << (fls(size) - 1));
	val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
	val &= ~PCIEM_CTL_MAX_READ_REQUEST;
	val \|= (fls(size) - 8) << 12;
	pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2);
	return (size);
	}

	uint32_t
	pcie_read_config(device_t dev, int reg, int width)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	int cap;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0) {
	if (width == 2)
	return (0xffff);
	return (0xffffffff);
	}

	return (pci_read_config(dev, cap + reg, width));
	}

	void
	pcie_write_config(device_t dev, int reg, uint32_t value, int width)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	int cap;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return;
	pci_write_config(dev, cap + reg, value, width);
	}

	/*
	* Adjusts a PCI-e capability register by clearing the bits in mask
	* and setting the bits in (value & mask). Bits not set in mask are
	* not adjusted.
	*
	* Returns the old value on success or all ones on failure.
	*/
	uint32_t
	pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
	int width)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	uint32_t old, new;
	int cap;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0) {
	if (width == 2)
	return (0xffff);
	return (0xffffffff);
	}

	old = pci_read_config(dev, cap + reg, width);
	new = old & ~mask;
	new \|= (value & mask);
	pci_write_config(dev, cap + reg, new, width);
	return (old);
	}

	/*
	* Support for MSI message signalled interrupts.
	*/
	void
	pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
	uint16_t data)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msi *msi = &dinfo->cfg.msi;

	/* Write data and address values. */
	pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
	address & 0xffffffff, 4);
	if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
	pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
	address >> 32, 4);
	pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
	data, 2);
	} else
	pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
	2);

	/* Enable MSI in the control register. */
	msi->msi_ctrl \|= PCIM_MSICTRL_MSI_ENABLE;
	pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
	msi->msi_ctrl, 2);

	/* Enable MSI -> HT mapping. */
	pci_ht_map_msi(child, address);
	}

	void
	pci_disable_msi_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msi *msi = &dinfo->cfg.msi;

	/* Disable MSI -> HT mapping. */
	pci_ht_map_msi(child, 0);

	/* Disable MSI in the control register. */
	msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
	pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
	msi->msi_ctrl, 2);
	}

	/*
	* Restore MSI registers during resume. If MSI is enabled then
	* restore the data and address registers in addition to the control
	* register.
	*/
	static void
	pci_resume_msi(device_t dev)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	struct pcicfg_msi *msi = &dinfo->cfg.msi;
	uint64_t address;
	uint16_t data;

	if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
	address = msi->msi_addr;
	data = msi->msi_data;
	pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
	address & 0xffffffff, 4);
	if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
	pci_write_config(dev, msi->msi_location +
	PCIR_MSI_ADDR_HIGH, address >> 32, 4);
	pci_write_config(dev, msi->msi_location +
	PCIR_MSI_DATA_64BIT, data, 2);
	} else
	pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
	data, 2);
	}
	pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
	2);
	}

	static int
	pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	pcicfgregs *cfg = &dinfo->cfg;
	struct resource_list_entry *rle;
	struct msix_table_entry *mte;
	struct msix_vector *mv;
	uint64_t addr;
	uint32_t data;
	int error, i, j;

	/*
	* Handle MSI first. We try to find this IRQ among our list
	* of MSI IRQs. If we find it, we request updated address and
	* data registers and apply the results.
	*/
	if (cfg->msi.msi_alloc > 0) {

	/* If we don't have any active handlers, nothing to do. */
	if (cfg->msi.msi_handlers == 0)
	return (0);
	for (i = 0; i < cfg->msi.msi_alloc; i++) {
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
	i + 1);
	if (rle->start == irq) {
	error = PCIB_MAP_MSI(device_get_parent(bus),
	dev, irq, &addr, &data);
	if (error)
	return (error);
	pci_disable_msi(dev);
	dinfo->cfg.msi.msi_addr = addr;
	dinfo->cfg.msi.msi_data = data;
	pci_enable_msi(dev, addr, data);
	return (0);
	}
	}
	return (ENOENT);
	}

	/*
	* For MSI-X, we check to see if we have this IRQ. If we do,
	* we request the updated mapping info. If that works, we go
	* through all the slots that use this IRQ and update them.
	*/
	if (cfg->msix.msix_alloc > 0) {
	for (i = 0; i < cfg->msix.msix_alloc; i++) {
	mv = &cfg->msix.msix_vectors[i];
	if (mv->mv_irq == irq) {
	error = PCIB_MAP_MSI(device_get_parent(bus),
	dev, irq, &addr, &data);
	if (error)
	return (error);
	mv->mv_address = addr;
	mv->mv_data = data;
	for (j = 0; j < cfg->msix.msix_table_len; j++) {
	mte = &cfg->msix.msix_table[j];
	if (mte->mte_vector != i + 1)
	continue;
	if (mte->mte_handlers == 0)
	continue;
	pci_mask_msix(dev, j);
	pci_enable_msix(dev, j, addr, data);
	pci_unmask_msix(dev, j);
	}
	}
	}
	return (ENOENT);
	}

	return (ENOENT);
	}

	/*
	* Returns true if the specified device is blacklisted because MSI
	* doesn't work.
	*/
	int
	pci_msi_device_blacklisted(device_t dev)
	{

	if (!pci_honor_msi_blacklist)
	return (0);

	return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
	}

	/*
	* Determine if MSI is blacklisted globally on this system. Currently,
	* we just check for blacklisted chipsets as represented by the
	* host-PCI bridge at device 0:0:0. In the future, it may become
	* necessary to check other system attributes, such as the kenv values
	* that give the motherboard manufacturer and model number.
	*/
	static int
	pci_msi_blacklisted(void)
	{
	device_t dev;

	if (!pci_honor_msi_blacklist)
	return (0);

	/* Blacklist all non-PCI-express and non-PCI-X chipsets. */
	if (!(pcie_chipset \|\| pcix_chipset)) {
	if (vm_guest != VM_GUEST_NO) {
	/*
	* Whitelist older chipsets in virtual
	* machines known to support MSI.
	*/
	dev = pci_find_bsf(0, 0, 0);
	if (dev != NULL)
	return (!pci_has_quirk(pci_get_devid(dev),
	PCI_QUIRK_ENABLE_MSI_VM));
	}
	return (1);
	}

	dev = pci_find_bsf(0, 0, 0);
	if (dev != NULL)
	return (pci_msi_device_blacklisted(dev));
	return (0);
	}

	/*
	* Returns true if the specified device is blacklisted because MSI-X
	* doesn't work. Note that this assumes that if MSI doesn't work,
	* MSI-X doesn't either.
	*/
	int
	pci_msix_device_blacklisted(device_t dev)
	{

	if (!pci_honor_msi_blacklist)
	return (0);

	if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
	return (1);

	return (pci_msi_device_blacklisted(dev));
	}

	/*
	* Determine if MSI-X is blacklisted globally on this system. If MSI
	* is blacklisted, assume that MSI-X is as well. Check for additional
	* chipsets where MSI works but MSI-X does not.
	*/
	static int
	pci_msix_blacklisted(void)
	{
	device_t dev;

	if (!pci_honor_msi_blacklist)
	return (0);

	dev = pci_find_bsf(0, 0, 0);
	if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
	PCI_QUIRK_DISABLE_MSIX))
	return (1);

	return (pci_msi_blacklisted());
	}

	/*
	* Attempt to allocate *count MSI messages. The actual number allocated is
	* returned in *count. After this function returns, each message will be
	* available to the driver as SYS_RES_IRQ resources starting at a rid 1.
	*/
	int
	pci_alloc_msi_method(device_t dev, device_t child, int *count)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	struct resource_list_entry *rle;
	int actual, error, i, irqs[32];
	uint16_t ctrl;

	/* Don't let count == 0 get us into trouble. */
	if (*count == 0)
	return (EINVAL);

	/* If rid 0 is allocated, then fail. */
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
	if (rle != NULL && rle->res != NULL)
	return (ENXIO);

	/* Already have allocated messages? */
	if (cfg->msi.msi_alloc != 0 \|\| cfg->msix.msix_alloc != 0)
	return (ENXIO);

	/* If MSI is blacklisted for this system, fail. */
	if (pci_msi_blacklisted())
	return (ENXIO);

	/* MSI capability present? */
	if (cfg->msi.msi_location == 0 \|\| !pci_do_msi)
	return (ENODEV);

	if (bootverbose)
	device_printf(child,
	"attempting to allocate %d MSI vectors (%d supported)\n",
	*count, cfg->msi.msi_msgnum);

	/* Don't ask for more than the device supports. */
	actual = min(*count, cfg->msi.msi_msgnum);

	/* Don't ask for more than 32 messages. */
	actual = min(actual, 32);

	/* MSI requires power of 2 number of messages. */
	if (!powerof2(actual))
	return (EINVAL);

	for (;;) {
	/* Try to allocate N messages. */
	error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
	actual, irqs);
	if (error == 0)
	break;
	if (actual == 1)
	return (error);

	/* Try N / 2. */
	actual >>= 1;
	}

	/*
	* We now have N actual messages mapped onto SYS_RES_IRQ
	* resources in the irqs[] array, so add new resources
	* starting at rid 1.
	*/
	for (i = 0; i < actual; i++)
	resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
	irqs[i], irqs[i], 1);

	if (bootverbose) {
	if (actual == 1)
	device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
	else {
	int run;

	/*
	* Be fancy and try to print contiguous runs
	* of IRQ values as ranges. 'run' is true if
	* we are in a range.
	*/
	device_printf(child, "using IRQs %d", irqs[0]);
	run = 0;
	for (i = 1; i < actual; i++) {

	/* Still in a run? */
	if (irqs[i] == irqs[i - 1] + 1) {
	run = 1;
	continue;
	}

	/* Finish previous range. */
	if (run) {
	printf("-%d", irqs[i - 1]);
	run = 0;
	}

	/* Start new range. */
	printf(",%d", irqs[i]);
	}

	/* Unfinished range? */
	if (run)
	printf("-%d", irqs[actual - 1]);
	printf(" for MSI\n");
	}
	}

	/* Update control register with actual count. */
	ctrl = cfg->msi.msi_ctrl;
	ctrl &= ~PCIM_MSICTRL_MME_MASK;
	ctrl \|= (ffs(actual) - 1) << 4;
	cfg->msi.msi_ctrl = ctrl;
	pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);

	/* Update counts of alloc'd messages. */
	cfg->msi.msi_alloc = actual;
	cfg->msi.msi_handlers = 0;
	*count = actual;
	return (0);
	}

	/* Release the MSI messages associated with this device. */
	int
	pci_release_msi_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msi *msi = &dinfo->cfg.msi;
	struct resource_list_entry *rle;
	int error, i, irqs[32];

	/* Try MSI-X first. */
	error = pci_release_msix(dev, child);
	if (error != ENODEV)
	return (error);

	/* Do we have any messages to release? */
	if (msi->msi_alloc == 0)
	return (ENODEV);
	KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));

	/* Make sure none of the resources are allocated. */
	if (msi->msi_handlers > 0)
	return (EBUSY);
	for (i = 0; i < msi->msi_alloc; i++) {
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
	KASSERT(rle != NULL, ("missing MSI resource"));
	if (rle->res != NULL)
	return (EBUSY);
	irqs[i] = rle->start;
	}

	/* Update control register with 0 count. */
	KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
	("%s: MSI still enabled", __func__));
	msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
	pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
	msi->msi_ctrl, 2);

	/* Release the messages. */
	PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
	for (i = 0; i < msi->msi_alloc; i++)
	resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);

	/* Update alloc count. */
	msi->msi_alloc = 0;
	msi->msi_addr = 0;
	msi->msi_data = 0;
	return (0);
	}

	/*
	* Return the max supported MSI messages this device supports.
	* Basically, assuming the MD code can alloc messages, this function
	* should return the maximum value that pci_alloc_msi() can return.
	* Thus, it is subject to the tunables, etc.
	*/
	int
	pci_msi_count_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct pcicfg_msi *msi = &dinfo->cfg.msi;

	if (pci_do_msi && msi->msi_location != 0)
	return (msi->msi_msgnum);
	return (0);
	}

	/* free pcicfgregs structure and all depending data structures */

	int
	pci_freecfg(struct pci_devinfo *dinfo)
	{
	struct devlist *devlist_head;
	struct pci_map pm, next;
	int i;

	devlist_head = &pci_devq;

	if (dinfo->cfg.vpd.vpd_reg) {
	free(dinfo->cfg.vpd.vpd_ident, M_DEVBUF);
	for (i = 0; i < dinfo->cfg.vpd.vpd_rocnt; i++)
	free(dinfo->cfg.vpd.vpd_ros[i].value, M_DEVBUF);
	free(dinfo->cfg.vpd.vpd_ros, M_DEVBUF);
	for (i = 0; i < dinfo->cfg.vpd.vpd_wcnt; i++)
	free(dinfo->cfg.vpd.vpd_w[i].value, M_DEVBUF);
	free(dinfo->cfg.vpd.vpd_w, M_DEVBUF);
	}
	STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
	free(pm, M_DEVBUF);
	}
	STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
	free(dinfo, M_DEVBUF);

	/* increment the generation count */
	pci_generation++;

	/* we're losing one device */
	pci_numdevs--;
	return (0);
	}

	/*
	* PCI power manangement
	*/
	int
	pci_set_powerstate_method(device_t dev, device_t child, int state)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	uint16_t status;
	int oldstate, highest, delay;

	if (cfg->pp.pp_cap == 0)
	return (EOPNOTSUPP);

	/*
	* Optimize a no state change request away. While it would be OK to
	* write to the hardware in theory, some devices have shown odd
	* behavior when going from D3 -> D3.
	*/
	oldstate = pci_get_powerstate(child);
	if (oldstate == state)
	return (0);

	/*
	* The PCI power management specification states that after a state
	* transition between PCI power states, system software must
	* guarantee a minimal delay before the function accesses the device.
	* Compute the worst case delay that we need to guarantee before we
	* access the device. Many devices will be responsive much more
	* quickly than this delay, but there are some that don't respond
	* instantly to state changes. Transitions to/from D3 state require
	* 10ms, while D2 requires 200us, and D0/1 require none. The delay
	* is done below with DELAY rather than a sleeper function because
	* this function can be called from contexts where we cannot sleep.
	*/
	highest = (oldstate > state) ? oldstate : state;
	if (highest == PCI_POWERSTATE_D3)
	delay = 10000;
	else if (highest == PCI_POWERSTATE_D2)
	delay = 200;
	else
	delay = 0;
	status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2)
	& ~PCIM_PSTAT_DMASK;
	switch (state) {
	case PCI_POWERSTATE_D0:
	status \|= PCIM_PSTAT_D0;
	break;
	case PCI_POWERSTATE_D1:
	if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
	return (EOPNOTSUPP);
	status \|= PCIM_PSTAT_D1;
	break;
	case PCI_POWERSTATE_D2:
	if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
	return (EOPNOTSUPP);
	status \|= PCIM_PSTAT_D2;
	break;
	case PCI_POWERSTATE_D3:
	status \|= PCIM_PSTAT_D3;
	break;
	default:
	return (EINVAL);
	}

	if (bootverbose)
	pci_printf(cfg, "Transition from D%d to D%d\n", oldstate,
	state);

	PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2);
	if (delay)
	DELAY(delay);
	return (0);
	}

	int
	pci_get_powerstate_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;
	uint16_t status;
	int result;

	if (cfg->pp.pp_cap != 0) {
	status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2);
	switch (status & PCIM_PSTAT_DMASK) {
	case PCIM_PSTAT_D0:
	result = PCI_POWERSTATE_D0;
	break;
	case PCIM_PSTAT_D1:
	result = PCI_POWERSTATE_D1;
	break;
	case PCIM_PSTAT_D2:
	result = PCI_POWERSTATE_D2;
	break;
	case PCIM_PSTAT_D3:
	result = PCI_POWERSTATE_D3;
	break;
	default:
	result = PCI_POWERSTATE_UNKNOWN;
	break;
	}
	} else {
	/* No support, device is always at D0 */
	result = PCI_POWERSTATE_D0;
	}
	return (result);
	}

	/*
	* Some convenience functions for PCI device drivers.
	*/

	static __inline void
	pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
	{
	uint16_t command;

	command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
	command \|= bit;
	PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
	}

	static __inline void
	pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
	{
	uint16_t command;

	command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
	command &= ~bit;
	PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
	}

	int
	pci_enable_busmaster_method(device_t dev, device_t child)
	{
	pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
	return (0);
	}

	int
	pci_disable_busmaster_method(device_t dev, device_t child)
	{
	pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
	return (0);
	}

	int
	pci_enable_io_method(device_t dev, device_t child, int space)
	{
	uint16_t bit;

	switch(space) {
	case SYS_RES_IOPORT:
	bit = PCIM_CMD_PORTEN;
	break;
	case SYS_RES_MEMORY:
	bit = PCIM_CMD_MEMEN;
	break;
	default:
	return (EINVAL);
	}
	pci_set_command_bit(dev, child, bit);
	return (0);
	}

	int
	pci_disable_io_method(device_t dev, device_t child, int space)
	{
	uint16_t bit;

	switch(space) {
	case SYS_RES_IOPORT:
	bit = PCIM_CMD_PORTEN;
	break;
	case SYS_RES_MEMORY:
	bit = PCIM_CMD_MEMEN;
	break;
	default:
	return (EINVAL);
	}
	pci_clear_command_bit(dev, child, bit);
	return (0);
	}

	/*
	* New style pci driver. Parent device is either a pci-host-bridge or a
	* pci-pci-bridge. Both kinds are represented by instances of pcib.
	*/

	void
	pci_print_verbose(struct pci_devinfo *dinfo)
	{

	if (bootverbose) {
	pcicfgregs *cfg = &dinfo->cfg;

	printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
	cfg->vendor, cfg->device, cfg->revid);
	printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
	cfg->domain, cfg->bus, cfg->slot, cfg->func);
	printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
	cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
	cfg->mfdev);
	printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
	cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
	printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
	cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
	cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
	if (cfg->intpin > 0)
	printf("\tintpin=%c, irq=%d\n",
	cfg->intpin +'a' -1, cfg->intline);
	if (cfg->pp.pp_cap) {
	uint16_t status;

	status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2);
	printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
	cfg->pp.pp_cap & PCIM_PCAP_SPEC,
	cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
	cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
	status & PCIM_PSTAT_DMASK);
	}
	if (cfg->msi.msi_location) {
	int ctrl;

	ctrl = cfg->msi.msi_ctrl;
	printf("\tMSI supports %d message%s%s%s\n",
	cfg->msi.msi_msgnum,
	(cfg->msi.msi_msgnum == 1) ? "" : "s",
	(ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
	(ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
	}
	if (cfg->msix.msix_location) {
	printf("\tMSI-X supports %d message%s ",
	cfg->msix.msix_msgnum,
	(cfg->msix.msix_msgnum == 1) ? "" : "s");
	if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
	printf("in map 0x%x\n",
	cfg->msix.msix_table_bar);
	else
	printf("in maps 0x%x and 0x%x\n",
	cfg->msix.msix_table_bar,
	cfg->msix.msix_pba_bar);
	}
	}
	}

	static int
	pci_porten(device_t dev)
	{
	return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
	}

	static int
	pci_memen(device_t dev)
	{
	return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
	}

	void
	pci_read_bar(device_t dev, int reg, pci_addr_t mapp, pci_addr_t testvalp,
	int *bar64)
	{
	struct pci_devinfo *dinfo;
	pci_addr_t map, testval;
	int ln2range;
	uint16_t cmd;

	/*
	* The device ROM BAR is special. It is always a 32-bit
	* memory BAR. Bit 0 is special and should not be set when
	* sizing the BAR.
	*/
	dinfo = device_get_ivars(dev);
	if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
	map = pci_read_config(dev, reg, 4);
	pci_write_config(dev, reg, 0xfffffffe, 4);
	testval = pci_read_config(dev, reg, 4);
	pci_write_config(dev, reg, map, 4);
	*mapp = map;
	*testvalp = testval;
	if (bar64 != NULL)
	*bar64 = 0;
	return;
	}

	map = pci_read_config(dev, reg, 4);
	ln2range = pci_maprange(map);
	if (ln2range == 64)
	map \|= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;

	/*
	* Disable decoding via the command register before
	* determining the BAR's length since we will be placing it in
	* a weird state.
	*/
	cmd = pci_read_config(dev, PCIR_COMMAND, 2);
	pci_write_config(dev, PCIR_COMMAND,
	cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);

	/*
	* Determine the BAR's length by writing all 1's. The bottom
	* log_2(size) bits of the BAR will stick as 0 when we read
	* the value back.
	*
	* NB: according to the PCI Local Bus Specification, rev. 3.0:
	* "Software writes 0FFFFFFFFh to both registers, reads them back,
	* and combines the result into a 64-bit value." (section 6.2.5.1)
	*
	* Writes to both registers must be performed before attempting to
	* read back the size value.
	*/
	testval = 0;
	pci_write_config(dev, reg, 0xffffffff, 4);
	if (ln2range == 64) {
	pci_write_config(dev, reg + 4, 0xffffffff, 4);
	testval \|= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
	}
	testval \|= pci_read_config(dev, reg, 4);

	/*
	* Restore the original value of the BAR. We may have reprogrammed
	* the BAR of the low-level console device and when booting verbose,
	* we need the console device addressable.
	*/
	pci_write_config(dev, reg, map, 4);
	if (ln2range == 64)
	pci_write_config(dev, reg + 4, map >> 32, 4);
	pci_write_config(dev, PCIR_COMMAND, cmd, 2);

	*mapp = map;
	*testvalp = testval;
	if (bar64 != NULL)
	*bar64 = (ln2range == 64);
	}

	static void
	pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
	{
	struct pci_devinfo *dinfo;
	int ln2range;

	/* The device ROM BAR is always a 32-bit memory BAR. */
	dinfo = device_get_ivars(dev);
	if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
	ln2range = 32;
	else
	ln2range = pci_maprange(pm->pm_value);
	pci_write_config(dev, pm->pm_reg, base, 4);
	if (ln2range == 64)
	pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
	pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
	if (ln2range == 64)
	pm->pm_value \|= (pci_addr_t)pci_read_config(dev,
	pm->pm_reg + 4, 4) << 32;
	}

	struct pci_map *
	pci_find_bar(device_t dev, int reg)
	{
	struct pci_devinfo *dinfo;
	struct pci_map *pm;

	dinfo = device_get_ivars(dev);
	STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
	if (pm->pm_reg == reg)
	return (pm);
	}
	return (NULL);
	}

	int
	pci_bar_enabled(device_t dev, struct pci_map *pm)
	{
	struct pci_devinfo *dinfo;
	uint16_t cmd;

	dinfo = device_get_ivars(dev);
	if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
	!(pm->pm_value & PCIM_BIOS_ENABLE))
	return (0);
	cmd = pci_read_config(dev, PCIR_COMMAND, 2);
	if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) \|\| PCI_BAR_MEM(pm->pm_value))
	return ((cmd & PCIM_CMD_MEMEN) != 0);
	else
	return ((cmd & PCIM_CMD_PORTEN) != 0);
	}

	struct pci_map *
	pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
	{
	struct pci_devinfo *dinfo;
	struct pci_map pm, prev;

	dinfo = device_get_ivars(dev);
	pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK \| M_ZERO);
	pm->pm_reg = reg;
	pm->pm_value = value;
	pm->pm_size = size;
	STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
	KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
	reg));
	if (STAILQ_NEXT(prev, pm_link) == NULL \|\|
	STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
	break;
	}
	if (prev != NULL)
	STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
	else
	STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
	return (pm);
	}

	static void
	pci_restore_bars(device_t dev)
	{
	struct pci_devinfo *dinfo;
	struct pci_map *pm;
	int ln2range;

	dinfo = device_get_ivars(dev);
	STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
	if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
	ln2range = 32;
	else
	ln2range = pci_maprange(pm->pm_value);
	pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
	if (ln2range == 64)
	pci_write_config(dev, pm->pm_reg + 4,
	pm->pm_value >> 32, 4);
	}
	}

	/*
	* Add a resource based on a pci map register. Return 1 if the map
	* register is a 32bit map register or 2 if it is a 64bit register.
	*/
	static int
	pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
	int force, int prefetch)
	{
	struct pci_map *pm;
	pci_addr_t base, map, testval;
	pci_addr_t start, end, count;
	int barlen, basezero, flags, maprange, mapsize, type;
	uint16_t cmd;
	struct resource *res;

	/*
	* The BAR may already exist if the device is a CardBus card
	* whose CIS is stored in this BAR.
	*/
	pm = pci_find_bar(dev, reg);
	if (pm != NULL) {
	maprange = pci_maprange(pm->pm_value);
	barlen = maprange == 64 ? 2 : 1;
	return (barlen);
	}

	pci_read_bar(dev, reg, &map, &testval, NULL);
	if (PCI_BAR_MEM(map)) {
	type = SYS_RES_MEMORY;
	if (map & PCIM_BAR_MEM_PREFETCH)
	prefetch = 1;
	} else
	type = SYS_RES_IOPORT;
	mapsize = pci_mapsize(testval);
	base = pci_mapbase(map);
	#ifdef __PCI_BAR_ZERO_VALID
	basezero = 0;
	#else
	basezero = base == 0;
	#endif
	maprange = pci_maprange(map);
	barlen = maprange == 64 ? 2 : 1;

	/*
	* For I/O registers, if bottom bit is set, and the next bit up
	* isn't clear, we know we have a BAR that doesn't conform to the
	* spec, so ignore it. Also, sanity check the size of the data
	* areas to the type of memory involved. Memory must be at least
	* 16 bytes in size, while I/O ranges must be at least 4.
	*/
	if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
	return (barlen);
	if ((type == SYS_RES_MEMORY && mapsize < 4) \|\|
	(type == SYS_RES_IOPORT && mapsize < 2))
	return (barlen);

	/* Save a record of this BAR. */
	pm = pci_add_bar(dev, reg, map, mapsize);
	if (bootverbose) {
	printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
	reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
	if (type == SYS_RES_IOPORT && !pci_porten(dev))
	printf(", port disabled\n");
	else if (type == SYS_RES_MEMORY && !pci_memen(dev))
	printf(", memory disabled\n");
	else
	printf(", enabled\n");
	}

	/*
	* If base is 0, then we have problems if this architecture does
	* not allow that. It is best to ignore such entries for the
	* moment. These will be allocated later if the driver specifically
	* requests them. However, some removable buses look better when
	* all resources are allocated, so allow '0' to be overriden.
	*
	* Similarly treat maps whose values is the same as the test value
	* read back. These maps have had all f's written to them by the
	* BIOS in an attempt to disable the resources.
	*/
	if (!force && (basezero \|\| map == testval))
	return (barlen);
	if ((u_long)base != base) {
	device_printf(bus,
	"pci%d:%d:%d:%d bar %#x too many address bits",
	pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
	pci_get_function(dev), reg);
	return (barlen);
	}

	/*
	* This code theoretically does the right thing, but has
	* undesirable side effects in some cases where peripherals
	* respond oddly to having these bits enabled. Let the user
	* be able to turn them off (since pci_enable_io_modes is 1 by
	* default).
	*/
	if (pci_enable_io_modes) {
	/* Turn on resources that have been left off by a lazy BIOS */
	if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
	cmd = pci_read_config(dev, PCIR_COMMAND, 2);
	cmd \|= PCIM_CMD_PORTEN;
	pci_write_config(dev, PCIR_COMMAND, cmd, 2);
	}
	if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
	cmd = pci_read_config(dev, PCIR_COMMAND, 2);
	cmd \|= PCIM_CMD_MEMEN;
	pci_write_config(dev, PCIR_COMMAND, cmd, 2);
	}
	} else {
	if (type == SYS_RES_IOPORT && !pci_porten(dev))
	return (barlen);
	if (type == SYS_RES_MEMORY && !pci_memen(dev))
	return (barlen);
	}

	count = (pci_addr_t)1 << mapsize;
	flags = RF_ALIGNMENT_LOG2(mapsize);
	if (prefetch)
	flags \|= RF_PREFETCHABLE;
	if (basezero \|\| base == pci_mapbase(testval) \|\| pci_clear_bars) {
	start = 0; /* Let the parent decide. */
	end = ~0;
	} else {
	start = base;
	end = base + count - 1;
	}
	resource_list_add(rl, type, reg, start, end, count);

	/*
	* Try to allocate the resource for this BAR from our parent
	* so that this resource range is already reserved. The
	* driver for this device will later inherit this resource in
	* pci_alloc_resource().
	*/
	res = resource_list_reserve(rl, bus, dev, type, &reg, start, end, count,
	flags);
	if ((pci_do_realloc_bars
	\|\| pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
	&& res == NULL && (start != 0 \|\| end != ~0)) {
	/*
	* If the allocation fails, try to allocate a resource for
	* this BAR using any available range. The firmware felt
	* it was important enough to assign a resource, so don't
	* disable decoding if we can help it.
	*/
	resource_list_delete(rl, type, reg);
	resource_list_add(rl, type, reg, 0, ~0, count);
	res = resource_list_reserve(rl, bus, dev, type, &reg, 0, ~0,
	count, flags);
	}
	if (res == NULL) {
	/*
	* If the allocation fails, delete the resource list entry
	* and disable decoding for this device.
	*
	* If the driver requests this resource in the future,
	* pci_reserve_map() will try to allocate a fresh
	* resource range.
	*/
	resource_list_delete(rl, type, reg);
	pci_disable_io(dev, type);
	if (bootverbose)
	device_printf(bus,
	"pci%d:%d:%d:%d bar %#x failed to allocate\n",
	pci_get_domain(dev), pci_get_bus(dev),
	pci_get_slot(dev), pci_get_function(dev), reg);
	} else {
	start = rman_get_start(res);
	pci_write_bar(dev, pm, start);
	}
	return (barlen);
	}

	/*
	* For ATA devices we need to decide early what addressing mode to use.
	* Legacy demands that the primary and secondary ATA ports sits on the
	* same addresses that old ISA hardware did. This dictates that we use
	* those addresses and ignore the BAR's if we cannot set PCI native
	* addressing mode.
	*/
	static void
	pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
	uint32_t prefetchmask)
	{
	int rid, type, progif;
	#if 0
	/* if this device supports PCI native addressing use it */
	progif = pci_read_config(dev, PCIR_PROGIF, 1);
	if ((progif & 0x8a) == 0x8a) {
	if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
	pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
	printf("Trying ATA native PCI addressing mode\n");
	pci_write_config(dev, PCIR_PROGIF, progif \| 0x05, 1);
	}
	}
	#endif
	progif = pci_read_config(dev, PCIR_PROGIF, 1);
	type = SYS_RES_IOPORT;
	if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
	pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
	prefetchmask & (1 << 0));
	pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
	prefetchmask & (1 << 1));
	} else {
	rid = PCIR_BAR(0);
	resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
	(void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0,
	0x1f7, 8, 0);
	rid = PCIR_BAR(1);
	resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
	(void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6,
	0x3f6, 1, 0);
	}
	if (progif & PCIP_STORAGE_IDE_MODESEC) {
	pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
	prefetchmask & (1 << 2));
	pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
	prefetchmask & (1 << 3));
	} else {
	rid = PCIR_BAR(2);
	resource_list_add(rl, type, rid, 0x170, 0x177, 8);
	(void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170,
	0x177, 8, 0);
	rid = PCIR_BAR(3);
	resource_list_add(rl, type, rid, 0x376, 0x376, 1);
	(void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376,
	0x376, 1, 0);
	}
	pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
	prefetchmask & (1 << 4));
	pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
	prefetchmask & (1 << 5));
	}

	static void
	pci_assign_interrupt(device_t bus, device_t dev, int force_route)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	pcicfgregs *cfg = &dinfo->cfg;
	char tunable_name[64];
	int irq;

	/* Has to have an intpin to have an interrupt. */
	if (cfg->intpin == 0)
	return;

	/* Let the user override the IRQ with a tunable. */
	irq = PCI_INVALID_IRQ;
	snprintf(tunable_name, sizeof(tunable_name),
	"hw.pci%d.%d.%d.INT%c.irq",
	cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
	if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 \|\| irq <= 0))
	irq = PCI_INVALID_IRQ;

	/*
	* If we didn't get an IRQ via the tunable, then we either use the
	* IRQ value in the intline register or we ask the bus to route an
	* interrupt for us. If force_route is true, then we only use the
	* value in the intline register if the bus was unable to assign an
	* IRQ.
	*/
	if (!PCI_INTERRUPT_VALID(irq)) {
	if (!PCI_INTERRUPT_VALID(cfg->intline) \|\| force_route)
	irq = PCI_ASSIGN_INTERRUPT(bus, dev);
	if (!PCI_INTERRUPT_VALID(irq))
	irq = cfg->intline;
	}

	/* If after all that we don't have an IRQ, just bail. */
	if (!PCI_INTERRUPT_VALID(irq))
	return;

	/* Update the config register if it changed. */
	if (irq != cfg->intline) {
	cfg->intline = irq;
	pci_write_config(dev, PCIR_INTLINE, irq, 1);
	}

	/* Add this IRQ as rid 0 interrupt resource. */
	resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
	}

	/* Perform early OHCI takeover from SMM. */
	static void
	ohci_early_takeover(device_t self)
	{
	struct resource *res;
	uint32_t ctl;
	int rid;
	int i;

	rid = PCIR_BAR(0);
	res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (res == NULL)
	return;

	ctl = bus_read_4(res, OHCI_CONTROL);
	if (ctl & OHCI_IR) {
	if (bootverbose)
	printf("ohci early: "
	"SMM active, request owner change\n");
	bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
	for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
	DELAY(1000);
	ctl = bus_read_4(res, OHCI_CONTROL);
	}
	if (ctl & OHCI_IR) {
	if (bootverbose)
	printf("ohci early: "
	"SMM does not respond, resetting\n");
	bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
	}
	/* Disable interrupts */
	bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
	}

	bus_release_resource(self, SYS_RES_MEMORY, rid, res);
	}

	/* Perform early UHCI takeover from SMM. */
	static void
	uhci_early_takeover(device_t self)
	{
	struct resource *res;
	int rid;

	/*
	* Set the PIRQD enable bit and switch off all the others. We don't
	* want legacy support to interfere with us XXX Does this also mean
	* that the BIOS won't touch the keyboard anymore if it is connected
	* to the ports of the root hub?
	*/
	pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);

	/* Disable interrupts */
	rid = PCI_UHCI_BASE_REG;
	res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
	if (res != NULL) {
	bus_write_2(res, UHCI_INTR, 0);
	bus_release_resource(self, SYS_RES_IOPORT, rid, res);
	}
	}

	/* Perform early EHCI takeover from SMM. */
	static void
	ehci_early_takeover(device_t self)
	{
	struct resource *res;
	uint32_t cparams;
	uint32_t eec;
	uint8_t eecp;
	uint8_t bios_sem;
	uint8_t offs;
	int rid;
	int i;

	rid = PCIR_BAR(0);
	res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (res == NULL)
	return;

	cparams = bus_read_4(res, EHCI_HCCPARAMS);

	/* Synchronise with the BIOS if it owns the controller. */
	for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
	eecp = EHCI_EECP_NEXT(eec)) {
	eec = pci_read_config(self, eecp, 4);
	if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
	continue;
	}
	bios_sem = pci_read_config(self, eecp +
	EHCI_LEGSUP_BIOS_SEM, 1);
	if (bios_sem == 0) {
	continue;
	}
	if (bootverbose)
	printf("ehci early: "
	"SMM active, request owner change\n");

	pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);

	for (i = 0; (i < 100) && (bios_sem != 0); i++) {
	DELAY(1000);
	bios_sem = pci_read_config(self, eecp +
	EHCI_LEGSUP_BIOS_SEM, 1);
	}

	if (bios_sem != 0) {
	if (bootverbose)
	printf("ehci early: "
	"SMM does not respond\n");
	}
	/* Disable interrupts */
	offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
	bus_write_4(res, offs + EHCI_USBINTR, 0);
	}
	bus_release_resource(self, SYS_RES_MEMORY, rid, res);
	}

	/* Perform early XHCI takeover from SMM. */
	static void
	xhci_early_takeover(device_t self)
	{
	struct resource *res;
	uint32_t cparams;
	uint32_t eec;
	uint8_t eecp;
	uint8_t bios_sem;
	uint8_t offs;
	int rid;
	int i;

	rid = PCIR_BAR(0);
	res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (res == NULL)
	return;

	cparams = bus_read_4(res, XHCI_HCSPARAMS0);

	eec = -1;

	/* Synchronise with the BIOS if it owns the controller. */
	for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
	eecp += XHCI_XECP_NEXT(eec) << 2) {
	eec = bus_read_4(res, eecp);

	if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
	continue;

	bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
	if (bios_sem == 0)
	continue;

	if (bootverbose)
	printf("xhci early: "
	"SMM active, request owner change\n");

	bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);

	/* wait a maximum of 5 second */

	for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
	DELAY(1000);
	bios_sem = bus_read_1(res, eecp +
	XHCI_XECP_BIOS_SEM);
	}

	if (bios_sem != 0) {
	if (bootverbose)
	printf("xhci early: "
	"SMM does not respond\n");
	}

	/* Disable interrupts */
	offs = bus_read_1(res, XHCI_CAPLENGTH);
	bus_write_4(res, offs + XHCI_USBCMD, 0);
	bus_read_4(res, offs + XHCI_USBSTS);
	}
	bus_release_resource(self, SYS_RES_MEMORY, rid, res);
	}

	#if defined(NEW_PCIB) && defined(PCI_RES_BUS)
	static void
	pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
	struct resource_list *rl)
	{
	struct resource *res;
	char *cp;
	rman_res_t start, end, count;
	int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;

	switch (cfg->hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_BRIDGE:
	sec_reg = PCIR_SECBUS_1;
	sub_reg = PCIR_SUBBUS_1;
	break;
	case PCIM_HDRTYPE_CARDBUS:
	sec_reg = PCIR_SECBUS_2;
	sub_reg = PCIR_SUBBUS_2;
	break;
	default:
	return;
	}

	/*
	* If the existing bus range is valid, attempt to reserve it
	* from our parent. If this fails for any reason, clear the
	* secbus and subbus registers.
	*
	* XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
	* This would at least preserve the existing sec_bus if it is
	* valid.
	*/
	sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
	sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);

	/* Quirk handling. */
	switch (pci_get_devid(dev)) {
	case 0x12258086: /* Intel 82454KX/GX (Orion) */
	sup_bus = pci_read_config(dev, 0x41, 1);
	if (sup_bus != 0xff) {
	sec_bus = sup_bus + 1;
	sub_bus = sup_bus + 1;
	PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
	PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
	}
	break;

	case 0x00dd10de:
	/* Compaq R3000 BIOS sets wrong subordinate bus number. */
	if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
	break;
	if (strncmp(cp, "Compal", 6) != 0) {
	freeenv(cp);
	break;
	}
	freeenv(cp);
	if ((cp = kern_getenv("smbios.planar.product")) == NULL)
	break;
	if (strncmp(cp, "08A0", 4) != 0) {
	freeenv(cp);
	break;
	}
	freeenv(cp);
	if (sub_bus < 0xa) {
	sub_bus = 0xa;
	PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
	}
	break;
	}

	if (bootverbose)
	printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
	if (sec_bus > 0 && sub_bus >= sec_bus) {
	start = sec_bus;
	end = sub_bus;
	count = end - start + 1;

	resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);

	/*
	* If requested, clear secondary bus registers in
	* bridge devices to force a complete renumbering
	* rather than reserving the existing range. However,
	* preserve the existing size.
	*/
	if (pci_clear_buses)
	goto clear;

	rid = 0;
	res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid,
	start, end, count, 0);
	if (res != NULL)
	return;

	if (bootverbose)
	device_printf(bus,
	"pci%d:%d:%d:%d secbus failed to allocate\n",
	pci_get_domain(dev), pci_get_bus(dev),
	pci_get_slot(dev), pci_get_function(dev));
	}

	clear:
	PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
	PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
	}

	static struct resource *
	pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start,
	rman_res_t end, rman_res_t count, u_int flags)
	{
	struct pci_devinfo *dinfo;
	pcicfgregs *cfg;
	struct resource_list *rl;
	struct resource *res;
	int sec_reg, sub_reg;

	dinfo = device_get_ivars(child);
	cfg = &dinfo->cfg;
	rl = &dinfo->resources;
	switch (cfg->hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_BRIDGE:
	sec_reg = PCIR_SECBUS_1;
	sub_reg = PCIR_SUBBUS_1;
	break;
	case PCIM_HDRTYPE_CARDBUS:
	sec_reg = PCIR_SECBUS_2;
	sub_reg = PCIR_SUBBUS_2;
	break;
	default:
	return (NULL);
	}

	if (*rid != 0)
	return (NULL);

	if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL)
	resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count);
	if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) {
	res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
	start, end, count, flags & ~RF_ACTIVE);
	if (res == NULL) {
	resource_list_delete(rl, PCI_RES_BUS, *rid);
	device_printf(child, "allocating %ju bus%s failed\n",
	count, count == 1 ? "" : "es");
	return (NULL);
	}
	if (bootverbose)
	device_printf(child,
	"Lazy allocation of %ju bus%s at %ju\n", count,
	count == 1 ? "" : "es", rman_get_start(res));
	PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
	PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
	}
	return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
	end, count, flags));
	}
	#endif

	static int
	pci_ea_bei_to_rid(device_t dev, int bei)
	{
	#ifdef PCI_IOV
	struct pci_devinfo *dinfo;
	int iov_pos;
	struct pcicfg_iov *iov;

	dinfo = device_get_ivars(dev);
	iov = dinfo->cfg.iov;
	if (iov != NULL)
	iov_pos = iov->iov_pos;
	else
	iov_pos = 0;
	#endif

	/* Check if matches BAR */
	if ((bei >= PCIM_EA_BEI_BAR_0) &&
	(bei <= PCIM_EA_BEI_BAR_5))
	return (PCIR_BAR(bei));

	/* Check ROM */
	if (bei == PCIM_EA_BEI_ROM)
	return (PCIR_BIOS);

	#ifdef PCI_IOV
	/* Check if matches VF_BAR */
	if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
	(bei <= PCIM_EA_BEI_VF_BAR_5))
	return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
	iov_pos);
	#endif

	return (-1);
	}

	int
	pci_ea_is_enabled(device_t dev, int rid)
	{
	struct pci_ea_entry *ea;
	struct pci_devinfo *dinfo;

	dinfo = device_get_ivars(dev);

	STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
	if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
	return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
	}

	return (0);
	}

	void
	pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
	{
	struct pci_ea_entry *ea;
	struct pci_devinfo *dinfo;
	pci_addr_t start, end, count;
	struct resource_list *rl;
	int type, flags, rid;
	struct resource *res;
	uint32_t tmp;
	#ifdef PCI_IOV
	struct pcicfg_iov *iov;
	#endif

	dinfo = device_get_ivars(dev);
	rl = &dinfo->resources;
	flags = 0;

	#ifdef PCI_IOV
	iov = dinfo->cfg.iov;
	#endif

	if (dinfo->cfg.ea.ea_location == 0)
	return;

	STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {

	/*
	* TODO: Ignore EA-BAR if is not enabled.
	* Currently the EA implementation supports
	* only situation, where EA structure contains
	* predefined entries. In case they are not enabled
	* leave them unallocated and proceed with
	* a legacy-BAR mechanism.
	*/
	if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
	continue;

	switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
	case PCIM_EA_P_MEM_PREFETCH:
	case PCIM_EA_P_VF_MEM_PREFETCH:
	flags = RF_PREFETCHABLE;
	/* FALLTHROUGH */
	case PCIM_EA_P_VF_MEM:
	case PCIM_EA_P_MEM:
	type = SYS_RES_MEMORY;
	break;
	case PCIM_EA_P_IO:
	type = SYS_RES_IOPORT;
	break;
	default:
	continue;
	}

	if (alloc_iov != 0) {
	#ifdef PCI_IOV
	/* Allocating IOV, confirm BEI matches */
	if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) \|\|
	(ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
	continue;
	#else
	continue;
	#endif
	} else {
	/* Allocating BAR, confirm BEI matches */
	if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) \|\|
	(ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
	(ea->eae_bei != PCIM_EA_BEI_ROM))
	continue;
	}

	rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
	if (rid < 0)
	continue;

	/* Skip resources already allocated by EA */
	if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) \|\|
	(resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
	continue;

	start = ea->eae_base;
	count = ea->eae_max_offset + 1;
	#ifdef PCI_IOV
	if (iov != NULL)
	count = count * iov->iov_num_vfs;
	#endif
	end = start + count - 1;
	if (count == 0)
	continue;

	resource_list_add(rl, type, rid, start, end, count);
	res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count,
	flags);
	if (res == NULL) {
	resource_list_delete(rl, type, rid);

	/*
	* Failed to allocate using EA, disable entry.
	* Another attempt to allocation will be performed
	* further, but this time using legacy BAR registers
	*/
	tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
	tmp &= ~PCIM_EA_ENABLE;
	pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);

	/*
	* Disabling entry might fail in case it is hardwired.
	* Read flags again to match current status.
	*/
	ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);

	continue;
	}

	/* As per specification, fill BAR with zeros */
	pci_write_config(dev, rid, 0, 4);
	}
	}

	void
	pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
	{
	struct pci_devinfo *dinfo;
	pcicfgregs *cfg;
	struct resource_list *rl;
	const struct pci_quirk *q;
	uint32_t devid;
	int i;

	dinfo = device_get_ivars(dev);
	cfg = &dinfo->cfg;
	rl = &dinfo->resources;
	devid = (cfg->device << 16) \| cfg->vendor;

	/* Allocate resources using Enhanced Allocation */
	pci_add_resources_ea(bus, dev, 0);

	/* ATA devices needs special map treatment */
	if ((pci_get_class(dev) == PCIC_STORAGE) &&
	(pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
	((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) \|\|
	(!pci_read_config(dev, PCIR_BAR(0), 4) &&
	!pci_read_config(dev, PCIR_BAR(2), 4))) )
	pci_ata_maps(bus, dev, rl, force, prefetchmask);
	else
	for (i = 0; i < cfg->nummaps;) {
	/* Skip resources already managed by EA */
	if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) \|\|
	(resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) \|\|
	pci_ea_is_enabled(dev, PCIR_BAR(i))) {
	i++;
	continue;
	}

	/*
	* Skip quirked resources.
	*/
	for (q = &pci_quirks[0]; q->devid != 0; q++)
	if (q->devid == devid &&
	q->type == PCI_QUIRK_UNMAP_REG &&
	q->arg1 == PCIR_BAR(i))
	break;
	if (q->devid != 0) {
	i++;
	continue;
	}
	i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
	prefetchmask & (1 << i));
	}

	/*
	* Add additional, quirked resources.
	*/
	for (q = &pci_quirks[0]; q->devid != 0; q++)
	if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
	pci_add_map(bus, dev, q->arg1, rl, force, 0);

	if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) {
	#ifdef __PCI_REROUTE_INTERRUPT
	/*
	* Try to re-route interrupts. Sometimes the BIOS or
	* firmware may leave bogus values in these registers.
	* If the re-route fails, then just stick with what we
	* have.
	*/
	pci_assign_interrupt(bus, dev, 1);
	#else
	pci_assign_interrupt(bus, dev, 0);
	#endif
	}

	if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
	pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
	if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
	xhci_early_takeover(dev);
	else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
	ehci_early_takeover(dev);
	else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
	ohci_early_takeover(dev);
	else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
	uhci_early_takeover(dev);
	}

	#if defined(NEW_PCIB) && defined(PCI_RES_BUS)
	/*
	* Reserve resources for secondary bus ranges behind bridge
	* devices.
	*/
	pci_reserve_secbus(bus, dev, cfg, rl);
	#endif
	}

	static struct pci_devinfo *
	pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
	int slot, int func)
	{
	struct pci_devinfo *dinfo;

	dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
	if (dinfo != NULL)
	pci_add_child(dev, dinfo);

	return (dinfo);
	}

	void
	pci_add_children(device_t dev, int domain, int busno)
	{
	#define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
	device_t pcib = device_get_parent(dev);
	struct pci_devinfo *dinfo;
	int maxslots;
	int s, f, pcifunchigh;
	uint8_t hdrtype;
	int first_func;

	/*
	* Try to detect a device at slot 0, function 0. If it exists, try to
	* enable ARI. We must enable ARI before detecting the rest of the
	* functions on this bus as ARI changes the set of slots and functions
	* that are legal on this bus.
	*/
	dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
	if (dinfo != NULL && pci_enable_ari)
	PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);

	/*
	* Start looking for new devices on slot 0 at function 1 because we
	* just identified the device at slot 0, function 0.
	*/
	first_func = 1;

	maxslots = PCIB_MAXSLOTS(pcib);
	for (s = 0; s <= maxslots; s++, first_func = 0) {
	pcifunchigh = 0;
	f = 0;
	DELAY(1);
	hdrtype = REG(PCIR_HDRTYPE, 1);
	if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
	continue;
	if (hdrtype & PCIM_MFDEV)
	pcifunchigh = PCIB_MAXFUNCS(pcib);
	for (f = first_func; f <= pcifunchigh; f++)
	pci_identify_function(pcib, dev, domain, busno, s, f);
	}
	#undef REG
	}

	int
	pci_rescan_method(device_t dev)
	{
	#define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
	device_t pcib = device_get_parent(dev);
	device_t child, devlist, unchanged;
	int devcount, error, i, j, maxslots, oldcount;
	int busno, domain, s, f, pcifunchigh;
	uint8_t hdrtype;

	/* No need to check for ARI on a rescan. */
	error = device_get_children(dev, &devlist, &devcount);
	if (error)
	return (error);
	if (devcount != 0) {
	unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
	M_NOWAIT \| M_ZERO);
	if (unchanged == NULL) {
	free(devlist, M_TEMP);
	return (ENOMEM);
	}
	} else
	unchanged = NULL;

	domain = pcib_get_domain(dev);
	busno = pcib_get_bus(dev);
	maxslots = PCIB_MAXSLOTS(pcib);
	for (s = 0; s <= maxslots; s++) {
	/* If function 0 is not present, skip to the next slot. */
	f = 0;
	if (REG(PCIR_VENDOR, 2) == 0xffff)
	continue;
	pcifunchigh = 0;
	hdrtype = REG(PCIR_HDRTYPE, 1);
	if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
	continue;
	if (hdrtype & PCIM_MFDEV)
	pcifunchigh = PCIB_MAXFUNCS(pcib);
	for (f = 0; f <= pcifunchigh; f++) {
	if (REG(PCIR_VENDOR, 2) == 0xffff)
	continue;

	/*
	* Found a valid function. Check if a
	* device_t for this device already exists.
	*/
	for (i = 0; i < devcount; i++) {
	child = devlist[i];
	if (child == NULL)
	continue;
	if (pci_get_slot(child) == s &&
	pci_get_function(child) == f) {
	unchanged[i] = child;
	goto next_func;
	}
	}

	pci_identify_function(pcib, dev, domain, busno, s, f);
	next_func:;
	}
	}

	/* Remove devices that are no longer present. */
	for (i = 0; i < devcount; i++) {
	if (unchanged[i] != NULL)
	continue;
	device_delete_child(dev, devlist[i]);
	}

	free(devlist, M_TEMP);
	oldcount = devcount;

	/* Try to attach the devices just added. */
	error = device_get_children(dev, &devlist, &devcount);
	if (error) {
	free(unchanged, M_TEMP);
	return (error);
	}

	for (i = 0; i < devcount; i++) {
	for (j = 0; j < oldcount; j++) {
	if (devlist[i] == unchanged[j])
	goto next_device;
	}

	device_probe_and_attach(devlist[i]);
	next_device:;
	}

	free(unchanged, M_TEMP);
	free(devlist, M_TEMP);
	return (0);
	#undef REG
	}

	#ifdef PCI_IOV
	device_t
	pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
	uint16_t did)
	{
	struct pci_devinfo *vf_dinfo;
	device_t pcib;
	int busno, slot, func;

	pcib = device_get_parent(bus);

	PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);

	vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
	slot, func, vid, did);

	vf_dinfo->cfg.flags \|= PCICFG_VF;
	pci_add_child(bus, vf_dinfo);

	return (vf_dinfo->cfg.dev);
	}

	device_t
	pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
	uint16_t vid, uint16_t did)
	{

	return (pci_add_iov_child(bus, pf, rid, vid, did));
	}
	#endif

	static void
	pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
	{
	int aer;
	uint32_t r;
	uint16_t r2;

	if (dinfo->cfg.pcie.pcie_location != 0 &&
	dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
	r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
	PCIER_ROOT_CTL, 2);
	r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR \|
	PCIEM_ROOT_CTL_SERR_NONFATAL \| PCIEM_ROOT_CTL_SERR_FATAL);
	pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
	PCIER_ROOT_CTL, r2, 2);
	}
	if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
	r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
	pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
	if (r != 0 && bootverbose) {
	pci_printf(&dinfo->cfg,
	"clearing AER UC 0x%08x -> 0x%08x\n",
	r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
	4));
	}

	r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
	r &= ~(PCIM_AER_UC_TRAINING_ERROR \|
	PCIM_AER_UC_DL_PROTOCOL_ERROR \|
	PCIM_AER_UC_SURPRISE_LINK_DOWN \|
	PCIM_AER_UC_POISONED_TLP \|
	PCIM_AER_UC_FC_PROTOCOL_ERROR \|
	PCIM_AER_UC_COMPLETION_TIMEOUT \|
	PCIM_AER_UC_COMPLETER_ABORT \|
	PCIM_AER_UC_UNEXPECTED_COMPLETION \|
	PCIM_AER_UC_RECEIVER_OVERFLOW \|
	PCIM_AER_UC_MALFORMED_TLP \|
	PCIM_AER_UC_ECRC_ERROR \|
	PCIM_AER_UC_UNSUPPORTED_REQUEST \|
	PCIM_AER_UC_ACS_VIOLATION \|
	PCIM_AER_UC_INTERNAL_ERROR \|
	PCIM_AER_UC_MC_BLOCKED_TLP \|
	PCIM_AER_UC_ATOMIC_EGRESS_BLK \|
	PCIM_AER_UC_TLP_PREFIX_BLOCKED);
	pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);

	r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
	pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
	if (r != 0 && bootverbose) {
	pci_printf(&dinfo->cfg,
	"clearing AER COR 0x%08x -> 0x%08x\n",
	r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
	4));
	}

	r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
	r &= ~(PCIM_AER_COR_RECEIVER_ERROR \|
	PCIM_AER_COR_BAD_TLP \|
	PCIM_AER_COR_BAD_DLLP \|
	PCIM_AER_COR_REPLAY_ROLLOVER \|
	PCIM_AER_COR_REPLAY_TIMEOUT \|
	PCIM_AER_COR_ADVISORY_NF_ERROR \|
	PCIM_AER_COR_INTERNAL_ERROR \|
	PCIM_AER_COR_HEADER_LOG_OVFLOW);
	pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);

	r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
	PCIER_DEVICE_CTL, 2);
	r \|= PCIEM_CTL_COR_ENABLE \| PCIEM_CTL_NFER_ENABLE \|
	PCIEM_CTL_FER_ENABLE \| PCIEM_CTL_URR_ENABLE;
	pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
	PCIER_DEVICE_CTL, r, 2);
	}
	}

	void
	pci_add_child(device_t bus, struct pci_devinfo *dinfo)
	{
	device_t dev;

	dinfo->cfg.dev = dev = device_add_child(bus, NULL, -1);
	device_set_ivars(dev, dinfo);
	resource_list_init(&dinfo->resources);
	pci_cfg_save(dev, dinfo, 0);
	pci_cfg_restore(dev, dinfo);
	pci_print_verbose(dinfo);
	pci_add_resources(bus, dev, 0, 0);
	pci_child_added(dinfo->cfg.dev);

	if (pci_clear_aer_on_attach)
	pci_add_child_clear_aer(dev, dinfo);

	EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
	}

	void
	pci_child_added_method(device_t dev, device_t child)
	{

	}

	static int
	pci_probe(device_t dev)
	{

	device_set_desc(dev, "PCI bus");

	/* Allow other subclasses to override this driver. */
	return (BUS_PROBE_GENERIC);
	}

	int
	pci_attach_common(device_t dev)
	{
	struct pci_softc *sc;
	int busno, domain;
	#ifdef PCI_RES_BUS
	int rid;
	#endif

	sc = device_get_softc(dev);
	domain = pcib_get_domain(dev);
	busno = pcib_get_bus(dev);
	#ifdef PCI_RES_BUS
	rid = 0;
	sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
	1, 0);
	if (sc->sc_bus == NULL) {
	device_printf(dev, "failed to allocate bus number\n");
	return (ENXIO);
	}
	#endif
	if (bootverbose)
	device_printf(dev, "domain=%d, physical bus=%d\n",
	domain, busno);
	sc->sc_dma_tag = bus_get_dma_tag(dev);
	return (0);
	}

	int
	pci_attach(device_t dev)
	{
	int busno, domain, error;

	error = pci_attach_common(dev);
	if (error)
	return (error);

	/*
	* Since there can be multiple independently numbered PCI
	* buses on systems with multiple PCI domains, we can't use
	* the unit number to decide which bus we are probing. We ask
	* the parent pcib what our domain and bus numbers are.
	*/
	domain = pcib_get_domain(dev);
	busno = pcib_get_bus(dev);
	pci_add_children(dev, domain, busno);
	return (bus_generic_attach(dev));
	}

	int
	pci_detach(device_t dev)
	{
	#ifdef PCI_RES_BUS
	struct pci_softc *sc;
	#endif
	int error;

	error = bus_generic_detach(dev);
	if (error)
	return (error);
	#ifdef PCI_RES_BUS
	sc = device_get_softc(dev);
	error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
	if (error)
	return (error);
	#endif
	return (device_delete_children(dev));
	}

	static void
	pci_hint_device_unit(device_t dev, device_t child, const char name, int unitp)
	{
	int line, unit;
	const char *at;
	char me1[24], me2[32];
	uint8_t b, s, f;
	uint32_t d;

	d = pci_get_domain(child);
	b = pci_get_bus(child);
	s = pci_get_slot(child);
	f = pci_get_function(child);
	snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
	snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
	line = 0;
	while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
	resource_string_value(name, unit, "at", &at);
	if (strcmp(at, me1) != 0 && strcmp(at, me2) != 0)
	continue; /* No match, try next candidate */
	*unitp = unit;
	return;
	}
	}

	static void
	pci_set_power_child(device_t dev, device_t child, int state)
	{
	device_t pcib;
	int dstate;

	/*
	* Set the device to the given state. If the firmware suggests
	* a different power state, use it instead. If power management
	* is not present, the firmware is responsible for managing
	* device power. Skip children who aren't attached since they
	* are handled separately.
	*/
	pcib = device_get_parent(dev);
	dstate = state;
	if (device_is_attached(child) &&
	PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
	pci_set_powerstate(child, dstate);
	}

	int
	pci_suspend_child(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo;
	struct resource_list_entry *rle;
	int error;

	dinfo = device_get_ivars(child);

	/*
	* Save the PCI configuration space for the child and set the
	* device in the appropriate power state for this sleep state.
	*/
	pci_cfg_save(child, dinfo, 0);

	/* Suspend devices before potentially powering them down. */
	error = bus_generic_suspend_child(dev, child);

	if (error)
	return (error);

	if (pci_do_power_suspend) {
	/*
	* Make sure this device's interrupt handler is not invoked
	* in the case the device uses a shared interrupt that can
	* be raised by some other device.
	* This is applicable only to regular (legacy) PCI interrupts
	* as MSI/MSI-X interrupts are never shared.
	*/
	rle = resource_list_find(&dinfo->resources,
	SYS_RES_IRQ, 0);
	if (rle != NULL && rle->res != NULL)
	(void)bus_suspend_intr(child, rle->res);
	pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
	}

	return (0);
	}

	int
	pci_resume_child(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo;
	struct resource_list_entry *rle;

	if (pci_do_power_resume)
	pci_set_power_child(dev, child, PCI_POWERSTATE_D0);

	dinfo = device_get_ivars(child);
	pci_cfg_restore(child, dinfo);
	if (!device_is_attached(child))
	pci_cfg_save(child, dinfo, 1);

	bus_generic_resume_child(dev, child);

	/*
	* Allow interrupts only after fully resuming the driver and hardware.
	*/
	if (pci_do_power_suspend) {
	/* See pci_suspend_child for details. */
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
	if (rle != NULL && rle->res != NULL)
	(void)bus_resume_intr(child, rle->res);
	}

	return (0);
	}

	int
	pci_resume(device_t dev)
	{
	device_t child, *devlist;
	int error, i, numdevs;

	if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
	return (error);

	/*
	* Resume critical devices first, then everything else later.
	*/
	for (i = 0; i < numdevs; i++) {
	child = devlist[i];
	switch (pci_get_class(child)) {
	case PCIC_DISPLAY:
	case PCIC_MEMORY:
	case PCIC_BRIDGE:
	case PCIC_BASEPERIPH:
	BUS_RESUME_CHILD(dev, child);
	break;
	}
	}
	for (i = 0; i < numdevs; i++) {
	child = devlist[i];
	switch (pci_get_class(child)) {
	case PCIC_DISPLAY:
	case PCIC_MEMORY:
	case PCIC_BRIDGE:
	case PCIC_BASEPERIPH:
	break;
	default:
	BUS_RESUME_CHILD(dev, child);
	}
	}
	free(devlist, M_TEMP);
	return (0);
	}

	static void
	pci_load_vendor_data(void)
	{
	caddr_t data;
	void *ptr;
	size_t sz;

	data = preload_search_by_type("pci_vendor_data");
	if (data != NULL) {
	ptr = preload_fetch_addr(data);
	sz = preload_fetch_size(data);
	if (ptr != NULL && sz != 0) {
	pci_vendordata = ptr;
	pci_vendordata_size = sz;
	/* terminate the database */
	pci_vendordata[pci_vendordata_size] = '\n';
	}
	}
	}

	void
	pci_driver_added(device_t dev, driver_t *driver)
	{
	int numdevs;
	device_t *devlist;
	device_t child;
	struct pci_devinfo *dinfo;
	int i;

	if (bootverbose)
	device_printf(dev, "driver added\n");
	DEVICE_IDENTIFY(driver, dev);
	if (device_get_children(dev, &devlist, &numdevs) != 0)
	return;
	for (i = 0; i < numdevs; i++) {
	child = devlist[i];
	if (device_get_state(child) != DS_NOTPRESENT)
	continue;
	dinfo = device_get_ivars(child);
	pci_print_verbose(dinfo);
	if (bootverbose)
	pci_printf(&dinfo->cfg, "reprobing on driver added\n");
	pci_cfg_restore(child, dinfo);
	if (device_probe_and_attach(child) != 0)
	pci_child_detached(dev, child);
	}
	free(devlist, M_TEMP);
	}

	int
	pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
	driver_filter_t filter, driver_intr_t intr, void arg, void *cookiep)
	{
	struct pci_devinfo *dinfo;
	struct msix_table_entry *mte;
	struct msix_vector *mv;
	uint64_t addr;
	uint32_t data;
	void *cookie;
	int error, rid;

	error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
	arg, &cookie);
	if (error)
	return (error);

	/* If this is not a direct child, just bail out. */
	if (device_get_parent(child) != dev) {
	*cookiep = cookie;
	return(0);
	}

	rid = rman_get_rid(irq);
	if (rid == 0) {
	/* Make sure that INTx is enabled */
	pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
	} else {
	/*
	* Check to see if the interrupt is MSI or MSI-X.
	* Ask our parent to map the MSI and give
	* us the address and data register values.
	* If we fail for some reason, teardown the
	* interrupt handler.
	*/
	dinfo = device_get_ivars(child);
	if (dinfo->cfg.msi.msi_alloc > 0) {
	if (dinfo->cfg.msi.msi_addr == 0) {
	KASSERT(dinfo->cfg.msi.msi_handlers == 0,
	("MSI has handlers, but vectors not mapped"));
	error = PCIB_MAP_MSI(device_get_parent(dev),
	child, rman_get_start(irq), &addr, &data);
	if (error)
	goto bad;
	dinfo->cfg.msi.msi_addr = addr;
	dinfo->cfg.msi.msi_data = data;
	}
	if (dinfo->cfg.msi.msi_handlers == 0)
	pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
	dinfo->cfg.msi.msi_data);
	dinfo->cfg.msi.msi_handlers++;
	} else {
	KASSERT(dinfo->cfg.msix.msix_alloc > 0,
	("No MSI or MSI-X interrupts allocated"));
	KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
	("MSI-X index too high"));
	mte = &dinfo->cfg.msix.msix_table[rid - 1];
	KASSERT(mte->mte_vector != 0, ("no message vector"));
	mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
	KASSERT(mv->mv_irq == rman_get_start(irq),
	("IRQ mismatch"));
	if (mv->mv_address == 0) {
	KASSERT(mte->mte_handlers == 0,
	("MSI-X table entry has handlers, but vector not mapped"));
	error = PCIB_MAP_MSI(device_get_parent(dev),
	child, rman_get_start(irq), &addr, &data);
	if (error)
	goto bad;
	mv->mv_address = addr;
	mv->mv_data = data;
	}

	/*
	* The MSIX table entry must be made valid by
	* incrementing the mte_handlers before
	* calling pci_enable_msix() and
	* pci_resume_msix(). Else the MSIX rewrite
	* table quirk will not work as expected.
	*/
	mte->mte_handlers++;
	if (mte->mte_handlers == 1) {
	pci_enable_msix(child, rid - 1, mv->mv_address,
	mv->mv_data);
	pci_unmask_msix(child, rid - 1);
	}
	}

	/*
	* Make sure that INTx is disabled if we are using MSI/MSI-X,
	* unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
	* in which case we "enable" INTx so MSI/MSI-X actually works.
	*/
	if (!pci_has_quirk(pci_get_devid(child),
	PCI_QUIRK_MSI_INTX_BUG))
	pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
	else
	pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
	bad:
	if (error) {
	(void)bus_generic_teardown_intr(dev, child, irq,
	cookie);
	return (error);
	}
	}
	*cookiep = cookie;
	return (0);
	}

	int
	pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
	void *cookie)
	{
	struct msix_table_entry *mte;
	struct resource_list_entry *rle;
	struct pci_devinfo *dinfo;
	int error, rid;

	if (irq == NULL \|\| !(rman_get_flags(irq) & RF_ACTIVE))
	return (EINVAL);

	/* If this isn't a direct child, just bail out */
	if (device_get_parent(child) != dev)
	return(bus_generic_teardown_intr(dev, child, irq, cookie));

	rid = rman_get_rid(irq);
	if (rid == 0) {
	/* Mask INTx */
	pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
	} else {
	/*
	* Check to see if the interrupt is MSI or MSI-X. If so,
	* decrement the appropriate handlers count and mask the
	* MSI-X message, or disable MSI messages if the count
	* drops to 0.
	*/
	dinfo = device_get_ivars(child);
	rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
	if (rle->res != irq)
	return (EINVAL);
	if (dinfo->cfg.msi.msi_alloc > 0) {
	KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
	("MSI-X index too high"));
	if (dinfo->cfg.msi.msi_handlers == 0)
	return (EINVAL);
	dinfo->cfg.msi.msi_handlers--;
	if (dinfo->cfg.msi.msi_handlers == 0)
	pci_disable_msi(child);
	} else {
	KASSERT(dinfo->cfg.msix.msix_alloc > 0,
	("No MSI or MSI-X interrupts allocated"));
	KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
	("MSI-X index too high"));
	mte = &dinfo->cfg.msix.msix_table[rid - 1];
	if (mte->mte_handlers == 0)
	return (EINVAL);
	mte->mte_handlers--;
	if (mte->mte_handlers == 0)
	pci_mask_msix(child, rid - 1);
	}
	}
	error = bus_generic_teardown_intr(dev, child, irq, cookie);
	if (rid > 0)
	KASSERT(error == 0,
	("%s: generic teardown failed for MSI/MSI-X", __func__));
	return (error);
	}

	int
	pci_print_child(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo;
	struct resource_list *rl;
	int retval = 0;

	dinfo = device_get_ivars(child);
	rl = &dinfo->resources;

	retval += bus_print_child_header(dev, child);

	retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
	retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
	retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
	if (device_get_flags(dev))
	retval += printf(" flags %#x", device_get_flags(dev));

	retval += printf(" at device %d.%d", pci_get_slot(child),
	pci_get_function(child));

	retval += bus_print_child_domain(dev, child);
	retval += bus_print_child_footer(dev, child);

	return (retval);
	}

	static const struct
	{
	int class;
	int subclass;
	int report; /* 0 = bootverbose, 1 = always */
	const char *desc;
	} pci_nomatch_tab[] = {
	{PCIC_OLD, -1, 1, "old"},
	{PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
	{PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
	{PCIC_STORAGE, -1, 1, "mass storage"},
	{PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
	{PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
	{PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
	{PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
	{PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
	{PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
	{PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
	{PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
	{PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
	{PCIC_NETWORK, -1, 1, "network"},
	{PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
	{PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
	{PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
	{PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
	{PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
	{PCIC_DISPLAY, -1, 1, "display"},
	{PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
	{PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
	{PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
	{PCIC_MULTIMEDIA, -1, 1, "multimedia"},
	{PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
	{PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
	{PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
	{PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
	{PCIC_MEMORY, -1, 1, "memory"},
	{PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
	{PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
	{PCIC_BRIDGE, -1, 1, "bridge"},
	{PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
	{PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
	{PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
	{PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
	{PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
	{PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
	{PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
	{PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
	{PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
	{PCIC_SIMPLECOMM, -1, 1, "simple comms"},
	{PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
	{PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
	{PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
	{PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
	{PCIC_BASEPERIPH, -1, 0, "base peripheral"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
	{PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
	{PCIC_INPUTDEV, -1, 1, "input device"},
	{PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
	{PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
	{PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
	{PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
	{PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
	{PCIC_DOCKING, -1, 1, "docking station"},
	{PCIC_PROCESSOR, -1, 1, "processor"},
	{PCIC_SERIALBUS, -1, 1, "serial bus"},
	{PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
	{PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
	{PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
	{PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
	{PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
	{PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
	{PCIC_WIRELESS, -1, 1, "wireless controller"},
	{PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
	{PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
	{PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
	{PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
	{PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
	{PCIC_SATCOM, -1, 1, "satellite communication"},
	{PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
	{PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
	{PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
	{PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
	{PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
	{PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
	{PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
	{PCIC_DASP, -1, 0, "dasp"},
	{PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
	{PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
	{PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
	{PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
	{0, 0, 0, NULL}
	};

	void
	pci_probe_nomatch(device_t dev, device_t child)
	{
	int i, report;
	const char cp, scp;
	char *device;

	/*
	* Look for a listing for this device in a loaded device database.
	*/
	report = 1;
	if ((device = pci_describe_device(child)) != NULL) {
	device_printf(dev, "<%s>", device);
	free(device, M_DEVBUF);
	} else {
	/*
	* Scan the class/subclass descriptions for a general
	* description.
	*/
	cp = "unknown";
	scp = NULL;
	for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
	if (pci_nomatch_tab[i].class == pci_get_class(child)) {
	if (pci_nomatch_tab[i].subclass == -1) {
	cp = pci_nomatch_tab[i].desc;
	report = pci_nomatch_tab[i].report;
	} else if (pci_nomatch_tab[i].subclass ==
	pci_get_subclass(child)) {
	scp = pci_nomatch_tab[i].desc;
	report = pci_nomatch_tab[i].report;
	}
	}
	}
	if (report \|\| bootverbose) {
	device_printf(dev, "<%s%s%s>",
	cp ? cp : "",
	((cp != NULL) && (scp != NULL)) ? ", " : "",
	scp ? scp : "");
	}
	}
	if (report \|\| bootverbose) {
	printf(" at device %d.%d (no driver attached)\n",
	pci_get_slot(child), pci_get_function(child));
	}
	pci_cfg_save(child, device_get_ivars(child), 1);
	}

	void
	pci_child_detached(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo;
	struct resource_list *rl;

	dinfo = device_get_ivars(child);
	rl = &dinfo->resources;

	/*
	* Have to deallocate IRQs before releasing any MSI messages and
	* have to release MSI messages before deallocating any memory
	* BARs.
	*/
	if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
	pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
	if (dinfo->cfg.msi.msi_alloc != 0 \|\| dinfo->cfg.msix.msix_alloc != 0) {
	pci_printf(&dinfo->cfg, "Device leaked MSI vectors\n");
	(void)pci_release_msi(child);
	}
	if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
	pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
	if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
	pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
	#ifdef PCI_RES_BUS
	if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
	pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
	#endif

	pci_cfg_save(child, dinfo, 1);
	}

	/*
	* Parse the PCI device database, if loaded, and return a pointer to a
	* description of the device.
	*
	* The database is flat text formatted as follows:
	*
	* Any line not in a valid format is ignored.
	* Lines are terminated with newline '\n' characters.
	*
	* A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
	* the vendor name.
	*
	* A DEVICE line is entered immediately below the corresponding VENDOR ID.
	* - devices cannot be listed without a corresponding VENDOR line.
	* A DEVICE line consists of a TAB, the 4 digit (hex) device code,
	* another TAB, then the device name.
	*/

	/*
	* Assuming (ptr) points to the beginning of a line in the database,
	* return the vendor or device and description of the next entry.
	* The value of (vendor) or (device) inappropriate for the entry type
	* is set to -1. Returns nonzero at the end of the database.
	*
	* Note that this is slightly unrobust in the face of corrupt data;
	* we attempt to safeguard against this by spamming the end of the
	* database with a newline when we initialise.
	*/
	static int
	pci_describe_parse_line(char *ptr, int vendor, int device, char *desc)
	{
	char cp = ptr;
	int left;

	*device = -1;
	*vendor = -1;
	**desc = '\0';
	for (;;) {
	left = pci_vendordata_size - (cp - pci_vendordata);
	if (left <= 0) {
	*ptr = cp;
	return(1);
	}

	/* vendor entry? */
	if (*cp != '\t' &&
	sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
	break;
	/* device entry? */
	if (*cp == '\t' &&
	sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
	break;

	/* skip to next line */
	while (*cp != '\n' && left > 0) {
	cp++;
	left--;
	}
	if (*cp == '\n') {
	cp++;
	left--;
	}
	}
	/* skip to next line */
	while (*cp != '\n' && left > 0) {
	cp++;
	left--;
	}
	if (*cp == '\n' && left > 0)
	cp++;
	*ptr = cp;
	return(0);
	}

	static char *
	pci_describe_device(device_t dev)
	{
	int vendor, device;
	char desc, vp, dp, line;

	desc = vp = dp = NULL;

	/*
	* If we have no vendor data, we can't do anything.
	*/
	if (pci_vendordata == NULL)
	goto out;

	/*
	* Scan the vendor data looking for this device
	*/
	line = pci_vendordata;
	if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
	goto out;
	for (;;) {
	if (pci_describe_parse_line(&line, &vendor, &device, &vp))
	goto out;
	if (vendor == pci_get_vendor(dev))
	break;
	}
	if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
	goto out;
	for (;;) {
	if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
	*dp = 0;
	break;
	}
	if (vendor != -1) {
	*dp = 0;
	break;
	}
	if (device == pci_get_device(dev))
	break;
	}
	if (dp[0] == '\0')
	snprintf(dp, 80, "0x%x", pci_get_device(dev));
	if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
	NULL)
	sprintf(desc, "%s, %s", vp, dp);
	out:
	if (vp != NULL)
	free(vp, M_DEVBUF);
	if (dp != NULL)
	free(dp, M_DEVBUF);
	return(desc);
	}

	int
	pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
	{
	struct pci_devinfo *dinfo;
	pcicfgregs *cfg;

	dinfo = device_get_ivars(child);
	cfg = &dinfo->cfg;

	switch (which) {
	case PCI_IVAR_ETHADDR:
	/*
	* The generic accessor doesn't deal with failure, so
	* we set the return value, then return an error.
	*/
	((uint8_t *) result) = NULL;
	return (EINVAL);
	case PCI_IVAR_SUBVENDOR:
	*result = cfg->subvendor;
	break;
	case PCI_IVAR_SUBDEVICE:
	*result = cfg->subdevice;
	break;
	case PCI_IVAR_VENDOR:
	*result = cfg->vendor;
	break;
	case PCI_IVAR_DEVICE:
	*result = cfg->device;
	break;
	case PCI_IVAR_DEVID:
	*result = (cfg->device << 16) \| cfg->vendor;
	break;
	case PCI_IVAR_CLASS:
	*result = cfg->baseclass;
	break;
	case PCI_IVAR_SUBCLASS:
	*result = cfg->subclass;
	break;
	case PCI_IVAR_PROGIF:
	*result = cfg->progif;
	break;
	case PCI_IVAR_REVID:
	*result = cfg->revid;
	break;
	case PCI_IVAR_INTPIN:
	*result = cfg->intpin;
	break;
	case PCI_IVAR_IRQ:
	*result = cfg->intline;
	break;
	case PCI_IVAR_DOMAIN:
	*result = cfg->domain;
	break;
	case PCI_IVAR_BUS:
	*result = cfg->bus;
	break;
	case PCI_IVAR_SLOT:
	*result = cfg->slot;
	break;
	case PCI_IVAR_FUNCTION:
	*result = cfg->func;
	break;
	case PCI_IVAR_CMDREG:
	*result = cfg->cmdreg;
	break;
	case PCI_IVAR_CACHELNSZ:
	*result = cfg->cachelnsz;
	break;
	case PCI_IVAR_MINGNT:
	if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
	*result = -1;
	return (EINVAL);
	}
	*result = cfg->mingnt;
	break;
	case PCI_IVAR_MAXLAT:
	if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
	*result = -1;
	return (EINVAL);
	}
	*result = cfg->maxlat;
	break;
	case PCI_IVAR_LATTIMER:
	*result = cfg->lattimer;
	break;
	default:
	return (ENOENT);
	}
	return (0);
	}

	int
	pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
	{
	struct pci_devinfo *dinfo;

	dinfo = device_get_ivars(child);

	switch (which) {
	case PCI_IVAR_INTPIN:
	dinfo->cfg.intpin = value;
	return (0);
	case PCI_IVAR_ETHADDR:
	case PCI_IVAR_SUBVENDOR:
	case PCI_IVAR_SUBDEVICE:
	case PCI_IVAR_VENDOR:
	case PCI_IVAR_DEVICE:
	case PCI_IVAR_DEVID:
	case PCI_IVAR_CLASS:
	case PCI_IVAR_SUBCLASS:
	case PCI_IVAR_PROGIF:
	case PCI_IVAR_REVID:
	case PCI_IVAR_IRQ:
	case PCI_IVAR_DOMAIN:
	case PCI_IVAR_BUS:
	case PCI_IVAR_SLOT:
	case PCI_IVAR_FUNCTION:
	return (EINVAL); /* disallow for now */

	default:
	return (ENOENT);
	}
	}

	#include "opt_ddb.h"
	#ifdef DDB
	#include <ddb/ddb.h>
	#include <sys/cons.h>

	/*
	* List resources based on pci map registers, used for within ddb
	*/

	DB_SHOW_COMMAND(pciregs, db_pci_dump)
	{
	struct pci_devinfo *dinfo;
	struct devlist *devlist_head;
	struct pci_conf *p;
	const char *name;
	int i, error, none_count;

	none_count = 0;
	/* get the head of the device queue */
	devlist_head = &pci_devq;

	/*
	* Go through the list of devices and print out devices
	*/
	for (error = 0, i = 0,
	dinfo = STAILQ_FIRST(devlist_head);
	(dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
	dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {

	/* Populate pd_name and pd_unit */
	name = NULL;
	if (dinfo->cfg.dev)
	name = device_get_name(dinfo->cfg.dev);

	p = &dinfo->conf;
	db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
	"chip=0x%08x rev=0x%02x hdr=0x%02x\n",
	(name && *name) ? name : "none",
	(name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
	none_count++,
	p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
	p->pc_sel.pc_func, (p->pc_class << 16) \|
	(p->pc_subclass << 8) \| p->pc_progif,
	(p->pc_subdevice << 16) \| p->pc_subvendor,
	(p->pc_device << 16) \| p->pc_vendor,
	p->pc_revid, p->pc_hdr);
	}
	}
	#endif /* DDB */

	static struct resource *
	pci_reserve_map(device_t dev, device_t child, int type, int *rid,
	rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
	u_int flags)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	struct resource_list *rl = &dinfo->resources;
	struct resource *res;
	struct pci_map *pm;
	uint16_t cmd;
	pci_addr_t map, testval;
	int mapsize;

	res = NULL;

	/* If rid is managed by EA, ignore it */
	if (pci_ea_is_enabled(child, *rid))
	goto out;

	pm = pci_find_bar(child, *rid);
	if (pm != NULL) {
	/* This is a BAR that we failed to allocate earlier. */
	mapsize = pm->pm_size;
	map = pm->pm_value;
	} else {
	/*
	* Weed out the bogons, and figure out how large the
	* BAR/map is. BARs that read back 0 here are bogus
	* and unimplemented. Note: atapci in legacy mode are
	* special and handled elsewhere in the code. If you
	* have a atapci device in legacy mode and it fails
	* here, that other code is broken.
	*/
	pci_read_bar(child, *rid, &map, &testval, NULL);

	/*
	* Determine the size of the BAR and ignore BARs with a size
	* of 0. Device ROM BARs use a different mask value.
	*/
	if (PCIR_IS_BIOS(&dinfo->cfg, *rid))
	mapsize = pci_romsize(testval);
	else
	mapsize = pci_mapsize(testval);
	if (mapsize == 0)
	goto out;
	pm = pci_add_bar(child, *rid, map, mapsize);
	}

	if (PCI_BAR_MEM(map) \|\| PCIR_IS_BIOS(&dinfo->cfg, *rid)) {
	if (type != SYS_RES_MEMORY) {
	if (bootverbose)
	device_printf(dev,
	"child %s requested type %d for rid %#x,"
	" but the BAR says it is an memio\n",
	device_get_nameunit(child), type, *rid);
	goto out;
	}
	} else {
	if (type != SYS_RES_IOPORT) {
	if (bootverbose)
	device_printf(dev,
	"child %s requested type %d for rid %#x,"
	" but the BAR says it is an ioport\n",
	device_get_nameunit(child), type, *rid);
	goto out;
	}
	}

	/*
	* For real BARs, we need to override the size that
	* the driver requests, because that's what the BAR
	* actually uses and we would otherwise have a
	* situation where we might allocate the excess to
	* another driver, which won't work.
	*/
	count = ((pci_addr_t)1 << mapsize) * num;
	if (RF_ALIGNMENT(flags) < mapsize)
	flags = (flags & ~RF_ALIGNMENT_MASK) \| RF_ALIGNMENT_LOG2(mapsize);
	if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
	flags \|= RF_PREFETCHABLE;

	/*
	* Allocate enough resource, and then write back the
	* appropriate BAR for that resource.
	*/
	resource_list_add(rl, type, *rid, start, end, count);
	res = resource_list_reserve(rl, dev, child, type, rid, start, end,
	count, flags & ~RF_ACTIVE);
	if (res == NULL) {
	resource_list_delete(rl, type, *rid);
	device_printf(child,
	"%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
	count, *rid, type, start, end);
	goto out;
	}
	if (bootverbose)
	device_printf(child,
	"Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
	count, *rid, type, rman_get_start(res));

	/* Disable decoding via the CMD register before updating the BAR */
	cmd = pci_read_config(child, PCIR_COMMAND, 2);
	pci_write_config(child, PCIR_COMMAND,
	cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);

	map = rman_get_start(res);
	pci_write_bar(child, pm, map);

	/* Restore the original value of the CMD register */
	pci_write_config(child, PCIR_COMMAND, cmd, 2);
	out:
	return (res);
	}

	struct resource *
	pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid,
	rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
	u_int flags)
	{
	struct pci_devinfo *dinfo;
	struct resource_list *rl;
	struct resource_list_entry *rle;
	struct resource *res;
	pcicfgregs *cfg;

	/*
	* Perform lazy resource allocation
	*/
	dinfo = device_get_ivars(child);
	rl = &dinfo->resources;
	cfg = &dinfo->cfg;
	switch (type) {
	#if defined(NEW_PCIB) && defined(PCI_RES_BUS)
	case PCI_RES_BUS:
	return (pci_alloc_secbus(dev, child, rid, start, end, count,
	flags));
	#endif
	case SYS_RES_IRQ:
	/*
	* Can't alloc legacy interrupt once MSI messages have
	* been allocated.
	*/
	if (*rid == 0 && (cfg->msi.msi_alloc > 0 \|\|
	cfg->msix.msix_alloc > 0))
	return (NULL);

	/*
	* If the child device doesn't have an interrupt
	* routed and is deserving of an interrupt, try to
	* assign it one.
	*/
	if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
	(cfg->intpin != 0))
	pci_assign_interrupt(dev, child, 0);
	break;
	case SYS_RES_IOPORT:
	case SYS_RES_MEMORY:
	#ifdef NEW_PCIB
	/*
	* PCI-PCI bridge I/O window resources are not BARs.
	* For those allocations just pass the request up the
	* tree.
	*/
	if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
	switch (*rid) {
	case PCIR_IOBASEL_1:
	case PCIR_MEMBASE_1:
	case PCIR_PMBASEL_1:
	/*
	* XXX: Should we bother creating a resource
	* list entry?
	*/
	return (bus_generic_alloc_resource(dev, child,
	type, rid, start, end, count, flags));
	}
	}
	#endif
	/* Reserve resources for this BAR if needed. */
	rle = resource_list_find(rl, type, *rid);
	if (rle == NULL) {
	res = pci_reserve_map(dev, child, type, rid, start, end,
	count, num, flags);
	if (res == NULL)
	return (NULL);
	}
	}
	return (resource_list_alloc(rl, dev, child, type, rid,
	start, end, count, flags));
	}

	struct resource *
	pci_alloc_resource(device_t dev, device_t child, int type, int *rid,
	rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
	{
	#ifdef PCI_IOV
	struct pci_devinfo *dinfo;
	#endif

	if (device_get_parent(child) != dev)
	return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
	type, rid, start, end, count, flags));

	#ifdef PCI_IOV
	dinfo = device_get_ivars(child);
	if (dinfo->cfg.flags & PCICFG_VF) {
	switch (type) {
	/* VFs can't have I/O BARs. */
	case SYS_RES_IOPORT:
	return (NULL);
	case SYS_RES_MEMORY:
	return (pci_vf_alloc_mem_resource(dev, child, rid,
	start, end, count, flags));
	}

	/* Fall through for other types of resource allocations. */
	}
	#endif

	return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
	count, 1, flags));
	}

	int
	pci_release_resource(device_t dev, device_t child, int type, int rid,
	struct resource *r)
	{
	struct pci_devinfo *dinfo;
	struct resource_list *rl;
	pcicfgregs *cfg;

	if (device_get_parent(child) != dev)
	return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child,
	type, rid, r));

	dinfo = device_get_ivars(child);
	cfg = &dinfo->cfg;

	#ifdef PCI_IOV
	if (dinfo->cfg.flags & PCICFG_VF) {
	switch (type) {
	/* VFs can't have I/O BARs. */
	case SYS_RES_IOPORT:
	return (EDOOFUS);
	case SYS_RES_MEMORY:
	return (pci_vf_release_mem_resource(dev, child, rid,
	r));
	}

	/* Fall through for other types of resource allocations. */
	}
	#endif

	#ifdef NEW_PCIB
	/*
	* PCI-PCI bridge I/O window resources are not BARs. For
	* those allocations just pass the request up the tree.
	*/
	if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
	(type == SYS_RES_IOPORT \|\| type == SYS_RES_MEMORY)) {
	switch (rid) {
	case PCIR_IOBASEL_1:
	case PCIR_MEMBASE_1:
	case PCIR_PMBASEL_1:
	return (bus_generic_release_resource(dev, child, type,
	rid, r));
	}
	}
	#endif

	rl = &dinfo->resources;
	return (resource_list_release(rl, dev, child, type, rid, r));
	}

	int
	pci_activate_resource(device_t dev, device_t child, int type, int rid,
	struct resource *r)
	{
	struct pci_devinfo *dinfo;
	int error;

	error = bus_generic_activate_resource(dev, child, type, rid, r);
	if (error)
	return (error);

	/* Enable decoding in the command register when activating BARs. */
	if (device_get_parent(child) == dev) {
	/* Device ROMs need their decoding explicitly enabled. */
	dinfo = device_get_ivars(child);
	if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
	pci_write_bar(child, pci_find_bar(child, rid),
	rman_get_start(r) \| PCIM_BIOS_ENABLE);
	switch (type) {
	case SYS_RES_IOPORT:
	case SYS_RES_MEMORY:
	error = PCI_ENABLE_IO(dev, child, type);
	break;
	}
	}
	return (error);
	}

	int
	pci_deactivate_resource(device_t dev, device_t child, int type,
	int rid, struct resource *r)
	{
	struct pci_devinfo *dinfo;
	int error;

	error = bus_generic_deactivate_resource(dev, child, type, rid, r);
	if (error)
	return (error);

	/* Disable decoding for device ROMs. */
	if (device_get_parent(child) == dev) {
	dinfo = device_get_ivars(child);
	if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
	pci_write_bar(child, pci_find_bar(child, rid),
	rman_get_start(r));
	}
	return (0);
	}

	void
	pci_child_deleted(device_t dev, device_t child)
	{
	struct resource_list_entry *rle;
	struct resource_list *rl;
	struct pci_devinfo *dinfo;

	dinfo = device_get_ivars(child);
	rl = &dinfo->resources;

	EVENTHANDLER_INVOKE(pci_delete_device, child);

	/* Turn off access to resources we're about to free */
	if (bus_child_present(child) != 0) {
	pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
	PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN \| PCIM_CMD_PORTEN), 2);

	pci_disable_busmaster(child);
	}

	/* Free all allocated resources */
	STAILQ_FOREACH(rle, rl, link) {
	if (rle->res) {
	if (rman_get_flags(rle->res) & RF_ACTIVE \|\|
	resource_list_busy(rl, rle->type, rle->rid)) {
	pci_printf(&dinfo->cfg,
	"Resource still owned, oops. "
	"(type=%d, rid=%d, addr=%lx)\n",
	rle->type, rle->rid,
	rman_get_start(rle->res));
	bus_release_resource(child, rle->type, rle->rid,
	rle->res);
	}
	resource_list_unreserve(rl, dev, child, rle->type,
	rle->rid);
	}
	}
	resource_list_free(rl);

	pci_freecfg(dinfo);
	}

	void
	pci_delete_resource(device_t dev, device_t child, int type, int rid)
	{
	struct pci_devinfo *dinfo;
	struct resource_list *rl;
	struct resource_list_entry *rle;

	if (device_get_parent(child) != dev)
	return;

	dinfo = device_get_ivars(child);
	rl = &dinfo->resources;
	rle = resource_list_find(rl, type, rid);
	if (rle == NULL)
	return;

	if (rle->res) {
	if (rman_get_flags(rle->res) & RF_ACTIVE \|\|
	resource_list_busy(rl, type, rid)) {
	device_printf(dev, "delete_resource: "
	"Resource still owned by child, oops. "
	"(type=%d, rid=%d, addr=%jx)\n",
	type, rid, rman_get_start(rle->res));
	return;
	}
	resource_list_unreserve(rl, dev, child, type, rid);
	}
	resource_list_delete(rl, type, rid);
	}

	struct resource_list *
	pci_get_resource_list (device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);

	return (&dinfo->resources);
	}

	#ifdef ACPI_DMAR
	bus_dma_tag_t acpi_iommu_get_dma_tag(device_t dev, device_t child);
	bus_dma_tag_t
	pci_get_dma_tag(device_t bus, device_t dev)
	{
	bus_dma_tag_t tag;
	struct pci_softc *sc;

	if (device_get_parent(dev) == bus) {
	/* try iommu and return if it works */
	tag = acpi_iommu_get_dma_tag(bus, dev);
	} else
	tag = NULL;
	if (tag == NULL) {
	sc = device_get_softc(bus);
	tag = sc->sc_dma_tag;
	}
	return (tag);
	}
	#else
	bus_dma_tag_t
	pci_get_dma_tag(device_t bus, device_t dev)
	{
	struct pci_softc *sc = device_get_softc(bus);

	return (sc->sc_dma_tag);
	}
	#endif

	uint32_t
	pci_read_config_method(device_t dev, device_t child, int reg, int width)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;

	#ifdef PCI_IOV
	/*
	* SR-IOV VFs don't implement the VID or DID registers, so we have to
	* emulate them here.
	*/
	if (cfg->flags & PCICFG_VF) {
	if (reg == PCIR_VENDOR) {
	switch (width) {
	case 4:
	return (cfg->device << 16 \| cfg->vendor);
	case 2:
	return (cfg->vendor);
	case 1:
	return (cfg->vendor & 0xff);
	default:
	return (0xffffffff);
	}
	} else if (reg == PCIR_DEVICE) {
	switch (width) {
	/* Note that an unaligned 4-byte read is an error. */
	case 2:
	return (cfg->device);
	case 1:
	return (cfg->device & 0xff);
	default:
	return (0xffffffff);
	}
	}
	}
	#endif

	return (PCIB_READ_CONFIG(device_get_parent(dev),
	cfg->bus, cfg->slot, cfg->func, reg, width));
	}

	void
	pci_write_config_method(device_t dev, device_t child, int reg,
	uint32_t val, int width)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;

	PCIB_WRITE_CONFIG(device_get_parent(dev),
	cfg->bus, cfg->slot, cfg->func, reg, val, width);
	}

	int
	pci_child_location_str_method(device_t dev, device_t child, char *buf,
	size_t buflen)
	{

	snprintf(buf, buflen, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
	pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
	pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
	return (0);
	}

	int
	pci_child_pnpinfo_str_method(device_t dev, device_t child, char *buf,
	size_t buflen)
	{
	struct pci_devinfo *dinfo;
	pcicfgregs *cfg;

	dinfo = device_get_ivars(child);
	cfg = &dinfo->cfg;
	snprintf(buf, buflen, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
	"subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
	cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
	cfg->progif);
	return (0);
	}

	int
	pci_assign_interrupt_method(device_t dev, device_t child)
	{
	struct pci_devinfo *dinfo = device_get_ivars(child);
	pcicfgregs *cfg = &dinfo->cfg;

	return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
	cfg->intpin));
	}

	static void
	pci_lookup(void arg, const char name, device_t *dev)
	{
	long val;
	char *end;
	int domain, bus, slot, func;

	if (*dev != NULL)
	return;

	/*
	* Accept pciconf-style selectors of either pciD:B:S:F or
	* pciB:S:F. In the latter case, the domain is assumed to
	* be zero.
	*/
	if (strncmp(name, "pci", 3) != 0)
	return;
	val = strtol(name + 3, &end, 10);
	if (val < 0 \|\| val > INT_MAX \|\| *end != ':')
	return;
	domain = val;
	val = strtol(end + 1, &end, 10);
	if (val < 0 \|\| val > INT_MAX \|\| *end != ':')
	return;
	bus = val;
	val = strtol(end + 1, &end, 10);
	if (val < 0 \|\| val > INT_MAX)
	return;
	slot = val;
	if (*end == ':') {
	val = strtol(end + 1, &end, 10);
	if (val < 0 \|\| val > INT_MAX \|\| *end != '\0')
	return;
	func = val;
	} else if (*end == '\0') {
	func = slot;
	slot = bus;
	bus = domain;
	domain = 0;
	} else
	return;

	if (domain > PCI_DOMAINMAX \|\| bus > PCI_BUSMAX \|\| slot > PCI_SLOTMAX \|\|
	func > PCIE_ARI_FUNCMAX \|\| (slot != 0 && func > PCI_FUNCMAX))
	return;

	*dev = pci_find_dbsf(domain, bus, slot, func);
	}

	static int
	pci_modevent(module_t mod, int what, void *arg)
	{
	static struct cdev *pci_cdev;
	static eventhandler_tag tag;

	switch (what) {
	case MOD_LOAD:
	STAILQ_INIT(&pci_devq);
	pci_generation = 0;
	pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
	"pci");
	pci_load_vendor_data();
	tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
	1000);
	break;

	case MOD_UNLOAD:
	if (tag != NULL)
	EVENTHANDLER_DEREGISTER(dev_lookup, tag);
	destroy_dev(pci_cdev);
	break;
	}

	return (0);
	}

	static void
	pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
	{
	#define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
	struct pcicfg_pcie *cfg;
	int version, pos;

	cfg = &dinfo->cfg.pcie;
	pos = cfg->pcie_location;

	version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;

	WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);

	if (version > 1 \|\| cfg->pcie_type == PCIEM_TYPE_ROOT_PORT \|\|
	cfg->pcie_type == PCIEM_TYPE_ENDPOINT \|\|
	cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
	WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);

	if (version > 1 \|\| (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT \|\|
	(cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
	(cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
	WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);

	if (version > 1 \|\| cfg->pcie_type == PCIEM_TYPE_ROOT_PORT \|\|
	cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
	WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);

	if (version > 1) {
	WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
	WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
	WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
	}
	#undef WREG
	}

	static void
	pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
	{
	pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
	dinfo->cfg.pcix.pcix_command, 2);
	}

	void
	pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
	{

	/*
	* Restore the device to full power mode. We must do this
	* before we restore the registers because moving from D3 to
	* D0 will cause the chip's BARs and some other registers to
	* be reset to some unknown power on reset values. Cut down
	* the noise on boot by doing nothing if we are already in
	* state D0.
	*/
	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
	pci_set_powerstate(dev, PCI_POWERSTATE_D0);
	pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
	pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
	pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
	pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
	pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
	pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
	switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_NORMAL:
	pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
	pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
	break;
	case PCIM_HDRTYPE_BRIDGE:
	pci_write_config(dev, PCIR_SECLAT_1,
	dinfo->cfg.bridge.br_seclat, 1);
	pci_write_config(dev, PCIR_SUBBUS_1,
	dinfo->cfg.bridge.br_subbus, 1);
	pci_write_config(dev, PCIR_SECBUS_1,
	dinfo->cfg.bridge.br_secbus, 1);
	pci_write_config(dev, PCIR_PRIBUS_1,
	dinfo->cfg.bridge.br_pribus, 1);
	pci_write_config(dev, PCIR_BRIDGECTL_1,
	dinfo->cfg.bridge.br_control, 2);
	break;
	case PCIM_HDRTYPE_CARDBUS:
	pci_write_config(dev, PCIR_SECLAT_2,
	dinfo->cfg.bridge.br_seclat, 1);
	pci_write_config(dev, PCIR_SUBBUS_2,
	dinfo->cfg.bridge.br_subbus, 1);
	pci_write_config(dev, PCIR_SECBUS_2,
	dinfo->cfg.bridge.br_secbus, 1);
	pci_write_config(dev, PCIR_PRIBUS_2,
	dinfo->cfg.bridge.br_pribus, 1);
	pci_write_config(dev, PCIR_BRIDGECTL_2,
	dinfo->cfg.bridge.br_control, 2);
	break;
	}
	pci_restore_bars(dev);

	if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
	pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);

	/*
	* Restore extended capabilities for PCI-Express and PCI-X
	*/
	if (dinfo->cfg.pcie.pcie_location != 0)
	pci_cfg_restore_pcie(dev, dinfo);
	if (dinfo->cfg.pcix.pcix_location != 0)
	pci_cfg_restore_pcix(dev, dinfo);

	/* Restore MSI and MSI-X configurations if they are present. */
	if (dinfo->cfg.msi.msi_location != 0)
	pci_resume_msi(dev);
	if (dinfo->cfg.msix.msix_location != 0)
	pci_resume_msix(dev);

	#ifdef PCI_IOV
	if (dinfo->cfg.iov != NULL)
	pci_iov_cfg_restore(dev, dinfo);
	#endif
	}

	static void
	pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
	{
	#define RREG(n) pci_read_config(dev, pos + (n), 2)
	struct pcicfg_pcie *cfg;
	int version, pos;

	cfg = &dinfo->cfg.pcie;
	pos = cfg->pcie_location;

	cfg->pcie_flags = RREG(PCIER_FLAGS);

	version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;

	cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);

	if (version > 1 \|\| cfg->pcie_type == PCIEM_TYPE_ROOT_PORT \|\|
	cfg->pcie_type == PCIEM_TYPE_ENDPOINT \|\|
	cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
	cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);

	if (version > 1 \|\| (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT \|\|
	(cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
	(cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
	cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);

	if (version > 1 \|\| cfg->pcie_type == PCIEM_TYPE_ROOT_PORT \|\|
	cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
	cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);

	if (version > 1) {
	cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
	cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
	cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
	}
	#undef RREG
	}

	static void
	pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
	{
	dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
	dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
	}

	void
	pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
	{
	uint32_t cls;
	int ps;

	/*
	* Some drivers apparently write to these registers w/o updating our
	* cached copy. No harm happens if we update the copy, so do so here
	* so we can restore them. The COMMAND register is modified by the
	* bus w/o updating the cache. This should represent the normally
	* writable portion of the 'defined' part of type 0/1/2 headers.
	*/
	dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
	dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
	dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
	dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
	dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
	dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
	dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
	dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
	dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
	dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
	dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
	switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
	case PCIM_HDRTYPE_NORMAL:
	dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
	dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
	dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
	dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
	break;
	case PCIM_HDRTYPE_BRIDGE:
	dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
	PCIR_SECLAT_1, 1);
	dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
	PCIR_SUBBUS_1, 1);
	dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
	PCIR_SECBUS_1, 1);
	dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
	PCIR_PRIBUS_1, 1);
	dinfo->cfg.bridge.br_control = pci_read_config(dev,
	PCIR_BRIDGECTL_1, 2);
	break;
	case PCIM_HDRTYPE_CARDBUS:
	dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
	PCIR_SECLAT_2, 1);
	dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
	PCIR_SUBBUS_2, 1);
	dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
	PCIR_SECBUS_2, 1);
	dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
	PCIR_PRIBUS_2, 1);
	dinfo->cfg.bridge.br_control = pci_read_config(dev,
	PCIR_BRIDGECTL_2, 2);
	dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
	dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
	break;
	}

	if (dinfo->cfg.pcie.pcie_location != 0)
	pci_cfg_save_pcie(dev, dinfo);

	if (dinfo->cfg.pcix.pcix_location != 0)
	pci_cfg_save_pcix(dev, dinfo);

	#ifdef PCI_IOV
	if (dinfo->cfg.iov != NULL)
	pci_iov_cfg_save(dev, dinfo);
	#endif

	/*
	* don't set the state for display devices, base peripherals and
	* memory devices since bad things happen when they are powered down.
	* We should (a) have drivers that can easily detach and (b) use
	* generic drivers for these devices so that some device actually
	* attaches. We need to make sure that when we implement (a) we don't
	* power the device down on a reattach.
	*/
	cls = pci_get_class(dev);
	if (!setstate)
	return;
	switch (pci_do_power_nodriver)
	{
	case 0: /* NO powerdown at all */
	return;
	case 1: /* Conservative about what to power down */
	if (cls == PCIC_STORAGE)
	return;
	/FALLTHROUGH/
	case 2: /* Aggressive about what to power down */
	if (cls == PCIC_DISPLAY \|\| cls == PCIC_MEMORY \|\|
	cls == PCIC_BASEPERIPH)
	return;
	/FALLTHROUGH/
	case 3: /* Power down everything */
	break;
	}
	/*
	* PCI spec says we can only go into D3 state from D0 state.
	* Transition from D[12] into D0 before going to D3 state.
	*/
	ps = pci_get_powerstate(dev);
	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
	pci_set_powerstate(dev, PCI_POWERSTATE_D0);
	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
	pci_set_powerstate(dev, PCI_POWERSTATE_D3);
	}

	/* Wrapper APIs suitable for device driver use. */
	void
	pci_save_state(device_t dev)
	{
	struct pci_devinfo *dinfo;

	dinfo = device_get_ivars(dev);
	pci_cfg_save(dev, dinfo, 0);
	}

	void
	pci_restore_state(device_t dev)
	{
	struct pci_devinfo *dinfo;

	dinfo = device_get_ivars(dev);
	pci_cfg_restore(dev, dinfo);
	}

	static int
	pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
	uintptr_t *id)
	{

	return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
	}

	/* Find the upstream port of a given PCI device in a root complex. */
	device_t
	pci_find_pcie_root_port(device_t dev)
	{
	struct pci_devinfo *dinfo;
	devclass_t pci_class;
	device_t pcib, bus;

	pci_class = devclass_find("pci");
	KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class,
	("%s: non-pci device %s", __func__, device_get_nameunit(dev)));

	/*
	* Walk the bridge hierarchy until we find a PCI-e root
	* port or a non-PCI device.
	*/
	for (;;) {
	bus = device_get_parent(dev);
	KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
	device_get_nameunit(dev)));

	pcib = device_get_parent(bus);
	KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
	device_get_nameunit(bus)));

	/*
	* pcib's parent must be a PCI bus for this to be a
	* PCI-PCI bridge.
	*/
	if (device_get_devclass(device_get_parent(pcib)) != pci_class)
	return (NULL);

	dinfo = device_get_ivars(pcib);
	if (dinfo->cfg.pcie.pcie_location != 0 &&
	dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
	return (pcib);

	dev = pcib;
	}
	}

	/*
	* Wait for pending transactions to complete on a PCI-express function.
	*
	* The maximum delay is specified in milliseconds in max_delay. Note
	* that this function may sleep.
	*
	* Returns true if the function is idle and false if the timeout is
	* exceeded. If dev is not a PCI-express function, this returns true.
	*/
	bool
	pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	uint16_t sta;
	int cap;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return (true);

	sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
	while (sta & PCIEM_STA_TRANSACTION_PND) {
	if (max_delay == 0)
	return (false);

	/* Poll once every 100 milliseconds up to the timeout. */
	if (max_delay > 100) {
	pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
	max_delay -= 100;
	} else {
	pause_sbt("pcietp", max_delay * SBT_1MS, 0,
	C_HARDCLOCK);
	max_delay = 0;
	}
	sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
	}

	return (true);
	}

	/*
	* Determine the maximum Completion Timeout in microseconds.
	*
	* For non-PCI-express functions this returns 0.
	*/
	int
	pcie_get_max_completion_timeout(device_t dev)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	int cap;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return (0);

	/*
	* Functions using the 1.x spec use the default timeout range of
	* 50 microseconds to 50 milliseconds. Functions that do not
	* support programmable timeouts also use this range.
	*/
	if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 \|\|
	(pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
	PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
	return (50 * 1000);

	switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
	PCIEM_CTL2_COMP_TIMO_VAL) {
	case PCIEM_CTL2_COMP_TIMO_100US:
	return (100);
	case PCIEM_CTL2_COMP_TIMO_10MS:
	return (10 * 1000);
	case PCIEM_CTL2_COMP_TIMO_55MS:
	return (55 * 1000);
	case PCIEM_CTL2_COMP_TIMO_210MS:
	return (210 * 1000);
	case PCIEM_CTL2_COMP_TIMO_900MS:
	return (900 * 1000);
	case PCIEM_CTL2_COMP_TIMO_3500MS:
	return (3500 * 1000);
	case PCIEM_CTL2_COMP_TIMO_13S:
	return (13 * 1000 * 1000);
	case PCIEM_CTL2_COMP_TIMO_64S:
	return (64 * 1000 * 1000);
	default:
	return (50 * 1000);
	}
	}

	+void
	+pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
	+{
	+ struct pci_devinfo *dinfo = device_get_ivars(dev);
	+ const char *s;
	+ int aer;
	+ uint32_t r, r1;
	+ uint16_t rs;
	+
	+ if (sev == PCIEM_STA_CORRECTABLE_ERROR)
	+ s = "Correctable";
	+ else if (sev == PCIEM_STA_NON_FATAL_ERROR)
	+ s = "Uncorrectable (Non-Fatal)";
	+ else
	+ s = "Uncorrectable (Fatal)";
	+ device_printf(dev, "%s PCIe error reported by APEI\n", s);
	+ if (aerp) {
	+ if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
	+ r = le32dec(aerp + PCIR_AER_COR_STATUS);
	+ r1 = le32dec(aerp + PCIR_AER_COR_MASK);
	+ } else {
	+ r = le32dec(aerp + PCIR_AER_UC_STATUS);
	+ r1 = le32dec(aerp + PCIR_AER_UC_MASK);
	+ }
	+ device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
	+ if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
	+ r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
	+ rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
	+ printf(" severity 0x%08x first %d\n",
	+ r, rs & 0x1f);
	+ } else
	+ printf("\n");
	+ }
	+
	+ /* As kind of recovery just report and clear the error statuses. */
	+ if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
	+ r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
	+ if (r != 0) {
	+ pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
	+ device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
	+ }
	+
	+ r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
	+ if (r != 0) {
	+ pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
	+ device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
	+ }
	+ }
	+ if (dinfo->cfg.pcie.pcie_location != 0) {
	+ rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
	+ PCIER_DEVICE_STA, 2);
	+ if ((rs & (PCIEM_STA_CORRECTABLE_ERROR \|
	+ PCIEM_STA_NON_FATAL_ERROR \| PCIEM_STA_FATAL_ERROR \|
	+ PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
	+ pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
	+ PCIER_DEVICE_STA, rs, 2);
	+ device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
	+ }
	+ }
	+}
	+
	/*
	* Perform a Function Level Reset (FLR) on a device.
	*
	* This function first waits for any pending transactions to complete
	* within the timeout specified by max_delay. If transactions are
	* still pending, the function will return false without attempting a
	* reset.
	*
	* If dev is not a PCI-express function or does not support FLR, this
	* function returns false.
	*
	* Note that no registers are saved or restored. The caller is
	* responsible for saving and restoring any registers including
	* PCI-standard registers via pci_save_state() and
	* pci_restore_state().
	*/
	bool
	pcie_flr(device_t dev, u_int max_delay, bool force)
	{
	struct pci_devinfo *dinfo = device_get_ivars(dev);
	uint16_t cmd, ctl;
	int compl_delay;
	int cap;

	cap = dinfo->cfg.pcie.pcie_location;
	if (cap == 0)
	return (false);

	if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR))
	return (false);

	/*
	* Disable busmastering to prevent generation of new
	* transactions while waiting for the device to go idle. If
	* the idle timeout fails, the command register is restored
	* which will re-enable busmastering.
	*/
	cmd = pci_read_config(dev, PCIR_COMMAND, 2);
	pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
	if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
	if (!force) {
	pci_write_config(dev, PCIR_COMMAND, cmd, 2);
	return (false);
	}
	pci_printf(&dinfo->cfg,
	"Resetting with transactions pending after %d ms\n",
	max_delay);

	/*
	* Extend the post-FLR delay to cover the maximum
	* Completion Timeout delay of anything in flight
	* during the FLR delay. Enforce a minimum delay of
	* at least 10ms.
	*/
	compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
	if (compl_delay < 10)
	compl_delay = 10;
	} else
	compl_delay = 0;

	/* Initiate the reset. */
	ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
	pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl \|
	PCIEM_CTL_INITIATE_FLR, 2);

	/* Wait for 100ms. */
	pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);

	if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
	PCIEM_STA_TRANSACTION_PND)
	pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
	return (true);
	}

	/*
	* Attempt a power-management reset by cycling the device in/out of D3
	* state. PCI spec says we can only go into D3 state from D0 state.
	* Transition from D[12] into D0 before going to D3 state.
	*/
	int
	pci_power_reset(device_t dev)
	{
	int ps;

	ps = pci_get_powerstate(dev);
	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
	pci_set_powerstate(dev, PCI_POWERSTATE_D0);
	pci_set_powerstate(dev, PCI_POWERSTATE_D3);
	pci_set_powerstate(dev, ps);
	return (0);
	}

	/*
	* Try link drop and retrain of the downstream port of upstream
	* switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
	* cause Conventional Hot reset of the device in the slot.
	* Alternative, for PCIe, could be the secondary bus reset initiatied
	* on the upstream switch PCIR_BRIDGECTL_1, bit 6.
	*/
	int
	pcie_link_reset(device_t port, int pcie_location)
	{
	uint16_t v;

	v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
	v \|= PCIEM_LINK_CTL_LINK_DIS;
	pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
	pause_sbt("pcier1", mstosbt(20), 0, 0);
	v &= ~PCIEM_LINK_CTL_LINK_DIS;
	v \|= PCIEM_LINK_CTL_RETRAIN_LINK;
	pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
	pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
	v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
	return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
	}

	static int
	pci_reset_post(device_t dev, device_t child)
	{

	if (dev == device_get_parent(child))
	pci_restore_state(child);
	return (0);
	}

	static int
	pci_reset_prepare(device_t dev, device_t child)
	{

	if (dev == device_get_parent(child))
	pci_save_state(child);
	return (0);
	}

	static int
	pci_reset_child(device_t dev, device_t child, int flags)
	{
	int error;

	if (dev == NULL \|\| device_get_parent(child) != dev)
	return (0);
	if ((flags & DEVF_RESET_DETACH) != 0) {
	error = device_get_state(child) == DS_ATTACHED ?
	device_detach(child) : 0;
	} else {
	error = BUS_SUSPEND_CHILD(dev, child);
	}
	if (error == 0) {
	if (!pcie_flr(child, 1000, false)) {
	error = BUS_RESET_PREPARE(dev, child);
	if (error == 0)
	pci_power_reset(child);
	BUS_RESET_POST(dev, child);
	}
	if ((flags & DEVF_RESET_DETACH) != 0)
	device_probe_and_attach(child);
	else
	BUS_RESUME_CHILD(dev, child);
	}
	return (error);
	}

	const struct pci_device_table *
	pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
	{
	bool match;
	uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;

	vendor = pci_get_vendor(child);
	device = pci_get_device(child);
	subvendor = pci_get_subvendor(child);
	subdevice = pci_get_subdevice(child);
	class = pci_get_class(child);
	subclass = pci_get_subclass(child);
	revid = pci_get_revid(child);
	while (nelt-- > 0) {
	match = true;
	if (id->match_flag_vendor)
	match &= vendor == id->vendor;
	if (id->match_flag_device)
	match &= device == id->device;
	if (id->match_flag_subvendor)
	match &= subvendor == id->subvendor;
	if (id->match_flag_subdevice)
	match &= subdevice == id->subdevice;
	if (id->match_flag_class)
	match &= class == id->class_id;
	if (id->match_flag_subclass)
	match &= subclass == id->subclass;
	if (id->match_flag_revid)
	match &= revid == id->revid;
	if (match)
	return (id);
	id++;
	}
	return (NULL);
	}

	static void
	pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
	{
	const char *dev_name;
	device_t dev;

	dev = dinfo->cfg.dev;
	printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
	dinfo->cfg.slot, dinfo->cfg.func);
	dev_name = device_get_name(dev);
	if (dev_name != NULL)
	printf(" (%s%d)", dev_name, device_get_unit(dev));
	}

	void
	pci_print_faulted_dev(void)
	{
	struct pci_devinfo *dinfo;
	device_t dev;
	int aer, i;
	uint32_t r1, r2;
	uint16_t status;

	STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
	dev = dinfo->cfg.dev;
	status = pci_read_config(dev, PCIR_STATUS, 2);
	status &= PCIM_STATUS_MDPERR \| PCIM_STATUS_STABORT \|
	PCIM_STATUS_RTABORT \| PCIM_STATUS_RMABORT \|
	PCIM_STATUS_SERR \| PCIM_STATUS_PERR;
	if (status != 0) {
	pci_print_faulted_dev_name(dinfo);
	printf(" error 0x%04x\n", status);
	}
	if (dinfo->cfg.pcie.pcie_location != 0) {
	status = pci_read_config(dev,
	dinfo->cfg.pcie.pcie_location +
	PCIER_DEVICE_STA, 2);
	if ((status & (PCIEM_STA_CORRECTABLE_ERROR \|
	PCIEM_STA_NON_FATAL_ERROR \| PCIEM_STA_FATAL_ERROR \|
	PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
	pci_print_faulted_dev_name(dinfo);
	printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
	pci_read_config(dev,
	dinfo->cfg.pcie.pcie_location +
	PCIER_DEVICE_CTL, 2),
	status);
	}
	}
	if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
	r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
	r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
	if (r1 != 0 \|\| r2 != 0) {
	pci_print_faulted_dev_name(dinfo);
	printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
	" COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
	r1, pci_read_config(dev, aer +
	PCIR_AER_UC_MASK, 4),
	pci_read_config(dev, aer +
	PCIR_AER_UC_SEVERITY, 4),
	r2, pci_read_config(dev, aer +
	PCIR_AER_COR_MASK, 4),
	pci_read_config(dev, aer +
	PCIR_AER_CAP_CONTROL, 4));
	for (i = 0; i < 4; i++) {
	r1 = pci_read_config(dev, aer +
	PCIR_AER_HEADER_LOG + i * 4, 4);
	printf(" HL%d: 0x%08x\n", i, r1);
	}
	}
	}
	}
	}

	#ifdef DDB
	DB_SHOW_COMMAND(pcierr, pci_print_faulted_dev_db)
	{

	pci_print_faulted_dev();
	}

	static void
	db_clear_pcie_errors(const struct pci_devinfo *dinfo)
	{
	device_t dev;
	int aer;
	uint32_t r;

	dev = dinfo->cfg.dev;
	r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
	PCIER_DEVICE_STA, 2);
	pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
	PCIER_DEVICE_STA, r, 2);

	if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
	return;
	r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
	if (r != 0)
	pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
	r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
	if (r != 0)
	pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
	}

	DB_COMMAND(pci_clearerr, db_pci_clearerr)
	{
	struct pci_devinfo *dinfo;
	device_t dev;
	uint16_t status, status1;

	STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
	dev = dinfo->cfg.dev;
	status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
	status1 &= PCIM_STATUS_MDPERR \| PCIM_STATUS_STABORT \|
	PCIM_STATUS_RTABORT \| PCIM_STATUS_RMABORT \|
	PCIM_STATUS_SERR \| PCIM_STATUS_PERR;
	if (status1 != 0) {
	status &= ~status1;
	pci_write_config(dev, PCIR_STATUS, status, 2);
	}
	if (dinfo->cfg.pcie.pcie_location != 0)
	db_clear_pcie_errors(dinfo);
	}
	}
	#endif
	diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h
	index 37ad9242ebe1..29de6dad93f0 100644
	--- a/sys/dev/pci/pcivar.h
	+++ b/sys/dev/pci/pcivar.h
	@@ -1,731 +1,732 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright 1997, Stefan Esser <se@freebsd.org>
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice unmodified, this list of conditions, and the following
	* disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	* $FreeBSD$
	*
	*/

	#ifndef _PCIVAR_H_
	#define _PCIVAR_H_

	#include <sys/queue.h>
	#include <sys/_eventhandler.h>

	/* some PCI bus constants */
	#define PCI_MAXMAPS_0 6 /* max. no. of memory/port maps */
	#define PCI_MAXMAPS_1 2 /* max. no. of maps for PCI to PCI bridge */
	#define PCI_MAXMAPS_2 1 /* max. no. of maps for CardBus bridge */

	typedef uint64_t pci_addr_t;

	/* Config registers for PCI-PCI and PCI-Cardbus bridges. */
	struct pcicfg_bridge {
	uint8_t br_seclat;
	uint8_t br_subbus;
	uint8_t br_secbus;
	uint8_t br_pribus;
	uint16_t br_control;
	};

	/* Interesting values for PCI power management */
	struct pcicfg_pp {
	uint16_t pp_cap; /* PCI power management capabilities */
	uint8_t pp_status; /* conf. space addr. of PM control/status reg */
	uint8_t pp_bse; /* conf. space addr. of PM BSE reg */
	uint8_t pp_data; /* conf. space addr. of PM data reg */
	};

	struct pci_map {
	pci_addr_t pm_value; /* Raw BAR value */
	pci_addr_t pm_size;
	uint16_t pm_reg;
	STAILQ_ENTRY(pci_map) pm_link;
	};

	struct vpd_readonly {
	char keyword[2];
	char *value;
	int len;
	};

	struct vpd_write {
	char keyword[2];
	char *value;
	int start;
	int len;
	};

	struct pcicfg_vpd {
	uint8_t vpd_reg; /* base register, + 2 for addr, + 4 data */
	char vpd_cached;
	char vpd_ident; / string identifier */
	int vpd_rocnt;
	struct vpd_readonly *vpd_ros;
	int vpd_wcnt;
	struct vpd_write *vpd_w;
	};

	/* Interesting values for PCI MSI */
	struct pcicfg_msi {
	uint16_t msi_ctrl; /* Message Control */
	uint8_t msi_location; /* Offset of MSI capability registers. */
	uint8_t msi_msgnum; /* Number of messages */
	int msi_alloc; /* Number of allocated messages. */
	uint64_t msi_addr; /* Contents of address register. */
	uint16_t msi_data; /* Contents of data register. */
	u_int msi_handlers;
	};

	/* Interesting values for PCI MSI-X */
	struct msix_vector {
	uint64_t mv_address; /* Contents of address register. */
	uint32_t mv_data; /* Contents of data register. */
	int mv_irq;
	};

	struct msix_table_entry {
	u_int mte_vector; /* 1-based index into msix_vectors array. */
	u_int mte_handlers;
	};

	struct pcicfg_msix {
	uint16_t msix_ctrl; /* Message Control */
	uint16_t msix_msgnum; /* Number of messages */
	uint8_t msix_location; /* Offset of MSI-X capability registers. */
	uint8_t msix_table_bar; /* BAR containing vector table. */
	uint8_t msix_pba_bar; /* BAR containing PBA. */
	uint32_t msix_table_offset;
	uint32_t msix_pba_offset;
	int msix_alloc; /* Number of allocated vectors. */
	int msix_table_len; /* Length of virtual table. */
	struct msix_table_entry msix_table; / Virtual table. */
	struct msix_vector msix_vectors; / Array of allocated vectors. */
	struct resource msix_table_res; / Resource containing vector table. */
	struct resource msix_pba_res; / Resource containing PBA. */
	};

	/* Interesting values for HyperTransport */
	struct pcicfg_ht {
	uint8_t ht_slave; /* Non-zero if device is an HT slave. */
	uint8_t ht_msimap; /* Offset of MSI mapping cap registers. */
	uint16_t ht_msictrl; /* MSI mapping control */
	uint64_t ht_msiaddr; /* MSI mapping base address */
	};

	/* Interesting values for PCI-express */
	struct pcicfg_pcie {
	uint8_t pcie_location; /* Offset of PCI-e capability registers. */
	uint8_t pcie_type; /* Device type. */
	uint16_t pcie_flags; /* Device capabilities register. */
	uint16_t pcie_device_ctl; /* Device control register. */
	uint16_t pcie_link_ctl; /* Link control register. */
	uint16_t pcie_slot_ctl; /* Slot control register. */
	uint16_t pcie_root_ctl; /* Root control register. */
	uint16_t pcie_device_ctl2; /* Second device control register. */
	uint16_t pcie_link_ctl2; /* Second link control register. */
	uint16_t pcie_slot_ctl2; /* Second slot control register. */
	};

	struct pcicfg_pcix {
	uint16_t pcix_command;
	uint8_t pcix_location; /* Offset of PCI-X capability registers. */
	};

	struct pcicfg_vf {
	int index;
	};

	struct pci_ea_entry {
	int eae_bei;
	uint32_t eae_flags;
	uint64_t eae_base;
	uint64_t eae_max_offset;
	uint32_t eae_cfg_offset;
	STAILQ_ENTRY(pci_ea_entry) eae_link;
	};

	struct pcicfg_ea {
	int ea_location; /* Structure offset in Configuration Header */
	STAILQ_HEAD(, pci_ea_entry) ea_entries; /* EA entries */
	};

	#define PCICFG_VF 0x0001 /* Device is an SR-IOV Virtual Function */

	/* config header information common to all header types */
	typedef struct pcicfg {
	device_t dev; /* device which owns this */

	STAILQ_HEAD(, pci_map) maps; /* BARs */

	uint16_t subvendor; /* card vendor ID */
	uint16_t subdevice; /* card device ID, assigned by card vendor */
	uint16_t vendor; /* chip vendor ID */
	uint16_t device; /* chip device ID, assigned by chip vendor */

	uint16_t cmdreg; /* disable/enable chip and PCI options */
	uint16_t statreg; /* supported PCI features and error state */

	uint8_t baseclass; /* chip PCI class */
	uint8_t subclass; /* chip PCI subclass */
	uint8_t progif; /* chip PCI programming interface */
	uint8_t revid; /* chip revision ID */

	uint8_t hdrtype; /* chip config header type */
	uint8_t cachelnsz; /* cache line size in 4byte units */
	uint8_t intpin; /* PCI interrupt pin */
	uint8_t intline; /* interrupt line (IRQ for PC arch) */

	uint8_t mingnt; /* min. useful bus grant time in 250ns units */
	uint8_t maxlat; /* max. tolerated bus grant latency in 250ns */
	uint8_t lattimer; /* latency timer in units of 30ns bus cycles */

	uint8_t mfdev; /* multi-function device (from hdrtype reg) */
	uint8_t nummaps; /* actual number of PCI maps used */

	uint32_t domain; /* PCI domain */
	uint8_t bus; /* config space bus address */
	uint8_t slot; /* config space slot address */
	uint8_t func; /* config space function number */

	uint32_t flags; /* flags defined above */

	struct pcicfg_bridge bridge; /* Bridges */
	struct pcicfg_pp pp; /* Power management */
	struct pcicfg_vpd vpd; /* Vital product data */
	struct pcicfg_msi msi; /* PCI MSI */
	struct pcicfg_msix msix; /* PCI MSI-X */
	struct pcicfg_ht ht; /* HyperTransport */
	struct pcicfg_pcie pcie; /* PCI Express */
	struct pcicfg_pcix pcix; /* PCI-X */
	struct pcicfg_iov iov; / SR-IOV */
	struct pcicfg_vf vf; /* SR-IOV Virtual Function */
	struct pcicfg_ea ea; /* Enhanced Allocation */
	} pcicfgregs;

	/* additional type 1 device config header information (PCI to PCI bridge) */

	typedef struct {
	pci_addr_t pmembase; /* base address of prefetchable memory */
	pci_addr_t pmemlimit; /* topmost address of prefetchable memory */
	uint32_t membase; /* base address of memory window */
	uint32_t memlimit; /* topmost address of memory window */
	uint32_t iobase; /* base address of port window */
	uint32_t iolimit; /* topmost address of port window */
	uint16_t secstat; /* secondary bus status register */
	uint16_t bridgectl; /* bridge control register */
	uint8_t seclat; /* CardBus latency timer */
	} pcih1cfgregs;

	/* additional type 2 device config header information (CardBus bridge) */

	typedef struct {
	uint32_t membase0; /* base address of memory window */
	uint32_t memlimit0; /* topmost address of memory window */
	uint32_t membase1; /* base address of memory window */
	uint32_t memlimit1; /* topmost address of memory window */
	uint32_t iobase0; /* base address of port window */
	uint32_t iolimit0; /* topmost address of port window */
	uint32_t iobase1; /* base address of port window */
	uint32_t iolimit1; /* topmost address of port window */
	uint32_t pccardif; /* PC Card 16bit IF legacy more base addr. */
	uint16_t secstat; /* secondary bus status register */
	uint16_t bridgectl; /* bridge control register */
	uint8_t seclat; /* CardBus latency timer */
	} pcih2cfgregs;

	extern uint32_t pci_numdevs;
	extern int pci_enable_aspm;

	/*
	* The bitfield has to be stable and match the fields below (so that
	* match_flag_vendor must be bit 0) so we have to do the endian dance. We can't
	* use enums or #define constants because then the macros for subsetting matches
	* wouldn't work. These tables are parsed by devmatch and others to connect
	* modules with devices on the PCI bus.
	*/
	struct pci_device_table {
	#if BYTE_ORDER == LITTLE_ENDIAN
	uint16_t
	match_flag_vendor:1,
	match_flag_device:1,
	match_flag_subvendor:1,
	match_flag_subdevice:1,
	match_flag_class:1,
	match_flag_subclass:1,
	match_flag_revid:1,
	match_flag_unused:9;
	#else
	uint16_t
	match_flag_unused:9,
	match_flag_revid:1,
	match_flag_subclass:1,
	match_flag_class:1,
	match_flag_subdevice:1,
	match_flag_subvendor:1,
	match_flag_device:1,
	match_flag_vendor:1;
	#endif
	uint16_t vendor;
	uint16_t device;
	uint16_t subvendor;
	uint16_t subdevice;
	uint16_t class_id;
	uint16_t subclass;
	uint16_t revid;
	uint16_t unused;
	uintptr_t driver_data;
	char *descr;
	};

	#define PCI_DEV(v, d) \
	.match_flag_vendor = 1, .vendor = (v), \
	.match_flag_device = 1, .device = (d)
	#define PCI_SUBDEV(sv, sd) \
	.match_flag_subvendor = 1, .subvendor = (sv), \
	.match_flag_subdevice = 1, .subdevice = (sd)
	#define PCI_CLASS(x) \
	.match_flag_class = 1, .class_id = (x)
	#define PCI_SUBCLASS(x) \
	.match_flag_subclass = 1, .subclass = (x)
	#define PCI_REVID(x) \
	.match_flag_revid = 1, .revid = (x)
	#define PCI_DESCR(x) \
	.descr = (x)
	#define PCI_PNP_STR \
	"M16:mask;U16:vendor;U16:device;U16:subvendor;U16:subdevice;" \
	"U16:class;U16:subclass;U16:revid;"
	#define PCI_PNP_INFO(table) \
	MODULE_PNP_INFO(PCI_PNP_STR, pci, table, table, \
	sizeof(table) / sizeof(table[0]))

	const struct pci_device_table *pci_match_device(device_t child,
	const struct pci_device_table *id, size_t nelt);
	#define PCI_MATCH(child, table) \
	pci_match_device(child, (table), nitems(table));

	/* Only if the prerequisites are present */
	#if defined(_SYS_BUS_H_) && defined(_SYS_PCIIO_H_)
	struct pci_devinfo {
	STAILQ_ENTRY(pci_devinfo) pci_links;
	struct resource_list resources;
	pcicfgregs cfg;
	struct pci_conf conf;
	};
	#endif

	#ifdef _SYS_BUS_H_

	#include "pci_if.h"

	enum pci_device_ivars {
	PCI_IVAR_SUBVENDOR,
	PCI_IVAR_SUBDEVICE,
	PCI_IVAR_VENDOR,
	PCI_IVAR_DEVICE,
	PCI_IVAR_DEVID,
	PCI_IVAR_CLASS,
	PCI_IVAR_SUBCLASS,
	PCI_IVAR_PROGIF,
	PCI_IVAR_REVID,
	PCI_IVAR_INTPIN,
	PCI_IVAR_IRQ,
	PCI_IVAR_DOMAIN,
	PCI_IVAR_BUS,
	PCI_IVAR_SLOT,
	PCI_IVAR_FUNCTION,
	PCI_IVAR_ETHADDR,
	PCI_IVAR_CMDREG,
	PCI_IVAR_CACHELNSZ,
	PCI_IVAR_MINGNT,
	PCI_IVAR_MAXLAT,
	PCI_IVAR_LATTIMER
	};

	/*
	* Simplified accessors for pci devices
	*/
	#define PCI_ACCESSOR(var, ivar, type) \
	__BUS_ACCESSOR(pci, var, PCI, ivar, type)

	PCI_ACCESSOR(subvendor, SUBVENDOR, uint16_t)
	PCI_ACCESSOR(subdevice, SUBDEVICE, uint16_t)
	PCI_ACCESSOR(vendor, VENDOR, uint16_t)
	PCI_ACCESSOR(device, DEVICE, uint16_t)
	PCI_ACCESSOR(devid, DEVID, uint32_t)
	PCI_ACCESSOR(class, CLASS, uint8_t)
	PCI_ACCESSOR(subclass, SUBCLASS, uint8_t)
	PCI_ACCESSOR(progif, PROGIF, uint8_t)
	PCI_ACCESSOR(revid, REVID, uint8_t)
	PCI_ACCESSOR(intpin, INTPIN, uint8_t)
	PCI_ACCESSOR(irq, IRQ, uint8_t)
	PCI_ACCESSOR(domain, DOMAIN, uint32_t)
	PCI_ACCESSOR(bus, BUS, uint8_t)
	PCI_ACCESSOR(slot, SLOT, uint8_t)
	PCI_ACCESSOR(function, FUNCTION, uint8_t)
	PCI_ACCESSOR(ether, ETHADDR, uint8_t *)
	PCI_ACCESSOR(cmdreg, CMDREG, uint8_t)
	PCI_ACCESSOR(cachelnsz, CACHELNSZ, uint8_t)
	PCI_ACCESSOR(mingnt, MINGNT, uint8_t)
	PCI_ACCESSOR(maxlat, MAXLAT, uint8_t)
	PCI_ACCESSOR(lattimer, LATTIMER, uint8_t)

	#undef PCI_ACCESSOR

	/*
	* Operations on configuration space.
	*/
	static __inline uint32_t
	pci_read_config(device_t dev, int reg, int width)
	{
	return PCI_READ_CONFIG(device_get_parent(dev), dev, reg, width);
	}

	static __inline void
	pci_write_config(device_t dev, int reg, uint32_t val, int width)
	{
	PCI_WRITE_CONFIG(device_get_parent(dev), dev, reg, val, width);
	}

	/*
	* Ivars for pci bridges.
	*/

	/typedef enum pci_device_ivars pcib_device_ivars;/
	enum pcib_device_ivars {
	PCIB_IVAR_DOMAIN,
	PCIB_IVAR_BUS
	};

	#define PCIB_ACCESSOR(var, ivar, type) \
	__BUS_ACCESSOR(pcib, var, PCIB, ivar, type)

	PCIB_ACCESSOR(domain, DOMAIN, uint32_t)
	PCIB_ACCESSOR(bus, BUS, uint32_t)

	#undef PCIB_ACCESSOR

	/*
	* PCI interrupt validation. Invalid interrupt values such as 0 or 128
	* on i386 or other platforms should be mapped out in the MD pcireadconf
	* code and not here, since the only MI invalid IRQ is 255.
	*/
	#define PCI_INVALID_IRQ 255
	#define PCI_INTERRUPT_VALID(x) ((x) != PCI_INVALID_IRQ)

	/*
	* Convenience functions.
	*
	* These should be used in preference to manually manipulating
	* configuration space.
	*/
	static __inline int
	pci_enable_busmaster(device_t dev)
	{
	return(PCI_ENABLE_BUSMASTER(device_get_parent(dev), dev));
	}

	static __inline int
	pci_disable_busmaster(device_t dev)
	{
	return(PCI_DISABLE_BUSMASTER(device_get_parent(dev), dev));
	}

	static __inline int
	pci_enable_io(device_t dev, int space)
	{
	return(PCI_ENABLE_IO(device_get_parent(dev), dev, space));
	}

	static __inline int
	pci_disable_io(device_t dev, int space)
	{
	return(PCI_DISABLE_IO(device_get_parent(dev), dev, space));
	}

	static __inline int
	pci_get_vpd_ident(device_t dev, const char **identptr)
	{
	return(PCI_GET_VPD_IDENT(device_get_parent(dev), dev, identptr));
	}

	static __inline int
	pci_get_vpd_readonly(device_t dev, const char kw, const char *vptr)
	{
	return(PCI_GET_VPD_READONLY(device_get_parent(dev), dev, kw, vptr));
	}

	/*
	* Check if the address range falls within the VGA defined address range(s)
	*/
	static __inline int
	pci_is_vga_ioport_range(rman_res_t start, rman_res_t end)
	{

	return (((start >= 0x3b0 && end <= 0x3bb) \|\|
	(start >= 0x3c0 && end <= 0x3df)) ? 1 : 0);
	}

	static __inline int
	pci_is_vga_memory_range(rman_res_t start, rman_res_t end)
	{

	return ((start >= 0xa0000 && end <= 0xbffff) ? 1 : 0);
	}

	/*
	* PCI power states are as defined by ACPI:
	*
	* D0 State in which device is on and running. It is receiving full
	* power from the system and delivering full functionality to the user.
	* D1 Class-specific low-power state in which device context may or may not
	* be lost. Buses in D1 cannot do anything to the bus that would force
	* devices on that bus to lose context.
	* D2 Class-specific low-power state in which device context may or may
	* not be lost. Attains greater power savings than D1. Buses in D2
	* can cause devices on that bus to lose some context. Devices in D2
	* must be prepared for the bus to be in D2 or higher.
	* D3 State in which the device is off and not running. Device context is
	* lost. Power can be removed from the device.
	*/
	#define PCI_POWERSTATE_D0 0
	#define PCI_POWERSTATE_D1 1
	#define PCI_POWERSTATE_D2 2
	#define PCI_POWERSTATE_D3 3
	#define PCI_POWERSTATE_UNKNOWN -1

	static __inline int
	pci_set_powerstate(device_t dev, int state)
	{
	return PCI_SET_POWERSTATE(device_get_parent(dev), dev, state);
	}

	static __inline int
	pci_get_powerstate(device_t dev)
	{
	return PCI_GET_POWERSTATE(device_get_parent(dev), dev);
	}

	static __inline int
	pci_find_cap(device_t dev, int capability, int *capreg)
	{
	return (PCI_FIND_CAP(device_get_parent(dev), dev, capability, capreg));
	}

	static __inline int
	pci_find_next_cap(device_t dev, int capability, int start, int *capreg)
	{
	return (PCI_FIND_NEXT_CAP(device_get_parent(dev), dev, capability, start,
	capreg));
	}

	static __inline int
	pci_find_extcap(device_t dev, int capability, int *capreg)
	{
	return (PCI_FIND_EXTCAP(device_get_parent(dev), dev, capability, capreg));
	}

	static __inline int
	pci_find_next_extcap(device_t dev, int capability, int start, int *capreg)
	{
	return (PCI_FIND_NEXT_EXTCAP(device_get_parent(dev), dev, capability,
	start, capreg));
	}

	static __inline int
	pci_find_htcap(device_t dev, int capability, int *capreg)
	{
	return (PCI_FIND_HTCAP(device_get_parent(dev), dev, capability, capreg));
	}

	static __inline int
	pci_find_next_htcap(device_t dev, int capability, int start, int *capreg)
	{
	return (PCI_FIND_NEXT_HTCAP(device_get_parent(dev), dev, capability,
	start, capreg));
	}

	static __inline int
	pci_alloc_msi(device_t dev, int *count)
	{
	return (PCI_ALLOC_MSI(device_get_parent(dev), dev, count));
	}

	static __inline int
	pci_alloc_msix(device_t dev, int *count)
	{
	return (PCI_ALLOC_MSIX(device_get_parent(dev), dev, count));
	}

	static __inline void
	pci_enable_msi(device_t dev, uint64_t address, uint16_t data)
	{
	PCI_ENABLE_MSI(device_get_parent(dev), dev, address, data);
	}

	static __inline void
	pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
	{
	PCI_ENABLE_MSIX(device_get_parent(dev), dev, index, address, data);
	}

	static __inline void
	pci_disable_msi(device_t dev)
	{
	PCI_DISABLE_MSI(device_get_parent(dev), dev);
	}

	static __inline int
	pci_remap_msix(device_t dev, int count, const u_int *vectors)
	{
	return (PCI_REMAP_MSIX(device_get_parent(dev), dev, count, vectors));
	}

	static __inline int
	pci_release_msi(device_t dev)
	{
	return (PCI_RELEASE_MSI(device_get_parent(dev), dev));
	}

	static __inline int
	pci_msi_count(device_t dev)
	{
	return (PCI_MSI_COUNT(device_get_parent(dev), dev));
	}

	static __inline int
	pci_msix_count(device_t dev)
	{
	return (PCI_MSIX_COUNT(device_get_parent(dev), dev));
	}

	static __inline int
	pci_msix_pba_bar(device_t dev)
	{
	return (PCI_MSIX_PBA_BAR(device_get_parent(dev), dev));
	}

	static __inline int
	pci_msix_table_bar(device_t dev)
	{
	return (PCI_MSIX_TABLE_BAR(device_get_parent(dev), dev));
	}

	static __inline int
	pci_get_id(device_t dev, enum pci_id_type type, uintptr_t *id)
	{
	return (PCI_GET_ID(device_get_parent(dev), dev, type, id));
	}

	/*
	* This is the deprecated interface, there is no way to tell the difference
	* between a failure and a valid value that happens to be the same as the
	* failure value.
	*/
	static __inline uint16_t
	pci_get_rid(device_t dev)
	{
	uintptr_t rid;

	if (pci_get_id(dev, PCI_ID_RID, &rid) != 0)
	return (0);

	return (rid);
	}

	static __inline void
	pci_child_added(device_t dev)
	{

	return (PCI_CHILD_ADDED(device_get_parent(dev), dev));
	}

	device_t pci_find_bsf(uint8_t, uint8_t, uint8_t);
	device_t pci_find_dbsf(uint32_t, uint8_t, uint8_t, uint8_t);
	device_t pci_find_device(uint16_t, uint16_t);
	device_t pci_find_class(uint8_t class, uint8_t subclass);

	/* Can be used by drivers to manage the MSI-X table. */
	int pci_pending_msix(device_t dev, u_int index);

	int pci_msi_device_blacklisted(device_t dev);
	int pci_msix_device_blacklisted(device_t dev);

	void pci_ht_map_msi(device_t dev, uint64_t addr);

	device_t pci_find_pcie_root_port(device_t dev);
	int pci_get_max_payload(device_t dev);
	int pci_get_max_read_req(device_t dev);
	void pci_restore_state(device_t dev);
	void pci_save_state(device_t dev);
	int pci_set_max_read_req(device_t dev, int size);
	int pci_power_reset(device_t dev);
	uint32_t pcie_read_config(device_t dev, int reg, int width);
	void pcie_write_config(device_t dev, int reg, uint32_t value, int width);
	uint32_t pcie_adjust_config(device_t dev, int reg, uint32_t mask,
	uint32_t value, int width);
	+void pcie_apei_error(device_t dev, int sev, uint8_t *aer);
	bool pcie_flr(device_t dev, u_int max_delay, bool force);
	int pcie_get_max_completion_timeout(device_t dev);
	bool pcie_wait_for_pending_transactions(device_t dev, u_int max_delay);
	int pcie_link_reset(device_t port, int pcie_location);

	void pci_print_faulted_dev(void);

	#endif /* _SYS_BUS_H_ */

	/*
	* cdev switch for control device, initialised in generic PCI code
	*/
	extern struct cdevsw pcicdev;

	/*
	* List of all PCI devices, generation count for the list.
	*/
	STAILQ_HEAD(devlist, pci_devinfo);

	extern struct devlist pci_devq;
	extern uint32_t pci_generation;

	struct pci_map *pci_find_bar(device_t dev, int reg);
	int pci_bar_enabled(device_t dev, struct pci_map *pm);
	struct pcicfg_vpd *pci_fetch_vpd_list(device_t dev);

	#define VGA_PCI_BIOS_SHADOW_ADDR 0xC0000
	#define VGA_PCI_BIOS_SHADOW_SIZE 131072

	int vga_pci_is_boot_display(device_t dev);
	void * vga_pci_map_bios(device_t dev, size_t *size);
	void vga_pci_unmap_bios(device_t dev, void *bios);
	int vga_pci_repost(device_t dev);

	/**
	* Global eventhandlers invoked when PCI devices are added or removed
	* from the system.
	*/
	typedef void (pci_event_fn)(void arg, device_t dev);
	EVENTHANDLER_DECLARE(pci_add_device, pci_event_fn);
	EVENTHANDLER_DECLARE(pci_delete_device, pci_event_fn);

	#endif /* _PCIVAR_H_ */
	diff --git a/sys/dev/rc/rc.c b/sys/dev/rc/rc.c
	deleted file mode 100644
	index c4c4370af5fe..000000000000
	--- a/sys/dev/rc/rc.c
	+++ /dev/null
	@@ -1,1314 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (C) 1995 by Pavel Antonov, Moscow, Russia.
	- * Copyright (C) 1995 by Andrey A. Chernov, Moscow, Russia.
	- * All rights reserved.
	- * Copyright (C) 2002 by John Baldwin <jhb@FreeBSD.org>
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-/*
	- * SDL Communications Riscom/8 (based on Cirrus Logic CL-CD180) driver
	- *
	- */
	-
	-/#define RCDEBUG/
	-
	-#include "opt_tty.h"
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/bus.h>
	-#include <sys/conf.h>
	-#include <sys/fcntl.h>
	-#include <sys/interrupt.h>
	-#include <sys/kernel.h>
	-#include <sys/malloc.h>
	-#include <sys/module.h>
	-#include <sys/serial.h>
	-#include <sys/tty.h>
	-#include <machine/bus.h>
	-#include <machine/resource.h>
	-#include <sys/rman.h>
	-
	-#include <dev/ic/cd180.h>
	-#include <dev/rc/rcreg.h>
	-#include <isa/isavar.h>
	-
	-#define IOBASE_ADDRS 14
	-
	-#define rcin(sc, port) RC_IN(sc, port)
	-#define rcout(sc, port, v) RC_OUT(sc, port, v)
	-
	-#define WAITFORCCR(sc, chan) rc_wait0((sc), (chan), __LINE__)
	-
	-#define CCRCMD(sc, chan, cmd) do { \
	- WAITFORCCR((sc), (chan)); \
	- rcout((sc), CD180_CCR, (cmd)); \
	-} while (0)
	-
	-#define RC_IBUFSIZE 256
	-#define RB_I_HIGH_WATER (TTYHOG - 2 * RC_IBUFSIZE)
	-#define RC_OBUFSIZE 512
	-#define RC_IHIGHWATER (3 * RC_IBUFSIZE / 4)
	-#define INPUT_FLAGS_SHIFT (2 * RC_IBUFSIZE)
	-#define LOTS_OF_EVENTS 64
	-
	-#define RC_FAKEID 0x10
	-
	-/* Per-channel structure */
	-struct rc_chans {
	- struct rc_softc rc_rcb; / back ptr */
	- u_short rc_flags; /* Misc. flags */
	- int rc_chan; /* Channel # */
	- u_char rc_ier; /* intr. enable reg */
	- u_char rc_msvr; /* modem sig. status */
	- u_char rc_cor2; /* options reg */
	- u_char rc_pendcmd; /* special cmd pending */
	- u_int rc_dcdwaits; /* how many waits DCD in open */
	- struct tty rc_tp; / tty struct */
	- u_char rc_iptr; / Chars input buffer */
	- u_char rc_hiwat; / hi-water mark */
	- u_char rc_bufend; / end of buffer */
	- u_char rc_optr; / ptr in output buf */
	- u_char rc_obufend; / end of output buf */
	- u_char rc_ibuf[4 * RC_IBUFSIZE]; /* input buffer */
	- u_char rc_obuf[RC_OBUFSIZE]; /* output buffer */
	- struct callout rc_dtrcallout;
	-};
	-
	-/* Per-board structure */
	-struct rc_softc {
	- device_t sc_dev;
	- struct resource *sc_irq;
	- struct resource *sc_port[IOBASE_ADDRS];
	- int sc_irqrid;
	- void *sc_hwicookie;
	- bus_space_tag_t sc_bt;
	- bus_space_handle_t sc_bh;
	- u_int sc_unit; /* unit # */
	- u_char sc_dtr; /* DTR status */
	- int sc_scheduled_event;
	- void *sc_swicookie;
	- struct rc_chans sc_channels[CD180_NCHAN]; /* channels */
	-};
	-
	-/* Static prototypes */
	-static t_close_t rc_close;
	-static void rc_break(struct tty *, int);
	-static void rc_release_resources(device_t dev);
	-static void rc_intr(void *);
	-static void rc_hwreset(struct rc_softc *, unsigned int);
	-static int rc_test(struct rc_softc *);
	-static void rc_discard_output(struct rc_chans *);
	-static int rc_modem(struct tty *, int, int);
	-static void rc_start(struct tty *);
	-static void rc_stop(struct tty *, int rw);
	-static int rc_param(struct tty , struct termios );
	-static void rc_pollcard(void *);
	-static void rc_reinit(struct rc_softc *);
	-#ifdef RCDEBUG
	-static void printrcflags();
	-#endif
	-static void rc_wait0(struct rc_softc *sc, int chan, int line);
	-
	-static devclass_t rc_devclass;
	-
	-/* Flags */
	-#define RC_DTR_OFF 0x0001 /* DTR wait, for close/open */
	-#define RC_ACTOUT 0x0002 /* Dial-out port active */
	-#define RC_RTSFLOW 0x0004 /* RTS flow ctl enabled */
	-#define RC_CTSFLOW 0x0008 /* CTS flow ctl enabled */
	-#define RC_DORXFER 0x0010 /* RXFER event planned */
	-#define RC_DOXXFER 0x0020 /* XXFER event planned */
	-#define RC_MODCHG 0x0040 /* Modem status changed */
	-#define RC_OSUSP 0x0080 /* Output suspended */
	-#define RC_OSBUSY 0x0100 /* start() routine in progress */
	-#define RC_WAS_BUFOVFL 0x0200 /* low-level buffer ovferflow */
	-#define RC_WAS_SILOVFL 0x0400 /* silo buffer overflow */
	-#define RC_SEND_RDY 0x0800 /* ready to send */
	-
	-/* Table for translation of RCSR status bits to internal form */
	-static int rc_rcsrt[16] = {
	- 0, TTY_OE, TTY_FE,
	- TTY_FE\|TTY_OE, TTY_PE, TTY_PE\|TTY_OE,
	- TTY_PE\|TTY_FE, TTY_PE\|TTY_FE\|TTY_OE, TTY_BI,
	- TTY_BI\|TTY_OE, TTY_BI\|TTY_FE, TTY_BI\|TTY_FE\|TTY_OE,
	- TTY_BI\|TTY_PE, TTY_BI\|TTY_PE\|TTY_OE, TTY_BI\|TTY_PE\|TTY_FE,
	- TTY_BI\|TTY_PE\|TTY_FE\|TTY_OE
	-};
	-
	-static int rc_ports[] =
	- { 0x220, 0x240, 0x250, 0x260, 0x2a0, 0x2b0, 0x300, 0x320 };
	-static int iobase_addrs[IOBASE_ADDRS] =
	- { 0, 0x400, 0x800, 0xc00, 0x1400, 0x1800, 0x1c00, 0x2000,
	- 0x3000, 0x3400, 0x3800, 0x3c00, 0x4000, 0x8000 };
	-
	-/**********************************************/
	-
	-static int
	-rc_probe(device_t dev)
	-{
	- u_int port;
	- int i, found;
	-
	- /*
	- * We don't know of any PnP ID's for these cards.
	- */
	- if (isa_get_logicalid(dev) != 0)
	- return (ENXIO);
	-
	- /*
	- * We have to have an IO port hint that is valid.
	- */
	- port = isa_get_port(dev);
	- if (port == -1)
	- return (ENXIO);
	- found = 0;
	- for (i = 0; i < nitems(rc_ports); i++)
	- if (rc_ports[i] == port) {
	- found = 1;
	- break;
	- }
	- if (!found)
	- return (ENXIO);
	-
	- /*
	- * We have to have an IRQ hint.
	- */
	- if (isa_get_irq(dev) == -1)
	- return (ENXIO);
	-
	- device_set_desc(dev, "SDL Riscom/8");
	- return (0);
	-}
	-
	-static int
	-rc_attach(device_t dev)
	-{
	- struct rc_chans *rc;
	- struct tty *tp;
	- struct rc_softc *sc;
	- u_int port;
	- int base, chan, error, i, x;
	-
	- sc = device_get_softc(dev);
	- sc->sc_dev = dev;
	-
	- /*
	- * We need to have IO ports. Lots of them. We need
	- * the following ranges relative to the base port:
	- * 0x0 - 0x10
	- * 0x400 - 0x410
	- * 0x800 - 0x810
	- * 0xc00 - 0xc10
	- * 0x1400 - 0x1410
	- * 0x1800 - 0x1810
	- * 0x1c00 - 0x1c10
	- * 0x2000 - 0x2010
	- * 0x3000 - 0x3010
	- * 0x3400 - 0x3410
	- * 0x3800 - 0x3810
	- * 0x3c00 - 0x3c10
	- * 0x4000 - 0x4010
	- * 0x8000 - 0x8010
	- */
	- port = isa_get_port(dev);
	- for (i = 0; i < IOBASE_ADDRS; i++)
	- if (bus_set_resource(dev, SYS_RES_IOPORT, i,
	- port + iobase_addrs[i], 0x10) != 0)
	- return (ENXIO);
	- error = ENOMEM;
	- for (i = 0; i < IOBASE_ADDRS; i++) {
	- x = i;
	- sc->sc_port[i] = bus_alloc_resource_anywhere(dev,
	- SYS_RES_IOPORT, &x, 0x10, RF_ACTIVE);
	- if (x != i) {
	- device_printf(dev, "ioport %d was rid %d\n", i, x);
	- goto fail;
	- }
	- if (sc->sc_port[i] == NULL) {
	- device_printf(dev, "failed to alloc ioports %x-%x\n",
	- port + iobase_addrs[i],
	- port + iobase_addrs[i] + 0x10);
	- goto fail;
	- }
	- }
	- sc->sc_bt = rman_get_bustag(sc->sc_port[0]);
	- sc->sc_bh = rman_get_bushandle(sc->sc_port[0]);
	-
	- sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irqrid,
	- RF_ACTIVE);
	- if (sc->sc_irq == NULL) {
	- device_printf(dev, "failed to alloc IRQ\n");
	- goto fail;
	- }
	-
	- /*
	- * Now do some actual tests to make sure it works.
	- */
	- error = ENXIO;
	- rcout(sc, CD180_PPRL, 0x22); /* Random values to Prescale reg. */
	- rcout(sc, CD180_PPRH, 0x11);
	- if (rcin(sc, CD180_PPRL) != 0x22 \|\| rcin(sc, CD180_PPRH) != 0x11)
	- goto fail;
	- if (rc_test(sc))
	- goto fail;
	-
	- /*
	- * Ok, start actually hooking things up.
	- */
	- sc->sc_unit = device_get_unit(dev);
	- /sc->sc_chipid = 0x10 + device_get_unit(dev);/
	- device_printf(dev, "%d chans, firmware rev. %c\n",
	- CD180_NCHAN, (rcin(sc, CD180_GFRCR) & 0xF) + 'A');
	- rc = sc->sc_channels;
	- base = CD180_NCHAN * sc->sc_unit;
	- for (chan = 0; chan < CD180_NCHAN; chan++, rc++) {
	- rc->rc_rcb = sc;
	- rc->rc_chan = chan;
	- rc->rc_iptr = rc->rc_ibuf;
	- rc->rc_bufend = &rc->rc_ibuf[RC_IBUFSIZE];
	- rc->rc_hiwat = &rc->rc_ibuf[RC_IHIGHWATER];
	- rc->rc_optr = rc->rc_obufend = rc->rc_obuf;
	- callout_init(&rc->rc_dtrcallout, 0);
	- tp = rc->rc_tp = ttyalloc();
	- tp->t_sc = rc;
	- tp->t_oproc = rc_start;
	- tp->t_param = rc_param;
	- tp->t_modem = rc_modem;
	- tp->t_break = rc_break;
	- tp->t_close = rc_close;
	- tp->t_stop = rc_stop;
	- ttycreate(tp, TS_CALLOUT, "m%d", chan + base);
	- }
	-
	- error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_TTY, NULL, rc_intr,
	- sc, &sc->sc_hwicookie);
	- if (error) {
	- device_printf(dev, "failed to register interrupt handler\n");
	- goto fail;
	- }
	-
	- swi_add(&tty_intr_event, "rc", rc_pollcard, sc, SWI_TTY, 0,
	- &sc->sc_swicookie);
	- return (0);
	-
	-fail:
	- rc_release_resources(dev);
	- return (error);
	-}
	-
	-static int
	-rc_detach(device_t dev)
	-{
	- struct rc_softc *sc;
	- struct rc_chans *rc;
	- int error, i;
	-
	- sc = device_get_softc(dev);
	-
	- rc = sc->sc_channels;
	- for (i = 0; i < CD180_NCHAN; i++, rc++)
	- ttyfree(rc->rc_tp);
	-
	- error = bus_teardown_intr(dev, sc->sc_irq, sc->sc_hwicookie);
	- if (error)
	- device_printf(dev, "failed to deregister interrupt handler\n");
	- swi_remove(sc->sc_swicookie);
	- rc_release_resources(dev);
	-
	- return (0);
	-}
	-
	-static void
	-rc_release_resources(device_t dev)
	-{
	- struct rc_softc *sc;
	- int i;
	-
	- sc = device_get_softc(dev);
	- if (sc->sc_irq != NULL) {
	- bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqrid,
	- sc->sc_irq);
	- sc->sc_irq = NULL;
	- }
	- for (i = 0; i < IOBASE_ADDRS; i++) {
	- if (sc->sc_port[i] == NULL)
	- break;
	- bus_release_resource(dev, SYS_RES_IOPORT, i, sc->sc_port[i]);
	- sc->sc_port[i] = NULL;
	- }
	-}
	-
	-/* RC interrupt handling */
	-static void
	-rc_intr(void *arg)
	-{
	- struct rc_softc *sc;
	- struct rc_chans *rc;
	- int resid, chan;
	- u_char val, iack, bsr, ucnt, *optr;
	- int good_data, t_state;
	-
	- sc = (struct rc_softc *)arg;
	- bsr = ~(rcin(sc, RC_BSR));
	- if (!(bsr & (RC_BSR_TOUT\|RC_BSR_RXINT\|RC_BSR_TXINT\|RC_BSR_MOINT))) {
	- device_printf(sc->sc_dev, "extra interrupt\n");
	- rcout(sc, CD180_EOIR, 0);
	- return;
	- }
	-
	- while (bsr & (RC_BSR_TOUT\|RC_BSR_RXINT\|RC_BSR_TXINT\|RC_BSR_MOINT)) {
	-#ifdef RCDEBUG_DETAILED
	- device_printf(sc->sc_dev, "intr (%p) %s%s%s%s\n", arg, bsr,
	- (bsr & RC_BSR_TOUT)?"TOUT ":"",
	- (bsr & RC_BSR_RXINT)?"RXINT ":"",
	- (bsr & RC_BSR_TXINT)?"TXINT ":"",
	- (bsr & RC_BSR_MOINT)?"MOINT":"");
	-#endif
	- if (bsr & RC_BSR_TOUT) {
	- device_printf(sc->sc_dev,
	- "hardware failure, reset board\n");
	- rcout(sc, RC_CTOUT, 0);
	- rc_reinit(sc);
	- return;
	- }
	- if (bsr & RC_BSR_RXINT) {
	- iack = rcin(sc, RC_PILR_RX);
	- good_data = (iack == (GIVR_IT_RGDI \| RC_FAKEID));
	- if (!good_data && iack != (GIVR_IT_REI \| RC_FAKEID)) {
	- device_printf(sc->sc_dev,
	- "fake rxint: %02x\n", iack);
	- goto more_intrs;
	- }
	- chan = ((rcin(sc, CD180_GICR) & GICR_CHAN) >> GICR_LSH);
	- rc = &sc->sc_channels[chan];
	- t_state = rc->rc_tp->t_state;
	- /* Do RTS flow control stuff */
	- if ( (rc->rc_flags & RC_RTSFLOW)
	- \|\| !(t_state & TS_ISOPEN)
	- ) {
	- if ( ( !(t_state & TS_ISOPEN)
	- \|\| (t_state & TS_TBLOCK)
	- )
	- && (rc->rc_msvr & MSVR_RTS)
	- )
	- rcout(sc, CD180_MSVR,
	- rc->rc_msvr &= ~MSVR_RTS);
	- else if (!(rc->rc_msvr & MSVR_RTS))
	- rcout(sc, CD180_MSVR,
	- rc->rc_msvr \|= MSVR_RTS);
	- }
	- ucnt = rcin(sc, CD180_RDCR) & 0xF;
	- resid = 0;
	-
	- if (t_state & TS_ISOPEN) {
	- /* check for input buffer overflow */
	- if ((rc->rc_iptr + ucnt) >= rc->rc_bufend) {
	- resid = ucnt;
	- ucnt = rc->rc_bufend - rc->rc_iptr;
	- resid -= ucnt;
	- if (!(rc->rc_flags & RC_WAS_BUFOVFL)) {
	- rc->rc_flags \|= RC_WAS_BUFOVFL;
	- sc->sc_scheduled_event++;
	- }
	- }
	- optr = rc->rc_iptr;
	- /* check foor good data */
	- if (good_data) {
	- while (ucnt-- > 0) {
	- val = rcin(sc, CD180_RDR);
	- optr[0] = val;
	- optr[INPUT_FLAGS_SHIFT] = 0;
	- optr++;
	- sc->sc_scheduled_event++;
	- if (val != 0 && val == rc->rc_tp->t_hotchar)
	- swi_sched(sc->sc_swicookie, 0);
	- }
	- } else {
	- /* Store also status data */
	- while (ucnt-- > 0) {
	- iack = rcin(sc, CD180_RCSR);
	- if (iack & RCSR_Timeout)
	- break;
	- if ( (iack & RCSR_OE)
	- && !(rc->rc_flags & RC_WAS_SILOVFL)) {
	- rc->rc_flags \|= RC_WAS_SILOVFL;
	- sc->sc_scheduled_event++;
	- }
	- val = rcin(sc, CD180_RDR);
	- /*
	- Don't store PE if IGNPAR and BREAK if IGNBRK,
	- this hack allows "raw" tty optimization
	- works even if IGN* is set.
	- */
	- if ( !(iack & (RCSR_PE\|RCSR_FE\|RCSR_Break))
	- \|\| ((!(iack & (RCSR_PE\|RCSR_FE))
	- \|\| !(rc->rc_tp->t_iflag & IGNPAR))
	- && (!(iack & RCSR_Break)
	- \|\| !(rc->rc_tp->t_iflag & IGNBRK)))) {
	- if ( (iack & (RCSR_PE\|RCSR_FE))
	- && (t_state & TS_CAN_BYPASS_L_RINT)
	- && ((iack & RCSR_FE)
	- \|\| ((iack & RCSR_PE)
	- && (rc->rc_tp->t_iflag & INPCK))))
	- val = 0;
	- else if (val != 0 && val == rc->rc_tp->t_hotchar)
	- swi_sched(sc->sc_swicookie, 0);
	- optr[0] = val;
	- optr[INPUT_FLAGS_SHIFT] = iack;
	- optr++;
	- sc->sc_scheduled_event++;
	- }
	- }
	- }
	- rc->rc_iptr = optr;
	- rc->rc_flags \|= RC_DORXFER;
	- } else
	- resid = ucnt;
	- /* Clear FIFO if necessary */
	- while (resid-- > 0) {
	- if (!good_data)
	- iack = rcin(sc, CD180_RCSR);
	- else
	- iack = 0;
	- if (iack & RCSR_Timeout)
	- break;
	- (void) rcin(sc, CD180_RDR);
	- }
	- goto more_intrs;
	- }
	- if (bsr & RC_BSR_MOINT) {
	- iack = rcin(sc, RC_PILR_MODEM);
	- if (iack != (GIVR_IT_MSCI \| RC_FAKEID)) {
	- device_printf(sc->sc_dev, "fake moint: %02x\n",
	- iack);
	- goto more_intrs;
	- }
	- chan = ((rcin(sc, CD180_GICR) & GICR_CHAN) >> GICR_LSH);
	- rc = &sc->sc_channels[chan];
	- iack = rcin(sc, CD180_MCR);
	- rc->rc_msvr = rcin(sc, CD180_MSVR);
	- rcout(sc, CD180_MCR, 0);
	-#ifdef RCDEBUG
	- printrcflags(rc, "moint");
	-#endif
	- if (rc->rc_flags & RC_CTSFLOW) {
	- if (rc->rc_msvr & MSVR_CTS)
	- rc->rc_flags \|= RC_SEND_RDY;
	- else
	- rc->rc_flags &= ~RC_SEND_RDY;
	- } else
	- rc->rc_flags \|= RC_SEND_RDY;
	- if ((iack & MCR_CDchg) && !(rc->rc_flags & RC_MODCHG)) {
	- sc->sc_scheduled_event += LOTS_OF_EVENTS;
	- rc->rc_flags \|= RC_MODCHG;
	- swi_sched(sc->sc_swicookie, 0);
	- }
	- goto more_intrs;
	- }
	- if (bsr & RC_BSR_TXINT) {
	- iack = rcin(sc, RC_PILR_TX);
	- if (iack != (GIVR_IT_TDI \| RC_FAKEID)) {
	- device_printf(sc->sc_dev, "fake txint: %02x\n",
	- iack);
	- goto more_intrs;
	- }
	- chan = ((rcin(sc, CD180_GICR) & GICR_CHAN) >> GICR_LSH);
	- rc = &sc->sc_channels[chan];
	- if ( (rc->rc_flags & RC_OSUSP)
	- \|\| !(rc->rc_flags & RC_SEND_RDY)
	- )
	- goto more_intrs;
	- /* Handle breaks and other stuff */
	- if (rc->rc_pendcmd) {
	- rcout(sc, CD180_COR2, rc->rc_cor2 \|= COR2_ETC);
	- rcout(sc, CD180_TDR, CD180_C_ESC);
	- rcout(sc, CD180_TDR, rc->rc_pendcmd);
	- rcout(sc, CD180_COR2, rc->rc_cor2 &= ~COR2_ETC);
	- rc->rc_pendcmd = 0;
	- goto more_intrs;
	- }
	- optr = rc->rc_optr;
	- resid = rc->rc_obufend - optr;
	- if (resid > CD180_NFIFO)
	- resid = CD180_NFIFO;
	- while (resid-- > 0)
	- rcout(sc, CD180_TDR, *optr++);
	- rc->rc_optr = optr;
	-
	- /* output completed? */
	- if (optr >= rc->rc_obufend) {
	- rcout(sc, CD180_IER, rc->rc_ier &= ~IER_TxRdy);
	-#ifdef RCDEBUG
	- device_printf(sc->sc_dev,
	- "channel %d: output completed\n",
	- rc->rc_chan);
	-#endif
	- if (!(rc->rc_flags & RC_DOXXFER)) {
	- sc->sc_scheduled_event += LOTS_OF_EVENTS;
	- rc->rc_flags \|= RC_DOXXFER;
	- swi_sched(sc->sc_swicookie, 0);
	- }
	- }
	- }
	- more_intrs:
	- rcout(sc, CD180_EOIR, 0); /* end of interrupt */
	- rcout(sc, RC_CTOUT, 0);
	- bsr = ~(rcin(sc, RC_BSR));
	- }
	-}
	-
	-/* Feed characters to output buffer */
	-static void
	-rc_start(struct tty *tp)
	-{
	- struct rc_softc *sc;
	- struct rc_chans *rc;
	- int s;
	-
	- rc = tp->t_sc;
	- if (rc->rc_flags & RC_OSBUSY)
	- return;
	- sc = rc->rc_rcb;
	- s = spltty();
	- rc->rc_flags \|= RC_OSBUSY;
	- critical_enter();
	- if (tp->t_state & TS_TTSTOP)
	- rc->rc_flags \|= RC_OSUSP;
	- else
	- rc->rc_flags &= ~RC_OSUSP;
	- /* Do RTS flow control stuff */
	- if ( (rc->rc_flags & RC_RTSFLOW)
	- && (tp->t_state & TS_TBLOCK)
	- && (rc->rc_msvr & MSVR_RTS)
	- ) {
	- rcout(sc, CD180_CAR, rc->rc_chan);
	- rcout(sc, CD180_MSVR, rc->rc_msvr &= ~MSVR_RTS);
	- } else if (!(rc->rc_msvr & MSVR_RTS)) {
	- rcout(sc, CD180_CAR, rc->rc_chan);
	- rcout(sc, CD180_MSVR, rc->rc_msvr \|= MSVR_RTS);
	- }
	- critical_exit();
	- if (tp->t_state & (TS_TIMEOUT\|TS_TTSTOP))
	- goto out;
	-#ifdef RCDEBUG
	- printrcflags(rc, "rcstart");
	-#endif
	- ttwwakeup(tp);
	-#ifdef RCDEBUG
	- printf("rcstart: outq = %d obuf = %d\n",
	- tp->t_outq.c_cc, rc->rc_obufend - rc->rc_optr);
	-#endif
	- if (tp->t_state & TS_BUSY)
	- goto out; /* output still in progress ... */
	-
	- if (tp->t_outq.c_cc > 0) {
	- u_int ocnt;
	-
	- tp->t_state \|= TS_BUSY;
	- ocnt = q_to_b(&tp->t_outq, rc->rc_obuf, sizeof rc->rc_obuf);
	- critical_enter();
	- rc->rc_optr = rc->rc_obuf;
	- rc->rc_obufend = rc->rc_optr + ocnt;
	- critical_exit();
	- if (!(rc->rc_ier & IER_TxRdy)) {
	-#ifdef RCDEBUG
	- device_printf(sc->sc_dev,
	- "channel %d: rcstart enable txint\n", rc->rc_chan);
	-#endif
	- rcout(sc, CD180_CAR, rc->rc_chan);
	- rcout(sc, CD180_IER, rc->rc_ier \|= IER_TxRdy);
	- }
	- }
	-out:
	- rc->rc_flags &= ~RC_OSBUSY;
	- (void) splx(s);
	-}
	-
	-/* Handle delayed events. */
	-void
	-rc_pollcard(void *arg)
	-{
	- struct rc_softc *sc;
	- struct rc_chans *rc;
	- struct tty *tp;
	- u_char tptr, eptr;
	- int chan, icnt;
	-
	- sc = (struct rc_softc *)arg;
	- if (sc->sc_scheduled_event == 0)
	- return;
	- do {
	- rc = sc->sc_channels;
	- for (chan = 0; chan < CD180_NCHAN; rc++, chan++) {
	- tp = rc->rc_tp;
	-#ifdef RCDEBUG
	- if (rc->rc_flags & (RC_DORXFER\|RC_DOXXFER\|RC_MODCHG\|
	- RC_WAS_BUFOVFL\|RC_WAS_SILOVFL))
	- printrcflags(rc, "rcevent");
	-#endif
	- if (rc->rc_flags & RC_WAS_BUFOVFL) {
	- critical_enter();
	- rc->rc_flags &= ~RC_WAS_BUFOVFL;
	- sc->sc_scheduled_event--;
	- critical_exit();
	- device_printf(sc->sc_dev,
	- "channel %d: interrupt-level buffer overflow\n",
	- chan);
	- }
	- if (rc->rc_flags & RC_WAS_SILOVFL) {
	- critical_enter();
	- rc->rc_flags &= ~RC_WAS_SILOVFL;
	- sc->sc_scheduled_event--;
	- critical_exit();
	- device_printf(sc->sc_dev,
	- "channel %d: silo overflow\n", chan);
	- }
	- if (rc->rc_flags & RC_MODCHG) {
	- critical_enter();
	- rc->rc_flags &= ~RC_MODCHG;
	- sc->sc_scheduled_event -= LOTS_OF_EVENTS;
	- critical_exit();
	- ttyld_modem(tp, !!(rc->rc_msvr & MSVR_CD));
	- }
	- if (rc->rc_flags & RC_DORXFER) {
	- critical_enter();
	- rc->rc_flags &= ~RC_DORXFER;
	- eptr = rc->rc_iptr;
	- if (rc->rc_bufend == &rc->rc_ibuf[2 * RC_IBUFSIZE])
	- tptr = &rc->rc_ibuf[RC_IBUFSIZE];
	- else
	- tptr = rc->rc_ibuf;
	- icnt = eptr - tptr;
	- if (icnt > 0) {
	- if (rc->rc_bufend == &rc->rc_ibuf[2 * RC_IBUFSIZE]) {
	- rc->rc_iptr = rc->rc_ibuf;
	- rc->rc_bufend = &rc->rc_ibuf[RC_IBUFSIZE];
	- rc->rc_hiwat = &rc->rc_ibuf[RC_IHIGHWATER];
	- } else {
	- rc->rc_iptr = &rc->rc_ibuf[RC_IBUFSIZE];
	- rc->rc_bufend = &rc->rc_ibuf[2 * RC_IBUFSIZE];
	- rc->rc_hiwat =
	- &rc->rc_ibuf[RC_IBUFSIZE + RC_IHIGHWATER];
	- }
	- if ( (rc->rc_flags & RC_RTSFLOW)
	- && (tp->t_state & TS_ISOPEN)
	- && !(tp->t_state & TS_TBLOCK)
	- && !(rc->rc_msvr & MSVR_RTS)
	- ) {
	- rcout(sc, CD180_CAR, chan);
	- rcout(sc, CD180_MSVR,
	- rc->rc_msvr \|= MSVR_RTS);
	- }
	- sc->sc_scheduled_event -= icnt;
	- }
	- critical_exit();
	-
	- if (icnt <= 0 \|\| !(tp->t_state & TS_ISOPEN))
	- goto done1;
	-
	- if ( (tp->t_state & TS_CAN_BYPASS_L_RINT)
	- && !(tp->t_state & TS_LOCAL)) {
	- if ((tp->t_rawq.c_cc + icnt) >= RB_I_HIGH_WATER
	- && ((rc->rc_flags & RC_RTSFLOW) \|\| (tp->t_iflag & IXOFF))
	- && !(tp->t_state & TS_TBLOCK))
	- ttyblock(tp);
	- tk_nin += icnt;
	- tk_rawcc += icnt;
	- tp->t_rawcc += icnt;
	- if (b_to_q(tptr, icnt, &tp->t_rawq))
	- device_printf(sc->sc_dev,
	- "channel %d: tty-level buffer overflow\n",
	- chan);
	- ttwakeup(tp);
	- if ((tp->t_state & TS_TTSTOP) && ((tp->t_iflag & IXANY)
	- \|\| (tp->t_cc[VSTART] == tp->t_cc[VSTOP]))) {
	- tp->t_state &= ~TS_TTSTOP;
	- tp->t_lflag &= ~FLUSHO;
	- rc_start(tp);
	- }
	- } else {
	- for (; tptr < eptr; tptr++)
	- ttyld_rint(tp,
	- (tptr[0] \|
	- rc_rcsrt[tptr[INPUT_FLAGS_SHIFT] & 0xF]));
	- }
	-done1: ;
	- }
	- if (rc->rc_flags & RC_DOXXFER) {
	- critical_enter();
	- sc->sc_scheduled_event -= LOTS_OF_EVENTS;
	- rc->rc_flags &= ~RC_DOXXFER;
	- rc->rc_tp->t_state &= ~TS_BUSY;
	- critical_exit();
	- ttyld_start(tp);
	- }
	- if (sc->sc_scheduled_event == 0)
	- break;
	- }
	- } while (sc->sc_scheduled_event >= LOTS_OF_EVENTS);
	-}
	-
	-static void
	-rc_stop(struct tty *tp, int rw)
	-{
	- struct rc_softc *sc;
	- struct rc_chans *rc;
	- u_char tptr, eptr;
	-
	- rc = tp->t_sc;
	- sc = rc->rc_rcb;
	-#ifdef RCDEBUG
	- device_printf(sc->sc_dev, "channel %d: rc_stop %s%s\n",
	- rc->rc_chan, (rw & FWRITE)?"FWRITE ":"", (rw & FREAD)?"FREAD":"");
	-#endif
	- if (rw & FWRITE)
	- rc_discard_output(rc);
	- critical_enter();
	- if (rw & FREAD) {
	- rc->rc_flags &= ~RC_DORXFER;
	- eptr = rc->rc_iptr;
	- if (rc->rc_bufend == &rc->rc_ibuf[2 * RC_IBUFSIZE]) {
	- tptr = &rc->rc_ibuf[RC_IBUFSIZE];
	- rc->rc_iptr = &rc->rc_ibuf[RC_IBUFSIZE];
	- } else {
	- tptr = rc->rc_ibuf;
	- rc->rc_iptr = rc->rc_ibuf;
	- }
	- sc->sc_scheduled_event -= eptr - tptr;
	- }
	- if (tp->t_state & TS_TTSTOP)
	- rc->rc_flags \|= RC_OSUSP;
	- else
	- rc->rc_flags &= ~RC_OSUSP;
	- critical_exit();
	-}
	-
	-static void
	-rc_close(struct tty *tp)
	-{
	- struct rc_chans *rc;
	- struct rc_softc *sc;
	- int s;
	-
	- rc = tp->t_sc;
	- sc = rc->rc_rcb;
	- s = spltty();
	- rcout(sc, CD180_CAR, rc->rc_chan);
	-
	- /* Disable rx/tx intrs */
	- rcout(sc, CD180_IER, rc->rc_ier = 0);
	- if ( (tp->t_cflag & HUPCL)
	- \|\| (!(rc->rc_flags & RC_ACTOUT)
	- && !(rc->rc_msvr & MSVR_CD)
	- && !(tp->t_cflag & CLOCAL))
	- \|\| !(tp->t_state & TS_ISOPEN)
	- ) {
	- CCRCMD(sc, rc->rc_chan, CCR_ResetChan);
	- WAITFORCCR(sc, rc->rc_chan);
	- (void) rc_modem(tp, SER_RTS, 0);
	- ttydtrwaitstart(tp);
	- }
	- rc->rc_flags &= ~RC_ACTOUT;
	- wakeup( &rc->rc_rcb); /* wake bi */
	- wakeup(TSA_CARR_ON(tp));
	- (void) splx(s);
	-}
	-
	-/* Reset the bastard */
	-static void
	-rc_hwreset(struct rc_softc *sc, u_int chipid)
	-{
	- CCRCMD(sc, -1, CCR_HWRESET); /* Hardware reset */
	- DELAY(20000);
	- WAITFORCCR(sc, -1);
	-
	- rcout(sc, RC_CTOUT, 0); /* Clear timeout */
	- rcout(sc, CD180_GIVR, chipid);
	- rcout(sc, CD180_GICR, 0);
	-
	- /* Set Prescaler Registers (1 msec) */
	- rcout(sc, CD180_PPRL, ((RC_OSCFREQ + 999) / 1000) & 0xFF);
	- rcout(sc, CD180_PPRH, ((RC_OSCFREQ + 999) / 1000) >> 8);
	-
	- /* Initialize Priority Interrupt Level Registers */
	- rcout(sc, CD180_PILR1, RC_PILR_MODEM);
	- rcout(sc, CD180_PILR2, RC_PILR_TX);
	- rcout(sc, CD180_PILR3, RC_PILR_RX);
	-
	- /* Reset DTR */
	- rcout(sc, RC_DTREG, ~0);
	-}
	-
	-/* Set channel parameters */
	-static int
	-rc_param(struct tty tp, struct termios ts)
	-{
	- struct rc_softc *sc;
	- struct rc_chans *rc;
	- int idivs, odivs, s, val, cflag, iflag, lflag, inpflow;
	-
	- if ( ts->c_ospeed < 0 \|\| ts->c_ospeed > 76800
	- \|\| ts->c_ispeed < 0 \|\| ts->c_ispeed > 76800
	- )
	- return (EINVAL);
	- if (ts->c_ispeed == 0)
	- ts->c_ispeed = ts->c_ospeed;
	- odivs = RC_BRD(ts->c_ospeed);
	- idivs = RC_BRD(ts->c_ispeed);
	-
	- rc = tp->t_sc;
	- sc = rc->rc_rcb;
	- s = spltty();
	-
	- /* Select channel */
	- rcout(sc, CD180_CAR, rc->rc_chan);
	-
	- /* If speed == 0, hangup line */
	- if (ts->c_ospeed == 0) {
	- CCRCMD(sc, rc->rc_chan, CCR_ResetChan);
	- WAITFORCCR(sc, rc->rc_chan);
	- (void) rc_modem(tp, 0, SER_DTR);
	- }
	-
	- tp->t_state &= ~TS_CAN_BYPASS_L_RINT;
	- cflag = ts->c_cflag;
	- iflag = ts->c_iflag;
	- lflag = ts->c_lflag;
	-
	- if (idivs > 0) {
	- rcout(sc, CD180_RBPRL, idivs & 0xFF);
	- rcout(sc, CD180_RBPRH, idivs >> 8);
	- }
	- if (odivs > 0) {
	- rcout(sc, CD180_TBPRL, odivs & 0xFF);
	- rcout(sc, CD180_TBPRH, odivs >> 8);
	- }
	-
	- /* set timeout value */
	- if (ts->c_ispeed > 0) {
	- int itm = ts->c_ispeed > 2400 ? 5 : 10000 / ts->c_ispeed + 1;
	-
	- if ( !(lflag & ICANON)
	- && ts->c_cc[VMIN] != 0 && ts->c_cc[VTIME] != 0
	- && ts->c_cc[VTIME] * 10 > itm)
	- itm = ts->c_cc[VTIME] * 10;
	-
	- rcout(sc, CD180_RTPR, itm <= 255 ? itm : 255);
	- }
	-
	- switch (cflag & CSIZE) {
	- case CS5: val = COR1_5BITS; break;
	- case CS6: val = COR1_6BITS; break;
	- case CS7: val = COR1_7BITS; break;
	- default:
	- case CS8: val = COR1_8BITS; break;
	- }
	- if (cflag & PARENB) {
	- val \|= COR1_NORMPAR;
	- if (cflag & PARODD)
	- val \|= COR1_ODDP;
	- if (!(cflag & INPCK))
	- val \|= COR1_Ignore;
	- } else
	- val \|= COR1_Ignore;
	- if (cflag & CSTOPB)
	- val \|= COR1_2SB;
	- rcout(sc, CD180_COR1, val);
	-
	- /* Set FIFO threshold */
	- val = ts->c_ospeed <= 4800 ? 1 : CD180_NFIFO / 2;
	- inpflow = 0;
	- if ( (iflag & IXOFF)
	- && ( ts->c_cc[VSTOP] != _POSIX_VDISABLE
	- && ( ts->c_cc[VSTART] != _POSIX_VDISABLE
	- \|\| (iflag & IXANY)
	- )
	- )
	- ) {
	- inpflow = 1;
	- val \|= COR3_SCDE\|COR3_FCT;
	- }
	- rcout(sc, CD180_COR3, val);
	-
	- /* Initialize on-chip automatic flow control */
	- val = 0;
	- rc->rc_flags &= ~(RC_CTSFLOW\|RC_SEND_RDY);
	- if (cflag & CCTS_OFLOW) {
	- rc->rc_flags \|= RC_CTSFLOW;
	- val \|= COR2_CtsAE;
	- } else
	- rc->rc_flags \|= RC_SEND_RDY;
	- if (tp->t_state & TS_TTSTOP)
	- rc->rc_flags \|= RC_OSUSP;
	- else
	- rc->rc_flags &= ~RC_OSUSP;
	- if (cflag & CRTS_IFLOW)
	- rc->rc_flags \|= RC_RTSFLOW;
	- else
	- rc->rc_flags &= ~RC_RTSFLOW;
	-
	- if (inpflow) {
	- if (ts->c_cc[VSTART] != _POSIX_VDISABLE)
	- rcout(sc, CD180_SCHR1, ts->c_cc[VSTART]);
	- rcout(sc, CD180_SCHR2, ts->c_cc[VSTOP]);
	- val \|= COR2_TxIBE;
	- if (iflag & IXANY)
	- val \|= COR2_IXM;
	- }
	-
	- rcout(sc, CD180_COR2, rc->rc_cor2 = val);
	-
	- CCRCMD(sc, rc->rc_chan, CCR_CORCHG1 \| CCR_CORCHG2 \| CCR_CORCHG3);
	-
	- ttyldoptim(tp);
	-
	- /* modem ctl */
	- val = cflag & CLOCAL ? 0 : MCOR1_CDzd;
	- if (cflag & CCTS_OFLOW)
	- val \|= MCOR1_CTSzd;
	- rcout(sc, CD180_MCOR1, val);
	-
	- val = cflag & CLOCAL ? 0 : MCOR2_CDod;
	- if (cflag & CCTS_OFLOW)
	- val \|= MCOR2_CTSod;
	- rcout(sc, CD180_MCOR2, val);
	-
	- /* enable i/o and interrupts */
	- CCRCMD(sc, rc->rc_chan,
	- CCR_XMTREN \| ((cflag & CREAD) ? CCR_RCVREN : CCR_RCVRDIS));
	- WAITFORCCR(sc, rc->rc_chan);
	-
	- rc->rc_ier = cflag & CLOCAL ? 0 : IER_CD;
	- if (cflag & CCTS_OFLOW)
	- rc->rc_ier \|= IER_CTS;
	- if (cflag & CREAD)
	- rc->rc_ier \|= IER_RxData;
	- if (tp->t_state & TS_BUSY)
	- rc->rc_ier \|= IER_TxRdy;
	- if (ts->c_ospeed != 0)
	- rc_modem(tp, SER_DTR, 0);
	- if ((cflag & CCTS_OFLOW) && (rc->rc_msvr & MSVR_CTS))
	- rc->rc_flags \|= RC_SEND_RDY;
	- rcout(sc, CD180_IER, rc->rc_ier);
	- (void) splx(s);
	- return 0;
	-}
	-
	-/* Re-initialize board after bogus interrupts */
	-static void
	-rc_reinit(struct rc_softc *sc)
	-{
	- struct rc_chans *rc;
	- int i;
	-
	- rc_hwreset(sc, RC_FAKEID);
	- rc = sc->sc_channels;
	- for (i = 0; i < CD180_NCHAN; i++, rc++)
	- (void) rc_param(rc->rc_tp, &rc->rc_tp->t_termios);
	-}
	-
	-/* Modem control routines */
	-
	-static int
	-rc_modem(struct tty *tp, int biton, int bitoff)
	-{
	- struct rc_chans *rc;
	- struct rc_softc *sc;
	- u_char *dtr;
	- u_char msvr;
	-
	- rc = tp->t_sc;
	- sc = rc->rc_rcb;
	- dtr = &sc->sc_dtr;
	- rcout(sc, CD180_CAR, rc->rc_chan);
	-
	- if (biton == 0 && bitoff == 0) {
	- msvr = rc->rc_msvr = rcin(sc, CD180_MSVR);
	-
	- if (msvr & MSVR_RTS)
	- biton \|= SER_RTS;
	- if (msvr & MSVR_CTS)
	- biton \|= SER_CTS;
	- if (msvr & MSVR_DSR)
	- biton \|= SER_DSR;
	- if (msvr & MSVR_DTR)
	- biton \|= SER_DTR;
	- if (msvr & MSVR_CD)
	- biton \|= SER_DCD;
	- if (~rcin(sc, RC_RIREG) & (1 << rc->rc_chan))
	- biton \|= SER_RI;
	- return biton;
	- }
	- if (biton & SER_DTR)
	- rcout(sc, RC_DTREG, ~(*dtr \|= 1 << rc->rc_chan));
	- if (bitoff & SER_DTR)
	- rcout(sc, RC_DTREG, ~(*dtr &= ~(1 << rc->rc_chan)));
	- msvr = rcin(sc, CD180_MSVR);
	- if (biton & SER_DTR)
	- msvr \|= MSVR_DTR;
	- if (bitoff & SER_DTR)
	- msvr &= ~MSVR_DTR;
	- if (biton & SER_RTS)
	- msvr \|= MSVR_RTS;
	- if (bitoff & SER_RTS)
	- msvr &= ~MSVR_RTS;
	- rcout(sc, CD180_MSVR, msvr);
	- return 0;
	-}
	-
	-static void
	-rc_break(struct tty *tp, int brk)
	-{
	- struct rc_chans *rc;
	-
	- rc = tp->t_sc;
	-
	- if (brk)
	- rc->rc_pendcmd = CD180_C_SBRK;
	- else
	- rc->rc_pendcmd = CD180_C_EBRK;
	-}
	-
	-#define ERR(s) do { \
	- device_printf(sc->sc_dev, "%s", ""); \
	- printf s ; \
	- printf("\n"); \
	- (void) splx(old_level); \
	- return 1; \
	-} while (0)
	-
	-/* Test the board. */
	-int
	-rc_test(struct rc_softc *sc)
	-{
	- int chan = 0;
	- int i = 0, rcnt, old_level;
	- unsigned int iack, chipid;
	- unsigned short divs;
	- static u_char ctest[] = "\377\125\252\045\244\0\377";
	-#define CTLEN 8
	-
	- struct rtest {
	- u_char txbuf[CD180_NFIFO]; /* TX buffer */
	- u_char rxbuf[CD180_NFIFO]; /* RX buffer */
	- int rxptr; /* RX pointer */
	- int txptr; /* TX pointer */
	- } tchans[CD180_NCHAN];
	-
	- old_level = spltty();
	-
	- chipid = RC_FAKEID;
	-
	- /* First, reset board to initial state */
	- rc_hwreset(sc, chipid);
	-
	- divs = RC_BRD(19200);
	-
	- /* Initialize channels */
	- for (chan = 0; chan < CD180_NCHAN; chan++) {
	-
	- /* Select and reset channel */
	- rcout(sc, CD180_CAR, chan);
	- CCRCMD(sc, chan, CCR_ResetChan);
	- WAITFORCCR(sc, chan);
	-
	- /* Set speed */
	- rcout(sc, CD180_RBPRL, divs & 0xFF);
	- rcout(sc, CD180_RBPRH, divs >> 8);
	- rcout(sc, CD180_TBPRL, divs & 0xFF);
	- rcout(sc, CD180_TBPRH, divs >> 8);
	-
	- /* set timeout value */
	- rcout(sc, CD180_RTPR, 0);
	-
	- /* Establish local loopback */
	- rcout(sc, CD180_COR1, COR1_NOPAR \| COR1_8BITS \| COR1_1SB);
	- rcout(sc, CD180_COR2, COR2_LLM);
	- rcout(sc, CD180_COR3, CD180_NFIFO);
	- CCRCMD(sc, chan, CCR_CORCHG1 \| CCR_CORCHG2 \| CCR_CORCHG3);
	- CCRCMD(sc, chan, CCR_RCVREN \| CCR_XMTREN);
	- WAITFORCCR(sc, chan);
	- rcout(sc, CD180_MSVR, MSVR_RTS);
	-
	- /* Fill TXBUF with test data */
	- for (i = 0; i < CD180_NFIFO; i++) {
	- tchans[chan].txbuf[i] = ctest[i];
	- tchans[chan].rxbuf[i] = 0;
	- }
	- tchans[chan].txptr = tchans[chan].rxptr = 0;
	-
	- /* Now, start transmit */
	- rcout(sc, CD180_IER, IER_TxMpty\|IER_RxData);
	- }
	- /* Pseudo-interrupt poll stuff */
	- for (rcnt = 10000; rcnt-- > 0; rcnt--) {
	- i = ~(rcin(sc, RC_BSR));
	- if (i & RC_BSR_TOUT)
	- ERR(("BSR timeout bit set\n"));
	- else if (i & RC_BSR_TXINT) {
	- iack = rcin(sc, RC_PILR_TX);
	- if (iack != (GIVR_IT_TDI \| chipid))
	- ERR(("Bad TX intr ack (%02x != %02x)\n",
	- iack, GIVR_IT_TDI \| chipid));
	- chan = (rcin(sc, CD180_GICR) & GICR_CHAN) >> GICR_LSH;
	- /* If no more data to transmit, disable TX intr */
	- if (tchans[chan].txptr >= CD180_NFIFO) {
	- iack = rcin(sc, CD180_IER);
	- rcout(sc, CD180_IER, iack & ~IER_TxMpty);
	- } else {
	- for (iack = tchans[chan].txptr;
	- iack < CD180_NFIFO; iack++)
	- rcout(sc, CD180_TDR,
	- tchans[chan].txbuf[iack]);
	- tchans[chan].txptr = iack;
	- }
	- rcout(sc, CD180_EOIR, 0);
	- } else if (i & RC_BSR_RXINT) {
	- u_char ucnt;
	-
	- iack = rcin(sc, RC_PILR_RX);
	- if (iack != (GIVR_IT_RGDI \| chipid) &&
	- iack != (GIVR_IT_REI \| chipid))
	- ERR(("Bad RX intr ack (%02x != %02x)\n",
	- iack, GIVR_IT_RGDI \| chipid));
	- chan = (rcin(sc, CD180_GICR) & GICR_CHAN) >> GICR_LSH;
	- ucnt = rcin(sc, CD180_RDCR) & 0xF;
	- while (ucnt-- > 0) {
	- iack = rcin(sc, CD180_RCSR);
	- if (iack & RCSR_Timeout)
	- break;
	- if (iack & 0xF)
	- ERR(("Bad char chan %d (RCSR = %02X)\n",
	- chan, iack));
	- if (tchans[chan].rxptr > CD180_NFIFO)
	- ERR(("Got extra chars chan %d\n",
	- chan));
	- tchans[chan].rxbuf[tchans[chan].rxptr++] =
	- rcin(sc, CD180_RDR);
	- }
	- rcout(sc, CD180_EOIR, 0);
	- }
	- rcout(sc, RC_CTOUT, 0);
	- for (iack = chan = 0; chan < CD180_NCHAN; chan++)
	- if (tchans[chan].rxptr >= CD180_NFIFO)
	- iack++;
	- if (iack == CD180_NCHAN)
	- break;
	- }
	- for (chan = 0; chan < CD180_NCHAN; chan++) {
	- /* Select and reset channel */
	- rcout(sc, CD180_CAR, chan);
	- CCRCMD(sc, chan, CCR_ResetChan);
	- }
	-
	- if (!rcnt)
	- ERR(("looses characters during local loopback\n"));
	- /* Now, check data */
	- for (chan = 0; chan < CD180_NCHAN; chan++)
	- for (i = 0; i < CD180_NFIFO; i++)
	- if (ctest[i] != tchans[chan].rxbuf[i])
	- ERR(("data mismatch chan %d ptr %d (%d != %d)\n",
	- chan, i, ctest[i], tchans[chan].rxbuf[i]));
	- (void) splx(old_level);
	- return 0;
	-}
	-
	-#ifdef RCDEBUG
	-static void
	-printrcflags(struct rc_chans rc, char comment)
	-{
	- struct rc_softc *sc;
	- u_short f = rc->rc_flags;
	-
	- sc = rc->rc_rcb;
	- printf("rc%d/%d: %s flags: %s%s%s%s%s%s%s%s%s%s%s%s\n",
	- rc->rc_rcb->rcb_unit, rc->rc_chan, comment,
	- (f & RC_DTR_OFF)?"DTR_OFF " :"",
	- (f & RC_ACTOUT) ?"ACTOUT " :"",
	- (f & RC_RTSFLOW)?"RTSFLOW " :"",
	- (f & RC_CTSFLOW)?"CTSFLOW " :"",
	- (f & RC_DORXFER)?"DORXFER " :"",
	- (f & RC_DOXXFER)?"DOXXFER " :"",
	- (f & RC_MODCHG) ?"MODCHG " :"",
	- (f & RC_OSUSP) ?"OSUSP " :"",
	- (f & RC_OSBUSY) ?"OSBUSY " :"",
	- (f & RC_WAS_BUFOVFL) ?"BUFOVFL " :"",
	- (f & RC_WAS_SILOVFL) ?"SILOVFL " :"",
	- (f & RC_SEND_RDY) ?"SEND_RDY":"");
	-
	- rcout(sc, CD180_CAR, rc->rc_chan);
	-
	- printf("rc%d/%d: msvr %02x ier %02x ccsr %02x\n",
	- rc->rc_rcb->rcb_unit, rc->rc_chan,
	- rcin(sc, CD180_MSVR),
	- rcin(sc, CD180_IER),
	- rcin(sc, CD180_CCSR));
	-}
	-#endif /* RCDEBUG */
	-
	-static void
	-rc_discard_output(struct rc_chans *rc)
	-{
	- critical_enter();
	- if (rc->rc_flags & RC_DOXXFER) {
	- rc->rc_rcb->sc_scheduled_event -= LOTS_OF_EVENTS;
	- rc->rc_flags &= ~RC_DOXXFER;
	- }
	- rc->rc_optr = rc->rc_obufend;
	- rc->rc_tp->t_state &= ~TS_BUSY;
	- critical_exit();
	- ttwwakeup(rc->rc_tp);
	-}
	-
	-static void
	-rc_wait0(struct rc_softc *sc, int chan, int line)
	-{
	- int rcnt;
	-
	- for (rcnt = 50; rcnt && rcin(sc, CD180_CCR); rcnt--)
	- DELAY(30);
	- if (rcnt == 0)
	- device_printf(sc->sc_dev,
	- "channel %d command timeout, rc.c line: %d\n", chan, line);
	-}
	-
	-static device_method_t rc_methods[] = {
	- /* Device interface */
	- DEVMETHOD(device_probe, rc_probe),
	- DEVMETHOD(device_attach, rc_attach),
	- DEVMETHOD(device_detach, rc_detach),
	- { 0, 0 }
	-};
	-
	-static driver_t rc_driver = {
	- "rc",
	- rc_methods, sizeof(struct rc_softc),
	-};
	-
	-DRIVER_MODULE(rc, isa, rc_driver, rc_devclass, 0, 0);
	diff --git a/sys/dev/rc/rcreg.h b/sys/dev/rc/rcreg.h
	deleted file mode 100644
	index 832a1cbe1e54..000000000000
	--- a/sys/dev/rc/rcreg.h
	+++ /dev/null
	@@ -1,72 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (C) 1995 by Pavel Antonov, Moscow, Russia.
	- * Copyright (C) 1995 by Andrey A. Chernov, Moscow, Russia.
	- * All rights reserved.
	- * Copyright (C) 2002 by John Baldwin <jhb@FreeBSD.org>
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-/*
	- * Cirrus Logic CD180 -based RISCom/8 board definitions
	- */
	-
	-/* Oscillator frequency - 19660.08Mhz / 2 */
	-#define RC_OSCFREQ 9830400
	-
	-#define RC_BRD(s) ((s) == 0 ? 0 : \
	- (((RC_OSCFREQ + (s) / 2) / (s)) + CD180_CTICKS/2) / CD180_CTICKS)
	-
	-/* Riscom/8 board ISA I/O mapping */
	-#define RC_IOMAP(r) ((((r) & 07) << 1) \| (((r) & ~07) << 7))
	-
	-/* I/O commands */
	-#define RC_OUT(sc, addr, value) \
	- bus_space_write_1((sc)->sc_bt, (sc)->sc_bh, RC_IOMAP(addr), (value))
	-#define RC_IN(sc, addr) \
	- bus_space_read_1((sc)->sc_bt, (sc)->sc_bh, RC_IOMAP(addr))
	-
	-/* Riscom on-board registers (mapping assumed) */
	-#define RC_RIREG 0x100 /* Ring Indicator Register (read-only) */
	-#define RC_DTREG 0x100 /* DTR Register (write-only) */
	-#define RC_BSR 0x101 /* Board Status Register (read-only) */
	-#define RC_CTOUT 0x101 /* Clear Timeout (write-only) */
	-
	-/* Board Status Register */
	-#define RC_BSR_TOUT 0x08 /* Timeout */
	-#define RC_BSR_RXINT 0x04 /* Receiver Interrupt */
	-#define RC_BSR_TXINT 0x02 /* Transmitter Interrupt */
	-#define RC_BSR_MOINT 0x01 /* Modem Control Interrupt */
	-
	-/* Interrupt groups */
	-#define RC_MODEMGRP 0x01 /* Modem interrupt group */
	-#define RC_RXGRP 0x02 /* Receiver interrupt group */
	-#define RC_TXGRP 0x04 /* Transmitter interrupt group */
	-
	-/* Priority Interrupt Level definitions */
	-#define RC_PILR_MODEM (0x80 \| RC_MODEMGRP)
	-#define RC_PILR_RX (0x80 \| RC_RXGRP )
	-#define RC_PILR_TX (0x80 \| RC_TXGRP )
	diff --git a/sys/dev/rp/rp.c b/sys/dev/rp/rp.c
	deleted file mode 100644
	index 390ab733d2ae..000000000000
	--- a/sys/dev/rp/rp.c
	+++ /dev/null
	@@ -1,1113 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	- *
	- * Copyright (c) Comtrol Corporation <support@comtrol.com>
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted prodived that the follwoing conditions
	- * are met.
	- * 1. Redistributions of source code must retain the above copyright
	- * notive, this list of conditions and the following disclainer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials prodided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Comtrol Corporation.
	- * 4. The name of Comtrol Corporation may not be used to endorse or
	- * promote products derived from this software without specific
	- * prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY COMTROL CORPORATION ``AS IS'' AND ANY
	- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL COMTROL CORPORATION BE LIABLE FOR
	- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-/*
	- * rp.c - for RocketPort FreeBSD
	- */
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/endian.h>
	-#include <sys/fcntl.h>
	-#include <sys/malloc.h>
	-#include <sys/serial.h>
	-#include <sys/tty.h>
	-#include <sys/conf.h>
	-#include <sys/kernel.h>
	-#include <machine/resource.h>
	-#include <machine/bus.h>
	-#include <sys/bus.h>
	-#include <sys/rman.h>
	-
	-#define ROCKET_C
	-#include <dev/rp/rpreg.h>
	-#include <dev/rp/rpvar.h>
	-
	-static const char RocketPortVersion[] = "3.02";
	-
	-static Byte_t RData[RDATASIZE] =
	-{
	- 0x00, 0x09, 0xf6, 0x82,
	- 0x02, 0x09, 0x86, 0xfb,
	- 0x04, 0x09, 0x00, 0x0a,
	- 0x06, 0x09, 0x01, 0x0a,
	- 0x08, 0x09, 0x8a, 0x13,
	- 0x0a, 0x09, 0xc5, 0x11,
	- 0x0c, 0x09, 0x86, 0x85,
	- 0x0e, 0x09, 0x20, 0x0a,
	- 0x10, 0x09, 0x21, 0x0a,
	- 0x12, 0x09, 0x41, 0xff,
	- 0x14, 0x09, 0x82, 0x00,
	- 0x16, 0x09, 0x82, 0x7b,
	- 0x18, 0x09, 0x8a, 0x7d,
	- 0x1a, 0x09, 0x88, 0x81,
	- 0x1c, 0x09, 0x86, 0x7a,
	- 0x1e, 0x09, 0x84, 0x81,
	- 0x20, 0x09, 0x82, 0x7c,
	- 0x22, 0x09, 0x0a, 0x0a
	-};
	-
	-static Byte_t RRegData[RREGDATASIZE]=
	-{
	- 0x00, 0x09, 0xf6, 0x82, /* 00: Stop Rx processor */
	- 0x08, 0x09, 0x8a, 0x13, /* 04: Tx software flow control */
	- 0x0a, 0x09, 0xc5, 0x11, /* 08: XON char */
	- 0x0c, 0x09, 0x86, 0x85, /* 0c: XANY */
	- 0x12, 0x09, 0x41, 0xff, /* 10: Rx mask char */
	- 0x14, 0x09, 0x82, 0x00, /* 14: Compare/Ignore #0 */
	- 0x16, 0x09, 0x82, 0x7b, /* 18: Compare #1 */
	- 0x18, 0x09, 0x8a, 0x7d, /* 1c: Compare #2 */
	- 0x1a, 0x09, 0x88, 0x81, /* 20: Interrupt #1 */
	- 0x1c, 0x09, 0x86, 0x7a, /* 24: Ignore/Replace #1 */
	- 0x1e, 0x09, 0x84, 0x81, /* 28: Interrupt #2 */
	- 0x20, 0x09, 0x82, 0x7c, /* 2c: Ignore/Replace #2 */
	- 0x22, 0x09, 0x0a, 0x0a /* 30: Rx FIFO Enable */
	-};
	-
	-#if 0
	-/* IRQ number to MUDBAC register 2 mapping */
	-Byte_t sIRQMap[16] =
	-{
	- 0,0,0,0x10,0x20,0x30,0,0,0,0x40,0x50,0x60,0x70,0,0,0x80
	-};
	-#endif
	-
	-Byte_t rp_sBitMapClrTbl[8] =
	-{
	- 0xfe,0xfd,0xfb,0xf7,0xef,0xdf,0xbf,0x7f
	-};
	-
	-Byte_t rp_sBitMapSetTbl[8] =
	-{
	- 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80
	-};
	-
	-static void rpfree(void *);
	-
	-/***************************************************************************
	-Function: sReadAiopID
	-Purpose: Read the AIOP idenfication number directly from an AIOP.
	-Call: sReadAiopID(CtlP, aiop)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- int aiop: AIOP index
	-Return: int: Flag AIOPID_XXXX if a valid AIOP is found, where X
	- is replace by an identifying number.
	- Flag AIOPID_NULL if no valid AIOP is found
	-Warnings: No context switches are allowed while executing this function.
	-
	-*/
	-int sReadAiopID(CONTROLLER_T *CtlP, int aiop)
	-{
	- Byte_t AiopID; /* ID byte from AIOP */
	-
	- rp_writeaiop1(CtlP, aiop, _CMD_REG, RESET_ALL); /* reset AIOP */
	- rp_writeaiop1(CtlP, aiop, _CMD_REG, 0x0);
	- AiopID = rp_readaiop1(CtlP, aiop, _CHN_STAT0) & 0x07;
	- if(AiopID == 0x06)
	- return(1);
	- else /* AIOP does not exist */
	- return(-1);
	-}
	-
	-/***************************************************************************
	-Function: sReadAiopNumChan
	-Purpose: Read the number of channels available in an AIOP directly from
	- an AIOP.
	-Call: sReadAiopNumChan(CtlP, aiop)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- int aiop: AIOP index
	-Return: int: The number of channels available
	-Comments: The number of channels is determined by write/reads from identical
	- offsets within the SRAM address spaces for channels 0 and 4.
	- If the channel 4 space is mirrored to channel 0 it is a 4 channel
	- AIOP, otherwise it is an 8 channel.
	-Warnings: No context switches are allowed while executing this function.
	-*/
	-int sReadAiopNumChan(CONTROLLER_T *CtlP, int aiop)
	-{
	- Word_t x, y;
	-
	- rp_writeaiop4(CtlP, aiop, _INDX_ADDR,0x12340000L); /* write to chan 0 SRAM */
	- rp_writeaiop2(CtlP, aiop, _INDX_ADDR,0); /* read from SRAM, chan 0 */
	- x = rp_readaiop2(CtlP, aiop, _INDX_DATA);
	- rp_writeaiop2(CtlP, aiop, _INDX_ADDR,0x4000); /* read from SRAM, chan 4 */
	- y = rp_readaiop2(CtlP, aiop, _INDX_DATA);
	- if(x != y) /* if different must be 8 chan */
	- return(8);
	- else
	- return(4);
	-}
	-
	-/***************************************************************************
	-Function: sInitChan
	-Purpose: Initialization of a channel and channel structure
	-Call: sInitChan(CtlP,ChP,AiopNum,ChanNum)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- CHANNEL_T *ChP; Ptr to channel structure
	- int AiopNum; AIOP number within controller
	- int ChanNum; Channel number within AIOP
	-Return: int: TRUE if initialization succeeded, FALSE if it fails because channel
	- number exceeds number of channels available in AIOP.
	-Comments: This function must be called before a channel can be used.
	-Warnings: No range checking on any of the parameters is done.
	-
	- No context switches are allowed while executing this function.
	-*/
	-int sInitChan( CONTROLLER_T *CtlP,
	- CHANNEL_T *ChP,
	- int AiopNum,
	- int ChanNum)
	-{
	- int i, ChOff;
	- Byte_t *ChR;
	- static Byte_t R[4];
	-
	- if(ChanNum >= CtlP->AiopNumChan[AiopNum])
	- return(FALSE); /* exceeds num chans in AIOP */
	-
	- /* Channel, AIOP, and controller identifiers */
	- ChP->CtlP = CtlP;
	- ChP->ChanID = CtlP->AiopID[AiopNum];
	- ChP->AiopNum = AiopNum;
	- ChP->ChanNum = ChanNum;
	-
	- /* Initialize the channel from the RData array */
	- for(i=0; i < RDATASIZE; i+=4)
	- {
	- R[0] = RData[i];
	- R[1] = RData[i+1] + 0x10 * ChanNum;
	- R[2] = RData[i+2];
	- R[3] = RData[i+3];
	- rp_writech4(ChP,_INDX_ADDR,le32dec(R));
	- }
	-
	- ChR = ChP->R;
	- for(i=0; i < RREGDATASIZE; i+=4)
	- {
	- ChR[i] = RRegData[i];
	- ChR[i+1] = RRegData[i+1] + 0x10 * ChanNum;
	- ChR[i+2] = RRegData[i+2];
	- ChR[i+3] = RRegData[i+3];
	- }
	-
	- /* Indexed registers */
	- ChOff = (Word_t)ChanNum * 0x1000;
	-
	- ChP->BaudDiv[0] = (Byte_t)(ChOff + _BAUD);
	- ChP->BaudDiv[1] = (Byte_t)((ChOff + _BAUD) >> 8);
	- ChP->BaudDiv[2] = (Byte_t)BRD9600;
	- ChP->BaudDiv[3] = (Byte_t)(BRD9600 >> 8);
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->BaudDiv));
	-
	- ChP->TxControl[0] = (Byte_t)(ChOff + _TX_CTRL);
	- ChP->TxControl[1] = (Byte_t)((ChOff + _TX_CTRL) >> 8);
	- ChP->TxControl[2] = 0;
	- ChP->TxControl[3] = 0;
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxControl));
	-
	- ChP->RxControl[0] = (Byte_t)(ChOff + _RX_CTRL);
	- ChP->RxControl[1] = (Byte_t)((ChOff + _RX_CTRL) >> 8);
	- ChP->RxControl[2] = 0;
	- ChP->RxControl[3] = 0;
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->RxControl));
	-
	- ChP->TxEnables[0] = (Byte_t)(ChOff + _TX_ENBLS);
	- ChP->TxEnables[1] = (Byte_t)((ChOff + _TX_ENBLS) >> 8);
	- ChP->TxEnables[2] = 0;
	- ChP->TxEnables[3] = 0;
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxEnables));
	-
	- ChP->TxCompare[0] = (Byte_t)(ChOff + _TXCMP1);
	- ChP->TxCompare[1] = (Byte_t)((ChOff + _TXCMP1) >> 8);
	- ChP->TxCompare[2] = 0;
	- ChP->TxCompare[3] = 0;
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxCompare));
	-
	- ChP->TxReplace1[0] = (Byte_t)(ChOff + _TXREP1B1);
	- ChP->TxReplace1[1] = (Byte_t)((ChOff + _TXREP1B1) >> 8);
	- ChP->TxReplace1[2] = 0;
	- ChP->TxReplace1[3] = 0;
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxReplace1));
	-
	- ChP->TxReplace2[0] = (Byte_t)(ChOff + _TXREP2);
	- ChP->TxReplace2[1] = (Byte_t)((ChOff + _TXREP2) >> 8);
	- ChP->TxReplace2[2] = 0;
	- ChP->TxReplace2[3] = 0;
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxReplace2));
	-
	- ChP->TxFIFOPtrs = ChOff + _TXF_OUTP;
	- ChP->TxFIFO = ChOff + _TX_FIFO;
	-
	- rp_writech1(ChP,_CMD_REG,(Byte_t)ChanNum \| RESTXFCNT); /* apply reset Tx FIFO count */
	- rp_writech1(ChP,_CMD_REG,(Byte_t)ChanNum); /* remove reset Tx FIFO count */
	- rp_writech2(ChP,_INDX_ADDR,ChP->TxFIFOPtrs); /* clear Tx in/out ptrs */
	- rp_writech2(ChP,_INDX_DATA,0);
	- ChP->RxFIFOPtrs = ChOff + _RXF_OUTP;
	- ChP->RxFIFO = ChOff + _RX_FIFO;
	-
	- rp_writech1(ChP,_CMD_REG,(Byte_t)ChanNum \| RESRXFCNT); /* apply reset Rx FIFO count */
	- rp_writech1(ChP,_CMD_REG,(Byte_t)ChanNum); /* remove reset Rx FIFO count */
	- rp_writech2(ChP,_INDX_ADDR,ChP->RxFIFOPtrs); /* clear Rx out ptr */
	- rp_writech2(ChP,_INDX_DATA,0);
	- rp_writech2(ChP,_INDX_ADDR,ChP->RxFIFOPtrs + 2); /* clear Rx in ptr */
	- rp_writech2(ChP,_INDX_DATA,0);
	- ChP->TxPrioCnt = ChOff + _TXP_CNT;
	- rp_writech2(ChP,_INDX_ADDR,ChP->TxPrioCnt);
	- rp_writech1(ChP,_INDX_DATA,0);
	- ChP->TxPrioPtr = ChOff + _TXP_PNTR;
	- rp_writech2(ChP,_INDX_ADDR,ChP->TxPrioPtr);
	- rp_writech1(ChP,_INDX_DATA,0);
	- ChP->TxPrioBuf = ChOff + _TXP_BUF;
	- sEnRxProcessor(ChP); /* start the Rx processor */
	-
	- return(TRUE);
	-}
	-
	-/***************************************************************************
	-Function: sStopRxProcessor
	-Purpose: Stop the receive processor from processing a channel.
	-Call: sStopRxProcessor(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-
	-Comments: The receive processor can be started again with sStartRxProcessor().
	- This function causes the receive processor to skip over the
	- stopped channel. It does not stop it from processing other channels.
	-
	-Warnings: No context switches are allowed while executing this function.
	-
	- Do not leave the receive processor stopped for more than one
	- character time.
	-
	- After calling this function a delay of 4 uS is required to ensure
	- that the receive processor is no longer processing this channel.
	-*/
	-void sStopRxProcessor(CHANNEL_T *ChP)
	-{
	- Byte_t R[4];
	-
	- R[0] = ChP->R[0];
	- R[1] = ChP->R[1];
	- R[2] = 0x0a;
	- R[3] = ChP->R[3];
	- rp_writech4(ChP,_INDX_ADDR,le32dec(R));
	-}
	-
	-/***************************************************************************
	-Function: sFlushRxFIFO
	-Purpose: Flush the Rx FIFO
	-Call: sFlushRxFIFO(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: void
	-Comments: To prevent data from being enqueued or dequeued in the Tx FIFO
	- while it is being flushed the receive processor is stopped
	- and the transmitter is disabled. After these operations a
	- 4 uS delay is done before clearing the pointers to allow
	- the receive processor to stop. These items are handled inside
	- this function.
	-Warnings: No context switches are allowed while executing this function.
	-*/
	-void sFlushRxFIFO(CHANNEL_T *ChP)
	-{
	- int i;
	- Byte_t Ch; /* channel number within AIOP */
	- int RxFIFOEnabled; /* TRUE if Rx FIFO enabled */
	-
	- if(sGetRxCnt(ChP) == 0) /* Rx FIFO empty */
	- return; /* don't need to flush */
	-
	- RxFIFOEnabled = FALSE;
	- if(ChP->R[0x32] == 0x08) /* Rx FIFO is enabled */
	- {
	- RxFIFOEnabled = TRUE;
	- sDisRxFIFO(ChP); /* disable it */
	- for(i=0; i < 2000/200; i++) /* delay 2 uS to allow proc to disable FIFO*/
	- rp_readch1(ChP,_INT_CHAN); /* depends on bus i/o timing */
	- }
	- sGetChanStatus(ChP); /* clear any pending Rx errors in chan stat */
	- Ch = (Byte_t)sGetChanNum(ChP);
	- rp_writech1(ChP,_CMD_REG,Ch \| RESRXFCNT); /* apply reset Rx FIFO count */
	- rp_writech1(ChP,_CMD_REG,Ch); /* remove reset Rx FIFO count */
	- rp_writech2(ChP,_INDX_ADDR,ChP->RxFIFOPtrs); /* clear Rx out ptr */
	- rp_writech2(ChP,_INDX_DATA,0);
	- rp_writech2(ChP,_INDX_ADDR,ChP->RxFIFOPtrs + 2); /* clear Rx in ptr */
	- rp_writech2(ChP,_INDX_DATA,0);
	- if(RxFIFOEnabled)
	- sEnRxFIFO(ChP); /* enable Rx FIFO */
	-}
	-
	-/***************************************************************************
	-Function: sFlushTxFIFO
	-Purpose: Flush the Tx FIFO
	-Call: sFlushTxFIFO(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: void
	-Comments: To prevent data from being enqueued or dequeued in the Tx FIFO
	- while it is being flushed the receive processor is stopped
	- and the transmitter is disabled. After these operations a
	- 4 uS delay is done before clearing the pointers to allow
	- the receive processor to stop. These items are handled inside
	- this function.
	-Warnings: No context switches are allowed while executing this function.
	-*/
	-void sFlushTxFIFO(CHANNEL_T *ChP)
	-{
	- int i;
	- Byte_t Ch; /* channel number within AIOP */
	- int TxEnabled; /* TRUE if transmitter enabled */
	-
	- if(sGetTxCnt(ChP) == 0) /* Tx FIFO empty */
	- return; /* don't need to flush */
	-
	- TxEnabled = FALSE;
	- if(ChP->TxControl[3] & TX_ENABLE)
	- {
	- TxEnabled = TRUE;
	- sDisTransmit(ChP); /* disable transmitter */
	- }
	- sStopRxProcessor(ChP); /* stop Rx processor */
	- for(i = 0; i < 4000/200; i++) /* delay 4 uS to allow proc to stop */
	- rp_readch1(ChP,_INT_CHAN); /* depends on bus i/o timing */
	- Ch = (Byte_t)sGetChanNum(ChP);
	- rp_writech1(ChP,_CMD_REG,Ch \| RESTXFCNT); /* apply reset Tx FIFO count */
	- rp_writech1(ChP,_CMD_REG,Ch); /* remove reset Tx FIFO count */
	- rp_writech2(ChP,_INDX_ADDR,ChP->TxFIFOPtrs); /* clear Tx in/out ptrs */
	- rp_writech2(ChP,_INDX_DATA,0);
	- if(TxEnabled)
	- sEnTransmit(ChP); /* enable transmitter */
	- sStartRxProcessor(ChP); /* restart Rx processor */
	-}
	-
	-/***************************************************************************
	-Function: sWriteTxPrioByte
	-Purpose: Write a byte of priority transmit data to a channel
	-Call: sWriteTxPrioByte(ChP,Data)
	- CHANNEL_T *ChP; Ptr to channel structure
	- Byte_t Data; The transmit data byte
	-
	-Return: int: 1 if the bytes is successfully written, otherwise 0.
	-
	-Comments: The priority byte is transmitted before any data in the Tx FIFO.
	-
	-Warnings: No context switches are allowed while executing this function.
	-*/
	-int sWriteTxPrioByte(CHANNEL_T *ChP, Byte_t Data)
	-{
	- Byte_t DWBuf[4]; /* buffer for double word writes */
	-
	- if(sGetTxCnt(ChP) > 1) /* write it to Tx priority buffer */
	- {
	- rp_writech2(ChP,_INDX_ADDR,ChP->TxPrioCnt); /* get priority buffer status */
	- if(rp_readch1(ChP,_INDX_DATA) & PRI_PEND) /* priority buffer busy */
	- return(0); /* nothing sent */
	-
	- le16enc(DWBuf,ChP->TxPrioBuf); /* data byte address */
	-
	- DWBuf[2] = Data; /* data byte value */
	- DWBuf[3] = 0; /* priority buffer pointer */
	- rp_writech4(ChP,_INDX_ADDR,le32dec(DWBuf)); /* write it out */
	-
	- le16enc(DWBuf,ChP->TxPrioCnt); /* Tx priority count address */
	-
	- DWBuf[2] = PRI_PEND + 1; /* indicate 1 byte pending */
	- DWBuf[3] = 0; /* priority buffer pointer */
	- rp_writech4(ChP,_INDX_ADDR,le32dec(DWBuf)); /* write it out */
	- }
	- else /* write it to Tx FIFO */
	- {
	- sWriteTxByte(ChP,sGetTxRxDataIO(ChP),Data);
	- }
	- return(1); /* 1 byte sent */
	-}
	-
	-/***************************************************************************
	-Function: sEnInterrupts
	-Purpose: Enable one or more interrupts for a channel
	-Call: sEnInterrupts(ChP,Flags)
	- CHANNEL_T *ChP; Ptr to channel structure
	- Word_t Flags: Interrupt enable flags, can be any combination
	- of the following flags:
	- TXINT_EN: Interrupt on Tx FIFO empty
	- RXINT_EN: Interrupt on Rx FIFO at trigger level (see
	- sSetRxTrigger())
	- SRCINT_EN: Interrupt on SRC (Special Rx Condition)
	- MCINT_EN: Interrupt on modem input change
	- CHANINT_EN: Allow channel interrupt signal to the AIOP's
	- Interrupt Channel Register.
	-Return: void
	-Comments: If an interrupt enable flag is set in Flags, that interrupt will be
	- enabled. If an interrupt enable flag is not set in Flags, that
	- interrupt will not be changed. Interrupts can be disabled with
	- function sDisInterrupts().
	-
	- This function sets the appropriate bit for the channel in the AIOP's
	- Interrupt Mask Register if the CHANINT_EN flag is set. This allows
	- this channel's bit to be set in the AIOP's Interrupt Channel Register.
	-
	- Interrupts must also be globally enabled before channel interrupts
	- will be passed on to the host. This is done with function
	- sEnGlobalInt().
	-
	- In some cases it may be desirable to disable interrupts globally but
	- enable channel interrupts. This would allow the global interrupt
	- status register to be used to determine which AIOPs need service.
	-*/
	-void sEnInterrupts(CHANNEL_T *ChP,Word_t Flags)
	-{
	- Byte_t Mask; /* Interrupt Mask Register */
	-
	- ChP->RxControl[2] \|=
	- ((Byte_t)Flags & (RXINT_EN \| SRCINT_EN \| MCINT_EN));
	-
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->RxControl));
	-
	- ChP->TxControl[2] \|= ((Byte_t)Flags & TXINT_EN);
	-
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxControl));
	-
	- if(Flags & CHANINT_EN)
	- {
	- Mask = rp_readch1(ChP,_INT_MASK) \| rp_sBitMapSetTbl[ChP->ChanNum];
	- rp_writech1(ChP,_INT_MASK,Mask);
	- }
	-}
	-
	-/***************************************************************************
	-Function: sDisInterrupts
	-Purpose: Disable one or more interrupts for a channel
	-Call: sDisInterrupts(ChP,Flags)
	- CHANNEL_T *ChP; Ptr to channel structure
	- Word_t Flags: Interrupt flags, can be any combination
	- of the following flags:
	- TXINT_EN: Interrupt on Tx FIFO empty
	- RXINT_EN: Interrupt on Rx FIFO at trigger level (see
	- sSetRxTrigger())
	- SRCINT_EN: Interrupt on SRC (Special Rx Condition)
	- MCINT_EN: Interrupt on modem input change
	- CHANINT_EN: Disable channel interrupt signal to the
	- AIOP's Interrupt Channel Register.
	-Return: void
	-Comments: If an interrupt flag is set in Flags, that interrupt will be
	- disabled. If an interrupt flag is not set in Flags, that
	- interrupt will not be changed. Interrupts can be enabled with
	- function sEnInterrupts().
	-
	- This function clears the appropriate bit for the channel in the AIOP's
	- Interrupt Mask Register if the CHANINT_EN flag is set. This blocks
	- this channel's bit from being set in the AIOP's Interrupt Channel
	- Register.
	-*/
	-void sDisInterrupts(CHANNEL_T *ChP,Word_t Flags)
	-{
	- Byte_t Mask; /* Interrupt Mask Register */
	-
	- ChP->RxControl[2] &=
	- ~((Byte_t)Flags & (RXINT_EN \| SRCINT_EN \| MCINT_EN));
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->RxControl));
	- ChP->TxControl[2] &= ~((Byte_t)Flags & TXINT_EN);
	- rp_writech4(ChP,_INDX_ADDR,le32dec(ChP->TxControl));
	-
	- if(Flags & CHANINT_EN)
	- {
	- Mask = rp_readch1(ChP,_INT_MASK) & rp_sBitMapClrTbl[ChP->ChanNum];
	- rp_writech1(ChP,_INT_MASK,Mask);
	- }
	-}
	-
	-/*********************************************************************
	- Begin FreeBsd-specific driver code
	-**********************************************************************/
	-
	-#define POLL_INTERVAL (hz / 100)
	-
	-#define RP_ISMULTIPORT(dev) ((dev)->id_flags & 0x1)
	-#define RP_MPMASTER(dev) (((dev)->id_flags >> 8) & 0xff)
	-#define RP_NOTAST4(dev) ((dev)->id_flags & 0x04)
	-
	-/*
	- * The top-level routines begin here
	- */
	-
	-static void rpclose(struct tty *tp);
	-static void rphardclose(struct tty *tp);
	-static int rpmodem(struct tty *, int, int);
	-static int rpparam(struct tty , struct termios );
	-static void rpstart(struct tty *);
	-static int rpioctl(struct tty , u_long, caddr_t, struct thread );
	-static int rpopen(struct tty *);
	-
	-static void rp_do_receive(struct rp_port rp, struct tty tp,
	- CHANNEL_t *cp, unsigned int ChanStatus)
	-{
	- unsigned int CharNStat;
	- int ToRecv, ch, err = 0;
	-
	- ToRecv = sGetRxCnt(cp);
	- if(ToRecv == 0)
	- return;
	-
	-/* If status indicates there are errored characters in the
	- FIFO, then enter status mode (a word in FIFO holds
	- characters and status)
	-*/
	-
	- if(ChanStatus & (RXFOVERFL \| RXBREAK \| RXFRAME \| RXPARITY)) {
	- if(!(ChanStatus & STATMODE)) {
	- ChanStatus \|= STATMODE;
	- sEnRxStatusMode(cp);
	- }
	- }
	-/*
	- if we previously entered status mode then read down the
	- FIFO one word at a time, pulling apart the character and
	- the status. Update error counters depending on status.
	-*/
	- tty_lock(tp);
	- if(ChanStatus & STATMODE) {
	- while(ToRecv) {
	- CharNStat = rp_readch2(cp,sGetTxRxDataIO(cp));
	- ch = CharNStat & 0xff;
	-
	- if((CharNStat & STMBREAK) \|\| (CharNStat & STMFRAMEH))
	- err \|= TRE_FRAMING;
	- else if (CharNStat & STMPARITYH)
	- err \|= TRE_PARITY;
	- else if (CharNStat & STMRCVROVRH) {
	- rp->rp_overflows++;
	- err \|= TRE_OVERRUN;
	- }
	-
	- ttydisc_rint(tp, ch, err);
	- ToRecv--;
	- }
	-/*
	- After emtying FIFO in status mode, turn off status mode
	-*/
	-
	- if(sGetRxCnt(cp) == 0) {
	- sDisRxStatusMode(cp);
	- }
	- } else {
	- ToRecv = sGetRxCnt(cp);
	- while (ToRecv) {
	- ch = rp_readch1(cp,sGetTxRxDataIO(cp));
	- ttydisc_rint(tp, ch & 0xff, err);
	- ToRecv--;
	- }
	- }
	- ttydisc_rint_done(tp);
	- tty_unlock(tp);
	-}
	-
	-static void rp_handle_port(struct rp_port *rp)
	-{
	- CHANNEL_t *cp;
	- struct tty *tp;
	- unsigned int IntMask, ChanStatus;
	-
	- if(!rp)
	- return;
	-
	- cp = &rp->rp_channel;
	- tp = rp->rp_tty;
	- IntMask = sGetChanIntID(cp);
	- IntMask = IntMask & rp->rp_intmask;
	- ChanStatus = sGetChanStatus(cp);
	- if(IntMask & RXF_TRIG)
	- rp_do_receive(rp, tp, cp, ChanStatus);
	- if(IntMask & DELTA_CD) {
	- if(ChanStatus & CD_ACT) {
	- (void)ttydisc_modem(tp, 1);
	- } else {
	- (void)ttydisc_modem(tp, 0);
	- }
	- }
	-/* oldcts = rp->rp_cts;
	- rp->rp_cts = ((ChanStatus & CTS_ACT) != 0);
	- if(oldcts != rp->rp_cts) {
	- printf("CTS change (now %s)... on port %d\n", rp->rp_cts ? "on" : "off", rp->rp_port);
	- }
	-*/
	-}
	-
	-static void rp_do_poll(void *arg)
	-{
	- CONTROLLER_t *ctl;
	- struct rp_port *rp;
	- struct tty *tp;
	- int count;
	- unsigned char CtlMask, AiopMask;
	-
	- rp = arg;
	- tp = rp->rp_tty;
	- tty_assert_locked(tp);
	- ctl = rp->rp_ctlp;
	- CtlMask = ctl->ctlmask(ctl);
	- if (CtlMask & (1 << rp->rp_aiop)) {
	- AiopMask = sGetAiopIntStatus(ctl, rp->rp_aiop);
	- if (AiopMask & (1 << rp->rp_chan)) {
	- rp_handle_port(rp);
	- }
	- }
	-
	- count = sGetTxCnt(&rp->rp_channel);
	- if (count >= 0 && (count <= rp->rp_restart)) {
	- rpstart(tp);
	- }
	- callout_schedule(&rp->rp_timer, POLL_INTERVAL);
	-}
	-
	-static struct ttydevsw rp_tty_class = {
	- .tsw_flags = TF_INITLOCK\|TF_CALLOUT,
	- .tsw_open = rpopen,
	- .tsw_close = rpclose,
	- .tsw_outwakeup = rpstart,
	- .tsw_ioctl = rpioctl,
	- .tsw_param = rpparam,
	- .tsw_modem = rpmodem,
	- .tsw_free = rpfree,
	-};
	-
	-
	-static void
	-rpfree(void *softc)
	-{
	- struct rp_port *rp = softc;
	- CONTROLLER_t *ctlp = rp->rp_ctlp;
	-
	- atomic_subtract_32(&ctlp->free, 1);
	-}
	-
	-int
	-rp_attachcommon(CONTROLLER_T *ctlp, int num_aiops, int num_ports)
	-{
	- int unit;
	- int num_chan;
	- int aiop, chan, port;
	- int ChanStatus;
	- int retval;
	- struct rp_port *rp;
	- struct tty *tp;
	-
	- unit = device_get_unit(ctlp->dev);
	-
	- printf("RocketPort%d (Version %s) %d ports.\n", unit,
	- RocketPortVersion, num_ports);
	-
	- ctlp->num_ports = num_ports;
	- ctlp->rp = rp = (struct rp_port *)
	- malloc(sizeof(struct rp_port) * num_ports, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (rp == NULL) {
	- device_printf(ctlp->dev, "rp_attachcommon: Could not malloc rp_ports structures.\n");
	- retval = ENOMEM;
	- goto nogo;
	- }
	-
	- port = 0;
	- for(aiop=0; aiop < num_aiops; aiop++) {
	- num_chan = sGetAiopNumChan(ctlp, aiop);
	- for(chan=0; chan < num_chan; chan++, port++, rp++) {
	- rp->rp_tty = tp = tty_alloc(&rp_tty_class, rp);
	- callout_init_mtx(&rp->rp_timer, tty_getlock(tp), 0);
	- rp->rp_port = port;
	- rp->rp_ctlp = ctlp;
	- rp->rp_unit = unit;
	- rp->rp_chan = chan;
	- rp->rp_aiop = aiop;
	-
	- rp->rp_intmask = RXF_TRIG \| TXFIFO_MT \| SRC_INT \|
	- DELTA_CD \| DELTA_CTS \| DELTA_DSR;
	-#ifdef notdef
	- ChanStatus = sGetChanStatus(&rp->rp_channel);
	-#endif /* notdef */
	- if(sInitChan(ctlp, &rp->rp_channel, aiop, chan) == 0) {
	- device_printf(ctlp->dev, "RocketPort sInitChan(%d, %d, %d) failed.\n",
	- unit, aiop, chan);
	- retval = ENXIO;
	- goto nogo;
	- }
	- ChanStatus = sGetChanStatus(&rp->rp_channel);
	- rp->rp_cts = (ChanStatus & CTS_ACT) != 0;
	- tty_makedev(tp, NULL, "R%r%r", unit, port);
	- }
	- }
	-
	- mtx_init(&ctlp->hwmtx, "rp_hwmtx", NULL, MTX_DEF);
	- ctlp->hwmtx_init = 1;
	- return (0);
	-
	-nogo:
	- rp_releaseresource(ctlp);
	-
	- return (retval);
	-}
	-
	-void
	-rp_releaseresource(CONTROLLER_t *ctlp)
	-{
	- struct rp_port *rp;
	- int i;
	-
	- if (ctlp->rp != NULL) {
	- for (i = 0; i < ctlp->num_ports; i++) {
	- rp = ctlp->rp + i;
	- atomic_add_32(&ctlp->free, 1);
	- tty_lock(rp->rp_tty);
	- tty_rel_gone(rp->rp_tty);
	- }
	- free(ctlp->rp, M_DEVBUF);
	- ctlp->rp = NULL;
	- }
	-
	- while (ctlp->free != 0) {
	- pause("rpwt", hz / 10);
	- }
	-
	- if (ctlp->hwmtx_init)
	- mtx_destroy(&ctlp->hwmtx);
	-}
	-
	-static int
	-rpopen(struct tty *tp)
	-{
	- struct rp_port *rp;
	- int flags;
	- unsigned int IntMask, ChanStatus;
	-
	- rp = tty_softc(tp);
	-
	- flags = 0;
	- flags \|= SET_RTS;
	- flags \|= SET_DTR;
	- rp->rp_channel.TxControl[3] =
	- ((rp->rp_channel.TxControl[3]
	- & ~(SET_RTS \| SET_DTR)) \| flags);
	- rp_writech4(&rp->rp_channel,_INDX_ADDR,
	- le32dec(rp->rp_channel.TxControl));
	- sSetRxTrigger(&rp->rp_channel, TRIG_1);
	- sDisRxStatusMode(&rp->rp_channel);
	- sFlushRxFIFO(&rp->rp_channel);
	- sFlushTxFIFO(&rp->rp_channel);
	-
	- sEnInterrupts(&rp->rp_channel,
	- (TXINT_EN\|MCINT_EN\|RXINT_EN\|SRCINT_EN\|CHANINT_EN));
	- sSetRxTrigger(&rp->rp_channel, TRIG_1);
	-
	- sDisRxStatusMode(&rp->rp_channel);
	- sClrTxXOFF(&rp->rp_channel);
	-
	-/* sDisRTSFlowCtl(&rp->rp_channel);
	- sDisCTSFlowCtl(&rp->rp_channel);
	-*/
	- sDisTxSoftFlowCtl(&rp->rp_channel);
	-
	- sStartRxProcessor(&rp->rp_channel);
	-
	- sEnRxFIFO(&rp->rp_channel);
	- sEnTransmit(&rp->rp_channel);
	-
	-/* sSetDTR(&rp->rp_channel);
	- sSetRTS(&rp->rp_channel);
	-*/
	-
	- IntMask = sGetChanIntID(&rp->rp_channel);
	- IntMask = IntMask & rp->rp_intmask;
	- ChanStatus = sGetChanStatus(&rp->rp_channel);
	-
	- callout_reset(&rp->rp_timer, POLL_INTERVAL, rp_do_poll, rp);
	-
	- device_busy(rp->rp_ctlp->dev);
	- return(0);
	-}
	-
	-static void
	-rpclose(struct tty *tp)
	-{
	- struct rp_port *rp;
	-
	- rp = tty_softc(tp);
	- callout_stop(&rp->rp_timer);
	- rphardclose(tp);
	- device_unbusy(rp->rp_ctlp->dev);
	-}
	-
	-static void
	-rphardclose(struct tty *tp)
	-{
	- struct rp_port *rp;
	- CHANNEL_t *cp;
	-
	- rp = tty_softc(tp);
	- cp = &rp->rp_channel;
	-
	- sFlushRxFIFO(cp);
	- sFlushTxFIFO(cp);
	- sDisTransmit(cp);
	- sDisInterrupts(cp, TXINT_EN\|MCINT_EN\|RXINT_EN\|SRCINT_EN\|CHANINT_EN);
	- sDisRTSFlowCtl(cp);
	- sDisCTSFlowCtl(cp);
	- sDisTxSoftFlowCtl(cp);
	- sClrTxXOFF(cp);
	-
	-#ifdef DJA
	- if(tp->t_cflag&HUPCL \|\| !(tp->t_state&TS_ISOPEN) \|\| !tp->t_actout) {
	- sClrDTR(cp);
	- }
	- if(ISCALLOUT(tp->t_dev)) {
	- sClrDTR(cp);
	- }
	- tp->t_actout = FALSE;
	- wakeup(&tp->t_actout);
	- wakeup(TSA_CARR_ON(tp));
	-#endif /* DJA */
	-}
	-
	-static int
	-rpioctl(struct tty tp, u_long cmd, caddr_t data, struct thread td)
	-{
	- struct rp_port *rp;
	-
	- rp = tty_softc(tp);
	- switch (cmd) {
	- case TIOCSBRK:
	- sSendBreak(&rp->rp_channel);
	- return (0);
	- case TIOCCBRK:
	- sClrBreak(&rp->rp_channel);
	- return (0);
	- default:
	- return ENOIOCTL;
	- }
	-}
	-
	-static int
	-rpmodem(struct tty *tp, int sigon, int sigoff)
	-{
	- struct rp_port *rp;
	- int i, j, k;
	-
	- rp = tty_softc(tp);
	- if (sigon != 0 \|\| sigoff != 0) {
	- i = j = 0;
	- if (sigon & SER_DTR)
	- i = SET_DTR;
	- if (sigoff & SER_DTR)
	- j = SET_DTR;
	- if (sigon & SER_RTS)
	- i = SET_RTS;
	- if (sigoff & SER_RTS)
	- j = SET_RTS;
	- rp->rp_channel.TxControl[3] &= ~i;
	- rp->rp_channel.TxControl[3] \|= j;
	- rp_writech4(&rp->rp_channel,_INDX_ADDR,
	- le32dec(rp->rp_channel.TxControl));
	- } else {
	- i = sGetChanStatusLo(&rp->rp_channel);
	- j = rp->rp_channel.TxControl[3];
	- k = 0;
	- if (j & SET_DTR)
	- k \|= SER_DTR;
	- if (j & SET_RTS)
	- k \|= SER_RTS;
	- if (i & CD_ACT)
	- k \|= SER_DCD;
	- if (i & DSR_ACT)
	- k \|= SER_DSR;
	- if (i & CTS_ACT)
	- k \|= SER_CTS;
	- return(k);
	- }
	- return (0);
	-}
	-
	-static struct
	-{
	- int baud;
	- int conversion;
	-} baud_table[] = {
	- {B0, 0}, {B50, BRD50}, {B75, BRD75},
	- {B110, BRD110}, {B134, BRD134}, {B150, BRD150},
	- {B200, BRD200}, {B300, BRD300}, {B600, BRD600},
	- {B1200, BRD1200}, {B1800, BRD1800}, {B2400, BRD2400},
	- {B4800, BRD4800}, {B9600, BRD9600}, {B19200, BRD19200},
	- {B38400, BRD38400}, {B7200, BRD7200}, {B14400, BRD14400},
	- {B57600, BRD57600}, {B76800, BRD76800},
	- {B115200, BRD115200}, {B230400, BRD230400},
	- {-1, -1}
	-};
	-
	-static int rp_convert_baud(int baud) {
	- int i;
	-
	- for (i = 0; baud_table[i].baud >= 0; i++) {
	- if (baud_table[i].baud == baud)
	- break;
	- }
	-
	- return baud_table[i].conversion;
	-}
	-
	-static int
	-rpparam(tp, t)
	- struct tty *tp;
	- struct termios *t;
	-{
	- struct rp_port *rp;
	- CHANNEL_t *cp;
	- int cflag, iflag, oflag, lflag;
	- int ospeed;
	-#ifdef RPCLOCAL
	- int devshift;
	-#endif
	-
	- rp = tty_softc(tp);
	- cp = &rp->rp_channel;
	-
	- cflag = t->c_cflag;
	-#ifdef RPCLOCAL
	- devshift = umynor / 32;
	- devshift = 1 << devshift;
	- if ( devshift & RPCLOCAL ) {
	- cflag \|= CLOCAL;
	- }
	-#endif
	- iflag = t->c_iflag;
	- oflag = t->c_oflag;
	- lflag = t->c_lflag;
	-
	- ospeed = rp_convert_baud(t->c_ispeed);
	- if(ospeed < 0 \|\| t->c_ispeed != t->c_ospeed)
	- return(EINVAL);
	-
	- if(t->c_ospeed == 0) {
	- sClrDTR(cp);
	- return(0);
	- }
	- rp->rp_fifo_lw = ((t->c_ospeed*2) / 1000) +1;
	-
	- /* Set baud rate ----- we only pay attention to ispeed */
	- sSetDTR(cp);
	- sSetRTS(cp);
	- sSetBaud(cp, ospeed);
	-
	- if(cflag & CSTOPB) {
	- sSetStop2(cp);
	- } else {
	- sSetStop1(cp);
	- }
	-
	- if(cflag & PARENB) {
	- sEnParity(cp);
	- if(cflag & PARODD) {
	- sSetOddParity(cp);
	- } else {
	- sSetEvenParity(cp);
	- }
	- }
	- else {
	- sDisParity(cp);
	- }
	- if((cflag & CSIZE) == CS8) {
	- sSetData8(cp);
	- rp->rp_imask = 0xFF;
	- } else {
	- sSetData7(cp);
	- rp->rp_imask = 0x7F;
	- }
	-
	- if(iflag & ISTRIP) {
	- rp->rp_imask &= 0x7F;
	- }
	-
	- if(cflag & CLOCAL) {
	- rp->rp_intmask &= ~DELTA_CD;
	- } else {
	- rp->rp_intmask \|= DELTA_CD;
	- }
	-
	- /* Put flow control stuff here */
	-
	- if(cflag & CCTS_OFLOW) {
	- sEnCTSFlowCtl(cp);
	- } else {
	- sDisCTSFlowCtl(cp);
	- }
	-
	- if(cflag & CRTS_IFLOW) {
	- rp->rp_rts_iflow = 1;
	- } else {
	- rp->rp_rts_iflow = 0;
	- }
	-
	- if(cflag & CRTS_IFLOW) {
	- sEnRTSFlowCtl(cp);
	- } else {
	- sDisRTSFlowCtl(cp);
	- }
	-
	- return(0);
	-}
	-
	-static void
	-rpstart(struct tty *tp)
	-{
	- struct rp_port *rp;
	- CHANNEL_t *cp;
	- char flags;
	- int xmit_fifo_room;
	- int i, count, wcount;
	-
	- rp = tty_softc(tp);
	- cp = &rp->rp_channel;
	- flags = rp->rp_channel.TxControl[3];
	-
	- if(rp->rp_xmit_stopped) {
	- sEnTransmit(cp);
	- rp->rp_xmit_stopped = 0;
	- }
	-
	- xmit_fifo_room = TXFIFO_SIZE - sGetTxCnt(cp);
	- count = ttydisc_getc(tp, &rp->TxBuf, xmit_fifo_room);
	- if(xmit_fifo_room > 0) {
	- for( i = 0, wcount = count >> 1; wcount > 0; i += 2, wcount-- ) {
	- rp_writech2(cp, sGetTxRxDataIO(cp), le16dec(&rp->TxBuf[i]));
	- }
	- if ( count & 1 ) {
	- rp_writech1(cp, sGetTxRxDataIO(cp), rp->TxBuf[(count-1)]);
	- }
	- }
	-}
	diff --git a/sys/dev/rp/rp_isa.c b/sys/dev/rp/rp_isa.c
	deleted file mode 100644
	index 7a96d54bd607..000000000000
	--- a/sys/dev/rp/rp_isa.c
	+++ /dev/null
	@@ -1,508 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	- *
	- * Copyright (c) Comtrol Corporation <support@comtrol.com>
	- * All rights reserved.
	- *
	- * ISA-specific part separated from:
	- * sys/i386/isa/rp.c,v 1.33 1999/09/28 11:45:27 phk Exp
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted prodived that the follwoing conditions
	- * are met.
	- * 1. Redistributions of source code must retain the above copyright
	- * notive, this list of conditions and the following disclainer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials prodided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Comtrol Corporation.
	- * 4. The name of Comtrol Corporation may not be used to endorse or
	- * promote products derived from this software without specific
	- * prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY COMTROL CORPORATION ``AS IS'' AND ANY
	- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL COMTROL CORPORATION BE LIABLE FOR
	- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/fcntl.h>
	-#include <sys/malloc.h>
	-#include <sys/conf.h>
	-#include <sys/kernel.h>
	-#include <sys/lock.h>
	-#include <sys/module.h>
	-#include <sys/mutex.h>
	-#include <machine/resource.h>
	-#include <machine/bus.h>
	-#include <sys/bus.h>
	-#include <sys/rman.h>
	-
	-#define ROCKET_C
	-#include <dev/rp/rpreg.h>
	-#include <dev/rp/rpvar.h>
	-
	-#include <isa/isavar.h>
	-
	-/* ISA-specific part of CONTROLLER_t */
	-struct ISACONTROLLER_T {
	- int MBaseIO; /* rid of the Mudbac controller for this controller */
	- int MReg0IO; /* offset0 of the Mudbac controller for this controller */
	- int MReg1IO; /* offset1 of the Mudbac controller for this controller */
	- int MReg2IO; /* offset2 of the Mudbac controller for this controller */
	- int MReg3IO; /* offset3 of the Mudbac controller for this controller */
	- Byte_t MReg2;
	- Byte_t MReg3;
	-};
	-typedef struct ISACONTROLLER_T ISACONTROLLER_t;
	-
	-#define ISACTL(ctlp) ((ISACONTROLLER_t *)((ctlp)->bus_ctlp))
	-
	-/***************************************************************************
	-Function: sControllerEOI
	-Purpose: Strobe the MUDBAC's End Of Interrupt bit.
	-Call: sControllerEOI(MudbacCtlP,CtlP)
	- CONTROLLER_T *MudbacCtlP; Ptr to Mudbac controller structure
	- CONTROLLER_T *CtlP; Ptr to controller structure
	-*/
	-#define sControllerEOI(MudbacCtlP,CtlP) \
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,ISACTL(CtlP)->MReg2IO,ISACTL(CtlP)->MReg2 \| INT_STROB)
	-
	-/***************************************************************************
	-Function: sDisAiop
	-Purpose: Disable I/O access to an AIOP
	-Call: sDisAiop(MudbacCtlP,CtlP)
	- CONTROLLER_T *MudbacCtlP; Ptr to Mudbac controller structure
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- int AiopNum; Number of AIOP on controller
	-*/
	-#define sDisAiop(MudbacCtlP,CtlP,AIOPNUM) \
	-{ \
	- ISACTL(CtlP)->MReg3 &= rp_sBitMapClrTbl[AIOPNUM]; \
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,ISACTL(CtlP)->MReg3IO,ISACTL(CtlP)->MReg3); \
	-}
	-
	-/***************************************************************************
	-Function: sEnAiop
	-Purpose: Enable I/O access to an AIOP
	-Call: sEnAiop(MudbacCtlP,CtlP)
	- CONTROLLER_T *MudbacCtlP; Ptr to Mudbac controller structure
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- int AiopNum; Number of AIOP on controller
	-*/
	-#define sEnAiop(MudbacCtlP,CtlP,AIOPNUM) \
	-{ \
	- ISACTL(CtlP)->MReg3 \|= rp_sBitMapSetTbl[AIOPNUM]; \
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,ISACTL(CtlP)->MReg3IO,ISACTL(CtlP)->MReg3); \
	-}
	-
	-/***************************************************************************
	-Function: sGetControllerIntStatus
	-Purpose: Get the controller interrupt status
	-Call: sGetControllerIntStatus(MudbacCtlP,CtlP)
	- CONTROLLER_T *MudbacCtlP; Ptr to Mudbac controller structure
	- CONTROLLER_T *CtlP; Ptr to controller structure
	-Return: Byte_t: The controller interrupt status in the lower 4
	- bits. Bits 0 through 3 represent AIOP's 0
	- through 3 respectively. If a bit is set that
	- AIOP is interrupting. Bits 4 through 7 will
	- always be cleared.
	-*/
	-#define sGetControllerIntStatus(MudbacCtlP,CtlP) \
	- (rp_readio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,ISACTL(CtlP)->MReg1IO) & 0x0f)
	-
	-static devclass_t rp_devclass;
	-static CONTROLLER_t *rp_controller;
	-static int rp_nisadevs;
	-
	-static int rp_probe(device_t dev);
	-static int rp_attach(device_t dev);
	-static void rp_isareleaseresource(CONTROLLER_t *ctlp);
	-static int sInitController(CONTROLLER_T *CtlP,
	- CONTROLLER_T *MudbacCtlP,
	- int AiopNum,
	- int IRQNum,
	- Byte_t Frequency,
	- int PeriodicOnly);
	-static rp_aiop2rid_t rp_isa_aiop2rid;
	-static rp_aiop2off_t rp_isa_aiop2off;
	-static rp_ctlmask_t rp_isa_ctlmask;
	-
	-static int
	-rp_probe(device_t dev)
	-{
	- int unit;
	- CONTROLLER_t *controller;
	- int num_aiops;
	- CONTROLLER_t *ctlp;
	- int retval;
	-
	- /*
	- * We have no PnP RocketPort cards.
	- * (At least according to LINT)
	- */
	- if (isa_get_logicalid(dev) != 0)
	- return (ENXIO);
	-
	- /* We need IO port resource to configure an ISA device. */
	- if (bus_get_resource_count(dev, SYS_RES_IOPORT, 0) == 0)
	- return (ENXIO);
	-
	- unit = device_get_unit(dev);
	- if (unit >= 4) {
	- device_printf(dev, "rpprobe: unit number %d invalid.\n", unit);
	- return (ENXIO);
	- }
	- device_printf(dev, "probing for RocketPort(ISA) unit %d.\n", unit);
	-
	- ctlp = device_get_softc(dev);
	- bzero(ctlp, sizeof(*ctlp));
	- ctlp->dev = dev;
	- ctlp->aiop2rid = rp_isa_aiop2rid;
	- ctlp->aiop2off = rp_isa_aiop2off;
	- ctlp->ctlmask = rp_isa_ctlmask;
	-
	- /* The IO ports of AIOPs for an ISA controller are discrete. */
	- ctlp->io_num = 1;
	- ctlp->io_rid = malloc(sizeof((ctlp->io_rid)) MAX_AIOPS_PER_BOARD, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- ctlp->io = malloc(sizeof((ctlp->io)) MAX_AIOPS_PER_BOARD, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (ctlp->io_rid == NULL \|\| ctlp->io == NULL) {
	- device_printf(dev, "rp_attach: Out of memory.\n");
	- retval = ENOMEM;
	- goto nogo;
	- }
	-
	- ctlp->bus_ctlp = malloc(sizeof(ISACONTROLLER_t) * 1, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (ctlp->bus_ctlp == NULL) {
	- device_printf(dev, "rp_attach: Out of memory.\n");
	- retval = ENOMEM;
	- goto nogo;
	- }
	-
	- ctlp->io_rid[0] = 0;
	- if (rp_controller != NULL) {
	- controller = rp_controller;
	- ctlp->io[0] = bus_alloc_resource_anywhere(dev, SYS_RES_IOPORT, &ctlp->io_rid[0], 0x40, RF_ACTIVE);
	- } else {
	- controller = rp_controller = ctlp;
	- ctlp->io[0] = bus_alloc_resource_anywhere(dev, SYS_RES_IOPORT, &ctlp->io_rid[0], 0x44, RF_ACTIVE);
	- }
	- if (ctlp->io[0] == NULL) {
	- device_printf(dev, "rp_attach: Resource not available.\n");
	- retval = ENXIO;
	- goto nogo;
	- }
	-
	- num_aiops = sInitController(ctlp,
	- controller,
	- MAX_AIOPS_PER_BOARD, 0,
	- FREQ_DIS, 0);
	- if (num_aiops <= 0) {
	- device_printf(dev, "board%d init failed.\n", unit);
	- retval = ENXIO;
	- goto nogo;
	- }
	-
	- if (rp_controller == NULL)
	- rp_controller = controller;
	- rp_nisadevs++;
	-
	- device_set_desc(dev, "RocketPort ISA");
	-
	- return (0);
	-
	-nogo:
	- rp_isareleaseresource(ctlp);
	-
	- return (retval);
	-}
	-
	-static int
	-rp_attach(device_t dev)
	-{
	- int unit;
	- int num_ports, num_aiops;
	- int aiop;
	- CONTROLLER_t *ctlp;
	- int retval;
	-
	- unit = device_get_unit(dev);
	-
	- ctlp = device_get_softc(dev);
	-
	-#ifdef notdef
	- num_aiops = sInitController(ctlp,
	- rp_controller,
	- MAX_AIOPS_PER_BOARD, 0,
	- FREQ_DIS, 0);
	-#else
	- num_aiops = ctlp->NumAiop;
	-#endif /* notdef */
	-
	- num_ports = 0;
	- for(aiop=0; aiop < num_aiops; aiop++) {
	- sResetAiopByNum(ctlp, aiop);
	- sEnAiop(rp_controller, ctlp, aiop);
	- num_ports += sGetAiopNumChan(ctlp, aiop);
	- }
	-
	- retval = rp_attachcommon(ctlp, num_aiops, num_ports);
	- if (retval != 0)
	- goto nogo;
	-
	- return (0);
	-
	-nogo:
	- rp_isareleaseresource(ctlp);
	-
	- return (retval);
	-}
	-
	-static void
	-rp_isareleaseresource(CONTROLLER_t *ctlp)
	-{
	- int i;
	-
	- rp_releaseresource(ctlp);
	-
	- if (ctlp == rp_controller)
	- rp_controller = NULL;
	- if (ctlp->io != NULL) {
	- for (i = 0 ; i < MAX_AIOPS_PER_BOARD ; i++)
	- if (ctlp->io[i] != NULL)
	- bus_release_resource(ctlp->dev, SYS_RES_IOPORT, ctlp->io_rid[i], ctlp->io[i]);
	- free(ctlp->io, M_DEVBUF);
	- }
	- if (ctlp->io_rid != NULL)
	- free(ctlp->io_rid, M_DEVBUF);
	- if (rp_controller != NULL && rp_controller->io[ISACTL(ctlp)->MBaseIO] != NULL) {
	- bus_release_resource(rp_controller->dev, SYS_RES_IOPORT, rp_controller->io_rid[ISACTL(ctlp)->MBaseIO], rp_controller->io[ISACTL(ctlp)->MBaseIO]);
	- rp_controller->io[ISACTL(ctlp)->MBaseIO] = NULL;
	- rp_controller->io_rid[ISACTL(ctlp)->MBaseIO] = 0;
	- }
	- if (ctlp->bus_ctlp != NULL)
	- free(ctlp->bus_ctlp, M_DEVBUF);
	-}
	-
	-/***************************************************************************
	-Function: sInitController
	-Purpose: Initialization of controller global registers and controller
	- structure.
	-Call: sInitController(CtlP,MudbacCtlP,AiopNum,
	- IRQNum,Frequency,PeriodicOnly)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- CONTROLLER_T *MudbacCtlP; Ptr to Mudbac controller structure
	- int AiopNum; Number of Aiops
	- int IRQNum; Interrupt Request number. Can be any of the following:
	- 0: Disable global interrupts
	- 3: IRQ 3
	- 4: IRQ 4
	- 5: IRQ 5
	- 9: IRQ 9
	- 10: IRQ 10
	- 11: IRQ 11
	- 12: IRQ 12
	- 15: IRQ 15
	- Byte_t Frequency: A flag identifying the frequency
	- of the periodic interrupt, can be any one of the following:
	- FREQ_DIS - periodic interrupt disabled
	- FREQ_137HZ - 137 Hertz
	- FREQ_69HZ - 69 Hertz
	- FREQ_34HZ - 34 Hertz
	- FREQ_17HZ - 17 Hertz
	- FREQ_9HZ - 9 Hertz
	- FREQ_4HZ - 4 Hertz
	- If IRQNum is set to 0 the Frequency parameter is
	- overidden, it is forced to a value of FREQ_DIS.
	- int PeriodicOnly: TRUE if all interrupts except the periodic
	- interrupt are to be blocked.
	- FALSE is both the periodic interrupt and
	- other channel interrupts are allowed.
	- If IRQNum is set to 0 the PeriodicOnly parameter is
	- overidden, it is forced to a value of FALSE.
	-Return: int: Number of AIOPs on the controller, or CTLID_NULL if controller
	- initialization failed.
	-
	-Comments:
	- If periodic interrupts are to be disabled but AIOP interrupts
	- are allowed, set Frequency to FREQ_DIS and PeriodicOnly to FALSE.
	-
	- If interrupts are to be completely disabled set IRQNum to 0.
	-
	- Setting Frequency to FREQ_DIS and PeriodicOnly to TRUE is an
	- invalid combination.
	-
	- This function performs initialization of global interrupt modes,
	- but it does not actually enable global interrupts. To enable
	- and disable global interrupts use functions sEnGlobalInt() and
	- sDisGlobalInt(). Enabling of global interrupts is normally not
	- done until all other initializations are complete.
	-
	- Even if interrupts are globally enabled, they must also be
	- individually enabled for each channel that is to generate
	- interrupts.
	-
	-Warnings: No range checking on any of the parameters is done.
	-
	- No context switches are allowed while executing this function.
	-
	- After this function all AIOPs on the controller are disabled,
	- they can be enabled with sEnAiop().
	-*/
	-static int
	-sInitController( CONTROLLER_T *CtlP,
	- CONTROLLER_T *MudbacCtlP,
	- int AiopNum,
	- int IRQNum,
	- Byte_t Frequency,
	- int PeriodicOnly)
	-{
	- int i;
	- int ctl_base, aiop_base, aiop_size;
	-
	- CtlP->CtlID = CTLID_0001; /* controller release 1 */
	-
	- ISACTL(CtlP)->MBaseIO = rp_nisadevs;
	- if (MudbacCtlP->io[ISACTL(CtlP)->MBaseIO] != NULL) {
	- ISACTL(CtlP)->MReg0IO = 0x40 + 0;
	- ISACTL(CtlP)->MReg1IO = 0x40 + 1;
	- ISACTL(CtlP)->MReg2IO = 0x40 + 2;
	- ISACTL(CtlP)->MReg3IO = 0x40 + 3;
	- } else {
	- MudbacCtlP->io_rid[ISACTL(CtlP)->MBaseIO] = ISACTL(CtlP)->MBaseIO;
	- ctl_base = rman_get_start(MudbacCtlP->io[0]) + 0x40 + 0x400 * rp_nisadevs;
	- MudbacCtlP->io[ISACTL(CtlP)->MBaseIO] = bus_alloc_resource(MudbacCtlP->dev, SYS_RES_IOPORT, &CtlP->io_rid[ISACTL(CtlP)->MBaseIO], ctl_base, ctl_base + 3, 4, RF_ACTIVE);
	- ISACTL(CtlP)->MReg0IO = 0;
	- ISACTL(CtlP)->MReg1IO = 1;
	- ISACTL(CtlP)->MReg2IO = 2;
	- ISACTL(CtlP)->MReg3IO = 3;
	- }
	-#if 1
	- ISACTL(CtlP)->MReg2 = 0; /* interrupt disable */
	- ISACTL(CtlP)->MReg3 = 0; /* no periodic interrupts */
	-#else
	- if(sIRQMap[IRQNum] == 0) /* interrupts globally disabled */
	- {
	- ISACTL(CtlP)->MReg2 = 0; /* interrupt disable */
	- ISACTL(CtlP)->MReg3 = 0; /* no periodic interrupts */
	- }
	- else
	- {
	- ISACTL(CtlP)->MReg2 = sIRQMap[IRQNum]; /* set IRQ number */
	- ISACTL(CtlP)->MReg3 = Frequency; /* set frequency */
	- if(PeriodicOnly) /* periodic interrupt only */
	- {
	- ISACTL(CtlP)->MReg3 \|= PERIODIC_ONLY;
	- }
	- }
	-#endif
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,ISACTL(CtlP)->MReg2IO,ISACTL(CtlP)->MReg2);
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,ISACTL(CtlP)->MReg3IO,ISACTL(CtlP)->MReg3);
	- sControllerEOI(MudbacCtlP,CtlP); /* clear EOI if warm init */
	-
	- /* Init AIOPs */
	- CtlP->NumAiop = 0;
	- for(i=0; i < AiopNum; i++)
	- {
	- if (CtlP->io[i] == NULL) {
	- CtlP->io_rid[i] = i;
	- aiop_base = rman_get_start(CtlP->io[0]) + 0x400 * i;
	- if (rp_nisadevs == 0)
	- aiop_size = 0x44;
	- else
	- aiop_size = 0x40;
	- CtlP->io[i] = bus_alloc_resource(CtlP->dev, SYS_RES_IOPORT, &CtlP->io_rid[i], aiop_base, aiop_base + aiop_size - 1, aiop_size, RF_ACTIVE);
	- } else
	- aiop_base = rman_get_start(CtlP->io[i]);
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,
	- ISACTL(CtlP)->MReg2IO,
	- ISACTL(CtlP)->MReg2 \| (i & 0x03)); /* AIOP index */
	- rp_writeio1(MudbacCtlP,ISACTL(CtlP)->MBaseIO,
	- ISACTL(CtlP)->MReg0IO,
	- (Byte_t)(aiop_base >> 6)); /* set up AIOP I/O in MUDBAC */
	- sEnAiop(MudbacCtlP,CtlP,i); /* enable the AIOP */
	-
	- CtlP->AiopID[i] = sReadAiopID(CtlP, i); /* read AIOP ID */
	- if(CtlP->AiopID[i] == AIOPID_NULL) /* if AIOP does not exist */
	- {
	- sDisAiop(MudbacCtlP,CtlP,i); /* disable AIOP */
	- bus_release_resource(CtlP->dev, SYS_RES_IOPORT, CtlP->io_rid[i], CtlP->io[i]);
	- CtlP->io[i] = NULL;
	- break; /* done looking for AIOPs */
	- }
	-
	- CtlP->AiopNumChan[i] = sReadAiopNumChan(CtlP, i); /* num channels in AIOP */
	- rp_writeaiop2(CtlP,i,_INDX_ADDR,_CLK_PRE); /* clock prescaler */
	- rp_writeaiop1(CtlP,i,_INDX_DATA,CLOCK_PRESC);
	- CtlP->NumAiop++; /* bump count of AIOPs */
	- sDisAiop(MudbacCtlP,CtlP,i); /* disable AIOP */
	- }
	-
	- if(CtlP->NumAiop == 0)
	- return(-1);
	- else
	- return(CtlP->NumAiop);
	-}
	-
	-/*
	- * ARGSUSED
	- * Maps (aiop, offset) to rid.
	- */
	-static int
	-rp_isa_aiop2rid(int aiop, int offset)
	-{
	- /* rid equals to aiop for an ISA controller. */
	- return aiop;
	-}
	-
	-/*
	- * ARGSUSED
	- * Maps (aiop, offset) to the offset of resource.
	- */
	-static int
	-rp_isa_aiop2off(int aiop, int offset)
	-{
	- /* Each aiop has its own resource. */
	- return offset;
	-}
	-
	-/* Read the int status for an ISA controller. */
	-static unsigned char
	-rp_isa_ctlmask(CONTROLLER_t *ctlp)
	-{
	- return sGetControllerIntStatus(rp_controller,ctlp);
	-}
	-
	-static device_method_t rp_methods[] = {
	- /* Device interface */
	- DEVMETHOD(device_probe, rp_probe),
	- DEVMETHOD(device_attach, rp_attach),
	-
	- { 0, 0 }
	-};
	-
	-static driver_t rp_driver = {
	- "rp",
	- rp_methods,
	- sizeof(CONTROLLER_t),
	-};
	-
	-/*
	- * rp can be attached to an isa bus.
	- */
	-DRIVER_MODULE(rp, isa, rp_driver, rp_devclass, 0, 0);
	diff --git a/sys/dev/rp/rp_pci.c b/sys/dev/rp/rp_pci.c
	deleted file mode 100644
	index afd378c3c119..000000000000
	--- a/sys/dev/rp/rp_pci.c
	+++ /dev/null
	@@ -1,368 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	- *
	- * Copyright (c) Comtrol Corporation <support@comtrol.com>
	- * All rights reserved.
	- *
	- * PCI-specific part separated from:
	- * sys/i386/isa/rp.c,v 1.33 1999/09/28 11:45:27 phk Exp
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted prodived that the follwoing conditions
	- * are met.
	- * 1. Redistributions of source code must retain the above copyright
	- * notive, this list of conditions and the following disclainer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials prodided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Comtrol Corporation.
	- * 4. The name of Comtrol Corporation may not be used to endorse or
	- * promote products derived from this software without specific
	- * prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY COMTROL CORPORATION ``AS IS'' AND ANY
	- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL COMTROL CORPORATION BE LIABLE FOR
	- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- */
	-
	-#include <sys/cdefs.h>
	-__FBSDID("$FreeBSD$");
	-
	-#include <sys/param.h>
	-#include <sys/systm.h>
	-#include <sys/fcntl.h>
	-#include <sys/malloc.h>
	-#include <sys/conf.h>
	-#include <sys/kernel.h>
	-#include <sys/lock.h>
	-#include <sys/module.h>
	-#include <sys/mutex.h>
	-#include <machine/resource.h>
	-#include <machine/bus.h>
	-#include <sys/bus.h>
	-#include <sys/rman.h>
	-
	-#define ROCKET_C
	-#include <dev/rp/rpreg.h>
	-#include <dev/rp/rpvar.h>
	-
	-#include <dev/pci/pcireg.h>
	-#include <dev/pci/pcivar.h>
	-
	-/* PCI IDs */
	-#define RP_VENDOR_ID 0x11FE
	-#define RP_DEVICE_ID_32I 0x0001
	-#define RP_DEVICE_ID_8I 0x0002
	-#define RP_DEVICE_ID_16I 0x0003
	-#define RP_DEVICE_ID_4Q 0x0004
	-#define RP_DEVICE_ID_8O 0x0005
	-#define RP_DEVICE_ID_8J 0x0006
	-#define RP_DEVICE_ID_4J 0x0007
	-#define RP_DEVICE_ID_6M 0x000C
	-#define RP_DEVICE_ID_4M 0x000D
	-#define RP_DEVICE_ID_UPCI_32 0x0801
	-#define RP_DEVICE_ID_UPCI_16 0x0803
	-#define RP_DEVICE_ID_UPCI_8O 0x0805
	-
	-/**************************************************************************
	- MUDBAC remapped for PCI
	-**************************************************************************/
	-
	-#define _CFG_INT_PCI 0x40
	-#define _PCI_INT_FUNC 0x3A
	-
	-#define PCI_STROB 0x2000
	-#define INTR_EN_PCI 0x0010
	-
	-/***************************************************************************
	-Function: sPCIControllerEOI
	-Purpose: Strobe the MUDBAC's End Of Interrupt bit.
	-Call: sPCIControllerEOI(CtlP)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	-*/
	-#define sPCIControllerEOI(CtlP) rp_writeio2(CtlP, 0, _PCI_INT_FUNC, PCI_STROB)
	-
	-/***************************************************************************
	-Function: sPCIGetControllerIntStatus
	-Purpose: Get the controller interrupt status
	-Call: sPCIGetControllerIntStatus(CtlP)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	-Return: Byte_t: The controller interrupt status in the lower 4
	- bits. Bits 0 through 3 represent AIOP's 0
	- through 3 respectively. If a bit is set that
	- AIOP is interrupting. Bits 4 through 7 will
	- always be cleared.
	-*/
	-#define sPCIGetControllerIntStatus(CTLP) ((rp_readio2(CTLP, 0, _PCI_INT_FUNC) >> 8) & 0x1f)
	-
	-static devclass_t rp_devclass;
	-
	-static int rp_pciprobe(device_t dev);
	-static int rp_pciattach(device_t dev);
	-#ifdef notdef
	-static int rp_pcidetach(device_t dev);
	-static int rp_pcishutdown(device_t dev);
	-#endif /* notdef */
	-static void rp_pcireleaseresource(CONTROLLER_t *ctlp);
	-static int sPCIInitController( CONTROLLER_t *CtlP,
	- int AiopNum,
	- int IRQNum,
	- Byte_t Frequency,
	- int PeriodicOnly,
	- int VendorDevice);
	-static rp_aiop2rid_t rp_pci_aiop2rid;
	-static rp_aiop2off_t rp_pci_aiop2off;
	-static rp_ctlmask_t rp_pci_ctlmask;
	-
	-/*
	- * The following functions are the pci-specific part
	- * of rp driver.
	- */
	-
	-static int
	-rp_pciprobe(device_t dev)
	-{
	- char *s;
	-
	- s = NULL;
	- if (pci_get_vendor(dev) == RP_VENDOR_ID)
	- s = "RocketPort PCI";
	-
	- if (s != NULL) {
	- device_set_desc(dev, s);
	- return (BUS_PROBE_DEFAULT);
	- }
	-
	- return (ENXIO);
	-}
	-
	-static int
	-rp_pciattach(device_t dev)
	-{
	- int num_ports, num_aiops;
	- int aiop;
	- CONTROLLER_t *ctlp;
	- int unit;
	- int retval;
	-
	- ctlp = device_get_softc(dev);
	- bzero(ctlp, sizeof(*ctlp));
	- ctlp->dev = dev;
	- unit = device_get_unit(dev);
	- ctlp->aiop2rid = rp_pci_aiop2rid;
	- ctlp->aiop2off = rp_pci_aiop2off;
	- ctlp->ctlmask = rp_pci_ctlmask;
	-
	- /* The IO ports of AIOPs for a PCI controller are continuous. */
	- ctlp->io_num = 1;
	- ctlp->io_rid = malloc(sizeof((ctlp->io_rid)) ctlp->io_num, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- ctlp->io = malloc(sizeof((ctlp->io)) ctlp->io_num, M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (ctlp->io_rid == NULL \|\| ctlp->io == NULL) {
	- device_printf(dev, "rp_pciattach: Out of memory.\n");
	- retval = ENOMEM;
	- goto nogo;
	- }
	-
	- ctlp->bus_ctlp = NULL;
	-
	- switch (pci_get_device(dev)) {
	- case RP_DEVICE_ID_UPCI_16:
	- case RP_DEVICE_ID_UPCI_32:
	- case RP_DEVICE_ID_UPCI_8O:
	- ctlp->io_rid[0] = PCIR_BAR(2);
	- break;
	- default:
	- ctlp->io_rid[0] = PCIR_BAR(0);
	- break;
	- }
	- ctlp->io[0] = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
	- &ctlp->io_rid[0], RF_ACTIVE);
	- if(ctlp->io[0] == NULL) {
	- device_printf(dev, "ioaddr mapping failed for RocketPort(PCI).\n");
	- retval = ENXIO;
	- goto nogo;
	- }
	-
	- num_aiops = sPCIInitController(ctlp,
	- MAX_AIOPS_PER_BOARD, 0,
	- FREQ_DIS, 0, pci_get_device(dev));
	-
	- num_ports = 0;
	- for(aiop=0; aiop < num_aiops; aiop++) {
	- sResetAiopByNum(ctlp, aiop);
	- num_ports += sGetAiopNumChan(ctlp, aiop);
	- }
	-
	- retval = rp_attachcommon(ctlp, num_aiops, num_ports);
	- if (retval != 0)
	- goto nogo;
	-
	- return (0);
	-
	-nogo:
	- rp_pcireleaseresource(ctlp);
	-
	- return (retval);
	-}
	-
	-static int
	-rp_pcidetach(device_t dev)
	-{
	- CONTROLLER_t *ctlp;
	-
	- ctlp = device_get_softc(dev);
	- rp_pcireleaseresource(ctlp);
	-
	- return (0);
	-}
	-
	-static int
	-rp_pcishutdown(device_t dev)
	-{
	- CONTROLLER_t *ctlp;
	-
	- ctlp = device_get_softc(dev);
	- rp_pcireleaseresource(ctlp);
	-
	- return (0);
	-}
	-
	-static void
	-rp_pcireleaseresource(CONTROLLER_t *ctlp)
	-{
	- rp_releaseresource(ctlp);
	- if (ctlp->io != NULL) {
	- if (ctlp->io[0] != NULL)
	- bus_release_resource(ctlp->dev, SYS_RES_IOPORT, ctlp->io_rid[0], ctlp->io[0]);
	- free(ctlp->io, M_DEVBUF);
	- ctlp->io = NULL;
	- }
	- if (ctlp->io_rid != NULL) {
	- free(ctlp->io_rid, M_DEVBUF);
	- ctlp->io = NULL;
	- }
	-}
	-
	-static int
	-sPCIInitController( CONTROLLER_t *CtlP,
	- int AiopNum,
	- int IRQNum,
	- Byte_t Frequency,
	- int PeriodicOnly,
	- int VendorDevice)
	-{
	- int i;
	-
	- CtlP->CtlID = CTLID_0001; /* controller release 1 */
	-
	- sPCIControllerEOI(CtlP);
	-
	- /* Init AIOPs */
	- CtlP->NumAiop = 0;
	- for(i=0; i < AiopNum; i++)
	- {
	- /device_printf(CtlP->dev, "aiop %d.\n", i);/
	- CtlP->AiopID[i] = sReadAiopID(CtlP, i); /* read AIOP ID */
	- /device_printf(CtlP->dev, "ID = %d.\n", CtlP->AiopID[i]);/
	- if(CtlP->AiopID[i] == AIOPID_NULL) /* if AIOP does not exist */
	- {
	- break; /* done looking for AIOPs */
	- }
	-
	- switch( VendorDevice ) {
	- case RP_DEVICE_ID_4Q:
	- case RP_DEVICE_ID_4J:
	- case RP_DEVICE_ID_4M:
	- CtlP->AiopNumChan[i] = 4;
	- break;
	- case RP_DEVICE_ID_6M:
	- CtlP->AiopNumChan[i] = 6;
	- break;
	- case RP_DEVICE_ID_8O:
	- case RP_DEVICE_ID_8J:
	- case RP_DEVICE_ID_8I:
	- case RP_DEVICE_ID_16I:
	- case RP_DEVICE_ID_32I:
	- CtlP->AiopNumChan[i] = 8;
	- break;
	- default:
	-#ifdef notdef
	- CtlP->AiopNumChan[i] = 8;
	-#else
	- CtlP->AiopNumChan[i] = sReadAiopNumChan(CtlP, i);
	-#endif /* notdef */
	- break;
	- }
	- /device_printf(CtlP->dev, "%d channels.\n", CtlP->AiopNumChan[i]);/
	- rp_writeaiop2(CtlP, i, _INDX_ADDR,_CLK_PRE); /* clock prescaler */
	- /device_printf(CtlP->dev, "configuring clock prescaler.\n");/
	- rp_writeaiop1(CtlP, i, _INDX_DATA,CLOCK_PRESC);
	- /device_printf(CtlP->dev, "configured clock prescaler.\n");/
	- CtlP->NumAiop++; /* bump count of AIOPs */
	- }
	-
	- if(CtlP->NumAiop == 0)
	- return(-1);
	- else
	- return(CtlP->NumAiop);
	-}
	-
	-/*
	- * ARGSUSED
	- * Maps (aiop, offset) to rid.
	- */
	-static int
	-rp_pci_aiop2rid(int aiop, int offset)
	-{
	- /* Always return zero for a PCI controller. */
	- return 0;
	-}
	-
	-/*
	- * ARGSUSED
	- * Maps (aiop, offset) to the offset of resource.
	- */
	-static int
	-rp_pci_aiop2off(int aiop, int offset)
	-{
	- /* Each AIOP reserves 0x40 bytes. */
	- return aiop * 0x40 + offset;
	-}
	-
	-/* Read the int status for a PCI controller. */
	-static unsigned char
	-rp_pci_ctlmask(CONTROLLER_t *ctlp)
	-{
	- return sPCIGetControllerIntStatus(ctlp);
	-}
	-
	-static device_method_t rp_pcimethods[] = {
	- /* Device interface */
	- DEVMETHOD(device_probe, rp_pciprobe),
	- DEVMETHOD(device_attach, rp_pciattach),
	- DEVMETHOD(device_detach, rp_pcidetach),
	- DEVMETHOD(device_shutdown, rp_pcishutdown),
	-
	- { 0, 0 }
	-};
	-
	-static driver_t rp_pcidriver = {
	- "rp",
	- rp_pcimethods,
	- sizeof(CONTROLLER_t),
	-};
	-
	-/*
	- * rp can be attached to a pci bus.
	- */
	-DRIVER_MODULE(rp, pci, rp_pcidriver, rp_devclass, 0, 0);
	diff --git a/sys/dev/rp/rpreg.h b/sys/dev/rp/rpreg.h
	deleted file mode 100644
	index bd1de316028d..000000000000
	--- a/sys/dev/rp/rpreg.h
	+++ /dev/null
	@@ -1,1033 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	- *
	- * Copyright (c) Comtrol Corporation <support@comtrol.com>
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted prodived that the follwoing conditions
	- * are met.
	- * 1. Redistributions of source code must retain the above copyright
	- * notive, this list of conditions and the following disclainer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials prodided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Comtrol Corporation.
	- * 4. The name of Comtrol Corporation may not be used to endorse or
	- * promote products derived from this software without specific
	- * prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY COMTROL CORPORATION ``AS IS'' AND ANY
	- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL COMTROL CORPORATION BE LIABLE FOR
	- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-/*
	- * Begin OS-specific defines - rpreg.h - for RocketPort FreeBSD
	- */
	-
	-typedef uint8_t Byte_t;
	-typedef uint8_t ByteIO_t;
	-
	-typedef uint16_t Word_t;
	-typedef uint16_t WordIO_t;
	-
	-typedef uint32_t DWord_t;
	-typedef uint32_t DWordIO_t;
	-
	-#define rp_readio(size, ctlp, rid, offset) \
	- (bus_read_##size(ctlp->io[rid], offset))
	-#define rp_readmultiio(size, ctlp, rid, offset, addr, count) \
	- (bus_read_multi_##size(ctlp->io[rid], offset, addr, count))
	-#define rp_writeio(size, ctlp, rid, offset, data) \
	- (bus_write_##size(ctlp->io[rid], offset, data))
	-#define rp_writemultiio(size, ctlp, rid, offset, addr, count) \
	- (bus_write_multi_##size(ctlp->io[rid], offset, addr, count))
	-
	-#define rp_readio1(ctlp, rid, offset) rp_readio(1, ctlp, rid, offset)
	-#define rp_readio2(ctlp, rid, offset) rp_readio(2, ctlp, rid, offset)
	-#define rp_readio4(ctlp, rid, offset) rp_readio(4, ctlp, rid, offset)
	-#define rp_writeio1(ctlp, rid, offset, data) rp_writeio(1, ctlp, rid, offset, data)
	-#define rp_writeio2(ctlp, rid, offset, data) rp_writeio(2, ctlp, rid, offset, data)
	-#define rp_writeio4(ctlp, rid, offset, data) rp_writeio(4, ctlp, rid, offset, data)
	-#define rp_readmultiio1(ctlp, rid, offset, addr, count) rp_readmultiio(1, ctlp, rid, offset, addr, count)
	-#define rp_readmultiio2(ctlp, rid, offset, addr, count) rp_readmultiio(2, ctlp, rid, offset, addr, count)
	-#define rp_readmultiio4(ctlp, rid, offset, addr, count) rp_readmultiio(4, ctlp, rid, offset, addr, count)
	-#define rp_writemultiio1(ctlp, rid, offset, addr, count) rp_writemultiio(1, ctlp, rid, offset, addr, count)
	-#define rp_writemultiio2(ctlp, rid, offset, addr, count) rp_writemultiio(2, ctlp, rid, offset, addr, count)
	-#define rp_writemultiio4(ctlp, rid, offset, addr, count) rp_writemultiio(4, ctlp, rid, offset, addr, count)
	-
	-#define rp_readaiop1(ctlp, aiop, offset) \
	- (rp_readio1((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset)))
	-#define rp_readaiop2(ctlp, aiop, offset) \
	- (rp_readio2((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset)))
	-#define rp_readaiop4(ctlp, aiop, offset) \
	- (rp_readio4((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset)))
	-#define rp_readmultiaiop1(ctlp, aiop, offset, addr, count) \
	- (rp_readmultiio1((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), addr, count))
	-#define rp_readmultiaiop2(ctlp, aiop, offset, addr, count) \
	- (rp_readmultiio2((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), addr, count))
	-#define rp_readmultiaiop4(ctlp, aiop, offset, addr, count) \
	- (rp_readmultiio4((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), addr, count))
	-#define rp_writeaiop1(ctlp, aiop, offset, data) \
	- (rp_writeio1((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), data))
	-#define rp_writeaiop2(ctlp, aiop, offset, data) \
	- (rp_writeio2((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), data))
	-#define rp_writeaiop4(ctlp, aiop, offset, data) \
	- (rp_writeio4((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), data))
	-#define rp_writemultiaiop1(ctlp, aiop, offset, addr, count) \
	- (rp_writemultiio1((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), addr, count))
	-#define rp_writemultiaiop2(ctlp, aiop, offset, addr, count) \
	- (rp_writemultiio2((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), addr, count))
	-#define rp_writemultiaiop4(ctlp, aiop, offset, addr, count) \
	- (rp_writemultiio4((ctlp), (ctlp)->aiop2rid(aiop, offset), (ctlp)->aiop2off(aiop, offset), addr, count))
	-
	-#define rp_readch1(chp, offset) \
	- (rp_readaiop1((chp)->CtlP, (chp)->AiopNum, offset))
	-#define rp_readch2(chp, offset) \
	- (rp_readaiop2((chp)->CtlP, (chp)->AiopNum, offset))
	-#define rp_readch4(chp, offset) \
	- (rp_readaiop4((chp)->CtlP, (chp)->AiopNum, offset))
	-#define rp_readmultich1(chp, offset, addr, count) \
	- (rp_readmultiaiop1((chp)->CtlP, (chp)->AiopNum, offset, addr, count))
	-#define rp_readmultich2(chp, offset, addr, count) \
	- (rp_readmultiaiop2((chp)->CtlP, (chp)->AiopNum, offset, addr, count))
	-#define rp_readmultich4(chp, offset, addr, count) \
	- (rp_readmultiaiop4((chp)->CtlP, (chp)->AiopNum, offset, addr, count))
	-#define rp_writech1(chp, offset, data) \
	- (rp_writeaiop1((chp)->CtlP, (chp)->AiopNum, offset, data))
	-#define rp_writech2(chp, offset, data) \
	- (rp_writeaiop2((chp)->CtlP, (chp)->AiopNum, offset, data))
	-#define rp_writech4(chp, offset, data) \
	- (rp_writeaiop4((chp)->CtlP, (chp)->AiopNum, offset, data))
	-#define rp_writemultich1(chp, offset, addr, count) \
	- (rp_writemultiaiop1((chp)->CtlP, (chp)->AiopNum, offset, addr, count))
	-#define rp_writemultich2(chp, offset, addr, count) \
	- (rp_writemultiaiop2((chp)->CtlP, (chp)->AiopNum, offset, addr, count))
	-#define rp_writemultich4(chp, offset, addr, count) \
	- (rp_writemultiaiop4((chp)->CtlP, (chp)->AiopNum, offset, addr, count))
	-
	-/*
	- * End of OS-specific defines
	- */
	-
	-#define ROCKET_H
	-
	-#define CTL_SIZE 4
	-#define AIOP_CTL_SIZE 4
	-#define CHAN_AIOP_SIZE 8
	-#define MAX_PORTS_PER_AIOP 8
	-#define MAX_AIOPS_PER_BOARD 4
	-#define MAX_PORTS_PER_BOARD 32
	-
	-/* Controller ID numbers */
	-#define CTLID_NULL -1 /* no controller exists */
	-#define CTLID_0001 0x0001 /* controller release 1 */
	-
	-/* AIOP ID numbers, identifies AIOP type implementing channel */
	-#define AIOPID_NULL -1 /* no AIOP or channel exists */
	-#define AIOPID_0001 0x0001 /* AIOP release 1 */
	-
	-#define NULLDEV -1 /* identifies non-existant device */
	-#define NULLCTL -1 /* identifies non-existant controller */
	-#define NULLCTLPTR (CONTROLLER_T )0 / identifies non-existant controller */
	-#define NULLAIOP -1 /* identifies non-existant AIOP */
	-#define NULLCHAN -1 /* identifies non-existant channel */
	-
	-/************************************************************************
	- Global Register Offsets - Direct Access - Fixed values
	-************************************************************************/
	-
	-#define _CMD_REG 0x38 /* Command Register 8 Write */
	-#define _INT_CHAN 0x39 /* Interrupt Channel Register 8 Read */
	-#define _INT_MASK 0x3A /* Interrupt Mask Register 8 Read / Write */
	-#define _UNUSED 0x3B /* Unused 8 */
	-#define _INDX_ADDR 0x3C /* Index Register Address 16 Write */
	-#define _INDX_DATA 0x3E /* Index Register Data 8/16 Read / Write */
	-
	-/************************************************************************
	- Channel Register Offsets for 1st channel in AIOP - Direct Access
	-************************************************************************/
	-#define _TD0 0x00 /* Transmit Data 16 Write */
	-#define _RD0 0x00 /* Receive Data 16 Read */
	-#define _CHN_STAT0 0x20 /* Channel Status 8/16 Read / Write */
	-#define _FIFO_CNT0 0x10 /* Transmit/Receive FIFO Count 16 Read */
	-#define _INT_ID0 0x30 /* Interrupt Identification 8 Read */
	-
	-/************************************************************************
	- Tx Control Register Offsets - Indexed - External - Fixed
	-************************************************************************/
	-#define _TX_ENBLS 0x980 /* Tx Processor Enables Register 8 Read / Write */
	-#define _TXCMP1 0x988 /* Transmit Compare Value #1 8 Read / Write */
	-#define _TXCMP2 0x989 /* Transmit Compare Value #2 8 Read / Write */
	-#define _TXREP1B1 0x98A /* Tx Replace Value #1 - Byte 1 8 Read / Write */
	-#define _TXREP1B2 0x98B /* Tx Replace Value #1 - Byte 2 8 Read / Write */
	-#define _TXREP2 0x98C /* Transmit Replace Value #2 8 Read / Write */
	-
	-/************************************************************************
	- Receive FIFO
	-************************************************************************/
	-#define RXFIFO_DATA 0x5f
	-#define RXFIFO_OUT 0x5c
	-#define RXFIFO_EN 0x08
	-#define RXFIFO_DIS 0xa7
	-
	-/************************************************************************
	-Memory Controller Register Offsets - Indexed - External - Fixed
	-************************************************************************/
	-#define _RX_FIFO 0x000 /* Rx FIFO */
	-#define _TX_FIFO 0x800 /* Tx FIFO */
	-#define _RXF_OUTP 0x990 /* Rx FIFO OUT pointer 16 Read / Write */
	-#define _RXF_INP 0x992 /* Rx FIFO IN pointer 16 Read / Write */
	-#define _TXF_OUTP 0x994 /* Tx FIFO OUT pointer 8 Read / Write */
	-#define _TXF_INP 0x995 /* Tx FIFO IN pointer 8 Read / Write */
	-#define _TXP_CNT 0x996 /* Tx Priority Count 8 Read / Write */
	-#define _TXP_PNTR 0x997 /* Tx Priority Pointer 8 Read / Write */
	-
	-#define PRI_PEND 0x80 /* Priority data pending (bit7, Tx pri cnt) */
	-#define TXFIFO_SIZE 255 /* size of Tx FIFO */
	-#define RXFIFO_SIZE 1023 /* size of Rx FIFO */
	-
	-/************************************************************************
	-Tx Priority Buffer - Indexed - External - Fixed
	-************************************************************************/
	-#define _TXP_BUF 0x9C0 /* Tx Priority Buffer 32 Bytes Read / Write */
	-#define TXP_SIZE 0x20 /* 32 bytes */
	-
	-/************************************************************************
	-Channel Register Offsets - Indexed - Internal - Fixed
	-************************************************************************/
	-
	-#define _TX_CTRL 0xFF0 /* Transmit Control 16 Write */
	-#define _RX_CTRL 0xFF2 /* Receive Control 8 Write */
	-#define _BAUD 0xFF4 /* Baud Rate 16 Write */
	-#define _CLK_PRE 0xFF6 /* Clock Prescaler 8 Write */
	-
	-#define CLOCK_PRESC 0x19 /* mod 9 (divide by 10) prescale */
	-
	-#define BRD50 4607
	-#define BRD75 3071
	-#define BRD110 2094
	-#define BRD134 1712
	-#define BRD150 1535
	-#define BRD200 1151
	-#define BRD300 767
	-#define BRD600 383
	-#define BRD1200 191
	-#define BRD1800 127
	-#define BRD2000 114
	-#define BRD2400 95
	-#define BRD3600 64
	-#define BRD4800 47
	-#define BRD7200 31
	-#define BRD9600 23
	-#define BRD14400 15
	-#define BRD19200 11
	-#define BRD38400 5
	-#define BRD57600 3
	-#define BRD76800 2
	-#define BRD115200 1
	-#define BRD230400 0
	-
	-#define STMBREAK 0x08 /* BREAK */
	-#define STMFRAME 0x04 /* framing error */
	-#define STMRCVROVR 0x02 /* receiver over run error */
	-#define STMPARITY 0x01 /* parity error */
	-#define STMERROR (STMBREAK \| STMFRAME \| STMPARITY)
	-#define STMBREAKH 0x800 /* BREAK */
	-#define STMFRAMEH 0x400 /* framing error */
	-#define STMRCVROVRH 0x200 /* receiver over run error */
	-#define STMPARITYH 0x100 /* parity error */
	-#define STMERRORH (STMBREAKH \| STMFRAMEH \| STMPARITYH)
	-
	-#define CTS_ACT 0x20 /* CTS input asserted */
	-#define DSR_ACT 0x10 /* DSR input asserted */
	-#define CD_ACT 0x08 /* CD input asserted */
	-#define TXFIFOMT 0x04 /* Tx FIFO is empty */
	-#define TXSHRMT 0x02 /* Tx shift register is empty */
	-#define RDA 0x01 /* Rx data available */
	-#define DRAINED (TXFIFOMT \| TXSHRMT) /* indicates Tx is drained */
	-
	-#define STATMODE 0x8000 /* status mode enable bit */
	-#define RXFOVERFL 0x2000 /* receive FIFO overflow */
	-#define RX2MATCH 0x1000 /* receive compare byte 2 match */
	-#define RX1MATCH 0x0800 /* receive compare byte 1 match */
	-#define RXBREAK 0x0400 /* received BREAK */
	-#define RXFRAME 0x0200 /* received framing error */
	-#define RXPARITY 0x0100 /* received parity error */
	-#define STATERROR (RXBREAK \| RXFRAME \| RXPARITY)
	-
	-#define CTSFC_EN 0x80 /* CTS flow control enable bit */
	-#define RTSTOG_EN 0x40 /* RTS toggle enable bit */
	-#define TXINT_EN 0x10 /* transmit interrupt enable */
	-#define STOP2 0x08 /* enable 2 stop bits (0 = 1 stop) */
	-#define PARITY_EN 0x04 /* enable parity (0 = no parity) */
	-#define EVEN_PAR 0x02 /* even parity (0 = odd parity) */
	-#define DATA8BIT 0x01 /* 8 bit data (0 = 7 bit data) */
	-
	-#define SETBREAK 0x10 /* send break condition (must clear) */
	-#define LOCALLOOP 0x08 /* local loopback set for test */
	-#define SET_DTR 0x04 /* assert DTR */
	-#define SET_RTS 0x02 /* assert RTS */
	-#define TX_ENABLE 0x01 /* enable transmitter */
	-
	-#define RTSFC_EN 0x40 /* RTS flow control enable */
	-#define RXPROC_EN 0x20 /* receive processor enable */
	-#define TRIG_NO 0x00 /* Rx FIFO trigger level 0 (no trigger) */
	-#define TRIG_1 0x08 /* trigger level 1 char */
	-#define TRIG_1_2 0x10 /* trigger level 1/2 */
	-#define TRIG_7_8 0x18 /* trigger level 7/8 */
	-#define TRIG_MASK 0x18 /* trigger level mask */
	-#define SRCINT_EN 0x04 /* special Rx condition interrupt enable */
	-#define RXINT_EN 0x02 /* Rx interrupt enable */
	-#define MCINT_EN 0x01 /* modem change interrupt enable */
	-
	-#define RXF_TRIG 0x20 /* Rx FIFO trigger level interrupt */
	-#define TXFIFO_MT 0x10 /* Tx FIFO empty interrupt */
	-#define SRC_INT 0x08 /* special receive condition interrupt */
	-#define DELTA_CD 0x04 /* CD change interrupt */
	-#define DELTA_CTS 0x02 /* CTS change interrupt */
	-#define DELTA_DSR 0x01 /* DSR change interrupt */
	-
	-#define REP1W2_EN 0x10 /* replace byte 1 with 2 bytes enable */
	-#define IGN2_EN 0x08 /* ignore byte 2 enable */
	-#define IGN1_EN 0x04 /* ignore byte 1 enable */
	-#define COMP2_EN 0x02 /* compare byte 2 enable */
	-#define COMP1_EN 0x01 /* compare byte 1 enable */
	-
	-#define RESET_ALL 0x80 /* reset AIOP (all channels) */
	-#define TXOVERIDE 0x40 /* Transmit software off override */
	-#define RESETUART 0x20 /* reset channel's UART */
	-#define RESTXFCNT 0x10 /* reset channel's Tx FIFO count register */
	-#define RESRXFCNT 0x08 /* reset channel's Rx FIFO count register */
	-
	-#define INTSTAT0 0x01 /* AIOP 0 interrupt status */
	-#define INTSTAT1 0x02 /* AIOP 1 interrupt status */
	-#define INTSTAT2 0x04 /* AIOP 2 interrupt status */
	-#define INTSTAT3 0x08 /* AIOP 3 interrupt status */
	-
	-#define INTR_EN 0x08 /* allow interrupts to host */
	-#define INT_STROB 0x04 /* strobe and clear interrupt line (EOI) */
	-
	-#define CHAN3_EN 0x08 /* enable AIOP 3 */
	-#define CHAN2_EN 0x04 /* enable AIOP 2 */
	-#define CHAN1_EN 0x02 /* enable AIOP 1 */
	-#define CHAN0_EN 0x01 /* enable AIOP 0 */
	-#define FREQ_DIS 0x00
	-#define FREQ_274HZ 0x60
	-#define FREQ_137HZ 0x50
	-#define FREQ_69HZ 0x40
	-#define FREQ_34HZ 0x30
	-#define FREQ_17HZ 0x20
	-#define FREQ_9HZ 0x10
	-#define PERIODIC_ONLY 0x80 /* only PERIODIC interrupt */
	-
	-#define CHANINT_EN 0x0100 /* flags to enable/disable channel ints */
	-
	-#define RDATASIZE 72
	-#define RREGDATASIZE 52
	-
	-#ifndef TRUE
	-#define TRUE 1
	-#endif
	-
	-#ifndef FALSE
	-#define FALSE 0
	-#endif
	-
	-struct CONTROLLER_str;
	-struct CHANNEL_str;
	-
	-/* The types of bus-specific methods */
	-typedef int rp_aiop2rid_t(int, int);
	-typedef int rp_aiop2off_t(int, int);
	-typedef unsigned char rp_ctlmask_t(struct CONTROLLER_str *);
	-
	-/* Controller level information structure */
	-struct CONTROLLER_str
	-{
	- int CtlID;
	- int NumAiop;
	- int AiopID[AIOP_CTL_SIZE];
	- int AiopNumChan[AIOP_CTL_SIZE];
	-
	- struct mtx hwmtx; /* Spinlock protecting hardware. */
	- int hwmtx_init;
	- int free;
	- int num_ports;
	-
	- /* Device and resource management */
	- device_t dev; /* device */
	- int io_num; /* Number of IO resources */
	- int io_rid; / IO resource IDs */
	- struct resource *io; / IO resources */
	-
	- struct rp_port rp; / port */
	-
	- /* Device nodes */
	- struct cdev **dev_nodes;
	-
	- /* Bus-specific properties */
	- void *bus_ctlp;
	-
	- /* Bus-specific methods */
	- rp_aiop2rid_t aiop2rid; / (aiop, offset) -> rid */
	- rp_aiop2off_t aiop2off; / (aiop, offset) -> off */
	- rp_ctlmask_t ctlmask; / Int status */
	-};
	-typedef struct CONTROLLER_str CONTROLLER_T;
	-typedef CONTROLLER_T CONTROLLER_t;
	-
	-/* Channel level information structure */
	-struct CHANNEL_str
	-{
	- CONTROLLER_t *CtlP;
	- int AiopNum;
	- int ChanID;
	- int ChanNum;
	-
	- Word_t TxFIFO;
	- Word_t TxFIFOPtrs;
	- Word_t RxFIFO;
	- Word_t RxFIFOPtrs;
	- Word_t TxPrioCnt;
	- Word_t TxPrioPtr;
	- Word_t TxPrioBuf;
	-
	- Byte_t R[RREGDATASIZE];
	-
	- Byte_t BaudDiv[4];
	- Byte_t TxControl[4];
	- Byte_t RxControl[4];
	- Byte_t TxEnables[4];
	- Byte_t TxCompare[4];
	- Byte_t TxReplace1[4];
	- Byte_t TxReplace2[4];
	-};
	-
	-typedef struct CHANNEL_str CHANNEL_T;
	-typedef CHANNEL_T CHANNEL_t;
	-typedef CHANNEL_T * CHANPTR_T;
	-
	-#define CHNOFF_TXRXDATA(chp) ((chp)->ChanNum * 2 + _TD0)
	-#define CHNOFF_CHANSTAT(chp) ((chp)->ChanNum * 2 + _CHN_STAT0)
	-#define CHNOFF_TXRXCOUNT(chp) ((chp)->ChanNum * 2 + _FIFO_CNT0)
	-#define CHNOFF_INTID(chp) ((chp)->ChanNum + _INT_ID0)
	-
	-/***************************************************************************
	-Function: sClrBreak
	-Purpose: Stop sending a transmit BREAK signal
	-Call: sClrBreak(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sClrBreak(ChP) \
	-{ \
	- (ChP)->TxControl[3] &= ~SETBREAK; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sClrDTR
	-Purpose: Clr the DTR output
	-Call: sClrDTR(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sClrDTR(ChP) \
	-{ \
	- (ChP)->TxControl[3] &= ~SET_DTR; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sClrRTS
	-Purpose: Clr the RTS output
	-Call: sClrRTS(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sClrRTS(ChP) \
	-{ \
	- (ChP)->TxControl[3] &= ~SET_RTS; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sClrTxXOFF
	-Purpose: Clear any existing transmit software flow control off condition
	-Call: sClrTxXOFF(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sClrTxXOFF(ChP) \
	-{ \
	- rp_writech1(ChP,_CMD_REG,TXOVERIDE \| (Byte_t)(ChP)->ChanNum); \
	- rp_writech1(ChP,_CMD_REG,(Byte_t)(ChP)->ChanNum); \
	-}
	-
	-/***************************************************************************
	-Function: sDisCTSFlowCtl
	-Purpose: Disable output flow control using CTS
	-Call: sDisCTSFlowCtl(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sDisCTSFlowCtl(ChP) \
	-{ \
	- (ChP)->TxControl[2] &= ~CTSFC_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: DisParity
	-Purpose: Disable parity
	-Call: sDisParity(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: Function sSetParity() can be used in place of functions sEnParity(),
	- sDisParity(), sSetOddParity(), and sSetEvenParity().
	-*/
	-#define sDisParity(ChP) \
	-{ \
	- (ChP)->TxControl[2] &= ~PARITY_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sDisRxFIFO
	-Purpose: Disable Rx FIFO
	-Call: sDisRxFIFO(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sDisRxFIFO(ChP) \
	-{ \
	- (ChP)->R[0x32] = 0x0a; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->R + 0x30)); \
	-}
	-
	-/***************************************************************************
	-Function: sDisRxStatusMode
	-Purpose: Disable the Rx status mode
	-Call: sDisRxStatusMode(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: This takes the channel out of the receive status mode. All
	- subsequent reads of receive data using sReadRxWord() will return
	- two data bytes.
	-*/
	-#define sDisRxStatusMode(ChP) rp_writech2(ChP,CHNOFF_CHANSTAT(ChP),0)
	-
	-/***************************************************************************
	-Function: sDisTransmit
	-Purpose: Disable transmit
	-Call: sDisTransmit(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	- This disables movement of Tx data from the Tx FIFO into the 1 byte
	- Tx buffer. Therefore there could be up to a 2 byte latency
	- between the time sDisTransmit() is called and the transmit buffer
	- and transmit shift register going completely empty.
	-*/
	-#define sDisTransmit(ChP) \
	-{ \
	- (ChP)->TxControl[3] &= ~TX_ENABLE; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sDisTxSoftFlowCtl
	-Purpose: Disable Tx Software Flow Control
	-Call: sDisTxSoftFlowCtl(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sDisTxSoftFlowCtl(ChP) \
	-{ \
	- (ChP)->R[0x06] = 0x8a; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->R + 0x04)); \
	-}
	-
	-/***************************************************************************
	-Function: sEnCTSFlowCtl
	-Purpose: Enable output flow control using CTS
	-Call: sEnCTSFlowCtl(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sEnCTSFlowCtl(ChP) \
	-{ \
	- (ChP)->TxControl[2] \|= CTSFC_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: EnParity
	-Purpose: Enable parity
	-Call: sEnParity(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: Function sSetParity() can be used in place of functions sEnParity(),
	- sDisParity(), sSetOddParity(), and sSetEvenParity().
	-
	-Warnings: Before enabling parity odd or even parity should be chosen using
	- functions sSetOddParity() or sSetEvenParity().
	-*/
	-#define sEnParity(ChP) \
	-{ \
	- (ChP)->TxControl[2] \|= PARITY_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sEnRTSFlowCtl
	-Return: void
	-*/
	-#define sEnRTSFlowCtl(ChP) \
	-{ \
	- (ChP)->TxControl[2] &= ~RTSTOG_EN; \
	- (ChP)->TxControl[3] &= ~SET_RTS; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	- (ChP)->RxControl[2] \|= RTSFC_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->RxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sDisRTSFlowCtl
	-Return: void
	-*/
	-#define sDisRTSFlowCtl(ChP) \
	-{ \
	- (ChP)->RxControl[2] &= ~RTSFC_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->RxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sEnRxFIFO
	-Purpose: Enable Rx FIFO
	-Call: sEnRxFIFO(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sEnRxFIFO(ChP) \
	-{ \
	- (ChP)->R[0x32] = 0x08; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->R + 0x30)); \
	-}
	-
	-/***************************************************************************
	-Function: sEnRxProcessor
	-Purpose: Enable the receive processor
	-Call: sEnRxProcessor(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: This function is used to start the receive processor. When
	- the channel is in the reset state the receive processor is not
	- running. This is done to prevent the receive processor from
	- executing invalid microcode instructions prior to the
	- downloading of the microcode.
	-
	-Warnings: This function must be called after valid microcode has been
	- downloaded to the AIOP, and it must not be called before the
	- microcode has been downloaded.
	-*/
	-#define sEnRxProcessor(ChP) \
	-{ \
	- (ChP)->RxControl[2] \|= RXPROC_EN; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->RxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sEnRxStatusMode
	-Purpose: Enable the Rx status mode
	-Call: sEnRxStatusMode(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: This places the channel in the receive status mode. All subsequent
	- reads of receive data using sReadRxWord() will return a data byte
	- in the low word and a status byte in the high word.
	-
	-*/
	-#define sEnRxStatusMode(ChP) rp_writech2(ChP,CHNOFF_CHANSTAT(ChP),STATMODE)
	-
	-/***************************************************************************
	-Function: sEnTransmit
	-Purpose: Enable transmit
	-Call: sEnTransmit(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sEnTransmit(ChP) \
	-{ \
	- (ChP)->TxControl[3] \|= TX_ENABLE; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sGetAiopIntStatus
	-Purpose: Get the AIOP interrupt status
	-Call: sGetAiopIntStatus(CtlP,AiopNum)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- int AiopNum; AIOP number
	-Return: Byte_t: The AIOP interrupt status. Bits 0 through 7
	- represent channels 0 through 7 respectively. If a
	- bit is set that channel is interrupting.
	-*/
	-#define sGetAiopIntStatus(CtlP,AIOPNUM) rp_readaiop1(CtlP,AIOPNUM,_INT_CHAN)
	-
	-/***************************************************************************
	-Function: sGetAiopNumChan
	-Purpose: Get the number of channels supported by an AIOP
	-Call: sGetAiopNumChan(CtlP,AiopNum)
	- CONTROLLER_T *CtlP; Ptr to controller structure
	- int AiopNum; AIOP number
	-Return: int: The number of channels supported by the AIOP
	-*/
	-#define sGetAiopNumChan(CtlP,AIOPNUM) CtlP->AiopNumChan[AIOPNUM]
	-
	-/***************************************************************************
	-Function: sGetChanIntID
	-Purpose: Get a channel's interrupt identification byte
	-Call: sGetChanIntID(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: Byte_t: The channel interrupt ID. Can be any
	- combination of the following flags:
	- RXF_TRIG: Rx FIFO trigger level interrupt
	- TXFIFO_MT: Tx FIFO empty interrupt
	- SRC_INT: Special receive condition interrupt
	- DELTA_CD: CD change interrupt
	- DELTA_CTS: CTS change interrupt
	- DELTA_DSR: DSR change interrupt
	-*/
	-#define sGetChanIntID(ChP) (rp_readch1(ChP,(ChP)->ChanNum+_INT_ID0) & (RXF_TRIG \| TXFIFO_MT \| SRC_INT \| DELTA_CD \| DELTA_CTS \| DELTA_DSR))
	-
	-/***************************************************************************
	-Function: sGetChanNum
	-Purpose: Get the number of a channel within an AIOP
	-Call: sGetChanNum(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: int: Channel number within AIOP, or NULLCHAN if channel does
	- not exist.
	-*/
	-#define sGetChanNum(ChP) (ChP)->ChanNum
	-
	-/***************************************************************************
	-Function: sGetChanStatus
	-Purpose: Get the channel status
	-Call: sGetChanStatus(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: Word_t: The channel status. Can be any combination of
	- the following flags:
	- LOW BYTE FLAGS
	- CTS_ACT: CTS input asserted
	- DSR_ACT: DSR input asserted
	- CD_ACT: CD input asserted
	- TXFIFOMT: Tx FIFO is empty
	- TXSHRMT: Tx shift register is empty
	- RDA: Rx data available
	-
	- HIGH BYTE FLAGS
	- STATMODE: status mode enable bit
	- RXFOVERFL: receive FIFO overflow
	- RX2MATCH: receive compare byte 2 match
	- RX1MATCH: receive compare byte 1 match
	- RXBREAK: received BREAK
	- RXFRAME: received framing error
	- RXPARITY: received parity error
	-Warnings: This function will clear the high byte flags in the Channel
	- Status Register.
	-*/
	-#define sGetChanStatus(ChP) rp_readch2(ChP,CHNOFF_CHANSTAT(ChP))
	-
	-/***************************************************************************
	-Function: sGetChanStatusLo
	-Purpose: Get the low byte only of the channel status
	-Call: sGetChanStatusLo(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: Byte_t: The channel status low byte. Can be any combination
	- of the following flags:
	- CTS_ACT: CTS input asserted
	- DSR_ACT: DSR input asserted
	- CD_ACT: CD input asserted
	- TXFIFOMT: Tx FIFO is empty
	- TXSHRMT: Tx shift register is empty
	- RDA: Rx data available
	-*/
	-#define sGetChanStatusLo(ChP) rp_readch1(ChP,CHNOFF_CHANSTAT(ChP))
	-
	-/***************************************************************************
	-Function: sGetRxCnt
	-Purpose: Get the number of data bytes in the Rx FIFO
	-Call: sGetRxCnt(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: int: The number of data bytes in the Rx FIFO.
	-Comments: Byte read of count register is required to obtain Rx count.
	-
	-*/
	-#define sGetRxCnt(ChP) rp_readch2(ChP,CHNOFF_TXRXCOUNT(ChP))
	-
	-/***************************************************************************
	-Function: sGetTxCnt
	-Purpose: Get the number of data bytes in the Tx FIFO
	-Call: sGetTxCnt(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: Byte_t: The number of data bytes in the Tx FIFO.
	-Comments: Byte read of count register is required to obtain Tx count.
	-
	-*/
	-#define sGetTxCnt(ChP) rp_readch1(ChP,CHNOFF_TXRXCOUNT(ChP))
	-
	-/*****************************************************************************
	-Function: sGetTxRxDataIO
	-Purpose: Get the offset of a channel's TxRx Data register
	-Call: sGetTxRxDataIO(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Return: WordIO_t: offset of a channel's TxRx Data register
	-*/
	-#define sGetTxRxDataIO(ChP) CHNOFF_TXRXDATA(ChP)
	-
	-/***************************************************************************
	-Function: sInitChanDefaults
	-Purpose: Initialize a channel structure to its default state.
	-Call: sInitChanDefaults(ChP)
	- CHANNEL_T *ChP; Ptr to the channel structure
	-Comments: This function must be called once for every channel structure
	- that exists before any other SSCI calls can be made.
	-
	-*/
	-#define sInitChanDefaults(ChP) \
	-{ \
	- (ChP)->CtlP = NULLCTLPTR; \
	- (ChP)->AiopNum = NULLAIOP; \
	- (ChP)->ChanID = AIOPID_NULL; \
	- (ChP)->ChanNum = NULLCHAN; \
	-}
	-
	-/***************************************************************************
	-Function: sResetAiopByNum
	-Purpose: Reset the AIOP by number
	-Call: sResetAiopByNum(CTLP,AIOPNUM)
	- CONTROLLER_T CTLP; Ptr to controller structure
	- AIOPNUM; AIOP index
	-*/
	-#define sResetAiopByNum(CTLP,AIOPNUM) \
	-{ \
	- rp_writeaiop1(CTLP,AIOPNUM,_CMD_REG,RESET_ALL); \
	- rp_writeaiop1(CTLP,AIOPNUM,_CMD_REG,0x0); \
	-}
	-
	-/***************************************************************************
	-Function: sSendBreak
	-Purpose: Send a transmit BREAK signal
	-Call: sSendBreak(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSendBreak(ChP) \
	-{ \
	- (ChP)->TxControl[3] \|= SETBREAK; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetBaud
	-Purpose: Set baud rate
	-Call: sSetBaud(ChP,Divisor)
	- CHANNEL_T *ChP; Ptr to channel structure
	- Word_t Divisor; 16 bit baud rate divisor for channel
	-*/
	-#define sSetBaud(ChP,DIVISOR) \
	-{ \
	- (ChP)->BaudDiv[2] = (Byte_t)(DIVISOR); \
	- (ChP)->BaudDiv[3] = (Byte_t)((DIVISOR) >> 8); \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->BaudDiv)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetData7
	-Purpose: Set data bits to 7
	-Call: sSetData7(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSetData7(ChP) \
	-{ \
	- (ChP)->TxControl[2] &= ~DATA8BIT; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetData8
	-Purpose: Set data bits to 8
	-Call: sSetData8(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSetData8(ChP) \
	-{ \
	- (ChP)->TxControl[2] \|= DATA8BIT; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetDTR
	-Purpose: Set the DTR output
	-Call: sSetDTR(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSetDTR(ChP) \
	-{ \
	- (ChP)->TxControl[3] \|= SET_DTR; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetEvenParity
	-Purpose: Set even parity
	-Call: sSetEvenParity(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: Function sSetParity() can be used in place of functions sEnParity(),
	- sDisParity(), sSetOddParity(), and sSetEvenParity().
	-
	-Warnings: This function has no effect unless parity is enabled with function
	- sEnParity().
	-*/
	-#define sSetEvenParity(ChP) \
	-{ \
	- (ChP)->TxControl[2] \|= EVEN_PAR; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetOddParity
	-Purpose: Set odd parity
	-Call: sSetOddParity(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: Function sSetParity() can be used in place of functions sEnParity(),
	- sDisParity(), sSetOddParity(), and sSetEvenParity().
	-
	-Warnings: This function has no effect unless parity is enabled with function
	- sEnParity().
	-*/
	-#define sSetOddParity(ChP) \
	-{ \
	- (ChP)->TxControl[2] &= ~EVEN_PAR; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetRTS
	-Purpose: Set the RTS output
	-Call: sSetRTS(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSetRTS(ChP) \
	-{ \
	- (ChP)->TxControl[3] \|= SET_RTS; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetRxTrigger
	-Purpose: Set the Rx FIFO trigger level
	-Call: sSetRxProcessor(ChP,Level)
	- CHANNEL_T *ChP; Ptr to channel structure
	- Byte_t Level; Number of characters in Rx FIFO at which the
	- interrupt will be generated. Can be any of the following flags:
	-
	- TRIG_NO: no trigger
	- TRIG_1: 1 character in FIFO
	- TRIG_1_2: FIFO 1/2 full
	- TRIG_7_8: FIFO 7/8 full
	-Comments: An interrupt will be generated when the trigger level is reached
	- only if function sEnInterrupt() has been called with flag
	- RXINT_EN set. The RXF_TRIG flag in the Interrupt Idenfification
	- register will be set whenever the trigger level is reached
	- regardless of the setting of RXINT_EN.
	-
	-*/
	-#define sSetRxTrigger(ChP,LEVEL) \
	-{ \
	- (ChP)->RxControl[2] &= ~TRIG_MASK; \
	- (ChP)->RxControl[2] \|= LEVEL; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->RxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetStop1
	-Purpose: Set stop bits to 1
	-Call: sSetStop1(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSetStop1(ChP) \
	-{ \
	- (ChP)->TxControl[2] &= ~STOP2; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sSetStop2
	-Purpose: Set stop bits to 2
	-Call: sSetStop2(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-*/
	-#define sSetStop2(ChP) \
	-{ \
	- (ChP)->TxControl[2] \|= STOP2; \
	- rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->TxControl)); \
	-}
	-
	-/***************************************************************************
	-Function: sStartRxProcessor
	-Purpose: Start a channel's receive processor
	-Call: sStartRxProcessor(ChP)
	- CHANNEL_T *ChP; Ptr to channel structure
	-Comments: This function is used to start a Rx processor after it was
	- stopped with sStopRxProcessor() or sStopSWInFlowCtl(). It
	- will restart both the Rx processor and software input flow control.
	-
	-*/
	-#define sStartRxProcessor(ChP) rp_writech4(ChP,_INDX_ADDR,le32dec((ChP)->R))
	-
	-/***************************************************************************
	-Function: sWriteTxByte
	-Purpose: Write a transmit data byte to a channel.
	- CHANNEL_T *ChP; Ptr to channel structure
	- ByteIO_t io: Channel transmit register I/O address. This can
	- be obtained with sGetTxRxDataIO().
	- Byte_t Data; The transmit data byte.
	-Warnings: This function writes the data byte without checking to see if
	- sMaxTxSize is exceeded in the Tx FIFO.
	-*/
	-#define sWriteTxByte(ChP,IO,DATA) rp_writech1(ChP,IO,DATA)
	-
	-int sReadAiopID(CONTROLLER_T *CtlP, int aiop);
	-int sReadAiopNumChan(CONTROLLER_T *CtlP, int aiop);
	-int sInitChan( CONTROLLER_T *CtlP,
	- CHANNEL_T *ChP,
	- int AiopNum,
	- int ChanNum);
	-Byte_t sGetRxErrStatus(CHANNEL_T *ChP);
	-void sStopRxProcessor(CHANNEL_T *ChP);
	-void sStopSWInFlowCtl(CHANNEL_T *ChP);
	-void sFlushRxFIFO(CHANNEL_T *ChP);
	-void sFlushTxFIFO(CHANNEL_T *ChP);
	-int sWriteTxPrioByte(CHANNEL_T *ChP, Byte_t Data);
	-void sEnInterrupts(CHANNEL_T *ChP,Word_t Flags);
	-void sDisInterrupts(CHANNEL_T *ChP,Word_t Flags);
	-int rp_attachcommon(CONTROLLER_T *ctlp, int num_aiops, int num_ports);
	-void rp_releaseresource(CONTROLLER_t *ctlp);
	-static __inline void
	-rp_lock(CONTROLLER_T *CtlP)
	-{
	- if (CtlP->hwmtx_init != 0)
	- mtx_lock(&CtlP->hwmtx);
	-}
	-static __inline void
	-rp_unlock(CONTROLLER_T *CtlP)
	-{
	- if (CtlP->hwmtx_init != 0)
	- mtx_unlock(&CtlP->hwmtx);
	-}
	-
	-#ifndef ROCKET_C
	-extern Byte_t R[RDATASIZE];
	-extern CONTROLLER_T sController[CTL_SIZE];
	-extern Byte_t sIRQMap[16];
	-#endif
	-extern Byte_t rp_sBitMapClrTbl[8];
	-extern Byte_t rp_sBitMapSetTbl[8];
	diff --git a/sys/dev/rp/rpvar.h b/sys/dev/rp/rpvar.h
	deleted file mode 100644
	index a93f48971469..000000000000
	--- a/sys/dev/rp/rpvar.h
	+++ /dev/null
	@@ -1,76 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-4-Clause
	- *
	- * Copyright (c) Comtrol Corporation <support@comtrol.com>
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted prodived that the follwoing conditions
	- * are met.
	- * 1. Redistributions of source code must retain the above copyright
	- * notive, this list of conditions and the following disclainer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials prodided with the distribution.
	- * 3. All advertising materials mentioning features or use of this software
	- * must display the following acknowledgement:
	- * This product includes software developed by Comtrol Corporation.
	- * 4. The name of Comtrol Corporation may not be used to endorse or
	- * promote products derived from this software without specific
	- * prior written permission.
	- *
	- * THIS SOFTWARE IS PROVIDED BY COMTROL CORPORATION ``AS IS'' AND ANY
	- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL COMTROL CORPORATION BE LIABLE FOR
	- * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, LIFE OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-/*
	- * rpvar.h --- RocketPort data structure includes for FreeBSD
	- */
	-
	-#define RP_UNIT(x) dv_unit(x)
	-#define RP_PORT(x) (dev2unit(x) & 0x3f)
	-#define MAX_RP_PORTS 128
	-
	-
	-struct rp_port {
	- struct tty * rp_tty; /* cross reference */
	- struct callout rp_timer;
	-
	- unsigned char state; /* state of dtr */
	-
	- int rp_port;
	- int rp_flags;
	- int rp_unit:2;
	- int rp_aiop:2;
	- int rp_chan:3;
	- int rp_intmask;
	- int rp_imask; /* Input mask */
	- int rp_fifo_lw;
	- int rp_restart;
	- int rp_overflows;
	- int rp_rts_iflow:1;
	- int rp_disable_writes:1;
	- int rp_cts:1;
	- int rp_waiting:1;
	- int rp_xmit_stopped:1;
	- CONTROLLER_t * rp_ctlp;
	- CHANNEL_t rp_channel;
	- unsigned char TxBuf[TXFIFO_SIZE];
	- unsigned char RxBuf[RXFIFO_SIZE];
	-};
	-
	-/* Actually not used */
	-#ifdef notdef
	-extern struct termios deftermios;
	-#endif /* notdef */
	diff --git a/sys/dev/smc/if_smc.c b/sys/dev/smc/if_smc.c
	index d3f911d8327c..627787eab641 100644
	--- a/sys/dev/smc/if_smc.c
	+++ b/sys/dev/smc/if_smc.c
	@@ -1,1329 +1,1352 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2008 Benno Rice. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/*
	* Driver for SMSC LAN91C111, may work for older variants.
	*/

	#ifdef HAVE_KERNEL_OPTION_HEADERS
	#include "opt_device_polling.h"
	#endif

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/errno.h>
	#include <sys/kernel.h>
	#include <sys/sockio.h>
	#include <sys/malloc.h>
	#include <sys/mbuf.h>
	#include <sys/queue.h>
	#include <sys/socket.h>
	#include <sys/syslog.h>
	#include <sys/taskqueue.h>

	#include <sys/module.h>
	#include <sys/bus.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/rman.h>

	#include <net/ethernet.h>
	#include <net/if.h>
	#include <net/if_var.h>
	#include <net/if_arp.h>
	#include <net/if_dl.h>
	#include <net/if_types.h>
	#include <net/if_mib.h>
	#include <net/if_media.h>

	#ifdef INET
	#include <netinet/in.h>
	#include <netinet/in_systm.h>
	#include <netinet/in_var.h>
	#include <netinet/ip.h>
	#endif

	#include <net/bpf.h>
	#include <net/bpfdesc.h>

	#include <dev/smc/if_smcreg.h>
	#include <dev/smc/if_smcvar.h>

	#include <dev/mii/mii.h>
	#include <dev/mii/mii_bitbang.h>
	#include <dev/mii/miivar.h>

	+#include "miibus_if.h"
	+
	#define SMC_LOCK(sc) mtx_lock(&(sc)->smc_mtx)
	#define SMC_UNLOCK(sc) mtx_unlock(&(sc)->smc_mtx)
	#define SMC_ASSERT_LOCKED(sc) mtx_assert(&(sc)->smc_mtx, MA_OWNED)

	#define SMC_INTR_PRIORITY 0
	#define SMC_RX_PRIORITY 5
	#define SMC_TX_PRIORITY 10

	devclass_t smc_devclass;

	static const char *smc_chip_ids[16] = {
	NULL, NULL, NULL,
	/* 3 */ "SMSC LAN91C90 or LAN91C92",
	/* 4 */ "SMSC LAN91C94",
	/* 5 */ "SMSC LAN91C95",
	/* 6 */ "SMSC LAN91C96",
	/* 7 */ "SMSC LAN91C100",
	/* 8 */ "SMSC LAN91C100FD",
	/* 9 */ "SMSC LAN91C110FD or LAN91C111FD",
	NULL, NULL, NULL,
	NULL, NULL, NULL
	};

	static void smc_init(void *);
	static void smc_start(struct ifnet *);
	static void smc_stop(struct smc_softc *);
	static int smc_ioctl(struct ifnet *, u_long, caddr_t);

	static void smc_init_locked(struct smc_softc *);
	static void smc_start_locked(struct ifnet *);
	static void smc_reset(struct smc_softc *);
	static int smc_mii_ifmedia_upd(struct ifnet *);
	static void smc_mii_ifmedia_sts(struct ifnet , struct ifmediareq );
	static void smc_mii_tick(void *);
	static void smc_mii_mediachg(struct smc_softc *);
	static int smc_mii_mediaioctl(struct smc_softc , struct ifreq , u_long);

	static void smc_task_intr(void *, int);
	static void smc_task_rx(void *, int);
	static void smc_task_tx(void *, int);

	static driver_filter_t smc_intr;
	static callout_func_t smc_watchdog;
	#ifdef DEVICE_POLLING
	static poll_handler_t smc_poll;
	#endif

	/*
	* MII bit-bang glue
	*/
	static uint32_t smc_mii_bitbang_read(device_t);
	static void smc_mii_bitbang_write(device_t, uint32_t);

	static const struct mii_bitbang_ops smc_mii_bitbang_ops = {
	smc_mii_bitbang_read,
	smc_mii_bitbang_write,
	{
	MGMT_MDO, /* MII_BIT_MDO */
	MGMT_MDI, /* MII_BIT_MDI */
	MGMT_MCLK, /* MII_BIT_MDC */
	MGMT_MDOE, /* MII_BIT_DIR_HOST_PHY */
	0, /* MII_BIT_DIR_PHY_HOST */
	}
	};

	static __inline void
	smc_select_bank(struct smc_softc *sc, uint16_t bank)
	{

	bus_barrier(sc->smc_reg, BSR, 2,
	BUS_SPACE_BARRIER_READ \| BUS_SPACE_BARRIER_WRITE);
	bus_write_2(sc->smc_reg, BSR, bank & BSR_BANK_MASK);
	bus_barrier(sc->smc_reg, BSR, 2,
	BUS_SPACE_BARRIER_READ \| BUS_SPACE_BARRIER_WRITE);
	}

	/* Never call this when not in bank 2. */
	static __inline void
	smc_mmu_wait(struct smc_softc *sc)
	{

	KASSERT((bus_read_2(sc->smc_reg, BSR) &
	BSR_BANK_MASK) == 2, ("%s: smc_mmu_wait called when not in bank 2",
	device_get_nameunit(sc->smc_dev)));
	while (bus_read_2(sc->smc_reg, MMUCR) & MMUCR_BUSY)
	;
	}

	static __inline uint8_t
	smc_read_1(struct smc_softc *sc, bus_size_t offset)
	{

	return (bus_read_1(sc->smc_reg, offset));
	}

	static __inline void
	smc_write_1(struct smc_softc *sc, bus_size_t offset, uint8_t val)
	{

	bus_write_1(sc->smc_reg, offset, val);
	}

	static __inline uint16_t
	smc_read_2(struct smc_softc *sc, bus_size_t offset)
	{

	return (bus_read_2(sc->smc_reg, offset));
	}

	static __inline void
	smc_write_2(struct smc_softc *sc, bus_size_t offset, uint16_t val)
	{

	bus_write_2(sc->smc_reg, offset, val);
	}

	static __inline void
	smc_read_multi_2(struct smc_softc sc, bus_size_t offset, uint16_t datap,
	bus_size_t count)
	{

	bus_read_multi_2(sc->smc_reg, offset, datap, count);
	}

	static __inline void
	smc_write_multi_2(struct smc_softc sc, bus_size_t offset, uint16_t datap,
	bus_size_t count)
	{

	bus_write_multi_2(sc->smc_reg, offset, datap, count);
	}

	static __inline void
	smc_barrier(struct smc_softc *sc, bus_size_t offset, bus_size_t length,
	int flags)
	{

	bus_barrier(sc->smc_reg, offset, length, flags);
	}

	int
	smc_probe(device_t dev)
	{
	int rid, type, error;
	uint16_t val;
	struct smc_softc *sc;
	struct resource *reg;

	sc = device_get_softc(dev);
	rid = 0;
	type = SYS_RES_IOPORT;
	error = 0;

	if (sc->smc_usemem)
	type = SYS_RES_MEMORY;

	reg = bus_alloc_resource_anywhere(dev, type, &rid, 16, RF_ACTIVE);
	if (reg == NULL) {
	if (bootverbose)
	device_printf(dev,
	"could not allocate I/O resource for probe\n");
	return (ENXIO);
	}

	/* Check for the identification value in the BSR. */
	val = bus_read_2(reg, BSR);
	if ((val & BSR_IDENTIFY_MASK) != BSR_IDENTIFY) {
	if (bootverbose)
	device_printf(dev, "identification value not in BSR\n");
	error = ENXIO;
	goto done;
	}

	/*
	* Try switching banks and make sure we still get the identification
	* value.
	*/
	bus_write_2(reg, BSR, 0);
	val = bus_read_2(reg, BSR);
	if ((val & BSR_IDENTIFY_MASK) != BSR_IDENTIFY) {
	if (bootverbose)
	device_printf(dev,
	"identification value not in BSR after write\n");
	error = ENXIO;
	goto done;
	}

	#if 0
	/* Check the BAR. */
	bus_write_2(reg, BSR, 1);
	val = bus_read_2(reg, BAR);
	val = BAR_ADDRESS(val);
	if (rman_get_start(reg) != val) {
	if (bootverbose)
	device_printf(dev, "BAR address %x does not match "
	"I/O resource address %lx\n", val,
	rman_get_start(reg));
	error = ENXIO;
	goto done;
	}
	#endif

	/* Compare REV against known chip revisions. */
	bus_write_2(reg, BSR, 3);
	val = bus_read_2(reg, REV);
	val = (val & REV_CHIP_MASK) >> REV_CHIP_SHIFT;
	if (smc_chip_ids[val] == NULL) {
	if (bootverbose)
	device_printf(dev, "Unknown chip revision: %d\n", val);
	error = ENXIO;
	goto done;
	}

	device_set_desc(dev, smc_chip_ids[val]);

	done:
	bus_release_resource(dev, type, rid, reg);
	return (error);
	}

	int
	smc_attach(device_t dev)
	{
	int type, error;
	uint16_t val;
	u_char eaddr[ETHER_ADDR_LEN];
	struct smc_softc *sc;
	struct ifnet *ifp;

	sc = device_get_softc(dev);
	error = 0;

	sc->smc_dev = dev;

	ifp = sc->smc_ifp = if_alloc(IFT_ETHER);
	if (ifp == NULL) {
	error = ENOSPC;
	goto done;
	}

	mtx_init(&sc->smc_mtx, device_get_nameunit(dev), NULL, MTX_DEF);

	/* Set up watchdog callout. */
	callout_init_mtx(&sc->smc_watchdog, &sc->smc_mtx, 0);

	type = SYS_RES_IOPORT;
	if (sc->smc_usemem)
	type = SYS_RES_MEMORY;

	sc->smc_reg_rid = 0;
	sc->smc_reg = bus_alloc_resource_anywhere(dev, type, &sc->smc_reg_rid,
	16, RF_ACTIVE);
	if (sc->smc_reg == NULL) {
	error = ENXIO;
	goto done;
	}

	sc->smc_irq = bus_alloc_resource_anywhere(dev, SYS_RES_IRQ,
	&sc->smc_irq_rid, 1, RF_ACTIVE \| RF_SHAREABLE);
	if (sc->smc_irq == NULL) {
	error = ENXIO;
	goto done;
	}

	SMC_LOCK(sc);
	smc_reset(sc);
	SMC_UNLOCK(sc);

	smc_select_bank(sc, 3);
	val = smc_read_2(sc, REV);
	sc->smc_chip = (val & REV_CHIP_MASK) >> REV_CHIP_SHIFT;
	sc->smc_rev = (val * REV_REV_MASK) >> REV_REV_SHIFT;
	if (bootverbose)
	device_printf(dev, "revision %x\n", sc->smc_rev);

	callout_init_mtx(&sc->smc_mii_tick_ch, &sc->smc_mtx,
	CALLOUT_RETURNUNLOCKED);
	if (sc->smc_chip >= REV_CHIP_91110FD) {
	(void)mii_attach(dev, &sc->smc_miibus, ifp,
	smc_mii_ifmedia_upd, smc_mii_ifmedia_sts, BMSR_DEFCAPMASK,
	MII_PHY_ANY, MII_OFFSET_ANY, 0);
	if (sc->smc_miibus != NULL) {
	sc->smc_mii_tick = smc_mii_tick;
	sc->smc_mii_mediachg = smc_mii_mediachg;
	sc->smc_mii_mediaioctl = smc_mii_mediaioctl;
	}
	}

	smc_select_bank(sc, 1);
	eaddr[0] = smc_read_1(sc, IAR0);
	eaddr[1] = smc_read_1(sc, IAR1);
	eaddr[2] = smc_read_1(sc, IAR2);
	eaddr[3] = smc_read_1(sc, IAR3);
	eaddr[4] = smc_read_1(sc, IAR4);
	eaddr[5] = smc_read_1(sc, IAR5);

	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
	ifp->if_softc = sc;
	ifp->if_flags = IFF_BROADCAST \| IFF_SIMPLEX \| IFF_MULTICAST;
	ifp->if_init = smc_init;
	ifp->if_ioctl = smc_ioctl;
	ifp->if_start = smc_start;
	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
	IFQ_SET_READY(&ifp->if_snd);

	ifp->if_capabilities = ifp->if_capenable = 0;

	#ifdef DEVICE_POLLING
	ifp->if_capabilities \|= IFCAP_POLLING;
	#endif

	ether_ifattach(ifp, eaddr);

	/* Set up taskqueue */
	TASK_INIT(&sc->smc_intr, SMC_INTR_PRIORITY, smc_task_intr, ifp);
	NET_TASK_INIT(&sc->smc_rx, SMC_RX_PRIORITY, smc_task_rx, ifp);
	TASK_INIT(&sc->smc_tx, SMC_TX_PRIORITY, smc_task_tx, ifp);
	sc->smc_tq = taskqueue_create_fast("smc_taskq", M_NOWAIT,
	taskqueue_thread_enqueue, &sc->smc_tq);
	taskqueue_start_threads(&sc->smc_tq, 1, PI_NET, "%s taskq",
	device_get_nameunit(sc->smc_dev));

	/* Mask all interrupts. */
	sc->smc_mask = 0;
	smc_write_1(sc, MSK, 0);

	/* Wire up interrupt */
	error = bus_setup_intr(dev, sc->smc_irq,
	INTR_TYPE_NET\|INTR_MPSAFE, smc_intr, NULL, sc, &sc->smc_ih);
	if (error != 0)
	goto done;

	done:
	if (error != 0)
	smc_detach(dev);
	return (error);
	}

	int
	smc_detach(device_t dev)
	{
	int type;
	struct smc_softc *sc;

	sc = device_get_softc(dev);
	SMC_LOCK(sc);
	smc_stop(sc);
	SMC_UNLOCK(sc);

	if (sc->smc_ifp != NULL) {
	ether_ifdetach(sc->smc_ifp);
	}

	callout_drain(&sc->smc_watchdog);
	callout_drain(&sc->smc_mii_tick_ch);

	#ifdef DEVICE_POLLING
	if (sc->smc_ifp->if_capenable & IFCAP_POLLING)
	ether_poll_deregister(sc->smc_ifp);
	#endif

	if (sc->smc_ih != NULL)
	bus_teardown_intr(sc->smc_dev, sc->smc_irq, sc->smc_ih);

	if (sc->smc_tq != NULL) {
	taskqueue_drain(sc->smc_tq, &sc->smc_intr);
	taskqueue_drain(sc->smc_tq, &sc->smc_rx);
	taskqueue_drain(sc->smc_tq, &sc->smc_tx);
	taskqueue_free(sc->smc_tq);
	sc->smc_tq = NULL;
	}

	if (sc->smc_ifp != NULL) {
	if_free(sc->smc_ifp);
	}

	if (sc->smc_miibus != NULL) {
	device_delete_child(sc->smc_dev, sc->smc_miibus);
	bus_generic_detach(sc->smc_dev);
	}

	if (sc->smc_reg != NULL) {
	type = SYS_RES_IOPORT;
	if (sc->smc_usemem)
	type = SYS_RES_MEMORY;

	bus_release_resource(sc->smc_dev, type, sc->smc_reg_rid,
	sc->smc_reg);
	}

	if (sc->smc_irq != NULL)
	bus_release_resource(sc->smc_dev, SYS_RES_IRQ, sc->smc_irq_rid,
	sc->smc_irq);

	if (mtx_initialized(&sc->smc_mtx))
	mtx_destroy(&sc->smc_mtx);

	return (0);
	}

	+static device_method_t smc_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_attach, smc_attach),
	+ DEVMETHOD(device_detach, smc_detach),
	+
	+ /* MII interface */
	+ DEVMETHOD(miibus_readreg, smc_miibus_readreg),
	+ DEVMETHOD(miibus_writereg, smc_miibus_writereg),
	+ DEVMETHOD(miibus_statchg, smc_miibus_statchg),
	+
	+ { 0, 0 }
	+};
	+
	+driver_t smc_driver = {
	+ "smc",
	+ smc_methods,
	+ sizeof(struct smc_softc),
	+};
	+
	+DRIVER_MODULE(miibus, smc, miibus_driver, miibus_devclass, 0, 0);
	+
	static void
	smc_start(struct ifnet *ifp)
	{
	struct smc_softc *sc;

	sc = ifp->if_softc;
	SMC_LOCK(sc);
	smc_start_locked(ifp);
	SMC_UNLOCK(sc);
	}

	static void
	smc_start_locked(struct ifnet *ifp)
	{
	struct smc_softc *sc;
	struct mbuf *m;
	u_int len, npages, spin_count;

	sc = ifp->if_softc;
	SMC_ASSERT_LOCKED(sc);

	if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
	return;
	if (IFQ_IS_EMPTY(&ifp->if_snd))
	return;

	/*
	* Grab the next packet. If it's too big, drop it.
	*/
	IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
	len = m_length(m, NULL);
	len += (len & 1);
	if (len > ETHER_MAX_LEN - ETHER_CRC_LEN) {
	if_printf(ifp, "large packet discarded\n");
	if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
	m_freem(m);
	return; /* XXX readcheck? */
	}

	/*
	* Flag that we're busy.
	*/
	ifp->if_drv_flags \|= IFF_DRV_OACTIVE;
	sc->smc_pending = m;

	/*
	* Work out how many 256 byte "pages" we need. We have to include the
	* control data for the packet in this calculation.
	*/
	npages = (len + PKT_CTRL_DATA_LEN) >> 8;
	if (npages == 0)
	npages = 1;

	/*
	* Request memory.
	*/
	smc_select_bank(sc, 2);
	smc_mmu_wait(sc);
	smc_write_2(sc, MMUCR, MMUCR_CMD_TX_ALLOC \| npages);

	/*
	* Spin briefly to see if the allocation succeeds.
	*/
	spin_count = TX_ALLOC_WAIT_TIME;
	do {
	if (smc_read_1(sc, IST) & ALLOC_INT) {
	smc_write_1(sc, ACK, ALLOC_INT);
	break;
	}
	} while (--spin_count);

	/*
	* If the allocation is taking too long, unmask the alloc interrupt
	* and wait.
	*/
	if (spin_count == 0) {
	sc->smc_mask \|= ALLOC_INT;
	if ((ifp->if_capenable & IFCAP_POLLING) == 0)
	smc_write_1(sc, MSK, sc->smc_mask);
	return;
	}

	taskqueue_enqueue(sc->smc_tq, &sc->smc_tx);
	}

	static void
	smc_task_tx(void *context, int pending)
	{
	struct ifnet *ifp;
	struct smc_softc *sc;
	struct mbuf m, m0;
	u_int packet, len;
	int last_len;
	uint8_t *data;

	(void)pending;
	ifp = (struct ifnet *)context;
	sc = ifp->if_softc;

	SMC_LOCK(sc);

	if (sc->smc_pending == NULL) {
	SMC_UNLOCK(sc);
	goto next_packet;
	}

	m = m0 = sc->smc_pending;
	sc->smc_pending = NULL;
	smc_select_bank(sc, 2);

	/*
	* Check the allocation result.
	*/
	packet = smc_read_1(sc, ARR);

	/*
	* If the allocation failed, requeue the packet and retry.
	*/
	if (packet & ARR_FAILED) {
	IFQ_DRV_PREPEND(&ifp->if_snd, m);
	if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
	smc_start_locked(ifp);
	SMC_UNLOCK(sc);
	return;
	}

	/*
	* Tell the device to write to our packet number.
	*/
	smc_write_1(sc, PNR, packet);
	smc_write_2(sc, PTR, 0 \| PTR_AUTO_INCR);

	/*
	* Tell the device how long the packet is (including control data).
	*/
	len = m_length(m, 0);
	len += PKT_CTRL_DATA_LEN;
	smc_write_2(sc, DATA0, 0);
	smc_write_2(sc, DATA0, len);

	/*
	* Push the data out to the device.
	*/
	data = NULL;
	last_len = 0;
	for (; m != NULL; m = m->m_next) {
	data = mtod(m, uint8_t *);
	smc_write_multi_2(sc, DATA0, (uint16_t *)data, m->m_len / 2);
	last_len = m->m_len;
	}

	/*
	* Push out the control byte and and the odd byte if needed.
	*/
	if ((len & 1) != 0 && data != NULL)
	smc_write_2(sc, DATA0, (CTRL_ODD << 8) \| data[last_len - 1]);
	else
	smc_write_2(sc, DATA0, 0);

	/*
	* Unmask the TX empty interrupt.
	*/
	sc->smc_mask \|= TX_EMPTY_INT;
	if ((ifp->if_capenable & IFCAP_POLLING) == 0)
	smc_write_1(sc, MSK, sc->smc_mask);

	/*
	* Enqueue the packet.
	*/
	smc_mmu_wait(sc);
	smc_write_2(sc, MMUCR, MMUCR_CMD_ENQUEUE);
	callout_reset(&sc->smc_watchdog, hz * 2, smc_watchdog, sc);

	/*
	* Finish up.
	*/
	if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
	SMC_UNLOCK(sc);
	BPF_MTAP(ifp, m0);
	m_freem(m0);

	next_packet:
	/*
	* See if there's anything else to do.
	*/
	smc_start(ifp);
	}

	static void
	smc_task_rx(void *context, int pending)
	{
	u_int packet, status, len;
	uint8_t *data;
	struct ifnet *ifp;
	struct smc_softc *sc;
	struct mbuf m, mhead, *mtail;

	(void)pending;
	ifp = (struct ifnet *)context;
	sc = ifp->if_softc;
	mhead = mtail = NULL;

	SMC_LOCK(sc);

	packet = smc_read_1(sc, FIFO_RX);
	while ((packet & FIFO_EMPTY) == 0) {
	/*
	* Grab an mbuf and attach a cluster.
	*/
	MGETHDR(m, M_NOWAIT, MT_DATA);
	if (m == NULL) {
	break;
	}
	if (!(MCLGET(m, M_NOWAIT))) {
	m_freem(m);
	break;
	}

	/*
	* Point to the start of the packet.
	*/
	smc_select_bank(sc, 2);
	smc_write_1(sc, PNR, packet);
	smc_write_2(sc, PTR, 0 \| PTR_READ \| PTR_RCV \| PTR_AUTO_INCR);

	/*
	* Grab status and packet length.
	*/
	status = smc_read_2(sc, DATA0);
	len = smc_read_2(sc, DATA0) & RX_LEN_MASK;
	len -= 6;
	if (status & RX_ODDFRM)
	len += 1;

	/*
	* Check for errors.
	*/
	if (status & (RX_TOOSHORT \| RX_TOOLNG \| RX_BADCRC \| RX_ALGNERR)) {
	smc_mmu_wait(sc);
	smc_write_2(sc, MMUCR, MMUCR_CMD_RELEASE);
	if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
	m_freem(m);
	break;
	}

	/*
	* Set the mbuf up the way we want it.
	*/
	m->m_pkthdr.rcvif = ifp;
	m->m_pkthdr.len = m->m_len = len + 2; /* XXX: Is this right? */
	m_adj(m, ETHER_ALIGN);

	/*
	* Pull the packet out of the device. Make sure we're in the
	* right bank first as things may have changed while we were
	* allocating our mbuf.
	*/
	smc_select_bank(sc, 2);
	smc_write_1(sc, PNR, packet);
	smc_write_2(sc, PTR, 4 \| PTR_READ \| PTR_RCV \| PTR_AUTO_INCR);
	data = mtod(m, uint8_t *);
	smc_read_multi_2(sc, DATA0, (uint16_t *)data, len >> 1);
	if (len & 1) {
	data += len & ~1;
	*data = smc_read_1(sc, DATA0);
	}

	/*
	* Tell the device we're done.
	*/
	smc_mmu_wait(sc);
	smc_write_2(sc, MMUCR, MMUCR_CMD_RELEASE);
	if (m == NULL) {
	break;
	}

	if (mhead == NULL) {
	mhead = mtail = m;
	m->m_next = NULL;
	} else {
	mtail->m_next = m;
	mtail = m;
	}
	packet = smc_read_1(sc, FIFO_RX);
	}

	sc->smc_mask \|= RCV_INT;
	if ((ifp->if_capenable & IFCAP_POLLING) == 0)
	smc_write_1(sc, MSK, sc->smc_mask);

	SMC_UNLOCK(sc);

	while (mhead != NULL) {
	m = mhead;
	mhead = mhead->m_next;
	m->m_next = NULL;
	if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
	(*ifp->if_input)(ifp, m);
	}
	}

	#ifdef DEVICE_POLLING
	static int
	smc_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
	{
	struct smc_softc *sc;

	sc = ifp->if_softc;

	SMC_LOCK(sc);
	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
	SMC_UNLOCK(sc);
	return (0);
	}
	SMC_UNLOCK(sc);

	if (cmd == POLL_AND_CHECK_STATUS)
	taskqueue_enqueue(sc->smc_tq, &sc->smc_intr);
	return (0);
	}
	#endif

	static int
	smc_intr(void *context)
	{
	struct smc_softc *sc;
	uint32_t curbank;

	sc = (struct smc_softc *)context;

	/*
	* Save current bank and restore later in this function
	*/
	curbank = (smc_read_2(sc, BSR) & BSR_BANK_MASK);

	/*
	* Block interrupts in order to let smc_task_intr to kick in
	*/
	smc_select_bank(sc, 2);
	smc_write_1(sc, MSK, 0);

	/* Restore bank */
	smc_select_bank(sc, curbank);

	taskqueue_enqueue(sc->smc_tq, &sc->smc_intr);
	return (FILTER_HANDLED);
	}

	static void
	smc_task_intr(void *context, int pending)
	{
	struct smc_softc *sc;
	struct ifnet *ifp;
	u_int status, packet, counter, tcr;

	(void)pending;
	ifp = (struct ifnet *)context;
	sc = ifp->if_softc;

	SMC_LOCK(sc);

	smc_select_bank(sc, 2);

	/*
	* Find out what interrupts are flagged.
	*/
	status = smc_read_1(sc, IST) & sc->smc_mask;

	/*
	* Transmit error
	*/
	if (status & TX_INT) {
	/*
	* Kill off the packet if there is one and re-enable transmit.
	*/
	packet = smc_read_1(sc, FIFO_TX);
	if ((packet & FIFO_EMPTY) == 0) {
	callout_stop(&sc->smc_watchdog);
	smc_select_bank(sc, 2);
	smc_write_1(sc, PNR, packet);
	smc_write_2(sc, PTR, 0 \| PTR_READ \|
	PTR_AUTO_INCR);
	smc_select_bank(sc, 0);
	tcr = smc_read_2(sc, EPHSR);
	#if 0
	if ((tcr & EPHSR_TX_SUC) == 0)
	device_printf(sc->smc_dev,
	"bad packet\n");
	#endif
	smc_select_bank(sc, 2);
	smc_mmu_wait(sc);
	smc_write_2(sc, MMUCR, MMUCR_CMD_RELEASE_PKT);

	smc_select_bank(sc, 0);
	tcr = smc_read_2(sc, TCR);
	tcr \|= TCR_TXENA \| TCR_PAD_EN;
	smc_write_2(sc, TCR, tcr);
	smc_select_bank(sc, 2);
	taskqueue_enqueue(sc->smc_tq, &sc->smc_tx);
	}

	/*
	* Ack the interrupt.
	*/
	smc_write_1(sc, ACK, TX_INT);
	}

	/*
	* Receive
	*/
	if (status & RCV_INT) {
	smc_write_1(sc, ACK, RCV_INT);
	sc->smc_mask &= ~RCV_INT;
	taskqueue_enqueue(sc->smc_tq, &sc->smc_rx);
	}

	/*
	* Allocation
	*/
	if (status & ALLOC_INT) {
	smc_write_1(sc, ACK, ALLOC_INT);
	sc->smc_mask &= ~ALLOC_INT;
	taskqueue_enqueue(sc->smc_tq, &sc->smc_tx);
	}

	/*
	* Receive overrun
	*/
	if (status & RX_OVRN_INT) {
	smc_write_1(sc, ACK, RX_OVRN_INT);
	if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
	}

	/*
	* Transmit empty
	*/
	if (status & TX_EMPTY_INT) {
	smc_write_1(sc, ACK, TX_EMPTY_INT);
	sc->smc_mask &= ~TX_EMPTY_INT;
	callout_stop(&sc->smc_watchdog);

	/*
	* Update collision stats.
	*/
	smc_select_bank(sc, 0);
	counter = smc_read_2(sc, ECR);
	smc_select_bank(sc, 2);
	if_inc_counter(ifp, IFCOUNTER_COLLISIONS,
	((counter & ECR_SNGLCOL_MASK) >> ECR_SNGLCOL_SHIFT) +
	((counter & ECR_MULCOL_MASK) >> ECR_MULCOL_SHIFT));

	/*
	* See if there are any packets to transmit.
	*/
	taskqueue_enqueue(sc->smc_tq, &sc->smc_tx);
	}

	/*
	* Update the interrupt mask.
	*/
	smc_select_bank(sc, 2);
	if ((ifp->if_capenable & IFCAP_POLLING) == 0)
	smc_write_1(sc, MSK, sc->smc_mask);

	SMC_UNLOCK(sc);
	}

	static uint32_t
	smc_mii_bitbang_read(device_t dev)
	{
	struct smc_softc *sc;
	uint32_t val;

	sc = device_get_softc(dev);

	SMC_ASSERT_LOCKED(sc);
	KASSERT((smc_read_2(sc, BSR) & BSR_BANK_MASK) == 3,
	("%s: smc_mii_bitbang_read called with bank %d (!= 3)",
	device_get_nameunit(sc->smc_dev),
	smc_read_2(sc, BSR) & BSR_BANK_MASK));

	val = smc_read_2(sc, MGMT);
	smc_barrier(sc, MGMT, 2,
	BUS_SPACE_BARRIER_READ \| BUS_SPACE_BARRIER_WRITE);

	return (val);
	}

	static void
	smc_mii_bitbang_write(device_t dev, uint32_t val)
	{
	struct smc_softc *sc;

	sc = device_get_softc(dev);

	SMC_ASSERT_LOCKED(sc);
	KASSERT((smc_read_2(sc, BSR) & BSR_BANK_MASK) == 3,
	("%s: smc_mii_bitbang_write called with bank %d (!= 3)",
	device_get_nameunit(sc->smc_dev),
	smc_read_2(sc, BSR) & BSR_BANK_MASK));

	smc_write_2(sc, MGMT, val);
	smc_barrier(sc, MGMT, 2,
	BUS_SPACE_BARRIER_READ \| BUS_SPACE_BARRIER_WRITE);
	}

	int
	smc_miibus_readreg(device_t dev, int phy, int reg)
	{
	struct smc_softc *sc;
	int val;

	sc = device_get_softc(dev);

	SMC_LOCK(sc);

	smc_select_bank(sc, 3);

	val = mii_bitbang_readreg(dev, &smc_mii_bitbang_ops, phy, reg);

	SMC_UNLOCK(sc);
	return (val);
	}

	int
	smc_miibus_writereg(device_t dev, int phy, int reg, int data)
	{
	struct smc_softc *sc;

	sc = device_get_softc(dev);

	SMC_LOCK(sc);

	smc_select_bank(sc, 3);

	mii_bitbang_writereg(dev, &smc_mii_bitbang_ops, phy, reg, data);

	SMC_UNLOCK(sc);
	return (0);
	}

	void
	smc_miibus_statchg(device_t dev)
	{
	struct smc_softc *sc;
	struct mii_data *mii;
	uint16_t tcr;

	sc = device_get_softc(dev);
	mii = device_get_softc(sc->smc_miibus);

	SMC_LOCK(sc);

	smc_select_bank(sc, 0);
	tcr = smc_read_2(sc, TCR);

	if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0)
	tcr \|= TCR_SWFDUP;
	else
	tcr &= ~TCR_SWFDUP;

	smc_write_2(sc, TCR, tcr);

	SMC_UNLOCK(sc);
	}

	static int
	smc_mii_ifmedia_upd(struct ifnet *ifp)
	{
	struct smc_softc *sc;
	struct mii_data *mii;

	sc = ifp->if_softc;
	if (sc->smc_miibus == NULL)
	return (ENXIO);

	mii = device_get_softc(sc->smc_miibus);
	return (mii_mediachg(mii));
	}

	static void
	smc_mii_ifmedia_sts(struct ifnet ifp, struct ifmediareq ifmr)
	{
	struct smc_softc *sc;
	struct mii_data *mii;

	sc = ifp->if_softc;
	if (sc->smc_miibus == NULL)
	return;

	mii = device_get_softc(sc->smc_miibus);
	mii_pollstat(mii);
	ifmr->ifm_active = mii->mii_media_active;
	ifmr->ifm_status = mii->mii_media_status;
	}

	static void
	smc_mii_tick(void *context)
	{
	struct smc_softc *sc;

	sc = (struct smc_softc *)context;

	if (sc->smc_miibus == NULL)
	return;

	SMC_UNLOCK(sc);

	mii_tick(device_get_softc(sc->smc_miibus));
	callout_reset(&sc->smc_mii_tick_ch, hz, smc_mii_tick, sc);
	}

	static void
	smc_mii_mediachg(struct smc_softc *sc)
	{

	if (sc->smc_miibus == NULL)
	return;
	mii_mediachg(device_get_softc(sc->smc_miibus));
	}

	static int
	smc_mii_mediaioctl(struct smc_softc sc, struct ifreq ifr, u_long command)
	{
	struct mii_data *mii;

	if (sc->smc_miibus == NULL)
	return (EINVAL);

	mii = device_get_softc(sc->smc_miibus);
	return (ifmedia_ioctl(sc->smc_ifp, ifr, &mii->mii_media, command));
	}

	static void
	smc_reset(struct smc_softc *sc)
	{
	u_int ctr;

	SMC_ASSERT_LOCKED(sc);

	smc_select_bank(sc, 2);

	/*
	* Mask all interrupts.
	*/
	smc_write_1(sc, MSK, 0);

	/*
	* Tell the device to reset.
	*/
	smc_select_bank(sc, 0);
	smc_write_2(sc, RCR, RCR_SOFT_RST);

	/*
	* Set up the configuration register.
	*/
	smc_select_bank(sc, 1);
	smc_write_2(sc, CR, CR_EPH_POWER_EN);
	DELAY(1);

	/*
	* Turn off transmit and receive.
	*/
	smc_select_bank(sc, 0);
	smc_write_2(sc, TCR, 0);
	smc_write_2(sc, RCR, 0);

	/*
	* Set up the control register.
	*/
	smc_select_bank(sc, 1);
	ctr = smc_read_2(sc, CTR);
	ctr \|= CTR_LE_ENABLE \| CTR_AUTO_RELEASE;
	smc_write_2(sc, CTR, ctr);

	/*
	* Reset the MMU.
	*/
	smc_select_bank(sc, 2);
	smc_mmu_wait(sc);
	smc_write_2(sc, MMUCR, MMUCR_CMD_MMU_RESET);
	}

	static void
	smc_enable(struct smc_softc *sc)
	{
	struct ifnet *ifp;

	SMC_ASSERT_LOCKED(sc);
	ifp = sc->smc_ifp;

	/*
	* Set up the receive/PHY control register.
	*/
	smc_select_bank(sc, 0);
	smc_write_2(sc, RPCR, RPCR_ANEG \| (RPCR_LED_LINK_ANY << RPCR_LSA_SHIFT)
	\| (RPCR_LED_ACT_ANY << RPCR_LSB_SHIFT));

	/*
	* Set up the transmit and receive control registers.
	*/
	smc_write_2(sc, TCR, TCR_TXENA \| TCR_PAD_EN);
	smc_write_2(sc, RCR, RCR_RXEN \| RCR_STRIP_CRC);

	/*
	* Set up the interrupt mask.
	*/
	smc_select_bank(sc, 2);
	sc->smc_mask = EPH_INT \| RX_OVRN_INT \| RCV_INT \| TX_INT;
	if ((ifp->if_capenable & IFCAP_POLLING) != 0)
	smc_write_1(sc, MSK, sc->smc_mask);
	}

	static void
	smc_stop(struct smc_softc *sc)
	{

	SMC_ASSERT_LOCKED(sc);

	/*
	* Turn off callouts.
	*/
	callout_stop(&sc->smc_watchdog);
	callout_stop(&sc->smc_mii_tick_ch);

	/*
	* Mask all interrupts.
	*/
	smc_select_bank(sc, 2);
	sc->smc_mask = 0;
	smc_write_1(sc, MSK, 0);
	#ifdef DEVICE_POLLING
	ether_poll_deregister(sc->smc_ifp);
	sc->smc_ifp->if_capenable &= ~IFCAP_POLLING;
	#endif

	/*
	* Disable transmit and receive.
	*/
	smc_select_bank(sc, 0);
	smc_write_2(sc, TCR, 0);
	smc_write_2(sc, RCR, 0);

	sc->smc_ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
	}

	static void
	smc_watchdog(void *arg)
	{
	struct smc_softc *sc;

	sc = (struct smc_softc *)arg;
	device_printf(sc->smc_dev, "watchdog timeout\n");
	taskqueue_enqueue(sc->smc_tq, &sc->smc_intr);
	}

	static void
	smc_init(void *context)
	{
	struct smc_softc *sc;

	sc = (struct smc_softc *)context;
	SMC_LOCK(sc);
	smc_init_locked(sc);
	SMC_UNLOCK(sc);
	}

	static void
	smc_init_locked(struct smc_softc *sc)
	{
	struct ifnet *ifp;

	SMC_ASSERT_LOCKED(sc);
	ifp = sc->smc_ifp;
	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
	return;

	smc_reset(sc);
	smc_enable(sc);

	ifp->if_drv_flags \|= IFF_DRV_RUNNING;
	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;

	smc_start_locked(ifp);

	if (sc->smc_mii_tick != NULL)
	callout_reset(&sc->smc_mii_tick_ch, hz, sc->smc_mii_tick, sc);

	#ifdef DEVICE_POLLING
	SMC_UNLOCK(sc);
	ether_poll_register(smc_poll, ifp);
	SMC_LOCK(sc);
	ifp->if_capenable \|= IFCAP_POLLING;
	#endif
	}

	static int
	smc_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
	{
	struct smc_softc *sc;
	int error;

	sc = ifp->if_softc;
	error = 0;

	switch (cmd) {
	case SIOCSIFFLAGS:
	if ((ifp->if_flags & IFF_UP) == 0 &&
	(ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
	SMC_LOCK(sc);
	smc_stop(sc);
	SMC_UNLOCK(sc);
	} else {
	smc_init(sc);
	if (sc->smc_mii_mediachg != NULL)
	sc->smc_mii_mediachg(sc);
	}
	break;

	case SIOCADDMULTI:
	case SIOCDELMULTI:
	/* XXX
	SMC_LOCK(sc);
	smc_setmcast(sc);
	SMC_UNLOCK(sc);
	*/
	error = EINVAL;
	break;

	case SIOCGIFMEDIA:
	case SIOCSIFMEDIA:
	if (sc->smc_mii_mediaioctl == NULL) {
	error = EINVAL;
	break;
	}
	sc->smc_mii_mediaioctl(sc, (struct ifreq *)data, cmd);
	break;

	default:
	error = ether_ioctl(ifp, cmd, data);
	break;
	}

	return (error);
	}
	diff --git a/sys/dev/smc/if_smc_fdt.c b/sys/dev/smc/if_smc_acpi.c
	similarity index 55%
	copy from sys/dev/smc/if_smc_fdt.c
	copy to sys/dev/smc/if_smc_acpi.c
	index baca0bd6113e..74f591493167 100644
	--- a/sys/dev/smc/if_smc_fdt.c
	+++ b/sys/dev/smc/if_smc_acpi.c
	@@ -1,126 +1,80 @@
	/*-
	* Copyright (c) 2008 Benno Rice
	* All rights reserved.
	+ * Copyright (c) 2020 Andrew Turner
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/socket.h>
	#include <sys/systm.h>
	#include <sys/taskqueue.h>

	-#include <machine/bus.h>
	-#include <machine/resource.h>
	-
	-#include <net/ethernet.h>
	#include <net/if.h>
	-#include <net/if_arp.h>
	-#include <net/if_media.h>

	#include <dev/smc/if_smcvar.h>

	-#include <dev/mii/mii.h>
	-#include <dev/mii/miivar.h>
	-
	-#include <dev/fdt/fdt_common.h>
	-#include <dev/ofw/openfirm.h>
	-#include <dev/ofw/ofw_bus.h>
	-#include <dev/ofw/ofw_bus_subr.h>
	-
	-#include "miibus_if.h"
	+#include <contrib/dev/acpica/include/acpi.h>
	+#include <dev/acpica/acpivar.h>

	-static int smc_fdt_probe(device_t);
	-static int smc_fdt_attach(device_t);
	-static int smc_fdt_detach(device_t);
	+static int smc_acpi_probe(device_t);

	static int
	-smc_fdt_probe(device_t dev)
	+smc_acpi_probe(device_t dev)
	{
	struct smc_softc *sc;
	+ ACPI_HANDLE h;

	- if (!ofw_bus_status_okay(dev))
	+ if ((h = acpi_get_handle(dev)) == NULL)
	return (ENXIO);

	- if (ofw_bus_is_compatible(dev, "smsc,lan91c111")) {
	- sc = device_get_softc(dev);
	- sc->smc_usemem = 1;
	-
	- if (smc_probe(dev) != 0) {
	- return (ENXIO);
	- }
	-
	- return (0);
	- }
	-
	- return (ENXIO);
	-}
	-
	-static int
	-smc_fdt_attach(device_t dev)
	-{
	-
	- return smc_attach(dev);
	-}
	-
	-static int
	-smc_fdt_detach(device_t dev)
	-{
	+ if (!acpi_MatchHid(h, "LNRO0003"))
	+ return (ENXIO);

	- smc_detach(dev);
	+ sc = device_get_softc(dev);
	+ sc->smc_usemem = 1;

	- return (0);
	+ return (smc_probe(dev));
	}

	-static device_method_t smc_fdt_methods[] = {
	+static device_method_t smc_acpi_methods[] = {
	/* Device interface */
	- DEVMETHOD(device_probe, smc_fdt_probe),
	- DEVMETHOD(device_attach, smc_fdt_attach),
	- DEVMETHOD(device_detach, smc_fdt_detach),
	-
	- /* MII interface */
	- DEVMETHOD(miibus_readreg, smc_miibus_readreg),
	- DEVMETHOD(miibus_writereg, smc_miibus_writereg),
	- DEVMETHOD(miibus_statchg, smc_miibus_statchg),
	-
	+ DEVMETHOD(device_probe, smc_acpi_probe),
	{ 0, 0 }
	};

	-static driver_t smc_fdt_driver = {
	- "smc",
	- smc_fdt_methods,
	- sizeof(struct smc_softc),
	-};
	+DEFINE_CLASS_1(smc, smc_acpi_driver, smc_acpi_methods,
	+ sizeof(struct smc_softc), smc_driver);

	extern devclass_t smc_devclass;

	-DRIVER_MODULE(smc, simplebus, smc_fdt_driver, smc_devclass, 0, 0);
	-DRIVER_MODULE(miibus, smc, miibus_driver, miibus_devclass, 0, 0);
	-MODULE_DEPEND(smc, fdt, 1, 1, 1);
	+DRIVER_MODULE(smc, acpi, smc_acpi_driver, smc_devclass, 0, 0);
	+MODULE_DEPEND(smc, acpi, 1, 1, 1);
	MODULE_DEPEND(smc, ether, 1, 1, 1);
	MODULE_DEPEND(smc, miibus, 1, 1, 1);
	diff --git a/sys/dev/smc/if_smc_fdt.c b/sys/dev/smc/if_smc_fdt.c
	index baca0bd6113e..2c63b9f94d49 100644
	--- a/sys/dev/smc/if_smc_fdt.c
	+++ b/sys/dev/smc/if_smc_fdt.c
	@@ -1,126 +1,85 @@
	/*-
	* Copyright (c) 2008 Benno Rice
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/socket.h>
	#include <sys/systm.h>
	#include <sys/taskqueue.h>

	-#include <machine/bus.h>
	-#include <machine/resource.h>
	-
	-#include <net/ethernet.h>
	#include <net/if.h>
	-#include <net/if_arp.h>
	-#include <net/if_media.h>

	#include <dev/smc/if_smcvar.h>

	-#include <dev/mii/mii.h>
	-#include <dev/mii/miivar.h>
	-
	#include <dev/fdt/fdt_common.h>
	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	-#include "miibus_if.h"
	-
	static int smc_fdt_probe(device_t);
	-static int smc_fdt_attach(device_t);
	-static int smc_fdt_detach(device_t);

	static int
	smc_fdt_probe(device_t dev)
	{
	struct smc_softc *sc;

	if (!ofw_bus_status_okay(dev))
	return (ENXIO);

	if (ofw_bus_is_compatible(dev, "smsc,lan91c111")) {
	sc = device_get_softc(dev);
	sc->smc_usemem = 1;

	if (smc_probe(dev) != 0) {
	return (ENXIO);
	}

	return (0);
	}

	return (ENXIO);
	}

	-static int
	-smc_fdt_attach(device_t dev)
	-{
	-
	- return smc_attach(dev);
	-}
	-
	-static int
	-smc_fdt_detach(device_t dev)
	-{
	-
	- smc_detach(dev);
	-
	- return (0);
	-}
	-
	static device_method_t smc_fdt_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, smc_fdt_probe),
	- DEVMETHOD(device_attach, smc_fdt_attach),
	- DEVMETHOD(device_detach, smc_fdt_detach),
	-
	- /* MII interface */
	- DEVMETHOD(miibus_readreg, smc_miibus_readreg),
	- DEVMETHOD(miibus_writereg, smc_miibus_writereg),
	- DEVMETHOD(miibus_statchg, smc_miibus_statchg),
	-
	{ 0, 0 }
	};

	-static driver_t smc_fdt_driver = {
	- "smc",
	- smc_fdt_methods,
	- sizeof(struct smc_softc),
	-};
	+DEFINE_CLASS_1(smc, smc_fdt_driver, smc_fdt_methods,
	+ sizeof(struct smc_softc), smc_driver);

	extern devclass_t smc_devclass;

	DRIVER_MODULE(smc, simplebus, smc_fdt_driver, smc_devclass, 0, 0);
	-DRIVER_MODULE(miibus, smc, miibus_driver, miibus_devclass, 0, 0);
	MODULE_DEPEND(smc, fdt, 1, 1, 1);
	MODULE_DEPEND(smc, ether, 1, 1, 1);
	MODULE_DEPEND(smc, miibus, 1, 1, 1);
	diff --git a/sys/dev/smc/if_smcvar.h b/sys/dev/smc/if_smcvar.h
	index a58eb0df0374..8de30706a0a5 100644
	--- a/sys/dev/smc/if_smcvar.h
	+++ b/sys/dev/smc/if_smcvar.h
	@@ -1,79 +1,81 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2008 Benno Rice. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	* $FreeBSD$
	*
	*/

	#ifndef _IF_SMCVAR_H_
	#define _IF_SMCVAR_H_

	struct smc_softc {
	struct ifnet *smc_ifp;
	device_t smc_dev;
	struct mtx smc_mtx;
	u_int smc_chip;
	u_int smc_rev;
	u_int smc_mask;

	/* Resources */
	int smc_usemem;
	int smc_reg_rid;
	int smc_irq_rid;
	struct resource *smc_reg;
	struct resource *smc_irq;
	void *smc_ih;

	/* Tasks */
	struct taskqueue *smc_tq;
	struct task smc_intr;
	struct task smc_rx;
	struct task smc_tx;
	struct mbuf *smc_pending;
	struct callout smc_watchdog;

	/* MII support */
	device_t smc_miibus;
	struct callout smc_mii_tick_ch;
	void (smc_mii_tick)(void );
	void (smc_mii_mediachg)(struct smc_softc );
	int (smc_mii_mediaioctl)(struct smc_softc ,
	struct ifreq *, u_long);

	/* DMA support */
	void (smc_read_packet)(struct smc_softc ,
	bus_addr_t, uint8_t *, bus_size_t);
	void *smc_read_arg;
	};

	+DECLARE_CLASS(smc_driver);
	+
	int smc_probe(device_t);
	int smc_attach(device_t);
	int smc_detach(device_t);

	int smc_miibus_readreg(device_t, int, int);
	int smc_miibus_writereg(device_t, int, int, int);
	void smc_miibus_statchg(device_t);

	#endif /* _IF_SMCVAR_H_ */
	diff --git a/sys/dev/uart/uart_dev_ti8250.c b/sys/dev/uart/uart_dev_ti8250.c
	index f5a230908da3..ebe777ed7fdb 100644
	--- a/sys/dev/uart/uart_dev_ti8250.c
	+++ b/sys/dev/uart/uart_dev_ti8250.c
	@@ -1,138 +1,129 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 Ian Lepore
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include "opt_platform.h"

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/conf.h>
	#include <sys/kernel.h>
	#include <sys/sysctl.h>
	#include <machine/bus.h>

	-#include <arm/ti/ti_prcm.h>
	-#include <arm/ti/ti_hwmods.h>
	-
	#include <dev/fdt/fdt_common.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#include <dev/uart/uart.h>
	#include <dev/uart/uart_cpu.h>
	#include <dev/uart/uart_cpu_fdt.h>
	#include <dev/uart/uart_bus.h>
	#include <dev/uart/uart_dev_ns8250.h>

	+#include <arm/ti/ti_sysc.h>
	+
	#include "uart_if.h"

	/*
	* High-level UART interface.
	*/
	struct ti8250_softc {
	struct ns8250_softc ns8250_base;
	/uint32_t mystuff;/
	};

	#define MDR1_REG 8
	#define MDR1_MODE_UART 0
	#define MDR1_MODE_DISABLE 7
	#define SYSCC_REG 15
	#define SYSCC_SOFTRESET (1 << 1)
	#define SYSS_REG 16
	#define SYSS_STATUS_RESETDONE (1 << 0)

	static int
	ti8250_bus_probe(struct uart_softc *sc)
	{
	int status;
	- clk_ident_t clkid;
	-
	- /* Enable clocks for this device. We can't continue if that fails. */
	- clkid = ti_hwmods_get_clock(sc->sc_dev);
	- if (clkid == INVALID_CLK_IDENT) {
	- device_printf(sc->sc_dev,
	- "failed to get clock based on hwmods\n");
	- clkid = UART1_CLK + device_get_unit(sc->sc_dev);
	- }
	- if ((status = ti_prcm_clk_enable(clkid)) != 0)
	+
	+ if ((status = ti_sysc_clock_enable(device_get_parent(sc->sc_dev))) != 0)
	return (status);

	/*
	* Set the hardware to disabled mode, do a full device reset, then set
	* it to uart mode. Most devices will be reset-and-disabled already,
	* but you never know what a bootloader might have done.
	*/
	uart_setreg(&sc->sc_bas, MDR1_REG, MDR1_MODE_DISABLE);
	uart_setreg(&sc->sc_bas, SYSCC_REG, SYSCC_SOFTRESET);
	while (uart_getreg(&sc->sc_bas, SYSS_REG) & SYSS_STATUS_RESETDONE)
	continue;
	uart_setreg(&sc->sc_bas, MDR1_REG, MDR1_MODE_UART);

	status = ns8250_bus_probe(sc);
	if (status == 0)
	device_set_desc(sc->sc_dev, "TI UART (16550 compatible)");

	return (status);
	}

	static kobj_method_t ti8250_methods[] = {
	KOBJMETHOD(uart_probe, ti8250_bus_probe),

	KOBJMETHOD(uart_attach, ns8250_bus_attach),
	KOBJMETHOD(uart_detach, ns8250_bus_detach),
	KOBJMETHOD(uart_flush, ns8250_bus_flush),
	KOBJMETHOD(uart_getsig, ns8250_bus_getsig),
	KOBJMETHOD(uart_ioctl, ns8250_bus_ioctl),
	KOBJMETHOD(uart_ipend, ns8250_bus_ipend),
	KOBJMETHOD(uart_param, ns8250_bus_param),
	KOBJMETHOD(uart_receive, ns8250_bus_receive),
	KOBJMETHOD(uart_setsig, ns8250_bus_setsig),
	KOBJMETHOD(uart_transmit, ns8250_bus_transmit),
	KOBJMETHOD_END
	};

	static struct uart_class uart_ti8250_class = {
	"ti8250",
	ti8250_methods,
	sizeof(struct ti8250_softc),
	.uc_ops = &uart_ns8250_ops,
	.uc_range = 0x88,
	.uc_rclk = 48000000,
	.uc_rshift = 2
	};
	static struct ofw_compat_data compat_data[] = {
	{"ti,ns16550", (uintptr_t)&uart_ti8250_class},
	{"ti,omap3-uart", (uintptr_t)&uart_ti8250_class},
	{"ti,omap4-uart", (uintptr_t)&uart_ti8250_class},
	{NULL, (uintptr_t)NULL},
	};
	UART_FDT_CLASS_AND_DEVICE(compat_data);
	diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c
	index aba35eb38ff5..05a632f526a8 100644
	--- a/sys/dev/virtio/pci/virtio_pci.c
	+++ b/sys/dev/virtio/pci/virtio_pci.c
	@@ -1,1356 +1,1347 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice unmodified, this list of conditions, and the following
	* disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	/* Driver for the VirtIO PCI interface. */

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/module.h>
	#include <sys/malloc.h>
	#include <sys/endian.h>

	#include <machine/bus.h>
	#include <machine/resource.h>
	#include <sys/bus.h>
	#include <sys/rman.h>

	#include <dev/pci/pcivar.h>
	#include <dev/pci/pcireg.h>

	#include <dev/virtio/virtio.h>
	#include <dev/virtio/virtqueue.h>
	#include <dev/virtio/pci/virtio_pci.h>

	#include "virtio_bus_if.h"
	#include "virtio_if.h"

	struct vtpci_interrupt {
	struct resource *vti_irq;
	int vti_rid;
	void *vti_handler;
	};

	struct vtpci_virtqueue {
	struct virtqueue *vtv_vq;
	int vtv_no_intr;
	};

	struct vtpci_softc {
	device_t vtpci_dev;
	struct resource *vtpci_res;
	struct resource *vtpci_msix_res;
	uint64_t vtpci_features;
	uint32_t vtpci_flags;
	#define VTPCI_FLAG_NO_MSI 0x0001
	#define VTPCI_FLAG_NO_MSIX 0x0002
	#define VTPCI_FLAG_LEGACY 0x1000
	#define VTPCI_FLAG_MSI 0x2000
	#define VTPCI_FLAG_MSIX 0x4000
	#define VTPCI_FLAG_SHARED_MSIX 0x8000
	#define VTPCI_FLAG_ITYPE_MASK 0xF000

	/* This "bus" will only ever have one child. */
	device_t vtpci_child_dev;
	struct virtio_feature_desc *vtpci_child_feat_desc;

	int vtpci_nvqs;
	struct vtpci_virtqueue *vtpci_vqs;

	/*
	* Ideally, each virtqueue that the driver provides a callback for will
	* receive its own MSIX vector. If there are not sufficient vectors
	* available, then attempt to have all the VQs share one vector. For
	* MSIX, the configuration changed notifications must be on their own
	* vector.
	*
	* If MSIX is not available, we will attempt to have the whole device
	* share one MSI vector, and then, finally, one legacy interrupt.
	*/
	struct vtpci_interrupt vtpci_device_interrupt;
	struct vtpci_interrupt *vtpci_msix_vq_interrupts;
	int vtpci_nmsix_resources;
	};

	static int vtpci_probe(device_t);
	static int vtpci_attach(device_t);
	static int vtpci_detach(device_t);
	static int vtpci_suspend(device_t);
	static int vtpci_resume(device_t);
	static int vtpci_shutdown(device_t);
	static void vtpci_driver_added(device_t, driver_t *);
	static void vtpci_child_detached(device_t, device_t);
	static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
	static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);

	static uint64_t vtpci_negotiate_features(device_t, uint64_t);
	static int vtpci_with_feature(device_t, uint64_t);
	static int vtpci_alloc_virtqueues(device_t, int, int,
	struct vq_alloc_info *);
	static int vtpci_setup_intr(device_t, enum intr_type);
	static void vtpci_stop(device_t);
	static int vtpci_reinit(device_t, uint64_t);
	static void vtpci_reinit_complete(device_t);
	static void vtpci_notify_virtqueue(device_t, uint16_t);
	static uint8_t vtpci_get_status(device_t);
	static void vtpci_set_status(device_t, uint8_t);
	static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
	static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);

	static void vtpci_describe_features(struct vtpci_softc , const char ,
	uint64_t);
	static void vtpci_probe_and_attach_child(struct vtpci_softc *);

	static int vtpci_alloc_msix(struct vtpci_softc *, int);
	static int vtpci_alloc_msi(struct vtpci_softc *);
	static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
	static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
	static int vtpci_alloc_intr_msi(struct vtpci_softc *);
	static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
	static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
	struct vtpci_interrupt *);
	static int vtpci_alloc_intr_resources(struct vtpci_softc *);

	static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
	enum intr_type);
	static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
	enum intr_type);
	static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
	enum intr_type);
	static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);

	static int vtpci_register_msix_vector(struct vtpci_softc *, int,
	struct vtpci_interrupt *);
	static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
	static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);

	static void vtpci_free_interrupt(struct vtpci_softc *,
	struct vtpci_interrupt *);
	static void vtpci_free_interrupts(struct vtpci_softc *);
	static void vtpci_free_virtqueues(struct vtpci_softc *);
	static void vtpci_release_child_resources(struct vtpci_softc *);
	static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
	static void vtpci_reset(struct vtpci_softc *);

	static void vtpci_select_virtqueue(struct vtpci_softc *, int);

	static void vtpci_legacy_intr(void *);
	static int vtpci_vq_shared_intr_filter(void *);
	static void vtpci_vq_shared_intr(void *);
	static int vtpci_vq_intr_filter(void *);
	static void vtpci_vq_intr(void *);
	static void vtpci_config_intr(void *);

	#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt

	#define VIRTIO_PCI_CONFIG(_sc) \
	VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)

	/*
	* I/O port read/write wrappers.
	*/
	#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
	-#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
	-#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
	#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
	-#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
	-#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))

	/*
	- * Legacy VirtIO header is always PCI endianness (little), so if we
	- * are in a BE machine we need to swap bytes from LE to BE when reading
	- * and from BE to LE when writing.
	- * If we are in a LE machine, there will be no swaps.
	+ * Virtio-pci specifies that PCI Configuration area is guest endian. However,
	+ * since PCI devices are inherently little-endian, on BE systems the bus layer
	+ * transparently converts it to BE. For virtio-legacy, this conversion is
	+ * undesired, so an extra byte swap is required to fix it.
	*/
	-#define vtpci_read_header_2(sc, o) le16toh(vtpci_read_config_2(sc, o))
	-#define vtpci_read_header_4(sc, o) le32toh(vtpci_read_config_4(sc, o))
	-#define vtpci_write_header_2(sc, o, v) vtpci_write_config_2(sc, o, (htole16(v)))
	-#define vtpci_write_header_4(sc, o, v) vtpci_write_config_4(sc, o, (htole32(v)))
	+#define vtpci_read_config_2(sc, o) le16toh(bus_read_2((sc)->vtpci_res, (o)))
	+#define vtpci_read_config_4(sc, o) le32toh(bus_read_4((sc)->vtpci_res, (o)))
	+#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (htole16(v)))
	+#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (htole32(v)))
	+
	+/* PCI Header LE. On BE systems the bus layer takes care of byte swapping */
	+#define vtpci_read_header_2(sc, o) (bus_read_2((sc)->vtpci_res, (o)))
	+#define vtpci_read_header_4(sc, o) (bus_read_4((sc)->vtpci_res, (o)))
	+#define vtpci_write_header_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
	+#define vtpci_write_header_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))

	/* Tunables. */
	static int vtpci_disable_msix = 0;
	TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);

	static device_method_t vtpci_methods[] = {
	/* Device interface. */
	DEVMETHOD(device_probe, vtpci_probe),
	DEVMETHOD(device_attach, vtpci_attach),
	DEVMETHOD(device_detach, vtpci_detach),
	DEVMETHOD(device_suspend, vtpci_suspend),
	DEVMETHOD(device_resume, vtpci_resume),
	DEVMETHOD(device_shutdown, vtpci_shutdown),

	/* Bus interface. */
	DEVMETHOD(bus_driver_added, vtpci_driver_added),
	DEVMETHOD(bus_child_detached, vtpci_child_detached),
	DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str),
	DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
	DEVMETHOD(bus_write_ivar, vtpci_write_ivar),

	/* VirtIO bus interface. */
	DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
	DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
	DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
	DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
	DEVMETHOD(virtio_bus_stop, vtpci_stop),
	DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
	DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
	DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
	DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),

	DEVMETHOD_END
	};

	static driver_t vtpci_driver = {
	"virtio_pci",
	vtpci_methods,
	sizeof(struct vtpci_softc)
	};

	devclass_t vtpci_devclass;

	DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
	MODULE_VERSION(virtio_pci, 1);
	MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
	MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);

	static int
	vtpci_probe(device_t dev)
	{
	char desc[36];
	const char *name;

	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
	return (ENXIO);

	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN \|\|
	pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
	return (ENXIO);

	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
	return (ENXIO);

	name = virtio_device_name(pci_get_subdevice(dev));
	if (name == NULL)
	name = "Unknown";

	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
	device_set_desc_copy(dev, desc);

	return (BUS_PROBE_DEFAULT);
	}

	static int
	vtpci_attach(device_t dev)
	{
	struct vtpci_softc *sc;
	device_t child;
	int rid;

	sc = device_get_softc(dev);
	sc->vtpci_dev = dev;

	pci_enable_busmaster(dev);

	rid = PCIR_BAR(0);
	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
	RF_ACTIVE);
	if (sc->vtpci_res == NULL) {
	device_printf(dev, "cannot map I/O space\n");
	return (ENXIO);
	}

	-/*
	- * For legacy VirtIO, the device-specific configuration is guest
	- * endian, while the common configuration header is always
	- * PCI (little) endian and will be handled specifically in
	- * other parts of this file via functions
	- * 'vtpci_[read\|write]_header_[2\|4]'
	- */
	-#if _BYTE_ORDER == _BIG_ENDIAN
	- rman_set_bustag(sc->vtpci_res, &bs_be_tag);
	-#endif
	-
	if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
	sc->vtpci_flags \|= VTPCI_FLAG_NO_MSI;

	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
	rid = PCIR_BAR(1);
	sc->vtpci_msix_res = bus_alloc_resource_any(dev,
	SYS_RES_MEMORY, &rid, RF_ACTIVE);
	}

	if (sc->vtpci_msix_res == NULL)
	sc->vtpci_flags \|= VTPCI_FLAG_NO_MSIX;

	vtpci_reset(sc);

	/* Tell the host we've noticed this device. */
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);

	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
	device_printf(dev, "cannot create child device\n");
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
	vtpci_detach(dev);
	return (ENOMEM);
	}

	sc->vtpci_child_dev = child;
	vtpci_probe_and_attach_child(sc);

	return (0);
	}

	static int
	vtpci_detach(device_t dev)
	{
	struct vtpci_softc *sc;
	device_t child;
	int error;

	sc = device_get_softc(dev);

	if ((child = sc->vtpci_child_dev) != NULL) {
	error = device_delete_child(dev, child);
	if (error)
	return (error);
	sc->vtpci_child_dev = NULL;
	}

	vtpci_reset(sc);

	if (sc->vtpci_msix_res != NULL) {
	bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
	sc->vtpci_msix_res);
	sc->vtpci_msix_res = NULL;
	}

	if (sc->vtpci_res != NULL) {
	bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
	sc->vtpci_res);
	sc->vtpci_res = NULL;
	}

	return (0);
	}

	static int
	vtpci_suspend(device_t dev)
	{

	return (bus_generic_suspend(dev));
	}

	static int
	vtpci_resume(device_t dev)
	{

	return (bus_generic_resume(dev));
	}

	static int
	vtpci_shutdown(device_t dev)
	{

	(void) bus_generic_shutdown(dev);
	/* Forcibly stop the host device. */
	vtpci_stop(dev);

	return (0);
	}

	static void
	vtpci_driver_added(device_t dev, driver_t *driver)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	vtpci_probe_and_attach_child(sc);
	}

	static void
	vtpci_child_detached(device_t dev, device_t child)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	vtpci_reset(sc);
	vtpci_release_child_resources(sc);
	}

	static int
	vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	if (sc->vtpci_child_dev != child)
	return (ENOENT);

	switch (index) {
	case VIRTIO_IVAR_DEVTYPE:
	case VIRTIO_IVAR_SUBDEVICE:
	*result = pci_get_subdevice(dev);
	break;
	case VIRTIO_IVAR_VENDOR:
	*result = pci_get_vendor(dev);
	break;
	case VIRTIO_IVAR_DEVICE:
	*result = pci_get_device(dev);
	break;
	case VIRTIO_IVAR_SUBVENDOR:
	*result = pci_get_subvendor(dev);
	break;
	default:
	return (ENOENT);
	}

	return (0);
	}

	static int
	vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	if (sc->vtpci_child_dev != child)
	return (ENOENT);

	switch (index) {
	case VIRTIO_IVAR_FEATURE_DESC:
	sc->vtpci_child_feat_desc = (void *) value;
	break;
	default:
	return (ENOENT);
	}

	return (0);
	}

	static uint64_t
	vtpci_negotiate_features(device_t dev, uint64_t child_features)
	{
	struct vtpci_softc *sc;
	uint64_t host_features, features;

	sc = device_get_softc(dev);

	host_features = vtpci_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES);
	vtpci_describe_features(sc, "host", host_features);

	/*
	* Limit negotiated features to what the driver, virtqueue, and
	* host all support.
	*/
	features = host_features & child_features;
	features = virtqueue_filter_features(features);
	sc->vtpci_features = features;

	vtpci_describe_features(sc, "negotiated", features);
	vtpci_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);

	return (features);
	}

	static int
	vtpci_with_feature(device_t dev, uint64_t feature)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	return ((sc->vtpci_features & feature) != 0);
	}

	static int
	vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
	struct vq_alloc_info *vq_info)
	{
	struct vtpci_softc *sc;
	struct virtqueue *vq;
	struct vtpci_virtqueue *vqx;
	struct vq_alloc_info *info;
	int idx, error;
	uint16_t size;

	sc = device_get_softc(dev);

	if (sc->vtpci_nvqs != 0)
	return (EALREADY);
	if (nvqs <= 0)
	return (EINVAL);

	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
	M_DEVBUF, M_NOWAIT \| M_ZERO);
	if (sc->vtpci_vqs == NULL)
	return (ENOMEM);

	for (idx = 0; idx < nvqs; idx++) {
	vqx = &sc->vtpci_vqs[idx];
	info = &vq_info[idx];

	vtpci_select_virtqueue(sc, idx);
	size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM);

	error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
	~(vm_paddr_t)0, info, &vq);
	if (error) {
	device_printf(dev,
	"cannot allocate virtqueue %d: %d\n", idx, error);
	break;
	}

	vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN,
	virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);

	vqx->vtv_vq = *info->vqai_vq = vq;
	vqx->vtv_no_intr = info->vqai_intr == NULL;

	sc->vtpci_nvqs++;
	}

	if (error)
	vtpci_free_virtqueues(sc);

	return (error);
	}

	static int
	vtpci_setup_intr(device_t dev, enum intr_type type)
	{
	struct vtpci_softc *sc;
	int attempt, error;

	sc = device_get_softc(dev);

	for (attempt = 0; attempt < 5; attempt++) {
	/*
	* Start with the most desirable interrupt configuration and
	* fallback towards less desirable ones.
	*/
	switch (attempt) {
	case 0:
	error = vtpci_alloc_intr_msix_pervq(sc);
	break;
	case 1:
	error = vtpci_alloc_intr_msix_shared(sc);
	break;
	case 2:
	error = vtpci_alloc_intr_msi(sc);
	break;
	case 3:
	error = vtpci_alloc_intr_legacy(sc);
	break;
	default:
	device_printf(dev,
	"exhausted all interrupt allocation attempts\n");
	return (ENXIO);
	}

	if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
	break;

	vtpci_cleanup_setup_intr_attempt(sc);
	}

	if (bootverbose) {
	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
	device_printf(dev, "using legacy interrupt\n");
	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
	device_printf(dev, "using MSI interrupt\n");
	else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
	device_printf(dev, "using shared MSIX interrupts\n");
	else
	device_printf(dev, "using per VQ MSIX interrupts\n");
	}

	return (0);
	}

	static void
	vtpci_stop(device_t dev)
	{

	vtpci_reset(device_get_softc(dev));
	}

	static int
	vtpci_reinit(device_t dev, uint64_t features)
	{
	struct vtpci_softc *sc;
	int idx, error;

	sc = device_get_softc(dev);

	/*
	* Redrive the device initialization. This is a bit of an abuse of
	* the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
	* play nice.
	*
	* We do not allow the host device to change from what was originally
	* negotiated beyond what the guest driver changed. MSIX state should
	* not change, number of virtqueues and their size remain the same, etc.
	* This will need to be rethought when we want to support migration.
	*/

	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
	vtpci_stop(dev);

	/*
	* Quickly drive the status through ACK and DRIVER. The device
	* does not become usable again until vtpci_reinit_complete().
	*/
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);

	vtpci_negotiate_features(dev, features);

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
	error = vtpci_reinit_virtqueue(sc, idx);
	if (error)
	return (error);
	}

	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
	error = vtpci_set_host_msix_vectors(sc);
	if (error)
	return (error);
	}

	return (0);
	}

	static void
	vtpci_reinit_complete(device_t dev)
	{

	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
	}

	static void
	vtpci_notify_virtqueue(device_t dev, uint16_t queue)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
	}

	static uint8_t
	vtpci_get_status(device_t dev)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
	}

	static void
	vtpci_set_status(device_t dev, uint8_t status)
	{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	if (status != VIRTIO_CONFIG_STATUS_RESET)
	status \|= vtpci_get_status(dev);

	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
	}

	static void
	vtpci_read_dev_config(device_t dev, bus_size_t offset,
	void *dst, int length)
	{
	struct vtpci_softc *sc;
	bus_size_t off;
	uint8_t *d;
	int size;

	sc = device_get_softc(dev);
	off = VIRTIO_PCI_CONFIG(sc) + offset;

	for (d = dst; length > 0; d += size, off += size, length -= size) {
	if (length >= 4) {
	size = 4;
	(uint32_t )d = vtpci_read_config_4(sc, off);
	} else if (length >= 2) {
	size = 2;
	(uint16_t )d = vtpci_read_config_2(sc, off);
	} else {
	size = 1;
	*d = vtpci_read_config_1(sc, off);
	}
	}
	}

	static void
	vtpci_write_dev_config(device_t dev, bus_size_t offset,
	void *src, int length)
	{
	struct vtpci_softc *sc;
	bus_size_t off;
	uint8_t *s;
	int size;

	sc = device_get_softc(dev);
	off = VIRTIO_PCI_CONFIG(sc) + offset;

	for (s = src; length > 0; s += size, off += size, length -= size) {
	if (length >= 4) {
	size = 4;
	vtpci_write_config_4(sc, off, (uint32_t )s);
	} else if (length >= 2) {
	size = 2;
	vtpci_write_config_2(sc, off, (uint16_t )s);
	} else {
	size = 1;
	vtpci_write_config_1(sc, off, *s);
	}
	}
	}

	static void
	vtpci_describe_features(struct vtpci_softc sc, const char msg,
	uint64_t features)
	{
	device_t dev, child;

	dev = sc->vtpci_dev;
	child = sc->vtpci_child_dev;

	if (device_is_attached(child) \|\| bootverbose == 0)
	return;

	virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
	}

	static void
	vtpci_probe_and_attach_child(struct vtpci_softc *sc)
	{
	device_t dev, child;

	dev = sc->vtpci_dev;
	child = sc->vtpci_child_dev;

	if (child == NULL)
	return;

	if (device_get_state(child) != DS_NOTPRESENT)
	return;

	if (device_probe(child) != 0)
	return;

	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
	if (device_attach(child) != 0) {
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
	vtpci_reset(sc);
	vtpci_release_child_resources(sc);
	/* Reset status for future attempt. */
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
	} else {
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
	VIRTIO_ATTACH_COMPLETED(child);
	}
	}

	static int
	vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
	{
	device_t dev;
	int nmsix, cnt, required;

	dev = sc->vtpci_dev;

	/* Allocate an additional vector for the config changes. */
	required = nvectors + 1;

	nmsix = pci_msix_count(dev);
	if (nmsix < required)
	return (1);

	cnt = required;
	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
	sc->vtpci_nmsix_resources = required;
	return (0);
	}

	pci_release_msi(dev);

	return (1);
	}

	static int
	vtpci_alloc_msi(struct vtpci_softc *sc)
	{
	device_t dev;
	int nmsi, cnt, required;

	dev = sc->vtpci_dev;
	required = 1;

	nmsi = pci_msi_count(dev);
	if (nmsi < required)
	return (1);

	cnt = required;
	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
	return (0);

	pci_release_msi(dev);

	return (1);
	}

	static int
	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
	{
	int i, nvectors, error;

	if (vtpci_disable_msix != 0 \|\|
	sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
	return (ENOTSUP);

	for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
	if (sc->vtpci_vqs[i].vtv_no_intr == 0)
	nvectors++;
	}

	error = vtpci_alloc_msix(sc, nvectors);
	if (error)
	return (error);

	sc->vtpci_flags \|= VTPCI_FLAG_MSIX;

	return (0);
	}

	static int
	vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
	{
	int error;

	if (vtpci_disable_msix != 0 \|\|
	sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
	return (ENOTSUP);

	error = vtpci_alloc_msix(sc, 1);
	if (error)
	return (error);

	sc->vtpci_flags \|= VTPCI_FLAG_MSIX \| VTPCI_FLAG_SHARED_MSIX;

	return (0);
	}

	static int
	vtpci_alloc_intr_msi(struct vtpci_softc *sc)
	{
	int error;

	/* Only BHyVe supports MSI. */
	if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
	return (ENOTSUP);

	error = vtpci_alloc_msi(sc);
	if (error)
	return (error);

	sc->vtpci_flags \|= VTPCI_FLAG_MSI;

	return (0);
	}

	static int
	vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
	{

	sc->vtpci_flags \|= VTPCI_FLAG_LEGACY;

	return (0);
	}

	static int
	vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
	struct vtpci_interrupt *intr)
	{
	struct resource *irq;

	irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
	if (irq == NULL)
	return (ENXIO);

	intr->vti_irq = irq;
	intr->vti_rid = rid;

	return (0);
	}

	static int
	vtpci_alloc_intr_resources(struct vtpci_softc *sc)
	{
	struct vtpci_interrupt *intr;
	int i, rid, flags, nvq_intrs, error;

	rid = 0;
	flags = RF_ACTIVE;

	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
	flags \|= RF_SHAREABLE;
	else
	rid = 1;

	/*
	* For legacy and MSI interrupts, this single resource handles all
	* interrupts. For MSIX, this resource is used for the configuration
	* changed interrupt.
	*/
	intr = &sc->vtpci_device_interrupt;
	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
	if (error \|\| sc->vtpci_flags & (VTPCI_FLAG_LEGACY \| VTPCI_FLAG_MSI))
	return (error);

	/* Subtract one for the configuration changed interrupt. */
	nvq_intrs = sc->vtpci_nmsix_resources - 1;

	intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
	sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT \| M_ZERO);
	if (sc->vtpci_msix_vq_interrupts == NULL)
	return (ENOMEM);

	for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
	if (error)
	return (error);
	}

	return (0);
	}

	static int
	vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
	{
	struct vtpci_interrupt *intr;
	int error;

	intr = &sc->vtpci_device_interrupt;
	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
	vtpci_legacy_intr, sc, &intr->vti_handler);

	return (error);
	}

	static int
	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
	{
	struct vtpci_virtqueue *vqx;
	struct vtpci_interrupt *intr;
	int i, error;

	intr = sc->vtpci_msix_vq_interrupts;

	for (i = 0; i < sc->vtpci_nvqs; i++) {
	vqx = &sc->vtpci_vqs[i];

	if (vqx->vtv_no_intr)
	continue;

	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
	vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
	&intr->vti_handler);
	if (error)
	return (error);

	intr++;
	}

	return (0);
	}

	static int
	vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
	{
	device_t dev;
	struct vtpci_interrupt *intr;
	int error;

	dev = sc->vtpci_dev;
	intr = &sc->vtpci_device_interrupt;

	error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
	vtpci_config_intr, sc, &intr->vti_handler);
	if (error)
	return (error);

	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
	intr = sc->vtpci_msix_vq_interrupts;
	error = bus_setup_intr(dev, intr->vti_irq, type,
	vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
	&intr->vti_handler);
	} else
	error = vtpci_setup_pervq_msix_interrupts(sc, type);

	return (error ? error : vtpci_set_host_msix_vectors(sc));
	}

	static int
	vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
	{
	int error;

	type \|= INTR_MPSAFE;
	KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
	("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));

	error = vtpci_alloc_intr_resources(sc);
	if (error)
	return (error);

	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
	error = vtpci_setup_legacy_interrupt(sc, type);
	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
	error = vtpci_setup_msi_interrupt(sc, type);
	else
	error = vtpci_setup_msix_interrupts(sc, type);

	return (error);
	}

	static int
	vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
	struct vtpci_interrupt *intr)
	{
	device_t dev;
	uint16_t vector;

	dev = sc->vtpci_dev;

	if (intr != NULL) {
	/* Map from guest rid to host vector. */
	vector = intr->vti_rid - 1;
	} else
	vector = VIRTIO_MSI_NO_VECTOR;

	vtpci_write_header_2(sc, offset, vector);

	/* Read vector to determine if the host had sufficient resources. */
	if (vtpci_read_header_2(sc, offset) != vector) {
	device_printf(dev,
	"insufficient host resources for MSIX interrupts\n");
	return (ENODEV);
	}

	return (0);
	}

	static int
	vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
	{
	struct vtpci_interrupt intr, tintr;
	int idx, offset, error;

	intr = &sc->vtpci_device_interrupt;
	offset = VIRTIO_MSI_CONFIG_VECTOR;

	error = vtpci_register_msix_vector(sc, offset, intr);
	if (error)
	return (error);

	intr = sc->vtpci_msix_vq_interrupts;
	offset = VIRTIO_MSI_QUEUE_VECTOR;

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
	vtpci_select_virtqueue(sc, idx);

	if (sc->vtpci_vqs[idx].vtv_no_intr)
	tintr = NULL;
	else
	tintr = intr;

	error = vtpci_register_msix_vector(sc, offset, tintr);
	if (error)
	break;

	/*
	* For shared MSIX, all the virtqueues share the first
	* interrupt.
	*/
	if (!sc->vtpci_vqs[idx].vtv_no_intr &&
	(sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
	intr++;
	}

	return (error);
	}

	static int
	vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
	{
	struct vtpci_virtqueue *vqx;
	struct virtqueue *vq;
	int error;
	uint16_t size;

	vqx = &sc->vtpci_vqs[idx];
	vq = vqx->vtv_vq;

	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));

	vtpci_select_virtqueue(sc, idx);
	size = vtpci_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM);

	error = virtqueue_reinit(vq, size);
	if (error)
	return (error);

	vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN,
	virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);

	return (0);
	}

	static void
	vtpci_free_interrupt(struct vtpci_softc sc, struct vtpci_interrupt intr)
	{
	device_t dev;

	dev = sc->vtpci_dev;

	if (intr->vti_handler != NULL) {
	bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
	intr->vti_handler = NULL;
	}

	if (intr->vti_irq != NULL) {
	bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
	intr->vti_irq);
	intr->vti_irq = NULL;
	intr->vti_rid = -1;
	}
	}

	static void
	vtpci_free_interrupts(struct vtpci_softc *sc)
	{
	struct vtpci_interrupt *intr;
	int i, nvq_intrs;

	vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);

	if (sc->vtpci_nmsix_resources != 0) {
	nvq_intrs = sc->vtpci_nmsix_resources - 1;
	sc->vtpci_nmsix_resources = 0;

	intr = sc->vtpci_msix_vq_interrupts;
	if (intr != NULL) {
	for (i = 0; i < nvq_intrs; i++, intr++)
	vtpci_free_interrupt(sc, intr);

	free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
	sc->vtpci_msix_vq_interrupts = NULL;
	}
	}

	if (sc->vtpci_flags & (VTPCI_FLAG_MSI \| VTPCI_FLAG_MSIX))
	pci_release_msi(sc->vtpci_dev);

	sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
	}

	static void
	vtpci_free_virtqueues(struct vtpci_softc *sc)
	{
	struct vtpci_virtqueue *vqx;
	int idx;

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
	vqx = &sc->vtpci_vqs[idx];

	vtpci_select_virtqueue(sc, idx);
	vtpci_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);

	virtqueue_free(vqx->vtv_vq);
	vqx->vtv_vq = NULL;
	}

	free(sc->vtpci_vqs, M_DEVBUF);
	sc->vtpci_vqs = NULL;
	sc->vtpci_nvqs = 0;
	}

	static void
	vtpci_release_child_resources(struct vtpci_softc *sc)
	{

	vtpci_free_interrupts(sc);
	vtpci_free_virtqueues(sc);
	}

	static void
	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
	{
	int idx;

	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
	vtpci_write_header_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
	VIRTIO_MSI_NO_VECTOR);

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
	vtpci_select_virtqueue(sc, idx);
	vtpci_write_header_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
	VIRTIO_MSI_NO_VECTOR);
	}
	}

	vtpci_free_interrupts(sc);
	}

	static void
	vtpci_reset(struct vtpci_softc *sc)
	{

	/*
	* Setting the status to RESET sets the host device to
	* the original, uninitialized state.
	*/
	vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
	}

	static void
	vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
	{

	vtpci_write_header_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
	}

	static void
	vtpci_legacy_intr(void *xsc)
	{
	struct vtpci_softc *sc;
	struct vtpci_virtqueue *vqx;
	int i;
	uint8_t isr;

	sc = xsc;
	vqx = &sc->vtpci_vqs[0];

	/* Reading the ISR also clears it. */
	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);

	if (isr & VIRTIO_PCI_ISR_CONFIG)
	vtpci_config_intr(sc);

	if (isr & VIRTIO_PCI_ISR_INTR) {
	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
	if (vqx->vtv_no_intr == 0)
	virtqueue_intr(vqx->vtv_vq);
	}
	}
	}

	static int
	vtpci_vq_shared_intr_filter(void *xsc)
	{
	struct vtpci_softc *sc;
	struct vtpci_virtqueue *vqx;
	int i, rc;

	rc = 0;
	sc = xsc;
	vqx = &sc->vtpci_vqs[0];

	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
	if (vqx->vtv_no_intr == 0)
	rc \|= virtqueue_intr_filter(vqx->vtv_vq);
	}

	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
	}

	static void
	vtpci_vq_shared_intr(void *xsc)
	{
	struct vtpci_softc *sc;
	struct vtpci_virtqueue *vqx;
	int i;

	sc = xsc;
	vqx = &sc->vtpci_vqs[0];

	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
	if (vqx->vtv_no_intr == 0)
	virtqueue_intr(vqx->vtv_vq);
	}
	}

	static int
	vtpci_vq_intr_filter(void *xvq)
	{
	struct virtqueue *vq;
	int rc;

	vq = xvq;
	rc = virtqueue_intr_filter(vq);

	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
	}

	static void
	vtpci_vq_intr(void *xvq)
	{
	struct virtqueue *vq;

	vq = xvq;
	virtqueue_intr(vq);
	}

	static void
	vtpci_config_intr(void *xsc)
	{
	struct vtpci_softc *sc;
	device_t child;

	sc = xsc;
	child = sc->vtpci_child_dev;

	if (child != NULL)
	VIRTIO_CONFIG_CHANGE(child);
	}
	diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
	index 51baee32616e..d9e03cf7b791 100644
	--- a/sys/fs/nfs/nfs_commonsubs.c
	+++ b/sys/fs/nfs/nfs_commonsubs.c
	@@ -1,4916 +1,4927 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* Rick Macklem at The University of Guelph.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	/*
	* These functions support the macros and help fiddle mbuf chains for
	* the nfs op functions. They do things like create the rpc header and
	* copy data between mbuf chains and uio lists.
	*/
	#include "opt_inet.h"
	#include "opt_inet6.h"

	#include <fs/nfs/nfsport.h>

	#include <sys/extattr.h>

	#include <security/mac/mac_framework.h>

	#include <vm/vm_param.h>

	/*
	* Data items converted to xdr at startup, since they are constant
	* This is kinda hokey, but may save a little time doing byte swaps
	*/
	u_int32_t newnfs_true, newnfs_false, newnfs_xdrneg1;

	/* And other global data */
	nfstype nfsv34_type[9] = { NFNON, NFREG, NFDIR, NFBLK, NFCHR, NFLNK, NFSOCK,
	NFFIFO, NFNON };
	enum vtype newnv2tov_type[8] = { VNON, VREG, VDIR, VBLK, VCHR, VLNK, VNON, VNON };
	enum vtype nv34tov_type[8]={ VNON, VREG, VDIR, VBLK, VCHR, VLNK, VSOCK, VFIFO };
	struct timeval nfsboottime; /* Copy boottime once, so it never changes */
	int nfscl_ticks;
	int nfsrv_useacl = 1;
	struct nfssockreq nfsrv_nfsuserdsock;
	nfsuserd_state nfsrv_nfsuserd = NOTRUNNING;
	static int nfsrv_userdupcalls = 0;
	struct nfsreqhead nfsd_reqq;
	uid_t nfsrv_defaultuid = UID_NOBODY;
	gid_t nfsrv_defaultgid = GID_NOGROUP;
	int nfsrv_lease = NFSRV_LEASE;
	int ncl_mbuf_mlen = MLEN;
	int nfsd_enable_stringtouid = 0;
	int nfsrv_doflexfile = 0;
	static int nfs_enable_uidtostring = 0;
	NFSNAMEIDMUTEX;
	NFSSOCKMUTEX;
	extern int nfsrv_lughashsize;
	extern struct mtx nfsrv_dslock_mtx;
	extern volatile int nfsrv_devidcnt;
	extern int nfscl_debuglevel;
	extern struct nfsdevicehead nfsrv_devidhead;
	extern struct nfsstatsv1 nfsstatsv1;

	SYSCTL_DECL(_vfs_nfs);
	SYSCTL_INT(_vfs_nfs, OID_AUTO, enable_uidtostring, CTLFLAG_RW,
	&nfs_enable_uidtostring, 0, "Make nfs always send numeric owner_names");

	int nfsrv_maxpnfsmirror = 1;
	SYSCTL_INT(_vfs_nfs, OID_AUTO, pnfsmirror, CTLFLAG_RD,
	&nfsrv_maxpnfsmirror, 0, "Mirror level for pNFS service");

	int nfs_maxcopyrange = 10 * 1024 * 1024;
	SYSCTL_INT(_vfs_nfs, OID_AUTO, maxcopyrange, CTLFLAG_RW,
	&nfs_maxcopyrange, 0, "Max size of a Copy so RPC times reasonable");

	/*
	* This array of structures indicates, for V4:
	* retfh - which of 3 types of calling args are used
	* 0 - doesn't change cfh or use a sfh
	* 1 - replaces cfh with a new one (unless it returns an error status)
	* 2 - uses cfh and sfh
	* needscfh - if the op wants a cfh and premtime
	* 0 - doesn't use a cfh
	* 1 - uses a cfh, but doesn't want pre-op attributes
	* 2 - uses a cfh and wants pre-op attributes
	* savereply - indicates a non-idempotent Op
	* 0 - not non-idempotent
	* 1 - non-idempotent
	* Ops that are ordered via seqid# are handled separately from these
	* non-idempotent Ops.
	* Define it here, since it is used by both the client and server.
	*/
	struct nfsv4_opflag nfsv4_opflag[NFSV42_NOPS] = {
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* undef */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* undef */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* undef */
	{ 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Access */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Close */
	{ 0, 2, 0, 1, LK_EXCLUSIVE, 1, 1 }, /* Commit */
	{ 1, 2, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Create */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Delegpurge */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Delegreturn */
	{ 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Getattr */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* GetFH */
	{ 2, 1, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Link */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Lock */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* LockT */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* LockU */
	{ 1, 2, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Lookup */
	{ 1, 2, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Lookupp */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* NVerify */
	{ 1, 1, 0, 1, LK_EXCLUSIVE, 1, 0 }, /* Open */
	{ 1, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* OpenAttr */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* OpenConfirm */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* OpenDowngrade */
	{ 1, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* PutFH */
	{ 1, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* PutPubFH */
	{ 1, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* PutRootFH */
	{ 0, 1, 0, 0, LK_SHARED, 1, 0 }, /* Read */
	{ 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Readdir */
	{ 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* ReadLink */
	{ 0, 2, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Remove */
	{ 2, 1, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Rename */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Renew */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* RestoreFH */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* SaveFH */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* SecInfo */
	{ 0, 2, 1, 1, LK_EXCLUSIVE, 1, 0 }, /* Setattr */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* SetClientID */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* SetClientIDConfirm */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Verify */
	{ 0, 2, 1, 1, LK_EXCLUSIVE, 1, 0 }, /* Write */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* ReleaseLockOwner */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Backchannel Ctrl */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0, 0 }, /* Bind Conn to Sess */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0, 0 }, /* Exchange ID */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0, 0 }, /* Create Session */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0, 0 }, /* Destroy Session */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Free StateID */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Get Dir Deleg */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Get Device Info */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Get Device List */
	{ 0, 1, 0, 1, LK_EXCLUSIVE, 1, 1 }, /* Layout Commit */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Layout Get */
	{ 0, 1, 0, 1, LK_EXCLUSIVE, 1, 0 }, /* Layout Return */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Secinfo No name */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Sequence */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Set SSV */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Test StateID */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Want Delegation */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0, 0 }, /* Destroy ClientID */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Reclaim Complete */
	{ 0, 1, 1, 1, LK_EXCLUSIVE, 1, 0 }, /* Allocate */
	{ 2, 1, 1, 0, LK_SHARED, 1, 0 }, /* Copy */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Copy Notify */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Deallocate */
	{ 0, 1, 0, 0, LK_SHARED, 1, 0 }, /* IO Advise */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Layout Error */
	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1, 0 }, /* Layout Stats */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Offload Cancel */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Offload Status */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Read Plus */
	{ 0, 1, 0, 0, LK_SHARED, 1, 0 }, /* Seek */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Write Same */
	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1, 1 }, /* Clone */
	{ 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Getxattr */
	{ 0, 1, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Setxattr */
	{ 0, 1, 0, 0, LK_SHARED, 1, 1 }, /* Listxattrs */
	{ 0, 1, 1, 1, LK_EXCLUSIVE, 1, 1 }, /* Removexattr */
	};

	static int ncl_mbuf_mhlen = MHLEN;
	static int nfsrv_usercnt = 0;
	static int nfsrv_dnsnamelen;
	static u_char *nfsrv_dnsname = NULL;
	static int nfsrv_usermax = 999999999;
	struct nfsrv_lughash {
	struct mtx mtx;
	struct nfsuserhashhead lughead;
	};
	static struct nfsrv_lughash *nfsuserhash;
	static struct nfsrv_lughash *nfsusernamehash;
	static struct nfsrv_lughash *nfsgrouphash;
	static struct nfsrv_lughash *nfsgroupnamehash;

	/*
	* This static array indicates whether or not the RPC generates a large
	* reply. This is used by nfs_reply() to decide whether or not an mbuf
	* cluster should be allocated. (If a cluster is required by an RPC
	* marked 0 in this array, the code will still work, just not quite as
	* efficiently.)
	*/
	static int nfs_bigreply[NFSV42_NPROCS] = { 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,
	1, 0, 0, 1 };

	/* local functions */
	static int nfsrv_skipace(struct nfsrv_descript nd, int acesizep);
	static void nfsv4_wanted(struct nfsv4lock *lp);
	static int nfsrv_cmpmixedcase(u_char cp, u_char cp2, int len);
	static int nfsrv_getuser(int procnum, uid_t uid, gid_t gid, char *name);
	static void nfsrv_removeuser(struct nfsusrgrp *usrp, int isuser);
	static int nfsrv_getrefstr(struct nfsrv_descript , u_char , u_char *,
	int , int );
	static void nfsrv_refstrbigenough(int, u_char , u_char , int *);

	static struct {
	int op;
	int opcnt;
	const u_char *tag;
	int taglen;
	} nfsv4_opmap[NFSV42_NPROCS] = {
	{ 0, 1, "Null", 4 },
	{ NFSV4OP_GETATTR, 1, "Getattr", 7, },
	{ NFSV4OP_SETATTR, 2, "Setattr", 7, },
	{ NFSV4OP_LOOKUP, 3, "Lookup", 6, },
	{ NFSV4OP_ACCESS, 2, "Access", 6, },
	{ NFSV4OP_READLINK, 2, "Readlink", 8, },
	{ NFSV4OP_READ, 1, "Read", 4, },
	{ NFSV4OP_WRITE, 2, "Write", 5, },
	{ NFSV4OP_OPEN, 5, "Open", 4, },
	{ NFSV4OP_CREATE, 5, "Create", 6, },
	{ NFSV4OP_CREATE, 1, "Create", 6, },
	{ NFSV4OP_CREATE, 3, "Create", 6, },
	{ NFSV4OP_REMOVE, 1, "Remove", 6, },
	{ NFSV4OP_REMOVE, 1, "Remove", 6, },
	{ NFSV4OP_SAVEFH, 5, "Rename", 6, },
	{ NFSV4OP_SAVEFH, 4, "Link", 4, },
	{ NFSV4OP_READDIR, 2, "Readdir", 7, },
	{ NFSV4OP_READDIR, 2, "Readdir", 7, },
	{ NFSV4OP_GETATTR, 1, "Getattr", 7, },
	{ NFSV4OP_GETATTR, 1, "Getattr", 7, },
	{ NFSV4OP_GETATTR, 1, "Getattr", 7, },
	{ NFSV4OP_COMMIT, 2, "Commit", 6, },
	{ NFSV4OP_LOOKUPP, 3, "Lookupp", 7, },
	{ NFSV4OP_SETCLIENTID, 1, "SetClientID", 11, },
	{ NFSV4OP_SETCLIENTIDCFRM, 1, "SetClientIDConfirm", 18, },
	{ NFSV4OP_LOCK, 1, "Lock", 4, },
	{ NFSV4OP_LOCKU, 1, "LockU", 5, },
	{ NFSV4OP_OPEN, 2, "Open", 4, },
	{ NFSV4OP_CLOSE, 1, "Close", 5, },
	{ NFSV4OP_OPENCONFIRM, 1, "Openconfirm", 11, },
	{ NFSV4OP_LOCKT, 1, "LockT", 5, },
	{ NFSV4OP_OPENDOWNGRADE, 1, "Opendowngrade", 13, },
	{ NFSV4OP_RENEW, 1, "Renew", 5, },
	{ NFSV4OP_PUTROOTFH, 1, "Dirpath", 7, },
	{ NFSV4OP_RELEASELCKOWN, 1, "Rellckown", 9, },
	{ NFSV4OP_DELEGRETURN, 1, "Delegret", 8, },
	{ NFSV4OP_DELEGRETURN, 3, "DelegRemove", 11, },
	{ NFSV4OP_DELEGRETURN, 7, "DelegRename1", 12, },
	{ NFSV4OP_DELEGRETURN, 9, "DelegRename2", 12, },
	{ NFSV4OP_GETATTR, 1, "Getacl", 6, },
	{ NFSV4OP_SETATTR, 1, "Setacl", 6, },
	{ NFSV4OP_EXCHANGEID, 1, "ExchangeID", 10, },
	{ NFSV4OP_CREATESESSION, 1, "CreateSession", 13, },
	{ NFSV4OP_DESTROYSESSION, 1, "DestroySession", 14, },
	{ NFSV4OP_DESTROYCLIENTID, 1, "DestroyClient", 13, },
	{ NFSV4OP_FREESTATEID, 1, "FreeStateID", 11, },
	{ NFSV4OP_LAYOUTGET, 1, "LayoutGet", 9, },
	{ NFSV4OP_GETDEVINFO, 1, "GetDeviceInfo", 13, },
	{ NFSV4OP_LAYOUTCOMMIT, 1, "LayoutCommit", 12, },
	{ NFSV4OP_LAYOUTRETURN, 1, "LayoutReturn", 12, },
	{ NFSV4OP_RECLAIMCOMPL, 1, "ReclaimComplete", 15, },
	{ NFSV4OP_WRITE, 1, "WriteDS", 7, },
	{ NFSV4OP_READ, 1, "ReadDS", 6, },
	{ NFSV4OP_COMMIT, 1, "CommitDS", 8, },
	{ NFSV4OP_OPEN, 3, "OpenLayoutGet", 13, },
	{ NFSV4OP_OPEN, 8, "CreateLayGet", 12, },
	{ NFSV4OP_IOADVISE, 1, "Advise", 6, },
	{ NFSV4OP_ALLOCATE, 2, "Allocate", 8, },
	{ NFSV4OP_SAVEFH, 5, "Copy", 4, },
	{ NFSV4OP_SEEK, 2, "Seek", 4, },
	{ NFSV4OP_SEEK, 1, "SeekDS", 6, },
	{ NFSV4OP_GETXATTR, 2, "Getxattr", 8, },
	{ NFSV4OP_SETXATTR, 2, "Setxattr", 8, },
	{ NFSV4OP_REMOVEXATTR, 2, "Rmxattr", 7, },
	{ NFSV4OP_LISTXATTRS, 2, "Listxattr", 9, },
	};

	/*
	* NFS RPCS that have large request message size.
	*/
	static int nfs_bigrequest[NFSV42_NPROCS] = {
	0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0
	};

	/*
	* Start building a request. Mostly just put the first file handle in
	* place.
	*/
	void
	nfscl_reqstart(struct nfsrv_descript nd, int procnum, struct nfsmount nmp,
	u_int8_t nfhp, int fhlen, u_int32_t opcntpp, struct nfsclsession sep,
	int vers, int minorvers, bool use_ext)
	{
	struct mbuf *mb;
	u_int32_t *tl;
	int opcnt;
	nfsattrbit_t attrbits;

	/*
	* First, fill in some of the fields of nd.
	*/
	nd->nd_slotseq = NULL;
	if (vers == NFS_VER4) {
	nd->nd_flag = ND_NFSV4 \| ND_NFSCL;
	if (minorvers == NFSV41_MINORVERSION)
	nd->nd_flag \|= ND_NFSV41;
	else if (minorvers == NFSV42_MINORVERSION)
	nd->nd_flag \|= (ND_NFSV41 \| ND_NFSV42);
	} else if (vers == NFS_VER3)
	nd->nd_flag = ND_NFSV3 \| ND_NFSCL;
	else {
	if (NFSHASNFSV4(nmp)) {
	nd->nd_flag = ND_NFSV4 \| ND_NFSCL;
	if (nmp->nm_minorvers == 1)
	nd->nd_flag \|= ND_NFSV41;
	else if (nmp->nm_minorvers == 2)
	nd->nd_flag \|= (ND_NFSV41 \| ND_NFSV42);
	} else if (NFSHASNFSV3(nmp))
	nd->nd_flag = ND_NFSV3 \| ND_NFSCL;
	else
	nd->nd_flag = ND_NFSV2 \| ND_NFSCL;
	}
	nd->nd_procnum = procnum;
	nd->nd_repstat = 0;
	nd->nd_maxextsiz = 16384;
	if (use_ext && mb_use_ext_pgs && PMAP_HAS_DMAP != 0)
	nd->nd_flag \|= ND_EXTPG;

	/*
	* Get the first mbuf for the request.
	*/
	if ((nd->nd_flag & ND_EXTPG) != 0) {
	mb = mb_alloc_ext_plus_pages(PAGE_SIZE, M_WAITOK);
	nd->nd_mreq = nd->nd_mb = mb;
	nfsm_set(nd, 0);
	} else {
	if (nfs_bigrequest[procnum])
	NFSMCLGET(mb, M_WAITOK);
	else
	NFSMGET(mb);
	mb->m_len = 0;
	nd->nd_mreq = nd->nd_mb = mb;
	nd->nd_bpos = mtod(mb, char *);
	}

	/*
	* And fill the first file handle into the request.
	*/
	if (nd->nd_flag & ND_NFSV4) {
	opcnt = nfsv4_opmap[procnum].opcnt +
	nfsv4_opflag[nfsv4_opmap[procnum].op].needscfh;
	if ((nd->nd_flag & ND_NFSV41) != 0) {
	opcnt += nfsv4_opflag[nfsv4_opmap[procnum].op].needsseq;
	if (procnum == NFSPROC_RENEW)
	/*
	* For the special case of Renew, just do a
	* Sequence Op.
	*/
	opcnt = 1;
	else if (procnum == NFSPROC_WRITEDS \|\|
	procnum == NFSPROC_COMMITDS)
	/*
	* For the special case of a Writeor Commit to
	* a DS, the opcnt == 3, for Sequence, PutFH,
	* Write/Commit.
	*/
	opcnt = 3;
	}
	/*
	* What should the tag really be?
	*/
	(void) nfsm_strtom(nd, nfsv4_opmap[procnum].tag,
	nfsv4_opmap[procnum].taglen);
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	if ((nd->nd_flag & ND_NFSV42) != 0)
	*tl++ = txdr_unsigned(NFSV42_MINORVERSION);
	else if ((nd->nd_flag & ND_NFSV41) != 0)
	*tl++ = txdr_unsigned(NFSV41_MINORVERSION);
	else
	*tl++ = txdr_unsigned(NFSV4_MINORVERSION);
	if (opcntpp != NULL)
	*opcntpp = tl;
	*tl = txdr_unsigned(opcnt);
	if ((nd->nd_flag & ND_NFSV41) != 0 &&
	nfsv4_opflag[nfsv4_opmap[procnum].op].needsseq > 0) {
	if (nfsv4_opflag[nfsv4_opmap[procnum].op].loopbadsess >
	0)
	nd->nd_flag \|= ND_LOOPBADSESS;
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4OP_SEQUENCE);
	if (sep == NULL) {
	sep = nfsmnt_mdssession(nmp);
	nfsv4_setsequence(nmp, nd, sep,
	nfs_bigreply[procnum]);
	} else
	nfsv4_setsequence(nmp, nd, sep,
	nfs_bigreply[procnum]);
	}
	if (nfsv4_opflag[nfsv4_opmap[procnum].op].needscfh > 0) {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4OP_PUTFH);
	(void) nfsm_fhtom(nd, nfhp, fhlen, 0);
	if (nfsv4_opflag[nfsv4_opmap[procnum].op].needscfh
	== 2 && procnum != NFSPROC_WRITEDS &&
	procnum != NFSPROC_COMMITDS) {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4OP_GETATTR);
	/*
	* For Lookup Ops, we want all the directory
	* attributes, so we can load the name cache.
	*/
	if (procnum == NFSPROC_LOOKUP \|\|
	procnum == NFSPROC_LOOKUPP)
	NFSGETATTR_ATTRBIT(&attrbits);
	else {
	NFSWCCATTR_ATTRBIT(&attrbits);
	nd->nd_flag \|= ND_V4WCCATTR;
	}
	(void) nfsrv_putattrbit(nd, &attrbits);
	}
	}
	if (procnum != NFSPROC_RENEW \|\|
	(nd->nd_flag & ND_NFSV41) == 0) {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(nfsv4_opmap[procnum].op);
	}
	} else {
	(void) nfsm_fhtom(nd, nfhp, fhlen, 0);
	}
	if (procnum < NFSV42_NPROCS)
	NFSINCRGLOBAL(nfsstatsv1.rpccnt[procnum]);
	}

	/*
	* Put a state Id in the mbuf list.
	*/
	void
	nfsm_stateidtom(struct nfsrv_descript nd, nfsv4stateid_t stateidp, int flag)
	{
	nfsv4stateid_t *st;

	NFSM_BUILD(st, nfsv4stateid_t *, NFSX_STATEID);
	if (flag == NFSSTATEID_PUTALLZERO) {
	st->seqid = 0;
	st->other[0] = 0;
	st->other[1] = 0;
	st->other[2] = 0;
	} else if (flag == NFSSTATEID_PUTALLONE) {
	st->seqid = 0xffffffff;
	st->other[0] = 0xffffffff;
	st->other[1] = 0xffffffff;
	st->other[2] = 0xffffffff;
	} else if (flag == NFSSTATEID_PUTSEQIDZERO) {
	st->seqid = 0;
	st->other[0] = stateidp->other[0];
	st->other[1] = stateidp->other[1];
	st->other[2] = stateidp->other[2];
	} else {
	st->seqid = stateidp->seqid;
	st->other[0] = stateidp->other[0];
	st->other[1] = stateidp->other[1];
	st->other[2] = stateidp->other[2];
	}
	}

	/*
	* Fill in the setable attributes. The full argument indicates whether
	* to fill in them all or just mode and time.
	*/
	void
	nfscl_fillsattr(struct nfsrv_descript nd, struct vattr vap,
	struct vnode *vp, int flags, u_int32_t rdev)
	{
	u_int32_t *tl;
	struct nfsv2_sattr *sp;
	nfsattrbit_t attrbits;
	+ struct nfsnode *np;

	switch (nd->nd_flag & (ND_NFSV2 \| ND_NFSV3 \| ND_NFSV4)) {
	case ND_NFSV2:
	NFSM_BUILD(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
	if (vap->va_mode == (mode_t)VNOVAL)
	sp->sa_mode = newnfs_xdrneg1;
	else
	sp->sa_mode = vtonfsv2_mode(vap->va_type, vap->va_mode);
	if (vap->va_uid == (uid_t)VNOVAL)
	sp->sa_uid = newnfs_xdrneg1;
	else
	sp->sa_uid = txdr_unsigned(vap->va_uid);
	if (vap->va_gid == (gid_t)VNOVAL)
	sp->sa_gid = newnfs_xdrneg1;
	else
	sp->sa_gid = txdr_unsigned(vap->va_gid);
	if (flags & NFSSATTR_SIZE0)
	sp->sa_size = 0;
	else if (flags & NFSSATTR_SIZENEG1)
	sp->sa_size = newnfs_xdrneg1;
	else if (flags & NFSSATTR_SIZERDEV)
	sp->sa_size = txdr_unsigned(rdev);
	else
	sp->sa_size = txdr_unsigned(vap->va_size);
	txdr_nfsv2time(&vap->va_atime, &sp->sa_atime);
	txdr_nfsv2time(&vap->va_mtime, &sp->sa_mtime);
	break;
	case ND_NFSV3:
	if (vap->va_mode != (mode_t)VNOVAL) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	*tl = txdr_unsigned(vap->va_mode);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_false;
	}
	if ((flags & NFSSATTR_FULL) && vap->va_uid != (uid_t)VNOVAL) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	*tl = txdr_unsigned(vap->va_uid);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_false;
	}
	if ((flags & NFSSATTR_FULL) && vap->va_gid != (gid_t)VNOVAL) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	*tl = txdr_unsigned(vap->va_gid);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_false;
	}
	if ((flags & NFSSATTR_FULL) && vap->va_size != VNOVAL) {
	NFSM_BUILD(tl, u_int32_t , 3 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	txdr_hyper(vap->va_size, tl);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_false;
	}
	if (vap->va_atime.tv_sec != VNOVAL) {
	if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
	NFSM_BUILD(tl, u_int32_t , 3 NFSX_UNSIGNED);
	*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
	txdr_nfsv3time(&vap->va_atime, tl);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV3SATTRTIME_TOSERVER);
	}
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE);
	}
	if (vap->va_mtime.tv_sec != VNOVAL) {
	if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
	NFSM_BUILD(tl, u_int32_t , 3 NFSX_UNSIGNED);
	*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
	txdr_nfsv3time(&vap->va_mtime, tl);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV3SATTRTIME_TOSERVER);
	}
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE);
	}
	break;
	case ND_NFSV4:
	NFSZERO_ATTRBIT(&attrbits);
	if (vap->va_mode != (mode_t)VNOVAL)
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_MODE);
	if ((flags & NFSSATTR_FULL) && vap->va_uid != (uid_t)VNOVAL)
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_OWNER);
	if ((flags & NFSSATTR_FULL) && vap->va_gid != (gid_t)VNOVAL)
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_OWNERGROUP);
	if ((flags & NFSSATTR_FULL) && vap->va_size != VNOVAL)
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SIZE);
	if (vap->va_atime.tv_sec != VNOVAL)
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEACCESSSET);
	if (vap->va_mtime.tv_sec != VNOVAL)
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEMODIFYSET);
	- if (vap->va_birthtime.tv_sec != VNOVAL)
	- NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMECREATE);
	+ if (vap->va_birthtime.tv_sec != VNOVAL &&
	+ strcmp(vp->v_mount->mnt_vfc->vfc_name, "nfs") == 0) {
	+ /*
	+ * We can only test for support of TimeCreate if
	+ * the "vp" argument is for an NFS vnode.
	+ */
	+ np = VTONFS(vp);
	+ if (NFSISSET_ATTRBIT(&np->n_vattr.na_suppattr,
	+ NFSATTRBIT_TIMECREATE))
	+ NFSSETBIT_ATTRBIT(&attrbits,
	+ NFSATTRBIT_TIMECREATE);
	+ }
	(void) nfsv4_fillattr(nd, vp->v_mount, vp, NULL, vap, NULL, 0,
	&attrbits, NULL, NULL, 0, 0, 0, 0, (uint64_t)0, NULL);
	break;
	}
	}

	#ifndef APPLE
	/*
	* copies mbuf chain to the uio scatter/gather list
	*/
	int
	nfsm_mbufuio(struct nfsrv_descript nd, struct uio uiop, int siz)
	{
	char mbufcp, uiocp;
	int xfer, left, len;
	struct mbuf *mp;
	long uiosiz, rem;
	int error = 0;

	mp = nd->nd_md;
	mbufcp = nd->nd_dpos;
	len = mtod(mp, caddr_t) + mp->m_len - mbufcp;
	rem = NFSM_RNDUP(siz) - siz;
	while (siz > 0) {
	if (uiop->uio_iovcnt <= 0 \|\| uiop->uio_iov == NULL) {
	error = EBADRPC;
	goto out;
	}
	left = uiop->uio_iov->iov_len;
	uiocp = uiop->uio_iov->iov_base;
	if (left > siz)
	left = siz;
	uiosiz = left;
	while (left > 0) {
	while (len == 0) {
	mp = mp->m_next;
	if (mp == NULL) {
	error = EBADRPC;
	goto out;
	}
	mbufcp = mtod(mp, caddr_t);
	len = mp->m_len;
	KASSERT(len >= 0,
	("len %d, corrupted mbuf?", len));
	}
	xfer = (left > len) ? len : left;
	#ifdef notdef
	/* Not Yet.. */
	if (uiop->uio_iov->iov_op != NULL)
	(*(uiop->uio_iov->iov_op))
	(mbufcp, uiocp, xfer);
	else
	#endif
	if (uiop->uio_segflg == UIO_SYSSPACE)
	NFSBCOPY(mbufcp, uiocp, xfer);
	else
	copyout(mbufcp, uiocp, xfer);
	left -= xfer;
	len -= xfer;
	mbufcp += xfer;
	uiocp += xfer;
	uiop->uio_offset += xfer;
	uiop->uio_resid -= xfer;
	}
	if (uiop->uio_iov->iov_len <= siz) {
	uiop->uio_iovcnt--;
	uiop->uio_iov++;
	} else {
	uiop->uio_iov->iov_base = (void *)
	((char *)uiop->uio_iov->iov_base + uiosiz);
	uiop->uio_iov->iov_len -= uiosiz;
	}
	siz -= uiosiz;
	}
	nd->nd_dpos = mbufcp;
	nd->nd_md = mp;
	if (rem > 0) {
	if (len < rem)
	error = nfsm_advance(nd, rem, len);
	else
	nd->nd_dpos += rem;
	}

	out:
	NFSEXITCODE2(error, nd);
	return (error);
	}
	#endif /* !APPLE */

	/*
	* Help break down an mbuf chain by setting the first siz bytes contiguous
	* pointed to by returned val.
	* This is used by the macro NFSM_DISSECT for tough
	* cases.
	*/
	void *
	nfsm_dissct(struct nfsrv_descript *nd, int siz, int how)
	{
	struct mbuf *mp2;
	int siz2, xfer;
	caddr_t p;
	int left;
	caddr_t retp;

	retp = NULL;
	left = mtod(nd->nd_md, caddr_t) + nd->nd_md->m_len - nd->nd_dpos;
	while (left == 0) {
	nd->nd_md = nd->nd_md->m_next;
	if (nd->nd_md == NULL)
	return (retp);
	left = nd->nd_md->m_len;
	nd->nd_dpos = mtod(nd->nd_md, caddr_t);
	}
	if (left >= siz) {
	retp = nd->nd_dpos;
	nd->nd_dpos += siz;
	} else if (nd->nd_md->m_next == NULL) {
	return (retp);
	} else if (siz > ncl_mbuf_mhlen) {
	panic("nfs S too big");
	} else {
	MGET(mp2, MT_DATA, how);
	if (mp2 == NULL)
	return (NULL);
	mp2->m_next = nd->nd_md->m_next;
	nd->nd_md->m_next = mp2;
	nd->nd_md->m_len -= left;
	nd->nd_md = mp2;
	retp = p = mtod(mp2, caddr_t);
	NFSBCOPY(nd->nd_dpos, p, left); /* Copy what was left */
	siz2 = siz - left;
	p += left;
	mp2 = mp2->m_next;
	/* Loop around copying up the siz2 bytes */
	while (siz2 > 0) {
	if (mp2 == NULL)
	return (NULL);
	xfer = (siz2 > mp2->m_len) ? mp2->m_len : siz2;
	if (xfer > 0) {
	NFSBCOPY(mtod(mp2, caddr_t), p, xfer);
	mp2->m_data += xfer;
	mp2->m_len -= xfer;
	p += xfer;
	siz2 -= xfer;
	}
	if (siz2 > 0)
	mp2 = mp2->m_next;
	}
	nd->nd_md->m_len = siz;
	nd->nd_md = mp2;
	nd->nd_dpos = mtod(mp2, caddr_t);
	}
	return (retp);
	}

	/*
	* Advance the position in the mbuf chain.
	* If offs == 0, this is a no-op, but it is simpler to just return from
	* here than check for offs > 0 for all calls to nfsm_advance.
	* If left == -1, it should be calculated here.
	*/
	int
	nfsm_advance(struct nfsrv_descript *nd, int offs, int left)
	{
	int error = 0;

	if (offs == 0)
	goto out;
	/*
	* A negative offs might indicate a corrupted mbuf chain and,
	* as such, a printf is logged.
	*/
	if (offs < 0) {
	printf("nfsrv_advance: negative offs\n");
	error = EBADRPC;
	goto out;
	}

	/*
	* If left == -1, calculate it here.
	*/
	if (left == -1)
	left = mtod(nd->nd_md, caddr_t) + nd->nd_md->m_len -
	nd->nd_dpos;

	/*
	* Loop around, advancing over the mbuf data.
	*/
	while (offs > left) {
	offs -= left;
	nd->nd_md = nd->nd_md->m_next;
	if (nd->nd_md == NULL) {
	error = EBADRPC;
	goto out;
	}
	left = nd->nd_md->m_len;
	nd->nd_dpos = mtod(nd->nd_md, caddr_t);
	}
	nd->nd_dpos += offs;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Copy a string into mbuf(s).
	* Return the number of bytes output, including XDR overheads.
	*/
	int
	nfsm_strtom(struct nfsrv_descript nd, const char cp, int siz)
	{
	struct mbuf *m2;
	int xfer, left;
	struct mbuf *m1;
	int rem, bytesize;
	u_int32_t *tl;
	char *cp2;

	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(siz);
	rem = NFSM_RNDUP(siz) - siz;
	bytesize = NFSX_UNSIGNED + siz + rem;
	m2 = nd->nd_mb;
	cp2 = nd->nd_bpos;
	if ((nd->nd_flag & ND_EXTPG) != 0)
	left = nd->nd_bextpgsiz;
	else
	left = M_TRAILINGSPACE(m2);

	KASSERT(((m2->m_flags & (M_EXT \| M_EXTPG)) ==
	(M_EXT \| M_EXTPG) && (nd->nd_flag & ND_EXTPG) != 0) \|\|
	((m2->m_flags & (M_EXT \| M_EXTPG)) !=
	(M_EXT \| M_EXTPG) && (nd->nd_flag & ND_EXTPG) == 0),
	("nfsm_strtom: ext_pgs and non-ext_pgs mbufs mixed"));
	/*
	* Loop around copying the string to mbuf(s).
	*/
	while (siz > 0) {
	if (left == 0) {
	if ((nd->nd_flag & ND_EXTPG) != 0) {
	m2 = nfsm_add_ext_pgs(m2,
	nd->nd_maxextsiz, &nd->nd_bextpg);
	cp2 = (char )(void )PHYS_TO_DMAP(
	m2->m_epg_pa[nd->nd_bextpg]);
	nd->nd_bextpgsiz = left = PAGE_SIZE;
	} else {
	if (siz > ncl_mbuf_mlen)
	NFSMCLGET(m1, M_WAITOK);
	else
	NFSMGET(m1);
	m1->m_len = 0;
	cp2 = mtod(m1, char *);
	left = M_TRAILINGSPACE(m1);
	m2->m_next = m1;
	m2 = m1;
	}
	}
	if (left >= siz)
	xfer = siz;
	else
	xfer = left;
	NFSBCOPY(cp, cp2, xfer);
	cp += xfer;
	cp2 += xfer;
	m2->m_len += xfer;
	siz -= xfer;
	left -= xfer;
	if ((nd->nd_flag & ND_EXTPG) != 0) {
	nd->nd_bextpgsiz -= xfer;
	m2->m_epg_last_len += xfer;
	}
	if (siz == 0 && rem) {
	if (left < rem)
	panic("nfsm_strtom");
	NFSBZERO(cp2, rem);
	m2->m_len += rem;
	cp2 += rem;
	if ((nd->nd_flag & ND_EXTPG) != 0) {
	nd->nd_bextpgsiz -= rem;
	m2->m_epg_last_len += rem;
	}
	}
	}
	nd->nd_mb = m2;
	if ((nd->nd_flag & ND_EXTPG) != 0)
	nd->nd_bpos = cp2;
	else
	nd->nd_bpos = mtod(m2, char *) + m2->m_len;
	return (bytesize);
	}

	/*
	* Called once to initialize data structures...
	*/
	void
	newnfs_init(void)
	{
	static int nfs_inited = 0;

	if (nfs_inited)
	return;
	nfs_inited = 1;

	newnfs_true = txdr_unsigned(TRUE);
	newnfs_false = txdr_unsigned(FALSE);
	newnfs_xdrneg1 = txdr_unsigned(-1);
	nfscl_ticks = (hz * NFS_TICKINTVL + 500) / 1000;
	if (nfscl_ticks < 1)
	nfscl_ticks = 1;
	NFSSETBOOTTIME(nfsboottime);

	/*
	* Initialize reply list and start timer
	*/
	TAILQ_INIT(&nfsd_reqq);
	NFS_TIMERINIT;
	}

	/*
	* Put a file handle in an mbuf list.
	* If the size argument == 0, just use the default size.
	* set_true == 1 if there should be an newnfs_true prepended on the file handle.
	* Return the number of bytes output, including XDR overhead.
	*/
	int
	nfsm_fhtom(struct nfsrv_descript nd, u_int8_t fhp, int size, int set_true)
	{
	u_int32_t *tl;
	u_int8_t *cp;
	int fullsiz, rem, bytesize = 0;

	if (size == 0)
	size = NFSX_MYFH;
	switch (nd->nd_flag & (ND_NFSV2 \| ND_NFSV3 \| ND_NFSV4)) {
	case ND_NFSV2:
	if (size > NFSX_V2FH)
	panic("fh size > NFSX_V2FH for NFSv2");
	NFSM_BUILD(cp, u_int8_t *, NFSX_V2FH);
	NFSBCOPY(fhp, cp, size);
	if (size < NFSX_V2FH)
	NFSBZERO(cp + size, NFSX_V2FH - size);
	bytesize = NFSX_V2FH;
	break;
	case ND_NFSV3:
	case ND_NFSV4:
	fullsiz = NFSM_RNDUP(size);
	rem = fullsiz - size;
	if (set_true) {
	bytesize = 2 * NFSX_UNSIGNED + fullsiz;
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_true;
	} else {
	bytesize = NFSX_UNSIGNED + fullsiz;
	}
	(void) nfsm_strtom(nd, fhp, size);
	break;
	}
	return (bytesize);
	}

	/*
	* This function compares two net addresses by family and returns TRUE
	* if they are the same host.
	* If there is any doubt, return FALSE.
	* The AF_INET family is handled as a special case so that address mbufs
	* don't need to be saved to store "struct in_addr", which is only 4 bytes.
	*/
	int
	nfsaddr_match(int family, union nethostaddr *haddr, NFSSOCKADDR_T nam)
	{
	#ifdef INET
	struct sockaddr_in *inetaddr;
	#endif

	switch (family) {
	#ifdef INET
	case AF_INET:
	inetaddr = NFSSOCKADDR(nam, struct sockaddr_in *);
	if (inetaddr->sin_family == AF_INET &&
	inetaddr->sin_addr.s_addr == haddr->had_inet.s_addr)
	return (1);
	break;
	#endif
	#ifdef INET6
	case AF_INET6:
	{
	struct sockaddr_in6 *inetaddr6;

	inetaddr6 = NFSSOCKADDR(nam, struct sockaddr_in6 *);
	/* XXX - should test sin6_scope_id ? */
	if (inetaddr6->sin6_family == AF_INET6 &&
	IN6_ARE_ADDR_EQUAL(&inetaddr6->sin6_addr,
	&haddr->had_inet6))
	return (1);
	}
	break;
	#endif
	}
	return (0);
	}

	/*
	* Similar to the above, but takes to NFSSOCKADDR_T args.
	*/
	int
	nfsaddr2_match(NFSSOCKADDR_T nam1, NFSSOCKADDR_T nam2)
	{
	struct sockaddr_in addr1, addr2;
	struct sockaddr *inaddr;

	inaddr = NFSSOCKADDR(nam1, struct sockaddr *);
	switch (inaddr->sa_family) {
	case AF_INET:
	addr1 = NFSSOCKADDR(nam1, struct sockaddr_in *);
	addr2 = NFSSOCKADDR(nam2, struct sockaddr_in *);
	if (addr2->sin_family == AF_INET &&
	addr1->sin_addr.s_addr == addr2->sin_addr.s_addr)
	return (1);
	break;
	#ifdef INET6
	case AF_INET6:
	{
	struct sockaddr_in6 inet6addr1, inet6addr2;

	inet6addr1 = NFSSOCKADDR(nam1, struct sockaddr_in6 *);
	inet6addr2 = NFSSOCKADDR(nam2, struct sockaddr_in6 *);
	/* XXX - should test sin6_scope_id ? */
	if (inet6addr2->sin6_family == AF_INET6 &&
	IN6_ARE_ADDR_EQUAL(&inet6addr1->sin6_addr,
	&inet6addr2->sin6_addr))
	return (1);
	}
	break;
	#endif
	}
	return (0);
	}

	/*
	* Trim trailing data off the mbuf list being built.
	*/
	void
	newnfs_trimtrailing(nd, mb, bpos)
	struct nfsrv_descript *nd;
	struct mbuf *mb;
	caddr_t bpos;
	{

	if (mb->m_next) {
	m_freem(mb->m_next);
	mb->m_next = NULL;
	}
	mb->m_len = bpos - mtod(mb, caddr_t);
	nd->nd_mb = mb;
	nd->nd_bpos = bpos;
	}

	/*
	* Dissect a file handle on the client.
	*/
	int
	nfsm_getfh(struct nfsrv_descript nd, struct nfsfh *nfhpp)
	{
	u_int32_t *tl;
	struct nfsfh *nfhp;
	int error, len;

	*nfhpp = NULL;
	if (nd->nd_flag & (ND_NFSV3 \| ND_NFSV4)) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if ((len = fxdr_unsigned(int, *tl)) <= 0 \|\|
	len > NFSX_FHMAX) {
	error = EBADRPC;
	goto nfsmout;
	}
	} else
	len = NFSX_V2FH;
	nfhp = malloc(sizeof (struct nfsfh) + len,
	M_NFSFH, M_WAITOK);
	error = nfsrv_mtostr(nd, nfhp->nfh_fh, len);
	if (error) {
	free(nfhp, M_NFSFH);
	goto nfsmout;
	}
	nfhp->nfh_len = len;
	*nfhpp = nfhp;
	nfsmout:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Break down the nfsv4 acl.
	* If the aclp == NULL or won't fit in an acl, just discard the acl info.
	*/
	int
	nfsrv_dissectacl(struct nfsrv_descript nd, NFSACL_T aclp, int *aclerrp,
	int aclsizep, __unused NFSPROC_T p)
	{
	u_int32_t *tl;
	int i, aclsize;
	int acecnt, error = 0, aceerr = 0, acesize;

	*aclerrp = 0;
	if (aclp)
	aclp->acl_cnt = 0;
	/*
	* Parse out the ace entries and expect them to conform to
	* what can be supported by R/W/X bits.
	*/
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	aclsize = NFSX_UNSIGNED;
	acecnt = fxdr_unsigned(int, *tl);
	if (acecnt > ACL_MAX_ENTRIES)
	aceerr = NFSERR_ATTRNOTSUPP;
	if (nfsrv_useacl == 0)
	aceerr = NFSERR_ATTRNOTSUPP;
	for (i = 0; i < acecnt; i++) {
	if (aclp && !aceerr)
	error = nfsrv_dissectace(nd, &aclp->acl_entry[i],
	&aceerr, &acesize, p);
	else
	error = nfsrv_skipace(nd, &acesize);
	if (error)
	goto nfsmout;
	aclsize += acesize;
	}
	if (aclp && !aceerr)
	aclp->acl_cnt = acecnt;
	if (aceerr)
	*aclerrp = aceerr;
	if (aclsizep)
	*aclsizep = aclsize;
	nfsmout:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Skip over an NFSv4 ace entry. Just dissect the xdr and discard it.
	*/
	static int
	nfsrv_skipace(struct nfsrv_descript nd, int acesizep)
	{
	u_int32_t *tl;
	int error, len = 0;

	NFSM_DISSECT(tl, u_int32_t , 4 NFSX_UNSIGNED);
	len = fxdr_unsigned(int, *(tl + 3));
	error = nfsm_advance(nd, NFSM_RNDUP(len), -1);
	nfsmout:
	acesizep = NFSM_RNDUP(len) + (4 NFSX_UNSIGNED);
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Get attribute bits from an mbuf list.
	* Returns EBADRPC for a parsing error, 0 otherwise.
	* If the clearinvalid flag is set, clear the bits not supported.
	*/
	int
	nfsrv_getattrbits(struct nfsrv_descript nd, nfsattrbit_t attrbitp, int *cntp,
	int *retnotsupp)
	{
	u_int32_t *tl;
	int cnt, i, outcnt;
	int error = 0;

	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	cnt = fxdr_unsigned(int, *tl);
	if (cnt < 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	if (cnt > NFSATTRBIT_MAXWORDS)
	outcnt = NFSATTRBIT_MAXWORDS;
	else
	outcnt = cnt;
	NFSZERO_ATTRBIT(attrbitp);
	if (outcnt > 0) {
	NFSM_DISSECT(tl, u_int32_t , outcnt NFSX_UNSIGNED);
	for (i = 0; i < outcnt; i++)
	attrbitp->bits[i] = fxdr_unsigned(u_int32_t, *tl++);
	}
	for (i = 0; i < (cnt - outcnt); i++) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (retnotsupp != NULL && *tl != 0)
	*retnotsupp = NFSERR_ATTRNOTSUPP;
	}
	if (cntp)
	cntp = NFSX_UNSIGNED + (cnt NFSX_UNSIGNED);
	nfsmout:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Get the attributes for V4.
	* If the compare flag is true, test for any attribute changes,
	* otherwise return the attribute values.
	* These attributes cover fields in "struct vattr", "struct statfs",
	* "struct nfsfsinfo", the file handle and the lease duration.
	* The value of retcmpp is set to 1 if all attributes are the same,
	* and 0 otherwise.
	* Returns EBADRPC if it can't be parsed, 0 otherwise.
	*/
	int
	nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
	struct nfsvattr nap, struct nfsfh nfhpp, fhandle_t fhp, int fhsize,
	struct nfsv3_pathconf pc, struct statfs sbp, struct nfsstatfs *sfp,
	struct nfsfsinfo fsp, NFSACL_T aclp, int compare, int *retcmpp,
	u_int32_t leasep, u_int32_t rderrp, NFSPROC_T p, struct ucred cred)
	{
	u_int32_t *tl;
	int i = 0, j, k, l = 0, m, bitpos, attrsum = 0;
	int error, tfhsize, aceerr, attrsize, cnt, retnotsup;
	u_char cp, cp2, namestr[NFSV4_SMALLSTR + 1];
	nfsattrbit_t attrbits, retattrbits, checkattrbits;
	struct nfsfh *tnfhp;
	struct nfsreferral *refp;
	u_quad_t tquad;
	nfsquad_t tnfsquad;
	struct timespec temptime;
	uid_t uid;
	gid_t gid;
	u_int32_t freenum = 0, tuint;
	u_int64_t uquad = 0, thyp, thyp2;
	#ifdef QUOTA
	struct dqblk dqb;
	uid_t savuid;
	#endif

	CTASSERT(sizeof(ino_t) == sizeof(uint64_t));
	if (compare) {
	retnotsup = 0;
	error = nfsrv_getattrbits(nd, &attrbits, NULL, &retnotsup);
	} else {
	error = nfsrv_getattrbits(nd, &attrbits, NULL, NULL);
	}
	if (error)
	goto nfsmout;

	if (compare) {
	*retcmpp = retnotsup;
	} else {
	/*
	* Just set default values to some of the important ones.
	*/
	if (nap != NULL) {
	nap->na_type = VREG;
	nap->na_mode = 0;
	nap->na_rdev = (NFSDEV_T)0;
	nap->na_mtime.tv_sec = 0;
	nap->na_mtime.tv_nsec = 0;
	nap->na_gen = 0;
	nap->na_flags = 0;
	nap->na_blocksize = NFS_FABLKSIZE;
	}
	if (sbp != NULL) {
	sbp->f_bsize = NFS_FABLKSIZE;
	sbp->f_blocks = 0;
	sbp->f_bfree = 0;
	sbp->f_bavail = 0;
	sbp->f_files = 0;
	sbp->f_ffree = 0;
	}
	if (fsp != NULL) {
	fsp->fs_rtmax = 8192;
	fsp->fs_rtpref = 8192;
	fsp->fs_maxname = NFS_MAXNAMLEN;
	fsp->fs_wtmax = 8192;
	fsp->fs_wtpref = 8192;
	fsp->fs_wtmult = NFS_FABLKSIZE;
	fsp->fs_dtpref = 8192;
	fsp->fs_maxfilesize = 0xffffffffffffffffull;
	fsp->fs_timedelta.tv_sec = 0;
	fsp->fs_timedelta.tv_nsec = 1;
	fsp->fs_properties = (NFSV3_FSFLINK \| NFSV3_FSFSYMLINK \|
	NFSV3_FSFHOMOGENEOUS \| NFSV3_FSFCANSETTIME);
	}
	if (pc != NULL) {
	pc->pc_linkmax = NFS_LINK_MAX;
	pc->pc_namemax = NAME_MAX;
	pc->pc_notrunc = 0;
	pc->pc_chownrestricted = 0;
	pc->pc_caseinsensitive = 0;
	pc->pc_casepreserving = 1;
	}
	if (sfp != NULL) {
	sfp->sf_ffiles = UINT64_MAX;
	sfp->sf_tfiles = UINT64_MAX;
	sfp->sf_afiles = UINT64_MAX;
	sfp->sf_fbytes = UINT64_MAX;
	sfp->sf_tbytes = UINT64_MAX;
	sfp->sf_abytes = UINT64_MAX;
	}
	}

	/*
	* Loop around getting the attributes.
	*/
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsize = fxdr_unsigned(int, *tl);
	for (bitpos = 0; bitpos < NFSATTRBIT_MAX; bitpos++) {
	if (attrsum > attrsize) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	if (NFSISSET_ATTRBIT(&attrbits, bitpos))
	switch (bitpos) {
	case NFSATTRBIT_SUPPORTEDATTRS:
	retnotsup = 0;
	if (compare \|\| nap == NULL)
	error = nfsrv_getattrbits(nd, &retattrbits,
	&cnt, &retnotsup);
	else
	error = nfsrv_getattrbits(nd, &nap->na_suppattr,
	&cnt, &retnotsup);
	if (error)
	goto nfsmout;
	if (compare && !(*retcmpp)) {
	NFSSETSUPP_ATTRBIT(&checkattrbits, nd);

	/* Some filesystem do not support NFSv4ACL */
	if (nfsrv_useacl == 0 \|\| nfs_supportsnfsv4acls(vp) == 0) {
	NFSCLRBIT_ATTRBIT(&checkattrbits, NFSATTRBIT_ACL);
	NFSCLRBIT_ATTRBIT(&checkattrbits, NFSATTRBIT_ACLSUPPORT);
	}
	if (!NFSEQUAL_ATTRBIT(&retattrbits, &checkattrbits)
	\|\| retnotsup)
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += cnt;
	break;
	case NFSATTRBIT_TYPE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (nap->na_type != nfsv34tov_type(*tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_type = nfsv34tov_type(*tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_FHEXPIRETYPE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp)) {
	if (fxdr_unsigned(int, *tl) !=
	NFSV4FHTYPE_PERSISTENT)
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CHANGE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp)) {
	if (nap->na_filerev != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_filerev = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SIZE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp)) {
	if (nap->na_size != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_size = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_LINKSUPPORT:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_properties & NFSV3_FSFLINK) {
	if (*tl == newnfs_false)
	*retcmpp = NFSERR_NOTSAME;
	} else {
	if (*tl == newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	}
	} else if (fsp != NULL) {
	if (*tl == newnfs_true)
	fsp->fs_properties \|= NFSV3_FSFLINK;
	else
	fsp->fs_properties &= ~NFSV3_FSFLINK;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_SYMLINKSUPPORT:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_properties & NFSV3_FSFSYMLINK) {
	if (*tl == newnfs_false)
	*retcmpp = NFSERR_NOTSAME;
	} else {
	if (*tl == newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	}
	} else if (fsp != NULL) {
	if (*tl == newnfs_true)
	fsp->fs_properties \|= NFSV3_FSFSYMLINK;
	else
	fsp->fs_properties &= ~NFSV3_FSFSYMLINK;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_NAMEDATTR:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp)) {
	if (*tl != newnfs_false)
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_FSID:
	NFSM_DISSECT(tl, u_int32_t , 4 NFSX_UNSIGNED);
	thyp = fxdr_hyper(tl);
	tl += 2;
	thyp2 = fxdr_hyper(tl);
	if (compare) {
	if (*retcmpp == 0) {
	if (thyp != (u_int64_t)
	vp->v_mount->mnt_stat.f_fsid.val[0] \|\|
	thyp2 != (u_int64_t)
	vp->v_mount->mnt_stat.f_fsid.val[1])
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_filesid[0] = thyp;
	nap->na_filesid[1] = thyp2;
	}
	attrsum += (4 * NFSX_UNSIGNED);
	break;
	case NFSATTRBIT_UNIQUEHANDLES:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp)) {
	if (*tl != newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_LEASETIME:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (fxdr_unsigned(int, *tl) != nfsrv_lease &&
	!(*retcmpp))
	*retcmpp = NFSERR_NOTSAME;
	} else if (leasep != NULL) {
	leasep = fxdr_unsigned(u_int32_t, tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_RDATTRERROR:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp))
	*retcmpp = NFSERR_INVAL;
	} else if (rderrp != NULL) {
	rderrp = fxdr_unsigned(u_int32_t, tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_ACL:
	if (compare) {
	if (!(*retcmpp)) {
	if (nfsrv_useacl && nfs_supportsnfsv4acls(vp)) {
	NFSACL_T *naclp;

	naclp = acl_alloc(M_WAITOK);
	error = nfsrv_dissectacl(nd, naclp, &aceerr,
	&cnt, p);
	if (error) {
	acl_free(naclp);
	goto nfsmout;
	}
	if (aceerr \|\| aclp == NULL \|\|
	nfsrv_compareacl(aclp, naclp))
	*retcmpp = NFSERR_NOTSAME;
	acl_free(naclp);
	} else {
	error = nfsrv_dissectacl(nd, NULL, &aceerr,
	&cnt, p);
	*retcmpp = NFSERR_ATTRNOTSUPP;
	}
	}
	} else {
	if (vp != NULL && aclp != NULL)
	error = nfsrv_dissectacl(nd, aclp, &aceerr,
	&cnt, p);
	else
	error = nfsrv_dissectacl(nd, NULL, &aceerr,
	&cnt, p);
	if (error)
	goto nfsmout;
	}

	attrsum += cnt;
	break;
	case NFSATTRBIT_ACLSUPPORT:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp)) {
	if (nfsrv_useacl && nfs_supportsnfsv4acls(vp)) {
	if (fxdr_unsigned(u_int32_t, *tl) !=
	NFSV4ACE_SUPTYPES)
	*retcmpp = NFSERR_NOTSAME;
	} else {
	*retcmpp = NFSERR_ATTRNOTSUPP;
	}
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_ARCHIVE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CANSETTIME:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_properties & NFSV3_FSFCANSETTIME) {
	if (*tl == newnfs_false)
	*retcmpp = NFSERR_NOTSAME;
	} else {
	if (*tl == newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	}
	} else if (fsp != NULL) {
	if (*tl == newnfs_true)
	fsp->fs_properties \|= NFSV3_FSFCANSETTIME;
	else
	fsp->fs_properties &= ~NFSV3_FSFCANSETTIME;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CASEINSENSITIVE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (*tl != newnfs_false)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (pc != NULL) {
	pc->pc_caseinsensitive =
	fxdr_unsigned(u_int32_t, *tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CASEPRESERVING:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (*tl != newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (pc != NULL) {
	pc->pc_casepreserving =
	fxdr_unsigned(u_int32_t, *tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CHOWNRESTRICTED:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (*tl != newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (pc != NULL) {
	pc->pc_chownrestricted =
	fxdr_unsigned(u_int32_t, *tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_FILEHANDLE:
	error = nfsm_getfh(nd, &tnfhp);
	if (error)
	goto nfsmout;
	tfhsize = tnfhp->nfh_len;
	if (compare) {
	if (!(*retcmpp) &&
	!NFSRV_CMPFH(tnfhp->nfh_fh, tfhsize,
	fhp, fhsize))
	*retcmpp = NFSERR_NOTSAME;
	free(tnfhp, M_NFSFH);
	} else if (nfhpp != NULL) {
	*nfhpp = tnfhp;
	} else {
	free(tnfhp, M_NFSFH);
	}
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(tfhsize));
	break;
	case NFSATTRBIT_FILEID:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	thyp = fxdr_hyper(tl);
	if (compare) {
	if (!(*retcmpp)) {
	if (nap->na_fileid != thyp)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL)
	nap->na_fileid = thyp;
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FILESAVAIL:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp) &&
	sfp->sf_afiles != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	} else if (sfp != NULL) {
	sfp->sf_afiles = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FILESFREE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp) &&
	sfp->sf_ffiles != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	} else if (sfp != NULL) {
	sfp->sf_ffiles = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FILESTOTAL:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp) &&
	sfp->sf_tfiles != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	} else if (sfp != NULL) {
	sfp->sf_tfiles = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FSLOCATIONS:
	error = nfsrv_getrefstr(nd, &cp, &cp2, &l, &m);
	if (error)
	goto nfsmout;
	attrsum += l;
	if (compare && !(*retcmpp)) {
	refp = nfsv4root_getreferral(vp, NULL, 0);
	if (refp != NULL) {
	if (cp == NULL \|\| cp2 == NULL \|\|
	strcmp(cp, "/") \|\|
	strcmp(cp2, refp->nfr_srvlist))
	*retcmpp = NFSERR_NOTSAME;
	} else if (m == 0) {
	*retcmpp = NFSERR_NOTSAME;
	}
	}
	if (cp != NULL)
	free(cp, M_NFSSTRING);
	if (cp2 != NULL)
	free(cp2, M_NFSSTRING);
	break;
	case NFSATTRBIT_HIDDEN:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_HOMOGENEOUS:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_properties &
	NFSV3_FSFHOMOGENEOUS) {
	if (*tl == newnfs_false)
	*retcmpp = NFSERR_NOTSAME;
	} else {
	if (*tl == newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	}
	} else if (fsp != NULL) {
	if (*tl == newnfs_true)
	fsp->fs_properties \|= NFSV3_FSFHOMOGENEOUS;
	else
	fsp->fs_properties &= ~NFSV3_FSFHOMOGENEOUS;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MAXFILESIZE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	tnfsquad.qval = fxdr_hyper(tl);
	if (compare) {
	if (!(*retcmpp)) {
	tquad = NFSRV_MAXFILESIZE;
	if (tquad != tnfsquad.qval)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (fsp != NULL) {
	fsp->fs_maxfilesize = tnfsquad.qval;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_MAXLINK:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (fxdr_unsigned(int, *tl) != NFS_LINK_MAX)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (pc != NULL) {
	pc->pc_linkmax = fxdr_unsigned(u_int32_t, *tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MAXNAME:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_maxname !=
	fxdr_unsigned(u_int32_t, *tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else {
	tuint = fxdr_unsigned(u_int32_t, *tl);
	/*
	* Some Linux NFSv4 servers report this
	* as 0 or 4billion, so I'll set it to
	* NFS_MAXNAMLEN. If a server actually creates
	* a name longer than NFS_MAXNAMLEN, it will
	* get an error back.
	*/
	if (tuint == 0 \|\| tuint > NFS_MAXNAMLEN)
	tuint = NFS_MAXNAMLEN;
	if (fsp != NULL)
	fsp->fs_maxname = tuint;
	if (pc != NULL)
	pc->pc_namemax = tuint;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MAXREAD:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_rtmax != fxdr_unsigned(u_int32_t,
	(tl + 1)) \|\| tl != 0)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (fsp != NULL) {
	fsp->fs_rtmax = fxdr_unsigned(u_int32_t, *++tl);
	fsp->fs_rtpref = fsp->fs_rtmax;
	fsp->fs_dtpref = fsp->fs_rtpref;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_MAXWRITE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp)) {
	if (fsp->fs_wtmax != fxdr_unsigned(u_int32_t,
	(tl + 1)) \|\| tl != 0)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (fsp != NULL) {
	fsp->fs_wtmax = fxdr_unsigned(int, *++tl);
	fsp->fs_wtpref = fsp->fs_wtmax;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_MIMETYPE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	i = fxdr_unsigned(int, *tl);
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(i));
	error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
	if (error)
	goto nfsmout;
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_ATTRNOTSUPP;
	break;
	case NFSATTRBIT_MODE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (nap->na_mode != nfstov_mode(*tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_mode = nfstov_mode(*tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_NOTRUNC:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare) {
	if (!(*retcmpp)) {
	if (*tl != newnfs_true)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (pc != NULL) {
	pc->pc_notrunc = fxdr_unsigned(u_int32_t, *tl);
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_NUMLINKS:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	tuint = fxdr_unsigned(u_int32_t, *tl);
	if (compare) {
	if (!(*retcmpp)) {
	if ((u_int32_t)nap->na_nlink != tuint)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_nlink = tuint;
	}
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_OWNER:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	j = fxdr_unsigned(int, *tl);
	if (j < 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(j));
	if (j > NFSV4_SMALLSTR)
	cp = malloc(j + 1, M_NFSSTRING, M_WAITOK);
	else
	cp = namestr;
	error = nfsrv_mtostr(nd, cp, j);
	if (error) {
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	goto nfsmout;
	}
	if (compare) {
	if (!(*retcmpp)) {
	if (nfsv4_strtouid(nd, cp, j, &uid) \|\|
	nap->na_uid != uid)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	if (nfsv4_strtouid(nd, cp, j, &uid))
	nap->na_uid = nfsrv_defaultuid;
	else
	nap->na_uid = uid;
	}
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	break;
	case NFSATTRBIT_OWNERGROUP:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	j = fxdr_unsigned(int, *tl);
	if (j < 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(j));
	if (j > NFSV4_SMALLSTR)
	cp = malloc(j + 1, M_NFSSTRING, M_WAITOK);
	else
	cp = namestr;
	error = nfsrv_mtostr(nd, cp, j);
	if (error) {
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	goto nfsmout;
	}
	if (compare) {
	if (!(*retcmpp)) {
	if (nfsv4_strtogid(nd, cp, j, &gid) \|\|
	nap->na_gid != gid)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	if (nfsv4_strtogid(nd, cp, j, &gid))
	nap->na_gid = nfsrv_defaultgid;
	else
	nap->na_gid = gid;
	}
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	break;
	case NFSATTRBIT_QUOTAHARD:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (sbp != NULL) {
	if (priv_check_cred(cred, PRIV_VFS_EXCEEDQUOTA))
	freenum = sbp->f_bfree;
	else
	freenum = sbp->f_bavail;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(vp->v_mount,QCMD(Q_GETQUOTA,
	USRQUOTA), cred->cr_uid, (caddr_t)&dqb))
	freenum = min(dqb.dqb_bhardlimit, freenum);
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	uquad = (u_int64_t)freenum;
	NFSQUOTABLKTOBYTE(uquad, sbp->f_bsize);
	}
	if (compare && !(*retcmpp)) {
	if (uquad != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_QUOTASOFT:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (sbp != NULL) {
	if (priv_check_cred(cred, PRIV_VFS_EXCEEDQUOTA))
	freenum = sbp->f_bfree;
	else
	freenum = sbp->f_bavail;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(vp->v_mount,QCMD(Q_GETQUOTA,
	USRQUOTA), cred->cr_uid, (caddr_t)&dqb))
	freenum = min(dqb.dqb_bsoftlimit, freenum);
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	uquad = (u_int64_t)freenum;
	NFSQUOTABLKTOBYTE(uquad, sbp->f_bsize);
	}
	if (compare && !(*retcmpp)) {
	if (uquad != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_QUOTAUSED:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (sbp != NULL) {
	freenum = 0;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(vp->v_mount,QCMD(Q_GETQUOTA,
	USRQUOTA), cred->cr_uid, (caddr_t)&dqb))
	freenum = dqb.dqb_curblocks;
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	uquad = (u_int64_t)freenum;
	NFSQUOTABLKTOBYTE(uquad, sbp->f_bsize);
	}
	if (compare && !(*retcmpp)) {
	if (uquad != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_RAWDEV:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4SPECDATA);
	j = fxdr_unsigned(int, *tl++);
	k = fxdr_unsigned(int, *tl);
	if (compare) {
	if (!(*retcmpp)) {
	if (nap->na_rdev != NFSMAKEDEV(j, k))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_rdev = NFSMAKEDEV(j, k);
	}
	attrsum += NFSX_V4SPECDATA;
	break;
	case NFSATTRBIT_SPACEAVAIL:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp) &&
	sfp->sf_abytes != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	} else if (sfp != NULL) {
	sfp->sf_abytes = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SPACEFREE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp) &&
	sfp->sf_fbytes != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	} else if (sfp != NULL) {
	sfp->sf_fbytes = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SPACETOTAL:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (compare) {
	if (!(*retcmpp) &&
	sfp->sf_tbytes != fxdr_hyper(tl))
	*retcmpp = NFSERR_NOTSAME;
	} else if (sfp != NULL) {
	sfp->sf_tbytes = fxdr_hyper(tl);
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SPACEUSED:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	thyp = fxdr_hyper(tl);
	if (compare) {
	if (!(*retcmpp)) {
	if ((u_int64_t)nap->na_bytes != thyp)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_bytes = thyp;
	}
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SYSTEM:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_TIMEACCESS:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	fxdr_nfsv4time(tl, &temptime);
	if (compare) {
	if (!(*retcmpp)) {
	if (!NFS_CMPTIME(temptime, nap->na_atime))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_atime = temptime;
	}
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEACCESSSET:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsum += NFSX_UNSIGNED;
	i = fxdr_unsigned(int, *tl);
	if (i == NFSV4SATTRTIME_TOCLIENT) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	attrsum += NFSX_V4TIME;
	}
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_INVAL;
	break;
	case NFSATTRBIT_TIMEBACKUP:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMECREATE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	fxdr_nfsv4time(tl, &temptime);
	if (compare) {
	if (!(*retcmpp)) {
	if (!NFS_CMPTIME(temptime, nap->na_btime))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_btime = temptime;
	}
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEDELTA:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	if (fsp != NULL) {
	if (compare) {
	if (!(*retcmpp)) {
	if ((u_int32_t)fsp->fs_timedelta.tv_sec !=
	fxdr_unsigned(u_int32_t, *(tl + 1)) \|\|
	(u_int32_t)fsp->fs_timedelta.tv_nsec !=
	(fxdr_unsigned(u_int32_t, *(tl + 2)) %
	1000000000) \|\|
	*tl != 0)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else {
	fxdr_nfsv4time(tl, &fsp->fs_timedelta);
	}
	}
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMETADATA:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	fxdr_nfsv4time(tl, &temptime);
	if (compare) {
	if (!(*retcmpp)) {
	if (!NFS_CMPTIME(temptime, nap->na_ctime))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_ctime = temptime;
	}
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMODIFY:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	fxdr_nfsv4time(tl, &temptime);
	if (compare) {
	if (!(*retcmpp)) {
	if (!NFS_CMPTIME(temptime, nap->na_mtime))
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL) {
	nap->na_mtime = temptime;
	}
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMODIFYSET:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsum += NFSX_UNSIGNED;
	i = fxdr_unsigned(int, *tl);
	if (i == NFSV4SATTRTIME_TOCLIENT) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	attrsum += NFSX_V4TIME;
	}
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_INVAL;
	break;
	case NFSATTRBIT_MOUNTEDONFILEID:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	thyp = fxdr_hyper(tl);
	if (compare) {
	if (!(*retcmpp)) {
	if (!vp \|\| !nfsrv_atroot(vp, &thyp2))
	thyp2 = nap->na_fileid;
	if (thyp2 != thyp)
	*retcmpp = NFSERR_NOTSAME;
	}
	} else if (nap != NULL)
	nap->na_mntonfileno = thyp;
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SUPPATTREXCLCREAT:
	retnotsup = 0;
	error = nfsrv_getattrbits(nd, &retattrbits,
	&cnt, &retnotsup);
	if (error)
	goto nfsmout;
	if (compare && !(*retcmpp)) {
	NFSSETSUPP_ATTRBIT(&checkattrbits, nd);
	NFSCLRNOTSETABLE_ATTRBIT(&checkattrbits, nd);
	NFSCLRBIT_ATTRBIT(&checkattrbits,
	NFSATTRBIT_TIMEACCESSSET);
	if (!NFSEQUAL_ATTRBIT(&retattrbits, &checkattrbits)
	\|\| retnotsup)
	*retcmpp = NFSERR_NOTSAME;
	}
	attrsum += cnt;
	break;
	case NFSATTRBIT_FSLAYOUTTYPE:
	case NFSATTRBIT_LAYOUTTYPE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsum += NFSX_UNSIGNED;
	i = fxdr_unsigned(int, *tl);
	if (i > 0) {
	NFSM_DISSECT(tl, u_int32_t , i
	NFSX_UNSIGNED);
	attrsum += i * NFSX_UNSIGNED;
	j = fxdr_unsigned(int, *tl);
	if (i == 1 && compare && !(*retcmpp) &&
	(((nfsrv_doflexfile != 0 \|\|
	nfsrv_maxpnfsmirror > 1) &&
	j != NFSLAYOUT_FLEXFILE) \|\|
	(nfsrv_doflexfile == 0 &&
	j != NFSLAYOUT_NFSV4_1_FILES)))
	*retcmpp = NFSERR_NOTSAME;
	}
	if (nfsrv_devidcnt == 0) {
	if (compare && !(*retcmpp) && i > 0)
	*retcmpp = NFSERR_NOTSAME;
	} else {
	if (compare && !(*retcmpp) && i != 1)
	*retcmpp = NFSERR_NOTSAME;
	}
	break;
	case NFSATTRBIT_LAYOUTALIGNMENT:
	case NFSATTRBIT_LAYOUTBLKSIZE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsum += NFSX_UNSIGNED;
	i = fxdr_unsigned(int, *tl);
	if (compare && !(*retcmpp) && i != NFS_SRVMAXIO)
	*retcmpp = NFSERR_NOTSAME;
	break;
	default:
	printf("EEK! nfsv4_loadattr unknown attr=%d\n",
	bitpos);
	if (compare && !(*retcmpp))
	*retcmpp = NFSERR_ATTRNOTSUPP;
	/*
	* and get out of the loop, since we can't parse
	* the unknown attrbute data.
	*/
	bitpos = NFSATTRBIT_MAX;
	break;
	}
	}

	/*
	* some clients pad the attrlist, so we need to skip over the
	* padding.
	*/
	if (attrsum > attrsize) {
	error = NFSERR_BADXDR;
	} else {
	attrsize = NFSM_RNDUP(attrsize);
	if (attrsum < attrsize)
	error = nfsm_advance(nd, attrsize - attrsum, -1);
	}
	nfsmout:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Implement sleep locks for newnfs. The nfslock_usecnt allows for a
	* shared lock and the NFSXXX_LOCK flag permits an exclusive lock.
	* The first argument is a pointer to an nfsv4lock structure.
	* The second argument is 1 iff a blocking lock is wanted.
	* If this argument is 0, the call waits until no thread either wants nor
	* holds an exclusive lock.
	* It returns 1 if the lock was acquired, 0 otherwise.
	* If several processes call this function concurrently wanting the exclusive
	* lock, one will get the lock and the rest will return without getting the
	* lock. (If the caller must have the lock, it simply calls this function in a
	* loop until the function returns 1 to indicate the lock was acquired.)
	* Any usecnt must be decremented by calling nfsv4_relref() before
	* calling nfsv4_lock(). It was done this way, so nfsv4_lock() could
	* be called in a loop.
	* The isleptp argument is set to indicate if the call slept, iff not NULL
	* and the mp argument indicates to check for a forced dismount, iff not
	* NULL.
	*/
	int
	nfsv4_lock(struct nfsv4lock lp, int iwantlock, int isleptp,
	void mutex, struct mount mp)
	{

	if (isleptp)
	*isleptp = 0;
	/*
	* If a lock is wanted, loop around until the lock is acquired by
	* someone and then released. If I want the lock, try to acquire it.
	* For a lock to be issued, no lock must be in force and the usecnt
	* must be zero.
	*/
	if (iwantlock) {
	if (!(lp->nfslock_lock & NFSV4LOCK_LOCK) &&
	lp->nfslock_usecnt == 0) {
	lp->nfslock_lock &= ~NFSV4LOCK_LOCKWANTED;
	lp->nfslock_lock \|= NFSV4LOCK_LOCK;
	return (1);
	}
	lp->nfslock_lock \|= NFSV4LOCK_LOCKWANTED;
	}
	while (lp->nfslock_lock & (NFSV4LOCK_LOCK \| NFSV4LOCK_LOCKWANTED)) {
	if (mp != NULL && NFSCL_FORCEDISM(mp)) {
	lp->nfslock_lock &= ~NFSV4LOCK_LOCKWANTED;
	return (0);
	}
	lp->nfslock_lock \|= NFSV4LOCK_WANTED;
	if (isleptp)
	*isleptp = 1;
	(void) nfsmsleep(&lp->nfslock_lock, mutex,
	PZERO - 1, "nfsv4lck", NULL);
	if (iwantlock && !(lp->nfslock_lock & NFSV4LOCK_LOCK) &&
	lp->nfslock_usecnt == 0) {
	lp->nfslock_lock &= ~NFSV4LOCK_LOCKWANTED;
	lp->nfslock_lock \|= NFSV4LOCK_LOCK;
	return (1);
	}
	}
	return (0);
	}

	/*
	* Release the lock acquired by nfsv4_lock().
	* The second argument is set to 1 to indicate the nfslock_usecnt should be
	* incremented, as well.
	*/
	void
	nfsv4_unlock(struct nfsv4lock *lp, int incref)
	{

	lp->nfslock_lock &= ~NFSV4LOCK_LOCK;
	if (incref)
	lp->nfslock_usecnt++;
	nfsv4_wanted(lp);
	}

	/*
	* Release a reference cnt.
	*/
	void
	nfsv4_relref(struct nfsv4lock *lp)
	{

	if (lp->nfslock_usecnt <= 0)
	panic("nfsv4root ref cnt");
	lp->nfslock_usecnt--;
	if (lp->nfslock_usecnt == 0)
	nfsv4_wanted(lp);
	}

	/*
	* Get a reference cnt.
	* This function will wait for any exclusive lock to be released, but will
	* not wait for threads that want the exclusive lock. If priority needs
	* to be given to threads that need the exclusive lock, a call to nfsv4_lock()
	* with the 2nd argument == 0 should be done before calling nfsv4_getref().
	* If the mp argument is not NULL, check for NFSCL_FORCEDISM() being set and
	* return without getting a refcnt for that case.
	*/
	void
	nfsv4_getref(struct nfsv4lock lp, int isleptp, void *mutex,
	struct mount *mp)
	{

	if (isleptp)
	*isleptp = 0;

	/*
	* Wait for a lock held.
	*/
	while (lp->nfslock_lock & NFSV4LOCK_LOCK) {
	if (mp != NULL && NFSCL_FORCEDISM(mp))
	return;
	lp->nfslock_lock \|= NFSV4LOCK_WANTED;
	if (isleptp)
	*isleptp = 1;
	(void) nfsmsleep(&lp->nfslock_lock, mutex,
	PZERO - 1, "nfsv4gr", NULL);
	}
	if (mp != NULL && NFSCL_FORCEDISM(mp))
	return;

	lp->nfslock_usecnt++;
	}

	/*
	* Get a reference as above, but return failure instead of sleeping if
	* an exclusive lock is held.
	*/
	int
	nfsv4_getref_nonblock(struct nfsv4lock *lp)
	{

	if ((lp->nfslock_lock & NFSV4LOCK_LOCK) != 0)
	return (0);

	lp->nfslock_usecnt++;
	return (1);
	}

	/*
	* Test for a lock. Return 1 if locked, 0 otherwise.
	*/
	int
	nfsv4_testlock(struct nfsv4lock *lp)
	{

	if ((lp->nfslock_lock & NFSV4LOCK_LOCK) == 0 &&
	lp->nfslock_usecnt == 0)
	return (0);
	return (1);
	}

	/*
	* Wake up anyone sleeping, waiting for this lock.
	*/
	static void
	nfsv4_wanted(struct nfsv4lock *lp)
	{

	if (lp->nfslock_lock & NFSV4LOCK_WANTED) {
	lp->nfslock_lock &= ~NFSV4LOCK_WANTED;
	wakeup((caddr_t)&lp->nfslock_lock);
	}
	}

	/*
	* Copy a string from an mbuf list into a character array.
	* Return EBADRPC if there is an mbuf error,
	* 0 otherwise.
	*/
	int
	nfsrv_mtostr(struct nfsrv_descript nd, char str, int siz)
	{
	char *cp;
	int xfer, len;
	struct mbuf *mp;
	int rem, error = 0;

	mp = nd->nd_md;
	cp = nd->nd_dpos;
	len = mtod(mp, caddr_t) + mp->m_len - cp;
	rem = NFSM_RNDUP(siz) - siz;
	while (siz > 0) {
	if (len > siz)
	xfer = siz;
	else
	xfer = len;
	NFSBCOPY(cp, str, xfer);
	str += xfer;
	siz -= xfer;
	if (siz > 0) {
	mp = mp->m_next;
	if (mp == NULL) {
	error = EBADRPC;
	goto out;
	}
	cp = mtod(mp, caddr_t);
	len = mp->m_len;
	} else {
	cp += xfer;
	len -= xfer;
	}
	}
	*str = '\0';
	nd->nd_dpos = cp;
	nd->nd_md = mp;
	if (rem > 0) {
	if (len < rem)
	error = nfsm_advance(nd, rem, len);
	else
	nd->nd_dpos += rem;
	}

	out:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Fill in the attributes as marked by the bitmap (V4).
	*/
	int
	nfsv4_fillattr(struct nfsrv_descript nd, struct mount mp, vnode_t vp,
	NFSACL_T saclp, struct vattr vap, fhandle_t *fhp, int rderror,
	nfsattrbit_t attrbitp, struct ucred cred, NFSPROC_T *p, int isdgram,
	int reterr, int supports_nfsv4acls, int at_root, uint64_t mounted_on_fileno,
	struct statfs *pnfssf)
	{
	int bitpos, retnum = 0;
	u_int32_t *tl;
	int siz, prefixnum, error;
	u_char *cp, namestr[NFSV4_SMALLSTR];
	nfsattrbit_t attrbits, retbits;
	nfsattrbit_t *retbitp = &retbits;
	u_int32_t freenum, *retnump;
	u_int64_t uquad;
	struct statfs *fs;
	struct nfsfsinfo fsinf;
	struct timespec temptime;
	NFSACL_T aclp, naclp = NULL;
	size_t atsiz;
	bool xattrsupp;
	#ifdef QUOTA
	struct dqblk dqb;
	uid_t savuid;
	#endif

	/*
	* First, set the bits that can be filled and get fsinfo.
	*/
	NFSSET_ATTRBIT(retbitp, attrbitp);
	/*
	* If both p and cred are NULL, it is a client side setattr call.
	* If both p and cred are not NULL, it is a server side reply call.
	* If p is not NULL and cred is NULL, it is a client side callback
	* reply call.
	*/
	if (p == NULL && cred == NULL) {
	NFSCLRNOTSETABLE_ATTRBIT(retbitp, nd);
	aclp = saclp;
	} else {
	NFSCLRNOTFILLABLE_ATTRBIT(retbitp, nd);
	naclp = acl_alloc(M_WAITOK);
	aclp = naclp;
	}
	nfsvno_getfs(&fsinf, isdgram);
	#ifndef APPLE
	/*
	* Get the VFS_STATFS(), since some attributes need them.
	*/
	fs = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	if (NFSISSETSTATFS_ATTRBIT(retbitp)) {
	error = VFS_STATFS(mp, fs);
	if (error != 0) {
	if (reterr) {
	nd->nd_repstat = NFSERR_ACCES;
	free(fs, M_STATFS);
	return (0);
	}
	NFSCLRSTATFS_ATTRBIT(retbitp);
	}
	}
	#endif

	/*
	* And the NFSv4 ACL...
	*/
	if (NFSISSET_ATTRBIT(retbitp, NFSATTRBIT_ACLSUPPORT) &&
	(nfsrv_useacl == 0 \|\| ((cred != NULL \|\| p != NULL) &&
	supports_nfsv4acls == 0))) {
	NFSCLRBIT_ATTRBIT(retbitp, NFSATTRBIT_ACLSUPPORT);
	}
	if (NFSISSET_ATTRBIT(retbitp, NFSATTRBIT_ACL)) {
	if (nfsrv_useacl == 0 \|\| ((cred != NULL \|\| p != NULL) &&
	supports_nfsv4acls == 0)) {
	NFSCLRBIT_ATTRBIT(retbitp, NFSATTRBIT_ACL);
	} else if (naclp != NULL) {
	if (NFSVOPLOCK(vp, LK_SHARED) == 0) {
	error = VOP_ACCESSX(vp, VREAD_ACL, cred, p);
	if (error == 0)
	error = VOP_GETACL(vp, ACL_TYPE_NFS4,
	naclp, cred, p);
	NFSVOPUNLOCK(vp);
	} else
	error = NFSERR_PERM;
	if (error != 0) {
	if (reterr) {
	nd->nd_repstat = NFSERR_ACCES;
	free(fs, M_STATFS);
	return (0);
	}
	NFSCLRBIT_ATTRBIT(retbitp, NFSATTRBIT_ACL);
	}
	}
	}

	/* Check to see if Extended Attributes are supported. */
	xattrsupp = false;
	if (NFSISSET_ATTRBIT(retbitp, NFSATTRBIT_XATTRSUPPORT)) {
	if (NFSVOPLOCK(vp, LK_SHARED) == 0) {
	error = VOP_GETEXTATTR(vp, EXTATTR_NAMESPACE_USER,
	"xxx", NULL, &atsiz, cred, p);
	NFSVOPUNLOCK(vp);
	if (error != EOPNOTSUPP)
	xattrsupp = true;
	}
	}

	/*
	* Put out the attribute bitmap for the ones being filled in
	* and get the field for the number of attributes returned.
	*/
	prefixnum = nfsrv_putattrbit(nd, retbitp);
	NFSM_BUILD(retnump, u_int32_t *, NFSX_UNSIGNED);
	prefixnum += NFSX_UNSIGNED;

	/*
	* Now, loop around filling in the attributes for each bit set.
	*/
	for (bitpos = 0; bitpos < NFSATTRBIT_MAX; bitpos++) {
	if (NFSISSET_ATTRBIT(retbitp, bitpos)) {
	switch (bitpos) {
	case NFSATTRBIT_SUPPORTEDATTRS:
	NFSSETSUPP_ATTRBIT(&attrbits, nd);
	if (nfsrv_useacl == 0 \|\| ((cred != NULL \|\| p != NULL)
	&& supports_nfsv4acls == 0)) {
	NFSCLRBIT_ATTRBIT(&attrbits,NFSATTRBIT_ACLSUPPORT);
	NFSCLRBIT_ATTRBIT(&attrbits,NFSATTRBIT_ACL);
	}
	retnum += nfsrv_putattrbit(nd, &attrbits);
	break;
	case NFSATTRBIT_TYPE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = vtonfsv34_type(vap->va_type);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_FHEXPIRETYPE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4FHTYPE_PERSISTENT);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CHANGE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	txdr_hyper(vap->va_filerev, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SIZE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	txdr_hyper(vap->va_size, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_LINKSUPPORT:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	if (fsinf.fs_properties & NFSV3FSINFO_LINK)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_SYMLINKSUPPORT:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	if (fsinf.fs_properties & NFSV3FSINFO_SYMLINK)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_NAMEDATTR:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_FSID:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4FSID);
	*tl++ = 0;
	*tl++ = txdr_unsigned(mp->mnt_stat.f_fsid.val[0]);
	*tl++ = 0;
	*tl = txdr_unsigned(mp->mnt_stat.f_fsid.val[1]);
	retnum += NFSX_V4FSID;
	break;
	case NFSATTRBIT_UNIQUEHANDLES:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_true;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_LEASETIME:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(nfsrv_lease);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_RDATTRERROR:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(rderror);
	retnum += NFSX_UNSIGNED;
	break;
	/*
	* Recommended Attributes. (Only the supported ones.)
	*/
	case NFSATTRBIT_ACL:
	retnum += nfsrv_buildacl(nd, aclp, vnode_vtype(vp), p);
	break;
	case NFSATTRBIT_ACLSUPPORT:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4ACE_SUPTYPES);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CANSETTIME:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	if (fsinf.fs_properties & NFSV3FSINFO_CANSETTIME)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CASEINSENSITIVE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CASEPRESERVING:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_true;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_CHOWNRESTRICTED:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_true;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_FILEHANDLE:
	retnum += nfsm_fhtom(nd, (u_int8_t *)fhp, 0, 0);
	break;
	case NFSATTRBIT_FILEID:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	uquad = vap->va_fileid;
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FILESAVAIL:
	/*
	* Check quota and use min(quota, f_ffree).
	*/
	freenum = fs->f_ffree;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
	cred->cr_uid, (caddr_t)&dqb))
	freenum = min(dqb.dqb_isoftlimit-dqb.dqb_curinodes,
	freenum);
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	*tl++ = 0;
	*tl = txdr_unsigned(freenum);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FILESFREE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	*tl++ = 0;
	*tl = txdr_unsigned(fs->f_ffree);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FILESTOTAL:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	*tl++ = 0;
	*tl = txdr_unsigned(fs->f_files);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_FSLOCATIONS:
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = 0;
	*tl = 0;
	retnum += 2 * NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_HOMOGENEOUS:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	if (fsinf.fs_properties & NFSV3FSINFO_HOMOGENEOUS)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MAXFILESIZE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	uquad = NFSRV_MAXFILESIZE;
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_MAXLINK:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFS_LINK_MAX);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MAXNAME:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFS_MAXNAMLEN);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MAXREAD:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	*tl++ = 0;
	*tl = txdr_unsigned(fsinf.fs_rtmax);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_MAXWRITE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	*tl++ = 0;
	*tl = txdr_unsigned(fsinf.fs_wtmax);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_MODE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = vtonfsv34_mode(vap->va_mode);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_NOTRUNC:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = newnfs_true;
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_NUMLINKS:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(vap->va_nlink);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_OWNER:
	cp = namestr;
	nfsv4_uidtostr(vap->va_uid, &cp, &siz);
	retnum += nfsm_strtom(nd, cp, siz);
	if (cp != namestr)
	free(cp, M_NFSSTRING);
	break;
	case NFSATTRBIT_OWNERGROUP:
	cp = namestr;
	nfsv4_gidtostr(vap->va_gid, &cp, &siz);
	retnum += nfsm_strtom(nd, cp, siz);
	if (cp != namestr)
	free(cp, M_NFSSTRING);
	break;
	case NFSATTRBIT_QUOTAHARD:
	if (priv_check_cred(cred, PRIV_VFS_EXCEEDQUOTA))
	freenum = fs->f_bfree;
	else
	freenum = fs->f_bavail;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
	cred->cr_uid, (caddr_t)&dqb))
	freenum = min(dqb.dqb_bhardlimit, freenum);
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	uquad = (u_int64_t)freenum;
	NFSQUOTABLKTOBYTE(uquad, fs->f_bsize);
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_QUOTASOFT:
	if (priv_check_cred(cred, PRIV_VFS_EXCEEDQUOTA))
	freenum = fs->f_bfree;
	else
	freenum = fs->f_bavail;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
	cred->cr_uid, (caddr_t)&dqb))
	freenum = min(dqb.dqb_bsoftlimit, freenum);
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	uquad = (u_int64_t)freenum;
	NFSQUOTABLKTOBYTE(uquad, fs->f_bsize);
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_QUOTAUSED:
	freenum = 0;
	#ifdef QUOTA
	/*
	* ufs_quotactl() insists that the uid argument
	* equal p_ruid for non-root quota access, so
	* we'll just make sure that's the case.
	*/
	savuid = p->p_cred->p_ruid;
	p->p_cred->p_ruid = cred->cr_uid;
	if (!VFS_QUOTACTL(mp, QCMD(Q_GETQUOTA,USRQUOTA),
	cred->cr_uid, (caddr_t)&dqb))
	freenum = dqb.dqb_curblocks;
	p->p_cred->p_ruid = savuid;
	#endif /* QUOTA */
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	uquad = (u_int64_t)freenum;
	NFSQUOTABLKTOBYTE(uquad, fs->f_bsize);
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_RAWDEV:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4SPECDATA);
	*tl++ = txdr_unsigned(NFSMAJOR(vap->va_rdev));
	*tl = txdr_unsigned(NFSMINOR(vap->va_rdev));
	retnum += NFSX_V4SPECDATA;
	break;
	case NFSATTRBIT_SPACEAVAIL:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE)) {
	if (pnfssf != NULL)
	uquad = (u_int64_t)pnfssf->f_bfree;
	else
	uquad = (u_int64_t)fs->f_bfree;
	} else {
	if (pnfssf != NULL)
	uquad = (u_int64_t)pnfssf->f_bavail;
	else
	uquad = (u_int64_t)fs->f_bavail;
	}
	if (pnfssf != NULL)
	uquad *= pnfssf->f_bsize;
	else
	uquad *= fs->f_bsize;
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SPACEFREE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	if (pnfssf != NULL) {
	uquad = (u_int64_t)pnfssf->f_bfree;
	uquad *= pnfssf->f_bsize;
	} else {
	uquad = (u_int64_t)fs->f_bfree;
	uquad *= fs->f_bsize;
	}
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SPACETOTAL:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	if (pnfssf != NULL) {
	uquad = (u_int64_t)pnfssf->f_blocks;
	uquad *= pnfssf->f_bsize;
	} else {
	uquad = (u_int64_t)fs->f_blocks;
	uquad *= fs->f_bsize;
	}
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SPACEUSED:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	txdr_hyper(vap->va_bytes, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_TIMEACCESS:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4TIME);
	txdr_nfsv4time(&vap->va_atime, tl);
	retnum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEACCESSSET:
	if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4SETTIME);
	*tl++ = txdr_unsigned(NFSV4SATTRTIME_TOCLIENT);
	txdr_nfsv4time(&vap->va_atime, tl);
	retnum += NFSX_V4SETTIME;
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4SATTRTIME_TOSERVER);
	retnum += NFSX_UNSIGNED;
	}
	break;
	case NFSATTRBIT_TIMEDELTA:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4TIME);
	temptime.tv_sec = 0;
	temptime.tv_nsec = 1000000000 / hz;
	txdr_nfsv4time(&temptime, tl);
	retnum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMETADATA:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4TIME);
	txdr_nfsv4time(&vap->va_ctime, tl);
	retnum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMODIFY:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4TIME);
	txdr_nfsv4time(&vap->va_mtime, tl);
	retnum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMECREATE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4TIME);
	txdr_nfsv4time(&vap->va_birthtime, tl);
	retnum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMODIFYSET:
	if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
	NFSM_BUILD(tl, u_int32_t *, NFSX_V4SETTIME);
	*tl++ = txdr_unsigned(NFSV4SATTRTIME_TOCLIENT);
	txdr_nfsv4time(&vap->va_mtime, tl);
	retnum += NFSX_V4SETTIME;
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4SATTRTIME_TOSERVER);
	retnum += NFSX_UNSIGNED;
	}
	break;
	case NFSATTRBIT_MOUNTEDONFILEID:
	NFSM_BUILD(tl, u_int32_t *, NFSX_HYPER);
	if (at_root != 0)
	uquad = mounted_on_fileno;
	else
	uquad = vap->va_fileid;
	txdr_hyper(uquad, tl);
	retnum += NFSX_HYPER;
	break;
	case NFSATTRBIT_SUPPATTREXCLCREAT:
	NFSSETSUPP_ATTRBIT(&attrbits, nd);
	NFSCLRNOTSETABLE_ATTRBIT(&attrbits, nd);
	NFSCLRBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEACCESSSET);
	retnum += nfsrv_putattrbit(nd, &attrbits);
	break;
	case NFSATTRBIT_FSLAYOUTTYPE:
	case NFSATTRBIT_LAYOUTTYPE:
	if (nfsrv_devidcnt == 0)
	siz = 1;
	else
	siz = 2;
	if (siz == 2) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	tl++ = txdr_unsigned(1); / One entry. */
	if (nfsrv_doflexfile != 0 \|\|
	nfsrv_maxpnfsmirror > 1)
	*tl = txdr_unsigned(NFSLAYOUT_FLEXFILE);
	else
	*tl = txdr_unsigned(
	NFSLAYOUT_NFSV4_1_FILES);
	} else {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = 0;
	}
	retnum += siz * NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_LAYOUTALIGNMENT:
	case NFSATTRBIT_LAYOUTBLKSIZE:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFS_SRVMAXIO);
	retnum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_XATTRSUPPORT:
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	if (xattrsupp)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	retnum += NFSX_UNSIGNED;
	break;
	default:
	printf("EEK! Bad V4 attribute bitpos=%d\n", bitpos);
	}
	}
	}
	if (naclp != NULL)
	acl_free(naclp);
	free(fs, M_STATFS);
	*retnump = txdr_unsigned(retnum);
	return (retnum + prefixnum);
	}

	/*
	* Put the attribute bits onto an mbuf list.
	* Return the number of bytes of output generated.
	*/
	int
	nfsrv_putattrbit(struct nfsrv_descript nd, nfsattrbit_t attrbitp)
	{
	u_int32_t *tl;
	int cnt, i, bytesize;

	for (cnt = NFSATTRBIT_MAXWORDS; cnt > 0; cnt--)
	if (attrbitp->bits[cnt - 1])
	break;
	bytesize = (cnt + 1) * NFSX_UNSIGNED;
	NFSM_BUILD(tl, u_int32_t *, bytesize);
	*tl++ = txdr_unsigned(cnt);
	for (i = 0; i < cnt; i++)
	*tl++ = txdr_unsigned(attrbitp->bits[i]);
	return (bytesize);
	}

	/*
	* Convert a uid to a string.
	* If the lookup fails, just output the digits.
	* uid - the user id
	* cpp - points to a buffer of size NFSV4_SMALLSTR
	* (malloc a larger one, as required)
	* retlenp - pointer to length to be returned
	*/
	void
	nfsv4_uidtostr(uid_t uid, u_char *cpp, int retlenp)
	{
	int i;
	struct nfsusrgrp *usrp;
	u_char cp = cpp;
	uid_t tmp;
	int cnt, hasampersand, len = NFSV4_SMALLSTR, ret;
	struct nfsrv_lughash *hp;

	cnt = 0;
	tryagain:
	if (nfsrv_dnsnamelen > 0 && !nfs_enable_uidtostring) {
	/*
	* Always map nfsrv_defaultuid to "nobody".
	*/
	if (uid == nfsrv_defaultuid) {
	i = nfsrv_dnsnamelen + 7;
	if (i > len) {
	if (len > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	cp = malloc(i, M_NFSSTRING, M_WAITOK);
	*cpp = cp;
	len = i;
	goto tryagain;
	}
	*retlenp = i;
	NFSBCOPY("nobody@", cp, 7);
	cp += 7;
	NFSBCOPY(nfsrv_dnsname, cp, nfsrv_dnsnamelen);
	return;
	}
	hasampersand = 0;
	hp = NFSUSERHASH(uid);
	mtx_lock(&hp->mtx);
	TAILQ_FOREACH(usrp, &hp->lughead, lug_numhash) {
	if (usrp->lug_uid == uid) {
	if (usrp->lug_expiry < NFSD_MONOSEC)
	break;
	/*
	* If the name doesn't already have an '@'
	* in it, append @domainname to it.
	*/
	for (i = 0; i < usrp->lug_namelen; i++) {
	if (usrp->lug_name[i] == '@') {
	hasampersand = 1;
	break;
	}
	}
	if (hasampersand)
	i = usrp->lug_namelen;
	else
	i = usrp->lug_namelen +
	nfsrv_dnsnamelen + 1;
	if (i > len) {
	mtx_unlock(&hp->mtx);
	if (len > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	cp = malloc(i, M_NFSSTRING, M_WAITOK);
	*cpp = cp;
	len = i;
	goto tryagain;
	}
	*retlenp = i;
	NFSBCOPY(usrp->lug_name, cp, usrp->lug_namelen);
	if (!hasampersand) {
	cp += usrp->lug_namelen;
	*cp++ = '@';
	NFSBCOPY(nfsrv_dnsname, cp, nfsrv_dnsnamelen);
	}
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	TAILQ_INSERT_TAIL(&hp->lughead, usrp,
	lug_numhash);
	mtx_unlock(&hp->mtx);
	return;
	}
	}
	mtx_unlock(&hp->mtx);
	cnt++;
	ret = nfsrv_getuser(RPCNFSUSERD_GETUID, uid, (gid_t)0, NULL);
	if (ret == 0 && cnt < 2)
	goto tryagain;
	}

	/*
	* No match, just return a string of digits.
	*/
	tmp = uid;
	i = 0;
	while (tmp \|\| i == 0) {
	tmp /= 10;
	i++;
	}
	len = (i > len) ? len : i;
	*retlenp = len;
	cp += (len - 1);
	tmp = uid;
	for (i = 0; i < len; i++) {
	*cp-- = '0' + (tmp % 10);
	tmp /= 10;
	}
	return;
	}

	/*
	* Get a credential for the uid with the server's group list.
	* If none is found, just return the credential passed in after
	* logging a warning message.
	*/
	struct ucred *
	nfsrv_getgrpscred(struct ucred *oldcred)
	{
	struct nfsusrgrp *usrp;
	struct ucred *newcred;
	int cnt, ret;
	uid_t uid;
	struct nfsrv_lughash *hp;

	cnt = 0;
	uid = oldcred->cr_uid;
	tryagain:
	if (nfsrv_dnsnamelen > 0) {
	hp = NFSUSERHASH(uid);
	mtx_lock(&hp->mtx);
	TAILQ_FOREACH(usrp, &hp->lughead, lug_numhash) {
	if (usrp->lug_uid == uid) {
	if (usrp->lug_expiry < NFSD_MONOSEC)
	break;
	if (usrp->lug_cred != NULL) {
	newcred = crhold(usrp->lug_cred);
	crfree(oldcred);
	} else
	newcred = oldcred;
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	TAILQ_INSERT_TAIL(&hp->lughead, usrp,
	lug_numhash);
	mtx_unlock(&hp->mtx);
	return (newcred);
	}
	}
	mtx_unlock(&hp->mtx);
	cnt++;
	ret = nfsrv_getuser(RPCNFSUSERD_GETUID, uid, (gid_t)0, NULL);
	if (ret == 0 && cnt < 2)
	goto tryagain;
	}
	return (oldcred);
	}

	/*
	* Convert a string to a uid.
	* If no conversion is possible return NFSERR_BADOWNER, otherwise
	* return 0.
	* If this is called from a client side mount using AUTH_SYS and the
	* string is made up entirely of digits, just convert the string to
	* a number.
	*/
	int
	nfsv4_strtouid(struct nfsrv_descript nd, u_char str, int len, uid_t *uidp)
	{
	int i;
	char cp, endstr, *str0;
	struct nfsusrgrp *usrp;
	int cnt, ret;
	int error = 0;
	uid_t tuid;
	struct nfsrv_lughash hp, hp2;

	if (len == 0) {
	error = NFSERR_BADOWNER;
	goto out;
	}
	/* If a string of digits and an AUTH_SYS mount, just convert it. */
	str0 = str;
	tuid = (uid_t)strtoul(str0, &endstr, 10);
	if ((endstr - str0) == len) {
	/* A numeric string. */
	if ((nd->nd_flag & ND_KERBV) == 0 &&
	((nd->nd_flag & ND_NFSCL) != 0 \|\|
	nfsd_enable_stringtouid != 0))
	*uidp = tuid;
	else
	error = NFSERR_BADOWNER;
	goto out;
	}
	/*
	* Look for an '@'.
	*/
	cp = strchr(str0, '@');
	if (cp != NULL)
	i = (int)(cp++ - str0);
	else
	i = len;

	cnt = 0;
	tryagain:
	if (nfsrv_dnsnamelen > 0) {
	/*
	* If an '@' is found and the domain name matches, search for
	* the name with dns stripped off.
	* Mixed case alpahbetics will match for the domain name, but
	* all upper case will not.
	*/
	if (cnt == 0 && i < len && i > 0 &&
	(len - 1 - i) == nfsrv_dnsnamelen &&
	!nfsrv_cmpmixedcase(cp, nfsrv_dnsname, nfsrv_dnsnamelen)) {
	len -= (nfsrv_dnsnamelen + 1);
	*(cp - 1) = '\0';
	}

	/*
	* Check for the special case of "nobody".
	*/
	if (len == 6 && !NFSBCMP(str, "nobody", 6)) {
	*uidp = nfsrv_defaultuid;
	error = 0;
	goto out;
	}

	hp = NFSUSERNAMEHASH(str, len);
	mtx_lock(&hp->mtx);
	TAILQ_FOREACH(usrp, &hp->lughead, lug_namehash) {
	if (usrp->lug_namelen == len &&
	!NFSBCMP(usrp->lug_name, str, len)) {
	if (usrp->lug_expiry < NFSD_MONOSEC)
	break;
	hp2 = NFSUSERHASH(usrp->lug_uid);
	mtx_lock(&hp2->mtx);
	TAILQ_REMOVE(&hp2->lughead, usrp, lug_numhash);
	TAILQ_INSERT_TAIL(&hp2->lughead, usrp,
	lug_numhash);
	*uidp = usrp->lug_uid;
	mtx_unlock(&hp2->mtx);
	mtx_unlock(&hp->mtx);
	error = 0;
	goto out;
	}
	}
	mtx_unlock(&hp->mtx);
	cnt++;
	ret = nfsrv_getuser(RPCNFSUSERD_GETUSER, (uid_t)0, (gid_t)0,
	str);
	if (ret == 0 && cnt < 2)
	goto tryagain;
	}
	error = NFSERR_BADOWNER;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Convert a gid to a string.
	* gid - the group id
	* cpp - points to a buffer of size NFSV4_SMALLSTR
	* (malloc a larger one, as required)
	* retlenp - pointer to length to be returned
	*/
	void
	nfsv4_gidtostr(gid_t gid, u_char *cpp, int retlenp)
	{
	int i;
	struct nfsusrgrp *usrp;
	u_char cp = cpp;
	gid_t tmp;
	int cnt, hasampersand, len = NFSV4_SMALLSTR, ret;
	struct nfsrv_lughash *hp;

	cnt = 0;
	tryagain:
	if (nfsrv_dnsnamelen > 0 && !nfs_enable_uidtostring) {
	/*
	* Always map nfsrv_defaultgid to "nogroup".
	*/
	if (gid == nfsrv_defaultgid) {
	i = nfsrv_dnsnamelen + 8;
	if (i > len) {
	if (len > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	cp = malloc(i, M_NFSSTRING, M_WAITOK);
	*cpp = cp;
	len = i;
	goto tryagain;
	}
	*retlenp = i;
	NFSBCOPY("nogroup@", cp, 8);
	cp += 8;
	NFSBCOPY(nfsrv_dnsname, cp, nfsrv_dnsnamelen);
	return;
	}
	hasampersand = 0;
	hp = NFSGROUPHASH(gid);
	mtx_lock(&hp->mtx);
	TAILQ_FOREACH(usrp, &hp->lughead, lug_numhash) {
	if (usrp->lug_gid == gid) {
	if (usrp->lug_expiry < NFSD_MONOSEC)
	break;
	/*
	* If the name doesn't already have an '@'
	* in it, append @domainname to it.
	*/
	for (i = 0; i < usrp->lug_namelen; i++) {
	if (usrp->lug_name[i] == '@') {
	hasampersand = 1;
	break;
	}
	}
	if (hasampersand)
	i = usrp->lug_namelen;
	else
	i = usrp->lug_namelen +
	nfsrv_dnsnamelen + 1;
	if (i > len) {
	mtx_unlock(&hp->mtx);
	if (len > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	cp = malloc(i, M_NFSSTRING, M_WAITOK);
	*cpp = cp;
	len = i;
	goto tryagain;
	}
	*retlenp = i;
	NFSBCOPY(usrp->lug_name, cp, usrp->lug_namelen);
	if (!hasampersand) {
	cp += usrp->lug_namelen;
	*cp++ = '@';
	NFSBCOPY(nfsrv_dnsname, cp, nfsrv_dnsnamelen);
	}
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	TAILQ_INSERT_TAIL(&hp->lughead, usrp,
	lug_numhash);
	mtx_unlock(&hp->mtx);
	return;
	}
	}
	mtx_unlock(&hp->mtx);
	cnt++;
	ret = nfsrv_getuser(RPCNFSUSERD_GETGID, (uid_t)0, gid, NULL);
	if (ret == 0 && cnt < 2)
	goto tryagain;
	}

	/*
	* No match, just return a string of digits.
	*/
	tmp = gid;
	i = 0;
	while (tmp \|\| i == 0) {
	tmp /= 10;
	i++;
	}
	len = (i > len) ? len : i;
	*retlenp = len;
	cp += (len - 1);
	tmp = gid;
	for (i = 0; i < len; i++) {
	*cp-- = '0' + (tmp % 10);
	tmp /= 10;
	}
	return;
	}

	/*
	* Convert a string to a gid.
	* If no conversion is possible return NFSERR_BADOWNER, otherwise
	* return 0.
	* If this is called from a client side mount using AUTH_SYS and the
	* string is made up entirely of digits, just convert the string to
	* a number.
	*/
	int
	nfsv4_strtogid(struct nfsrv_descript nd, u_char str, int len, gid_t *gidp)
	{
	int i;
	char cp, endstr, *str0;
	struct nfsusrgrp *usrp;
	int cnt, ret;
	int error = 0;
	gid_t tgid;
	struct nfsrv_lughash hp, hp2;

	if (len == 0) {
	error = NFSERR_BADOWNER;
	goto out;
	}
	/* If a string of digits and an AUTH_SYS mount, just convert it. */
	str0 = str;
	tgid = (gid_t)strtoul(str0, &endstr, 10);
	if ((endstr - str0) == len) {
	/* A numeric string. */
	if ((nd->nd_flag & ND_KERBV) == 0 &&
	((nd->nd_flag & ND_NFSCL) != 0 \|\|
	nfsd_enable_stringtouid != 0))
	*gidp = tgid;
	else
	error = NFSERR_BADOWNER;
	goto out;
	}
	/*
	* Look for an '@'.
	*/
	cp = strchr(str0, '@');
	if (cp != NULL)
	i = (int)(cp++ - str0);
	else
	i = len;

	cnt = 0;
	tryagain:
	if (nfsrv_dnsnamelen > 0) {
	/*
	* If an '@' is found and the dns name matches, search for the
	* name with the dns stripped off.
	*/
	if (cnt == 0 && i < len && i > 0 &&
	(len - 1 - i) == nfsrv_dnsnamelen &&
	!nfsrv_cmpmixedcase(cp, nfsrv_dnsname, nfsrv_dnsnamelen)) {
	len -= (nfsrv_dnsnamelen + 1);
	*(cp - 1) = '\0';
	}

	/*
	* Check for the special case of "nogroup".
	*/
	if (len == 7 && !NFSBCMP(str, "nogroup", 7)) {
	*gidp = nfsrv_defaultgid;
	error = 0;
	goto out;
	}

	hp = NFSGROUPNAMEHASH(str, len);
	mtx_lock(&hp->mtx);
	TAILQ_FOREACH(usrp, &hp->lughead, lug_namehash) {
	if (usrp->lug_namelen == len &&
	!NFSBCMP(usrp->lug_name, str, len)) {
	if (usrp->lug_expiry < NFSD_MONOSEC)
	break;
	hp2 = NFSGROUPHASH(usrp->lug_gid);
	mtx_lock(&hp2->mtx);
	TAILQ_REMOVE(&hp2->lughead, usrp, lug_numhash);
	TAILQ_INSERT_TAIL(&hp2->lughead, usrp,
	lug_numhash);
	*gidp = usrp->lug_gid;
	mtx_unlock(&hp2->mtx);
	mtx_unlock(&hp->mtx);
	error = 0;
	goto out;
	}
	}
	mtx_unlock(&hp->mtx);
	cnt++;
	ret = nfsrv_getuser(RPCNFSUSERD_GETGROUP, (uid_t)0, (gid_t)0,
	str);
	if (ret == 0 && cnt < 2)
	goto tryagain;
	}
	error = NFSERR_BADOWNER;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Cmp len chars, allowing mixed case in the first argument to match lower
	* case in the second, but not if the first argument is all upper case.
	* Return 0 for a match, 1 otherwise.
	*/
	static int
	nfsrv_cmpmixedcase(u_char cp, u_char cp2, int len)
	{
	int i;
	u_char tmp;
	int fndlower = 0;

	for (i = 0; i < len; i++) {
	if (cp >= 'A' && cp <= 'Z') {
	tmp = *cp++ + ('a' - 'A');
	} else {
	tmp = *cp++;
	if (tmp >= 'a' && tmp <= 'z')
	fndlower = 1;
	}
	if (tmp != *cp2++)
	return (1);
	}
	if (fndlower)
	return (0);
	else
	return (1);
	}

	/*
	* Set the port for the nfsuserd.
	*/
	int
	nfsrv_nfsuserdport(struct nfsuserd_args nargs, NFSPROC_T p)
	{
	struct nfssockreq *rp;
	#ifdef INET
	struct sockaddr_in *ad;
	#endif
	#ifdef INET6
	struct sockaddr_in6 *ad6;
	const struct in6_addr in6loopback = IN6ADDR_LOOPBACK_INIT;
	#endif
	int error;

	NFSLOCKNAMEID();
	if (nfsrv_nfsuserd != NOTRUNNING) {
	NFSUNLOCKNAMEID();
	error = EPERM;
	goto out;
	}
	nfsrv_nfsuserd = STARTSTOP;
	/*
	* Set up the socket record and connect.
	* Set nr_client NULL before unlocking, just to ensure that no other
	* process/thread/core will use a bogus old value. This could only
	* occur if the use of the nameid lock to protect nfsrv_nfsuserd is
	* broken.
	*/
	rp = &nfsrv_nfsuserdsock;
	rp->nr_client = NULL;
	NFSUNLOCKNAMEID();
	rp->nr_sotype = SOCK_DGRAM;
	rp->nr_soproto = IPPROTO_UDP;
	rp->nr_lock = (NFSR_RESERVEDPORT \| NFSR_LOCALHOST);
	rp->nr_cred = NULL;
	rp->nr_prog = RPCPROG_NFSUSERD;
	error = 0;
	switch (nargs->nuserd_family) {
	#ifdef INET
	case AF_INET:
	rp->nr_nam = malloc(sizeof(struct sockaddr_in), M_SONAME,
	M_WAITOK \| M_ZERO);
	ad = (struct sockaddr_in *)rp->nr_nam;
	ad->sin_len = sizeof(struct sockaddr_in);
	ad->sin_family = AF_INET;
	ad->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
	ad->sin_port = nargs->nuserd_port;
	break;
	#endif
	#ifdef INET6
	case AF_INET6:
	rp->nr_nam = malloc(sizeof(struct sockaddr_in6), M_SONAME,
	M_WAITOK \| M_ZERO);
	ad6 = (struct sockaddr_in6 *)rp->nr_nam;
	ad6->sin6_len = sizeof(struct sockaddr_in6);
	ad6->sin6_family = AF_INET6;
	ad6->sin6_addr = in6loopback;
	ad6->sin6_port = nargs->nuserd_port;
	break;
	#endif
	default:
	error = ENXIO;
	}
	rp->nr_vers = RPCNFSUSERD_VERS;
	if (error == 0)
	error = newnfs_connect(NULL, rp, NFSPROCCRED(p), p, 0);
	if (error == 0) {
	NFSLOCKNAMEID();
	nfsrv_nfsuserd = RUNNING;
	NFSUNLOCKNAMEID();
	} else {
	free(rp->nr_nam, M_SONAME);
	NFSLOCKNAMEID();
	nfsrv_nfsuserd = NOTRUNNING;
	NFSUNLOCKNAMEID();
	}
	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Delete the nfsuserd port.
	*/
	void
	nfsrv_nfsuserddelport(void)
	{

	NFSLOCKNAMEID();
	if (nfsrv_nfsuserd != RUNNING) {
	NFSUNLOCKNAMEID();
	return;
	}
	nfsrv_nfsuserd = STARTSTOP;
	/* Wait for all upcalls to complete. */
	while (nfsrv_userdupcalls > 0)
	msleep(&nfsrv_userdupcalls, NFSNAMEIDMUTEXPTR, PVFS,
	"nfsupcalls", 0);
	NFSUNLOCKNAMEID();
	newnfs_disconnect(&nfsrv_nfsuserdsock);
	free(nfsrv_nfsuserdsock.nr_nam, M_SONAME);
	NFSLOCKNAMEID();
	nfsrv_nfsuserd = NOTRUNNING;
	NFSUNLOCKNAMEID();
	}

	/*
	* Do upcalls to the nfsuserd, for cache misses of the owner/ownergroup
	* name<-->id cache.
	* Returns 0 upon success, non-zero otherwise.
	*/
	static int
	nfsrv_getuser(int procnum, uid_t uid, gid_t gid, char *name)
	{
	u_int32_t *tl;
	struct nfsrv_descript *nd;
	int len;
	struct nfsrv_descript nfsd;
	struct ucred *cred;
	int error;

	NFSLOCKNAMEID();
	if (nfsrv_nfsuserd != RUNNING) {
	NFSUNLOCKNAMEID();
	error = EPERM;
	goto out;
	}
	/*
	* Maintain a count of upcalls in progress, so that nfsrv_X()
	* can wait until no upcalls are in progress.
	*/
	nfsrv_userdupcalls++;
	NFSUNLOCKNAMEID();
	KASSERT(nfsrv_userdupcalls > 0,
	("nfsrv_getuser: non-positive upcalls"));
	nd = &nfsd;
	cred = newnfs_getcred();
	nd->nd_flag = ND_GSSINITREPLY;
	nfsrvd_rephead(nd);

	nd->nd_procnum = procnum;
	if (procnum == RPCNFSUSERD_GETUID \|\| procnum == RPCNFSUSERD_GETGID) {
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	if (procnum == RPCNFSUSERD_GETUID)
	*tl = txdr_unsigned(uid);
	else
	*tl = txdr_unsigned(gid);
	} else {
	len = strlen(name);
	(void) nfsm_strtom(nd, name, len);
	}
	error = newnfs_request(nd, NULL, NULL, &nfsrv_nfsuserdsock, NULL, NULL,
	cred, RPCPROG_NFSUSERD, RPCNFSUSERD_VERS, NULL, 0, NULL, NULL);
	NFSLOCKNAMEID();
	if (--nfsrv_userdupcalls == 0 && nfsrv_nfsuserd == STARTSTOP)
	wakeup(&nfsrv_userdupcalls);
	NFSUNLOCKNAMEID();
	NFSFREECRED(cred);
	if (!error) {
	m_freem(nd->nd_mrep);
	error = nd->nd_repstat;
	}
	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* This function is called from the nfssvc(2) system call, to update the
	* kernel user/group name list(s) for the V4 owner and ownergroup attributes.
	*/
	int
	nfssvc_idname(struct nfsd_idargs *nidp)
	{
	struct nfsusrgrp nusrp, usrp, *newusrp;
	struct nfsrv_lughash hp_name, hp_idnum, *thp;
	int i, group_locked, groupname_locked, user_locked, username_locked;
	int error = 0;
	u_char *cp;
	gid_t *grps;
	struct ucred *cr;
	static int onethread = 0;
	static time_t lasttime = 0;

	if (nidp->nid_namelen <= 0 \|\| nidp->nid_namelen > MAXHOSTNAMELEN) {
	error = EINVAL;
	goto out;
	}
	if (nidp->nid_flag & NFSID_INITIALIZE) {
	cp = malloc(nidp->nid_namelen + 1, M_NFSSTRING, M_WAITOK);
	error = copyin(nidp->nid_name, cp, nidp->nid_namelen);
	if (error != 0) {
	free(cp, M_NFSSTRING);
	goto out;
	}
	if (atomic_cmpset_acq_int(&nfsrv_dnsnamelen, 0, 0) == 0) {
	/*
	* Free up all the old stuff and reinitialize hash
	* lists. All mutexes for both lists must be locked,
	* with the user/group name ones before the uid/gid
	* ones, to avoid a LOR.
	*/
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsusernamehash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsuserhash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	TAILQ_FOREACH_SAFE(usrp,
	&nfsuserhash[i].lughead, lug_numhash, nusrp)
	nfsrv_removeuser(usrp, 1);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsuserhash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsusernamehash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsgroupnamehash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsgrouphash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	TAILQ_FOREACH_SAFE(usrp,
	&nfsgrouphash[i].lughead, lug_numhash,
	nusrp)
	nfsrv_removeuser(usrp, 0);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsgrouphash[i].mtx);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsgroupnamehash[i].mtx);
	free(nfsrv_dnsname, M_NFSSTRING);
	nfsrv_dnsname = NULL;
	}
	if (nfsuserhash == NULL) {
	/* Allocate the hash tables. */
	nfsuserhash = malloc(sizeof(struct nfsrv_lughash) *
	nfsrv_lughashsize, M_NFSUSERGROUP, M_WAITOK \|
	M_ZERO);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_init(&nfsuserhash[i].mtx, "nfsuidhash",
	NULL, MTX_DEF \| MTX_DUPOK);
	nfsusernamehash = malloc(sizeof(struct nfsrv_lughash) *
	nfsrv_lughashsize, M_NFSUSERGROUP, M_WAITOK \|
	M_ZERO);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_init(&nfsusernamehash[i].mtx,
	"nfsusrhash", NULL, MTX_DEF \|
	MTX_DUPOK);
	nfsgrouphash = malloc(sizeof(struct nfsrv_lughash) *
	nfsrv_lughashsize, M_NFSUSERGROUP, M_WAITOK \|
	M_ZERO);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_init(&nfsgrouphash[i].mtx, "nfsgidhash",
	NULL, MTX_DEF \| MTX_DUPOK);
	nfsgroupnamehash = malloc(sizeof(struct nfsrv_lughash) *
	nfsrv_lughashsize, M_NFSUSERGROUP, M_WAITOK \|
	M_ZERO);
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_init(&nfsgroupnamehash[i].mtx,
	"nfsgrphash", NULL, MTX_DEF \| MTX_DUPOK);
	}
	/* (Re)initialize the list heads. */
	for (i = 0; i < nfsrv_lughashsize; i++)
	TAILQ_INIT(&nfsuserhash[i].lughead);
	for (i = 0; i < nfsrv_lughashsize; i++)
	TAILQ_INIT(&nfsusernamehash[i].lughead);
	for (i = 0; i < nfsrv_lughashsize; i++)
	TAILQ_INIT(&nfsgrouphash[i].lughead);
	for (i = 0; i < nfsrv_lughashsize; i++)
	TAILQ_INIT(&nfsgroupnamehash[i].lughead);

	/*
	* Put name in "DNS" string.
	*/
	nfsrv_dnsname = cp;
	nfsrv_defaultuid = nidp->nid_uid;
	nfsrv_defaultgid = nidp->nid_gid;
	nfsrv_usercnt = 0;
	nfsrv_usermax = nidp->nid_usermax;
	atomic_store_rel_int(&nfsrv_dnsnamelen, nidp->nid_namelen);
	goto out;
	}

	/*
	* malloc the new one now, so any potential sleep occurs before
	* manipulation of the lists.
	*/
	newusrp = malloc(sizeof(struct nfsusrgrp) + nidp->nid_namelen,
	M_NFSUSERGROUP, M_WAITOK \| M_ZERO);
	error = copyin(nidp->nid_name, newusrp->lug_name,
	nidp->nid_namelen);
	if (error == 0 && nidp->nid_ngroup > 0 &&
	(nidp->nid_flag & NFSID_ADDUID) != 0) {
	grps = malloc(sizeof(gid_t) * nidp->nid_ngroup, M_TEMP,
	M_WAITOK);
	error = copyin(nidp->nid_grps, grps,
	sizeof(gid_t) * nidp->nid_ngroup);
	if (error == 0) {
	/*
	* Create a credential just like svc_getcred(),
	* but using the group list provided.
	*/
	cr = crget();
	cr->cr_uid = cr->cr_ruid = cr->cr_svuid = nidp->nid_uid;
	crsetgroups(cr, nidp->nid_ngroup, grps);
	cr->cr_rgid = cr->cr_svgid = cr->cr_groups[0];
	cr->cr_prison = &prison0;
	prison_hold(cr->cr_prison);
	#ifdef MAC
	mac_cred_associate_nfsd(cr);
	#endif
	newusrp->lug_cred = cr;
	}
	free(grps, M_TEMP);
	}
	if (error) {
	free(newusrp, M_NFSUSERGROUP);
	goto out;
	}
	newusrp->lug_namelen = nidp->nid_namelen;

	/*
	* The lock order is username[0]->[nfsrv_lughashsize - 1] followed
	* by uid[0]->[nfsrv_lughashsize - 1], with the same for group.
	* The flags user_locked, username_locked, group_locked and
	* groupname_locked are set to indicate all of those hash lists are
	* locked. hp_name != NULL and hp_idnum != NULL indicates that
	* the respective one mutex is locked.
	*/
	user_locked = username_locked = group_locked = groupname_locked = 0;
	hp_name = hp_idnum = NULL;

	/*
	* Delete old entries, as required.
	*/
	if (nidp->nid_flag & (NFSID_DELUID \| NFSID_ADDUID)) {
	/* Must lock all username hash lists first, to avoid a LOR. */
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsusernamehash[i].mtx);
	username_locked = 1;
	hp_idnum = NFSUSERHASH(nidp->nid_uid);
	mtx_lock(&hp_idnum->mtx);
	TAILQ_FOREACH_SAFE(usrp, &hp_idnum->lughead, lug_numhash,
	nusrp) {
	if (usrp->lug_uid == nidp->nid_uid)
	nfsrv_removeuser(usrp, 1);
	}
	} else if (nidp->nid_flag & (NFSID_DELUSERNAME \| NFSID_ADDUSERNAME)) {
	hp_name = NFSUSERNAMEHASH(newusrp->lug_name,
	newusrp->lug_namelen);
	mtx_lock(&hp_name->mtx);
	TAILQ_FOREACH_SAFE(usrp, &hp_name->lughead, lug_namehash,
	nusrp) {
	if (usrp->lug_namelen == newusrp->lug_namelen &&
	!NFSBCMP(usrp->lug_name, newusrp->lug_name,
	usrp->lug_namelen)) {
	thp = NFSUSERHASH(usrp->lug_uid);
	mtx_lock(&thp->mtx);
	nfsrv_removeuser(usrp, 1);
	mtx_unlock(&thp->mtx);
	}
	}
	hp_idnum = NFSUSERHASH(nidp->nid_uid);
	mtx_lock(&hp_idnum->mtx);
	} else if (nidp->nid_flag & (NFSID_DELGID \| NFSID_ADDGID)) {
	/* Must lock all groupname hash lists first, to avoid a LOR. */
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsgroupnamehash[i].mtx);
	groupname_locked = 1;
	hp_idnum = NFSGROUPHASH(nidp->nid_gid);
	mtx_lock(&hp_idnum->mtx);
	TAILQ_FOREACH_SAFE(usrp, &hp_idnum->lughead, lug_numhash,
	nusrp) {
	if (usrp->lug_gid == nidp->nid_gid)
	nfsrv_removeuser(usrp, 0);
	}
	} else if (nidp->nid_flag & (NFSID_DELGROUPNAME \| NFSID_ADDGROUPNAME)) {
	hp_name = NFSGROUPNAMEHASH(newusrp->lug_name,
	newusrp->lug_namelen);
	mtx_lock(&hp_name->mtx);
	TAILQ_FOREACH_SAFE(usrp, &hp_name->lughead, lug_namehash,
	nusrp) {
	if (usrp->lug_namelen == newusrp->lug_namelen &&
	!NFSBCMP(usrp->lug_name, newusrp->lug_name,
	usrp->lug_namelen)) {
	thp = NFSGROUPHASH(usrp->lug_gid);
	mtx_lock(&thp->mtx);
	nfsrv_removeuser(usrp, 0);
	mtx_unlock(&thp->mtx);
	}
	}
	hp_idnum = NFSGROUPHASH(nidp->nid_gid);
	mtx_lock(&hp_idnum->mtx);
	}

	/*
	* Now, we can add the new one.
	*/
	if (nidp->nid_usertimeout)
	newusrp->lug_expiry = NFSD_MONOSEC + nidp->nid_usertimeout;
	else
	newusrp->lug_expiry = NFSD_MONOSEC + 5;
	if (nidp->nid_flag & (NFSID_ADDUID \| NFSID_ADDUSERNAME)) {
	newusrp->lug_uid = nidp->nid_uid;
	thp = NFSUSERHASH(newusrp->lug_uid);
	mtx_assert(&thp->mtx, MA_OWNED);
	TAILQ_INSERT_TAIL(&thp->lughead, newusrp, lug_numhash);
	thp = NFSUSERNAMEHASH(newusrp->lug_name, newusrp->lug_namelen);
	mtx_assert(&thp->mtx, MA_OWNED);
	TAILQ_INSERT_TAIL(&thp->lughead, newusrp, lug_namehash);
	atomic_add_int(&nfsrv_usercnt, 1);
	} else if (nidp->nid_flag & (NFSID_ADDGID \| NFSID_ADDGROUPNAME)) {
	newusrp->lug_gid = nidp->nid_gid;
	thp = NFSGROUPHASH(newusrp->lug_gid);
	mtx_assert(&thp->mtx, MA_OWNED);
	TAILQ_INSERT_TAIL(&thp->lughead, newusrp, lug_numhash);
	thp = NFSGROUPNAMEHASH(newusrp->lug_name, newusrp->lug_namelen);
	mtx_assert(&thp->mtx, MA_OWNED);
	TAILQ_INSERT_TAIL(&thp->lughead, newusrp, lug_namehash);
	atomic_add_int(&nfsrv_usercnt, 1);
	} else {
	if (newusrp->lug_cred != NULL)
	crfree(newusrp->lug_cred);
	free(newusrp, M_NFSUSERGROUP);
	}

	/*
	* Once per second, allow one thread to trim the cache.
	*/
	if (lasttime < NFSD_MONOSEC &&
	atomic_cmpset_acq_int(&onethread, 0, 1) != 0) {
	/*
	* First, unlock the single mutexes, so that all entries
	* can be locked and any LOR is avoided.
	*/
	if (hp_name != NULL) {
	mtx_unlock(&hp_name->mtx);
	hp_name = NULL;
	}
	if (hp_idnum != NULL) {
	mtx_unlock(&hp_idnum->mtx);
	hp_idnum = NULL;
	}

	if ((nidp->nid_flag & (NFSID_DELUID \| NFSID_ADDUID \|
	NFSID_DELUSERNAME \| NFSID_ADDUSERNAME)) != 0) {
	if (username_locked == 0) {
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsusernamehash[i].mtx);
	username_locked = 1;
	}
	KASSERT(user_locked == 0,
	("nfssvc_idname: user_locked"));
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsuserhash[i].mtx);
	user_locked = 1;
	for (i = 0; i < nfsrv_lughashsize; i++) {
	TAILQ_FOREACH_SAFE(usrp,
	&nfsuserhash[i].lughead, lug_numhash,
	nusrp)
	if (usrp->lug_expiry < NFSD_MONOSEC)
	nfsrv_removeuser(usrp, 1);
	}
	for (i = 0; i < nfsrv_lughashsize; i++) {
	/*
	* Trim the cache using an approximate LRU
	* algorithm. This code deletes the least
	* recently used entry on each hash list.
	*/
	if (nfsrv_usercnt <= nfsrv_usermax)
	break;
	usrp = TAILQ_FIRST(&nfsuserhash[i].lughead);
	if (usrp != NULL)
	nfsrv_removeuser(usrp, 1);
	}
	} else {
	if (groupname_locked == 0) {
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsgroupnamehash[i].mtx);
	groupname_locked = 1;
	}
	KASSERT(group_locked == 0,
	("nfssvc_idname: group_locked"));
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_lock(&nfsgrouphash[i].mtx);
	group_locked = 1;
	for (i = 0; i < nfsrv_lughashsize; i++) {
	TAILQ_FOREACH_SAFE(usrp,
	&nfsgrouphash[i].lughead, lug_numhash,
	nusrp)
	if (usrp->lug_expiry < NFSD_MONOSEC)
	nfsrv_removeuser(usrp, 0);
	}
	for (i = 0; i < nfsrv_lughashsize; i++) {
	/*
	* Trim the cache using an approximate LRU
	* algorithm. This code deletes the least
	* recently user entry on each hash list.
	*/
	if (nfsrv_usercnt <= nfsrv_usermax)
	break;
	usrp = TAILQ_FIRST(&nfsgrouphash[i].lughead);
	if (usrp != NULL)
	nfsrv_removeuser(usrp, 0);
	}
	}
	lasttime = NFSD_MONOSEC;
	atomic_store_rel_int(&onethread, 0);
	}

	/* Now, unlock all locked mutexes. */
	if (hp_idnum != NULL)
	mtx_unlock(&hp_idnum->mtx);
	if (hp_name != NULL)
	mtx_unlock(&hp_name->mtx);
	if (user_locked != 0)
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsuserhash[i].mtx);
	if (username_locked != 0)
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsusernamehash[i].mtx);
	if (group_locked != 0)
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsgrouphash[i].mtx);
	if (groupname_locked != 0)
	for (i = 0; i < nfsrv_lughashsize; i++)
	mtx_unlock(&nfsgroupnamehash[i].mtx);
	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Remove a user/group name element.
	*/
	static void
	nfsrv_removeuser(struct nfsusrgrp *usrp, int isuser)
	{
	struct nfsrv_lughash *hp;

	if (isuser != 0) {
	hp = NFSUSERHASH(usrp->lug_uid);
	mtx_assert(&hp->mtx, MA_OWNED);
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	hp = NFSUSERNAMEHASH(usrp->lug_name, usrp->lug_namelen);
	mtx_assert(&hp->mtx, MA_OWNED);
	TAILQ_REMOVE(&hp->lughead, usrp, lug_namehash);
	} else {
	hp = NFSGROUPHASH(usrp->lug_gid);
	mtx_assert(&hp->mtx, MA_OWNED);
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	hp = NFSGROUPNAMEHASH(usrp->lug_name, usrp->lug_namelen);
	mtx_assert(&hp->mtx, MA_OWNED);
	TAILQ_REMOVE(&hp->lughead, usrp, lug_namehash);
	}
	atomic_add_int(&nfsrv_usercnt, -1);
	if (usrp->lug_cred != NULL)
	crfree(usrp->lug_cred);
	free(usrp, M_NFSUSERGROUP);
	}

	/*
	* Free up all the allocations related to the name<-->id cache.
	* This function should only be called when the nfsuserd daemon isn't
	* running, since it doesn't do any locking.
	* This function is meant to be used when the nfscommon module is unloaded.
	*/
	void
	nfsrv_cleanusergroup(void)
	{
	struct nfsrv_lughash hp, hp2;
	struct nfsusrgrp nusrp, usrp;
	int i;

	if (nfsuserhash == NULL)
	return;

	for (i = 0; i < nfsrv_lughashsize; i++) {
	hp = &nfsuserhash[i];
	TAILQ_FOREACH_SAFE(usrp, &hp->lughead, lug_numhash, nusrp) {
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	hp2 = NFSUSERNAMEHASH(usrp->lug_name,
	usrp->lug_namelen);
	TAILQ_REMOVE(&hp2->lughead, usrp, lug_namehash);
	if (usrp->lug_cred != NULL)
	crfree(usrp->lug_cred);
	free(usrp, M_NFSUSERGROUP);
	}
	hp = &nfsgrouphash[i];
	TAILQ_FOREACH_SAFE(usrp, &hp->lughead, lug_numhash, nusrp) {
	TAILQ_REMOVE(&hp->lughead, usrp, lug_numhash);
	hp2 = NFSGROUPNAMEHASH(usrp->lug_name,
	usrp->lug_namelen);
	TAILQ_REMOVE(&hp2->lughead, usrp, lug_namehash);
	if (usrp->lug_cred != NULL)
	crfree(usrp->lug_cred);
	free(usrp, M_NFSUSERGROUP);
	}
	mtx_destroy(&nfsuserhash[i].mtx);
	mtx_destroy(&nfsusernamehash[i].mtx);
	mtx_destroy(&nfsgroupnamehash[i].mtx);
	mtx_destroy(&nfsgrouphash[i].mtx);
	}
	free(nfsuserhash, M_NFSUSERGROUP);
	free(nfsusernamehash, M_NFSUSERGROUP);
	free(nfsgrouphash, M_NFSUSERGROUP);
	free(nfsgroupnamehash, M_NFSUSERGROUP);
	free(nfsrv_dnsname, M_NFSSTRING);
	}

	/*
	* This function scans a byte string and checks for UTF-8 compliance.
	* It returns 0 if it conforms and NFSERR_INVAL if not.
	*/
	int
	nfsrv_checkutf8(u_int8_t *cp, int len)
	{
	u_int32_t val = 0x0;
	int cnt = 0, gotd = 0, shift = 0;
	u_int8_t byte;
	static int utf8_shift[5] = { 7, 11, 16, 21, 26 };
	int error = 0;

	/*
	* Here are what the variables are used for:
	* val - the calculated value of a multibyte char, used to check
	* that it was coded with the correct range
	* cnt - the number of 10xxxxxx bytes to follow
	* gotd - set for a char of Dxxx, so D800<->DFFF can be checked for
	* shift - lower order bits of range (ie. "val >> shift" should
	* not be 0, in other words, dividing by the lower bound
	* of the range should get a non-zero value)
	* byte - used to calculate cnt
	*/
	while (len > 0) {
	if (cnt > 0) {
	/* This handles the 10xxxxxx bytes */
	if ((*cp & 0xc0) != 0x80 \|\|
	(gotd && (*cp & 0x20))) {
	error = NFSERR_INVAL;
	goto out;
	}
	gotd = 0;
	val <<= 6;
	val \|= (*cp & 0x3f);
	cnt--;
	if (cnt == 0 && (val >> shift) == 0x0) {
	error = NFSERR_INVAL;
	goto out;
	}
	} else if (*cp & 0x80) {
	/* first byte of multi byte char */
	byte = *cp;
	while ((byte & 0x40) && cnt < 6) {
	cnt++;
	byte <<= 1;
	}
	if (cnt == 0 \|\| cnt == 6) {
	error = NFSERR_INVAL;
	goto out;
	}
	val = (*cp & (0x3f >> cnt));
	shift = utf8_shift[cnt - 1];
	if (cnt == 2 && val == 0xd)
	/* Check for the 0xd800-0xdfff case */
	gotd = 1;
	}
	cp++;
	len--;
	}
	if (cnt > 0)
	error = NFSERR_INVAL;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Parse the xdr for an NFSv4 FsLocations attribute. Return two malloc'd
	* strings, one with the root path in it and the other with the list of
	* locations. The list is in the same format as is found in nfr_refs.
	* It is a "," separated list of entries, where each of them is of the
	* form <server>:<rootpath>. For example
	* "nfsv4-test:/sub2,nfsv4-test2:/user/mnt,nfsv4-test2:/user/mnt2"
	* The nilp argument is set to 1 for the special case of a null fs_root
	* and an empty server list.
	* It returns NFSERR_BADXDR, if the xdr can't be parsed and returns the
	* number of xdr bytes parsed in sump.
	*/
	static int
	nfsrv_getrefstr(struct nfsrv_descript nd, u_char fsrootp, u_char *srvp,
	int sump, int nilp)
	{
	u_int32_t *tl;
	u_char cp = NULL, cp2 = NULL, cp3, str;
	int i, j, len, stringlen, cnt, slen, siz, xdrsum, error = 0, nsrv;
	struct list {
	SLIST_ENTRY(list) next;
	int len;
	u_char host[1];
	} lsp, nlsp;
	SLIST_HEAD(, list) head;

	*fsrootp = NULL;
	*srvp = NULL;
	*nilp = 0;

	/*
	* Get the fs_root path and check for the special case of null path
	* and 0 length server list.
	*/
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	len = fxdr_unsigned(int, *tl);
	if (len < 0 \|\| len > 10240) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	if (len == 0) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (*tl != 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	*nilp = 1;
	sump = 2 NFSX_UNSIGNED;
	error = 0;
	goto nfsmout;
	}
	cp = malloc(len + 1, M_NFSSTRING, M_WAITOK);
	error = nfsrv_mtostr(nd, cp, len);
	if (!error) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	cnt = fxdr_unsigned(int, *tl);
	if (cnt <= 0)
	error = NFSERR_BADXDR;
	}
	if (error)
	goto nfsmout;

	/*
	* Now, loop through the location list and make up the srvlist.
	*/
	xdrsum = (2 * NFSX_UNSIGNED) + NFSM_RNDUP(len);
	cp2 = cp3 = malloc(1024, M_NFSSTRING, M_WAITOK);
	slen = 1024;
	siz = 0;
	for (i = 0; i < cnt; i++) {
	SLIST_INIT(&head);
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	nsrv = fxdr_unsigned(int, *tl);
	if (nsrv <= 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}

	/*
	* Handle the first server by putting it in the srvstr.
	*/
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	len = fxdr_unsigned(int, *tl);
	if (len <= 0 \|\| len > 1024) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	nfsrv_refstrbigenough(siz + len + 3, &cp2, &cp3, &slen);
	if (cp3 != cp2) {
	*cp3++ = ',';
	siz++;
	}
	error = nfsrv_mtostr(nd, cp3, len);
	if (error)
	goto nfsmout;
	cp3 += len;
	*cp3++ = ':';
	siz += (len + 1);
	xdrsum += (2 * NFSX_UNSIGNED) + NFSM_RNDUP(len);
	for (j = 1; j < nsrv; j++) {
	/*
	* Yuck, put them in an slist and process them later.
	*/
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	len = fxdr_unsigned(int, *tl);
	if (len <= 0 \|\| len > 1024) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	lsp = (struct list *)malloc(sizeof (struct list)
	+ len, M_TEMP, M_WAITOK);
	error = nfsrv_mtostr(nd, lsp->host, len);
	if (error)
	goto nfsmout;
	xdrsum += NFSX_UNSIGNED + NFSM_RNDUP(len);
	lsp->len = len;
	SLIST_INSERT_HEAD(&head, lsp, next);
	}

	/*
	* Finally, we can get the path.
	*/
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	len = fxdr_unsigned(int, *tl);
	if (len <= 0 \|\| len > 1024) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	nfsrv_refstrbigenough(siz + len + 1, &cp2, &cp3, &slen);
	error = nfsrv_mtostr(nd, cp3, len);
	if (error)
	goto nfsmout;
	xdrsum += NFSX_UNSIGNED + NFSM_RNDUP(len);
	str = cp3;
	stringlen = len;
	cp3 += len;
	siz += len;
	SLIST_FOREACH_SAFE(lsp, &head, next, nlsp) {
	nfsrv_refstrbigenough(siz + lsp->len + stringlen + 3,
	&cp2, &cp3, &slen);
	*cp3++ = ',';
	NFSBCOPY(lsp->host, cp3, lsp->len);
	cp3 += lsp->len;
	*cp3++ = ':';
	NFSBCOPY(str, cp3, stringlen);
	cp3 += stringlen;
	*cp3 = '\0';
	siz += (lsp->len + stringlen + 2);
	free(lsp, M_TEMP);
	}
	}
	*fsrootp = cp;
	*srvp = cp2;
	*sump = xdrsum;
	NFSEXITCODE2(0, nd);
	return (0);
	nfsmout:
	if (cp != NULL)
	free(cp, M_NFSSTRING);
	if (cp2 != NULL)
	free(cp2, M_NFSSTRING);
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Make the malloc'd space large enough. This is a pain, but the xdr
	* doesn't set an upper bound on the side, so...
	*/
	static void
	nfsrv_refstrbigenough(int siz, u_char cpp, u_char cpp2, int *slenp)
	{
	u_char *cp;
	int i;

	if (siz <= *slenp)
	return;
	cp = malloc(siz + 1024, M_NFSSTRING, M_WAITOK);
	NFSBCOPY(cpp, cp, slenp);
	free(*cpp, M_NFSSTRING);
	i = cpp2 - cpp;
	*cpp = cp;
	*cpp2 = cp + i;
	*slenp = siz + 1024;
	}

	/*
	* Initialize the reply header data structures.
	*/
	void
	nfsrvd_rephead(struct nfsrv_descript *nd)
	{
	struct mbuf *mreq;

	if ((nd->nd_flag & ND_EXTPG) != 0) {
	mreq = mb_alloc_ext_plus_pages(PAGE_SIZE, M_WAITOK);
	nd->nd_mreq = nd->nd_mb = mreq;
	nd->nd_bpos = (char )(void )
	PHYS_TO_DMAP(mreq->m_epg_pa[0]);
	nd->nd_bextpg = 0;
	nd->nd_bextpgsiz = PAGE_SIZE;
	} else {
	/*
	* If this is a big reply, use a cluster.
	*/
	if ((nd->nd_flag & ND_GSSINITREPLY) == 0 &&
	nfs_bigreply[nd->nd_procnum]) {
	NFSMCLGET(mreq, M_WAITOK);
	nd->nd_mreq = mreq;
	nd->nd_mb = mreq;
	} else {
	NFSMGET(mreq);
	nd->nd_mreq = mreq;
	nd->nd_mb = mreq;
	}
	nd->nd_bpos = mtod(mreq, char *);
	mreq->m_len = 0;
	}

	if ((nd->nd_flag & ND_GSSINITREPLY) == 0)
	NFSM_BUILD(nd->nd_errp, int *, NFSX_UNSIGNED);
	}

	/*
	* Lock a socket against others.
	* Currently used to serialize connect/disconnect attempts.
	*/
	int
	newnfs_sndlock(int *flagp)
	{
	struct timespec ts;

	NFSLOCKSOCK();
	while (*flagp & NFSR_SNDLOCK) {
	*flagp \|= NFSR_WANTSND;
	ts.tv_sec = 0;
	ts.tv_nsec = 0;
	(void) nfsmsleep((caddr_t)flagp, NFSSOCKMUTEXPTR,
	PZERO - 1, "nfsndlck", &ts);
	}
	*flagp \|= NFSR_SNDLOCK;
	NFSUNLOCKSOCK();
	return (0);
	}

	/*
	* Unlock the stream socket for others.
	*/
	void
	newnfs_sndunlock(int *flagp)
	{

	NFSLOCKSOCK();
	if ((*flagp & NFSR_SNDLOCK) == 0)
	panic("nfs sndunlock");
	*flagp &= ~NFSR_SNDLOCK;
	if (*flagp & NFSR_WANTSND) {
	*flagp &= ~NFSR_WANTSND;
	wakeup((caddr_t)flagp);
	}
	NFSUNLOCKSOCK();
	}

	int
	nfsv4_getipaddr(struct nfsrv_descript nd, struct sockaddr_in sin,
	struct sockaddr_in6 sin6, sa_family_t saf, int *isudp)
	{
	struct in_addr saddr;
	uint32_t portnum, *tl;
	int i, j, k;
	sa_family_t af = AF_UNSPEC;
	char addr[64], protocol[5], *cp;
	int cantparse = 0, error = 0;
	uint16_t portv;

	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	i = fxdr_unsigned(int, *tl);
	if (i >= 3 && i <= 4) {
	error = nfsrv_mtostr(nd, protocol, i);
	if (error)
	goto nfsmout;
	if (strcmp(protocol, "tcp") == 0) {
	af = AF_INET;
	*isudp = 0;
	} else if (strcmp(protocol, "udp") == 0) {
	af = AF_INET;
	*isudp = 1;
	} else if (strcmp(protocol, "tcp6") == 0) {
	af = AF_INET6;
	*isudp = 0;
	} else if (strcmp(protocol, "udp6") == 0) {
	af = AF_INET6;
	*isudp = 1;
	} else
	cantparse = 1;
	} else {
	cantparse = 1;
	if (i > 0) {
	error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
	if (error)
	goto nfsmout;
	}
	}
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	i = fxdr_unsigned(int, *tl);
	if (i < 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	} else if (cantparse == 0 && i >= 11 && i < 64) {
	/*
	* The shortest address is 11chars and the longest is < 64.
	*/
	error = nfsrv_mtostr(nd, addr, i);
	if (error)
	goto nfsmout;

	/* Find the port# at the end and extract that. */
	i = strlen(addr);
	k = 0;
	cp = &addr[i - 1];
	/* Count back two '.'s from end to get port# field. */
	for (j = 0; j < i; j++) {
	if (*cp == '.') {
	k++;
	if (k == 2)
	break;
	}
	cp--;
	}
	if (k == 2) {
	/*
	* The NFSv4 port# is appended as .N.N, where N is
	* a decimal # in the range 0-255, just like an inet4
	* address. Cheat and use inet_aton(), which will
	* return a Class A address and then shift the high
	* order 8bits over to convert it to the port#.
	*/
	*cp++ = '\0';
	if (inet_aton(cp, &saddr) == 1) {
	portnum = ntohl(saddr.s_addr);
	portv = (uint16_t)((portnum >> 16) \|
	(portnum & 0xff));
	} else
	cantparse = 1;
	} else
	cantparse = 1;
	if (cantparse == 0) {
	if (af == AF_INET) {
	if (inet_pton(af, addr, &sin->sin_addr) == 1) {
	sin->sin_len = sizeof(*sin);
	sin->sin_family = AF_INET;
	sin->sin_port = htons(portv);
	*saf = af;
	return (0);
	}
	} else {
	if (inet_pton(af, addr, &sin6->sin6_addr)
	== 1) {
	sin6->sin6_len = sizeof(*sin6);
	sin6->sin6_family = AF_INET6;
	sin6->sin6_port = htons(portv);
	*saf = af;
	return (0);
	}
	}
	}
	} else {
	if (i > 0) {
	error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
	if (error)
	goto nfsmout;
	}
	}
	error = EPERM;
	nfsmout:
	return (error);
	}

	/*
	* Handle an NFSv4.1 Sequence request for the session.
	* If reply != NULL, use it to return the cached reply, as required.
	* The client gets a cached reply via this call for callbacks, however the
	* server gets a cached reply via the nfsv4_seqsess_cachereply() call.
	*/
	int
	nfsv4_seqsession(uint32_t seqid, uint32_t slotid, uint32_t highslot,
	struct nfsslot slots, struct mbuf *reply, uint16_t maxslot)
	{
	int error;

	error = 0;
	if (reply != NULL)
	*reply = NULL;
	if (slotid > maxslot)
	return (NFSERR_BADSLOT);
	if (seqid == slots[slotid].nfssl_seq) {
	/* A retry. */
	if (slots[slotid].nfssl_inprog != 0)
	error = NFSERR_DELAY;
	else if (slots[slotid].nfssl_reply != NULL) {
	if (reply != NULL) {
	*reply = slots[slotid].nfssl_reply;
	slots[slotid].nfssl_reply = NULL;
	}
	slots[slotid].nfssl_inprog = 1;
	error = NFSERR_REPLYFROMCACHE;
	} else
	/* No reply cached, so just do it. */
	slots[slotid].nfssl_inprog = 1;
	} else if ((slots[slotid].nfssl_seq + 1) == seqid) {
	if (slots[slotid].nfssl_reply != NULL)
	m_freem(slots[slotid].nfssl_reply);
	slots[slotid].nfssl_reply = NULL;
	slots[slotid].nfssl_inprog = 1;
	slots[slotid].nfssl_seq++;
	} else
	error = NFSERR_SEQMISORDERED;
	return (error);
	}

	/*
	* Cache this reply for the slot.
	* Use the "rep" argument to return the cached reply if repstat is set to
	* NFSERR_REPLYFROMCACHE. The client never sets repstat to this value.
	*/
	void
	nfsv4_seqsess_cacherep(uint32_t slotid, struct nfsslot *slots, int repstat,
	struct mbuf **rep)
	{

	if (repstat == NFSERR_REPLYFROMCACHE) {
	*rep = slots[slotid].nfssl_reply;
	slots[slotid].nfssl_reply = NULL;
	} else {
	if (slots[slotid].nfssl_reply != NULL)
	m_freem(slots[slotid].nfssl_reply);
	slots[slotid].nfssl_reply = *rep;
	}
	slots[slotid].nfssl_inprog = 0;
	}

	/*
	* Generate the xdr for an NFSv4.1 Sequence Operation.
	*/
	void
	nfsv4_setsequence(struct nfsmount nmp, struct nfsrv_descript nd,
	struct nfsclsession *sep, int dont_replycache)
	{
	uint32_t *tl, slotseq = 0;
	int error, maxslot, slotpos;
	uint8_t sessionid[NFSX_V4SESSIONID];

	error = nfsv4_sequencelookup(nmp, sep, &slotpos, &maxslot, &slotseq,
	sessionid);
	nd->nd_maxreq = sep->nfsess_maxreq;
	nd->nd_maxresp = sep->nfsess_maxresp;

	/* Build the Sequence arguments. */
	NFSM_BUILD(tl, uint32_t , NFSX_V4SESSIONID + 4 NFSX_UNSIGNED);
	nd->nd_sequence = tl;
	bcopy(sessionid, tl, NFSX_V4SESSIONID);
	tl += NFSX_V4SESSIONID / NFSX_UNSIGNED;
	nd->nd_slotseq = tl;
	if (error == 0) {
	nd->nd_flag \|= ND_HASSLOTID;
	nd->nd_slotid = slotpos;
	*tl++ = txdr_unsigned(slotseq);
	*tl++ = txdr_unsigned(slotpos);
	*tl++ = txdr_unsigned(maxslot);
	if (dont_replycache == 0)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	} else {
	/*
	* There are two errors and the rest of the session can
	* just be zeros.
	* NFSERR_BADSESSION: This bad session should just generate
	* the same error again when the RPC is retried.
	* ESTALE: A forced dismount is in progress and will cause the
	* RPC to fail later.
	*/
	*tl++ = 0;
	*tl++ = 0;
	*tl++ = 0;
	*tl = 0;
	}
	nd->nd_flag \|= ND_HASSEQUENCE;
	}

	int
	nfsv4_sequencelookup(struct nfsmount nmp, struct nfsclsession sep,
	int slotposp, int maxslotp, uint32_t slotseqp, uint8_t sessionid)
	{
	int i, maxslot, slotpos;
	uint64_t bitval;

	/* Find an unused slot. */
	slotpos = -1;
	maxslot = -1;
	mtx_lock(&sep->nfsess_mtx);
	do {
	if (nmp != NULL && sep->nfsess_defunct != 0) {
	/* Just return the bad session. */
	bcopy(sep->nfsess_sessionid, sessionid,
	NFSX_V4SESSIONID);
	mtx_unlock(&sep->nfsess_mtx);
	return (NFSERR_BADSESSION);
	}
	bitval = 1;
	for (i = 0; i < sep->nfsess_foreslots; i++) {
	if ((bitval & sep->nfsess_slots) == 0) {
	slotpos = i;
	sep->nfsess_slots \|= bitval;
	sep->nfsess_slotseq[i]++;
	*slotseqp = sep->nfsess_slotseq[i];
	break;
	}
	bitval <<= 1;
	}
	if (slotpos == -1) {
	/*
	* If a forced dismount is in progress, just return.
	* This RPC attempt will fail when it calls
	* newnfs_request().
	*/
	if (nmp != NULL && NFSCL_FORCEDISM(nmp->nm_mountp)) {
	mtx_unlock(&sep->nfsess_mtx);
	return (ESTALE);
	}
	/* Wake up once/sec, to check for a forced dismount. */
	(void)mtx_sleep(&sep->nfsess_slots, &sep->nfsess_mtx,
	PZERO, "nfsclseq", hz);
	}
	} while (slotpos == -1);
	/* Now, find the highest slot in use. (nfsc_slots is 64bits) */
	bitval = 1;
	for (i = 0; i < 64; i++) {
	if ((bitval & sep->nfsess_slots) != 0)
	maxslot = i;
	bitval <<= 1;
	}
	bcopy(sep->nfsess_sessionid, sessionid, NFSX_V4SESSIONID);
	mtx_unlock(&sep->nfsess_mtx);
	*slotposp = slotpos;
	*maxslotp = maxslot;
	return (0);
	}

	/*
	* Free a session slot.
	*/
	void
	nfsv4_freeslot(struct nfsclsession *sep, int slot)
	{
	uint64_t bitval;

	bitval = 1;
	if (slot > 0)
	bitval <<= slot;
	mtx_lock(&sep->nfsess_mtx);
	if ((bitval & sep->nfsess_slots) == 0)
	printf("freeing free slot!!\n");
	sep->nfsess_slots &= ~bitval;
	wakeup(&sep->nfsess_slots);
	mtx_unlock(&sep->nfsess_mtx);
	}

	/*
	* Search for a matching pnfsd DS, based on the nmp arg.
	* Return one if found, NULL otherwise.
	*/
	struct nfsdevice *
	nfsv4_findmirror(struct nfsmount *nmp)
	{
	struct nfsdevice *ds;

	mtx_assert(NFSDDSMUTEXPTR, MA_OWNED);
	/*
	* Search the DS server list for a match with nmp.
	*/
	if (nfsrv_devidcnt == 0)
	return (NULL);
	TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) {
	if (ds->nfsdev_nmp == nmp) {
	NFSCL_DEBUG(4, "nfsv4_findmirror: fnd main ds\n");
	break;
	}
	}
	return (ds);
	}

	/*
	* Fill in the fields of "struct nfsrv_descript".
	*/
	void
	nfsm_set(struct nfsrv_descript *nd, u_int offs)
	{
	struct mbuf *m;
	int rlen;

	m = nd->nd_mb;
	if ((m->m_flags & M_EXTPG) != 0) {
	nd->nd_bextpg = 0;
	while (offs > 0) {
	if (nd->nd_bextpg == 0)
	rlen = m_epg_pagelen(m, 0, m->m_epg_1st_off);
	else
	rlen = m_epg_pagelen(m, nd->nd_bextpg, 0);
	if (offs <= rlen)
	break;
	offs -= rlen;
	nd->nd_bextpg++;
	if (nd->nd_bextpg == m->m_epg_npgs) {
	printf("nfsm_set: build offs "
	"out of range\n");
	nd->nd_bextpg--;
	break;
	}
	}
	nd->nd_bpos = (char )(void )
	PHYS_TO_DMAP(m->m_epg_pa[nd->nd_bextpg]);
	if (nd->nd_bextpg == 0)
	nd->nd_bpos += m->m_epg_1st_off;
	if (offs > 0) {
	nd->nd_bpos += offs;
	nd->nd_bextpgsiz = rlen - offs;
	} else if (nd->nd_bextpg == 0)
	nd->nd_bextpgsiz = PAGE_SIZE - m->m_epg_1st_off;
	else
	nd->nd_bextpgsiz = PAGE_SIZE;
	} else
	nd->nd_bpos = mtod(m, char *) + offs;
	}

	/*
	* Grow a ext_pgs mbuf list. Either allocate another page or add
	* an mbuf to the list.
	*/
	struct mbuf *
	nfsm_add_ext_pgs(struct mbuf m, int maxextsiz, int bextpg)
	{
	struct mbuf *mp;
	vm_page_t pg;

	if ((m->m_epg_npgs + 1) * PAGE_SIZE > maxextsiz) {
	mp = mb_alloc_ext_plus_pages(PAGE_SIZE, M_WAITOK);
	*bextpg = 0;
	m->m_next = mp;
	} else {
	do {
	pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \|
	VM_ALLOC_NOOBJ \| VM_ALLOC_NODUMP \|
	VM_ALLOC_WIRED);
	if (pg == NULL)
	vm_wait(NULL);
	} while (pg == NULL);
	m->m_epg_pa[m->m_epg_npgs] = VM_PAGE_TO_PHYS(pg);
	*bextpg = m->m_epg_npgs;
	m->m_epg_npgs++;
	m->m_epg_last_len = 0;
	mp = m;
	}
	return (mp);
	}
	diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
	index fbe77fcaf9b6..eb971d73d534 100644
	--- a/sys/fs/nfsserver/nfs_nfsdport.c
	+++ b/sys/fs/nfsserver/nfs_nfsdport.c
	@@ -1,6528 +1,6604 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* Rick Macklem at The University of Guelph.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/capsicum.h>
	#include <sys/extattr.h>

	/*
	* Functions that perform the vfs operations required by the routines in
	* nfsd_serv.c. It is hoped that this change will make the server more
	* portable.
	*/

	#include <fs/nfs/nfsport.h>
	#include <security/mac/mac_framework.h>
	#include <sys/filio.h>
	#include <sys/hash.h>
	#include <sys/sysctl.h>
	#include <nlm/nlm_prot.h>
	#include <nlm/nlm.h>

	FEATURE(nfsd, "NFSv4 server");

	extern u_int32_t newnfs_true, newnfs_false, newnfs_xdrneg1;
	extern int nfsrv_useacl;
	extern int newnfs_numnfsd;
	extern struct mount nfsv4root_mnt;
	extern struct nfsrv_stablefirst nfsrv_stablefirst;
	extern void (*nfsd_call_servertimer)(void);
	extern SVCPOOL *nfsrvd_pool;
	extern struct nfsv4lock nfsd_suspend_lock;
	extern struct nfsclienthashhead *nfsclienthash;
	extern struct nfslockhashhead *nfslockhash;
	extern struct nfssessionhash *nfssessionhash;
	extern int nfsrv_sessionhashsize;
	extern struct nfsstatsv1 nfsstatsv1;
	extern struct nfslayouthash *nfslayouthash;
	extern int nfsrv_layouthashsize;
	extern struct mtx nfsrv_dslock_mtx;
	extern int nfs_pnfsiothreads;
	extern struct nfsdontlisthead nfsrv_dontlisthead;
	extern volatile int nfsrv_dontlistlen;
	extern volatile int nfsrv_devidcnt;
	extern int nfsrv_maxpnfsmirror;
	struct vfsoptlist nfsv4root_opt, nfsv4root_newopt;
	NFSDLOCKMUTEX;
	NFSSTATESPINLOCK;
	struct nfsrchash_bucket nfsrchash_table[NFSRVCACHE_HASHSIZE];
	struct nfsrchash_bucket nfsrcahash_table[NFSRVCACHE_HASHSIZE];
	struct mtx nfsrc_udpmtx;
	struct mtx nfs_v4root_mutex;
	struct mtx nfsrv_dontlistlock_mtx;
	struct mtx nfsrv_recalllock_mtx;
	struct nfsrvfh nfs_rootfh, nfs_pubfh;
	int nfs_pubfhset = 0, nfs_rootfhset = 0;
	struct proc *nfsd_master_proc = NULL;
	int nfsd_debuglevel = 0;
	static pid_t nfsd_master_pid = (pid_t)-1;
	static char nfsd_master_comm[MAXCOMLEN + 1];
	static struct timeval nfsd_master_start;
	static uint32_t nfsv4_sysid = 0;
	static fhandle_t zerofh;

	static int nfssvc_srvcall(struct thread , struct nfssvc_args ,
	struct ucred *);

	int nfsrv_enable_crossmntpt = 1;
	static int nfs_commit_blks;
	static int nfs_commit_miss;
	extern int nfsrv_issuedelegs;
	extern int nfsrv_dolocallocks;
	extern int nfsd_enable_stringtouid;
	extern struct nfsdevicehead nfsrv_devidhead;

	static int nfsrv_createiovec(int, struct mbuf , struct mbuf ,
	struct iovec **);
	static int nfsrv_createiovecw(int, struct mbuf , char , struct iovec **,
	int *);
	static void nfsrv_pnfscreate(struct vnode , struct vattr , struct ucred *,
	NFSPROC_T *);
	static void nfsrv_pnfsremovesetup(struct vnode , NFSPROC_T , struct vnode **,
	int , char , fhandle_t *);
	static void nfsrv_pnfsremove(struct vnode *, int, char , fhandle_t *,
	NFSPROC_T *);
	static int nfsrv_proxyds(struct vnode , off_t, int, struct ucred ,
	struct thread , int, struct mbuf , char , struct mbuf **,
	struct nfsvattr , struct acl , off_t , int, bool );
	static int nfsrv_setextattr(struct vnode , struct nfsvattr , NFSPROC_T *);
	static int nfsrv_readdsrpc(fhandle_t , off_t, int, struct ucred ,
	NFSPROC_T , struct nfsmount , struct mbuf , struct mbuf );
	static int nfsrv_writedsrpc(fhandle_t , off_t, int, struct ucred ,
	NFSPROC_T , struct vnode , struct nfsmount , int, struct mbuf ,
	char , int );
	static int nfsrv_allocatedsrpc(fhandle_t , off_t, off_t, struct ucred ,
	NFSPROC_T , struct vnode , struct nfsmount *, int, int );
	static int nfsrv_setacldsrpc(fhandle_t , struct ucred , NFSPROC_T *,
	struct vnode , struct nfsmount , int, struct acl , int *);
	static int nfsrv_setattrdsrpc(fhandle_t , struct ucred , NFSPROC_T *,
	struct vnode , struct nfsmount , int, struct nfsvattr , int *);
	static int nfsrv_getattrdsrpc(fhandle_t , struct ucred , NFSPROC_T *,
	struct vnode , struct nfsmount , struct nfsvattr *);
	static int nfsrv_seekdsrpc(fhandle_t , off_t , int, bool , struct ucred ,
	NFSPROC_T , struct nfsmount );
	static int nfsrv_putfhname(fhandle_t , char );
	static int nfsrv_pnfslookupds(struct vnode , struct vnode ,
	struct pnfsdsfile , struct vnode , NFSPROC_T );
	static void nfsrv_pnfssetfh(struct vnode , struct pnfsdsfile , char , char ,
	struct vnode , NFSPROC_T );
	static int nfsrv_dsremove(struct vnode , char , struct ucred , NFSPROC_T );
	static int nfsrv_dssetacl(struct vnode , struct acl , struct ucred *,
	NFSPROC_T *);
	static int nfsrv_pnfsstatfs(struct statfs , struct mount );
	+static void nfsm_trimtrailing(struct nfsrv_descript , struct mbuf ,
	+ char *, int, int);

	int nfs_pnfsio(task_fn_t , void );

	SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"NFS server");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW,
	&nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, commit_blks, CTLFLAG_RW, &nfs_commit_blks,
	0, "");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, commit_miss, CTLFLAG_RW, &nfs_commit_miss,
	0, "");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, issue_delegations, CTLFLAG_RW,
	&nfsrv_issuedelegs, 0, "Enable nfsd to issue delegations");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, enable_locallocks, CTLFLAG_RW,
	&nfsrv_dolocallocks, 0, "Enable nfsd to acquire local locks on files");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, debuglevel, CTLFLAG_RW, &nfsd_debuglevel,
	0, "Debug level for NFS server");
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, enable_stringtouid, CTLFLAG_RW,
	&nfsd_enable_stringtouid, 0, "Enable nfsd to accept numeric owner_names");
	static int nfsrv_pnfsgetdsattr = 1;
	SYSCTL_INT(_vfs_nfsd, OID_AUTO, pnfsgetdsattr, CTLFLAG_RW,
	&nfsrv_pnfsgetdsattr, 0, "When set getattr gets DS attributes via RPC");

	/*
	* nfsrv_dsdirsize can only be increased and only when the nfsd threads are
	* not running.
	* The dsN subdirectories for the increased values must have been created
	* on all DS servers before this increase is done.
	*/
	u_int nfsrv_dsdirsize = 20;
	static int
	sysctl_dsdirsize(SYSCTL_HANDLER_ARGS)
	{
	int error, newdsdirsize;

	newdsdirsize = nfsrv_dsdirsize;
	error = sysctl_handle_int(oidp, &newdsdirsize, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	if (newdsdirsize <= nfsrv_dsdirsize \|\| newdsdirsize > 10000 \|\|
	newnfs_numnfsd != 0)
	return (EINVAL);
	nfsrv_dsdirsize = newdsdirsize;
	return (0);
	}
	SYSCTL_PROC(_vfs_nfsd, OID_AUTO, dsdirsize,
	CTLTYPE_UINT \| CTLFLAG_MPSAFE \| CTLFLAG_RW, 0, sizeof(nfsrv_dsdirsize),
	sysctl_dsdirsize, "IU", "Number of dsN subdirs on the DS servers");

	#define MAX_REORDERED_RPC 16
	#define NUM_HEURISTIC 1031
	#define NHUSE_INIT 64
	#define NHUSE_INC 16
	#define NHUSE_MAX 2048

	static struct nfsheur {
	struct vnode nh_vp; / vp to match (unreferenced pointer) */
	off_t nh_nextoff; /* next offset for sequential detection */
	int nh_use; /* use count for selection */
	int nh_seqcount; /* heuristic */
	} nfsheur[NUM_HEURISTIC];


	/*
	* Heuristic to detect sequential operation.
	*/
	static struct nfsheur *
	nfsrv_sequential_heuristic(struct uio uio, struct vnode vp)
	{
	struct nfsheur *nh;
	int hi, try;

	/* Locate best candidate. */
	try = 32;
	hi = ((int)(vm_offset_t)vp / sizeof(struct vnode)) % NUM_HEURISTIC;
	nh = &nfsheur[hi];
	while (try--) {
	if (nfsheur[hi].nh_vp == vp) {
	nh = &nfsheur[hi];
	break;
	}
	if (nfsheur[hi].nh_use > 0)
	--nfsheur[hi].nh_use;
	hi = (hi + 1) % NUM_HEURISTIC;
	if (nfsheur[hi].nh_use < nh->nh_use)
	nh = &nfsheur[hi];
	}

	/* Initialize hint if this is a new file. */
	if (nh->nh_vp != vp) {
	nh->nh_vp = vp;
	nh->nh_nextoff = uio->uio_offset;
	nh->nh_use = NHUSE_INIT;
	if (uio->uio_offset == 0)
	nh->nh_seqcount = 4;
	else
	nh->nh_seqcount = 1;
	}

	/* Calculate heuristic. */
	if ((uio->uio_offset == 0 && nh->nh_seqcount > 0) \|\|
	uio->uio_offset == nh->nh_nextoff) {
	/* See comments in vfs_vnops.c:sequential_heuristic(). */
	nh->nh_seqcount += howmany(uio->uio_resid, 16384);
	if (nh->nh_seqcount > IO_SEQMAX)
	nh->nh_seqcount = IO_SEQMAX;
	} else if (qabs(uio->uio_offset - nh->nh_nextoff) <= MAX_REORDERED_RPC *
	imax(vp->v_mount->mnt_stat.f_iosize, uio->uio_resid)) {
	/* Probably a reordered RPC, leave seqcount alone. */
	} else if (nh->nh_seqcount > 1) {
	nh->nh_seqcount /= 2;
	} else {
	nh->nh_seqcount = 0;
	}
	nh->nh_use += NHUSE_INC;
	if (nh->nh_use > NHUSE_MAX)
	nh->nh_use = NHUSE_MAX;
	return (nh);
	}

	/*
	* Get attributes into nfsvattr structure.
	*/
	int
	nfsvno_getattr(struct vnode vp, struct nfsvattr nvap,
	struct nfsrv_descript nd, struct thread p, int vpislocked,
	nfsattrbit_t *attrbitp)
	{
	int error, gotattr, lockedit = 0;
	struct nfsvattr na;

	if (vpislocked == 0) {
	/*
	* When vpislocked == 0, the vnode is either exclusively
	* locked by this thread or not locked by this thread.
	* As such, shared lock it, if not exclusively locked.
	*/
	if (NFSVOPISLOCKED(vp) != LK_EXCLUSIVE) {
	lockedit = 1;
	NFSVOPLOCK(vp, LK_SHARED \| LK_RETRY);
	}
	}

	/*
	* Acquire the Change, Size, TimeAccess, TimeModify and SpaceUsed
	* attributes, as required.
	* This needs to be done for regular files if:
	* - non-NFSv4 RPCs or
	* - when attrbitp == NULL or
	* - an NFSv4 RPC with any of the above attributes in attrbitp.
	* A return of 0 for nfsrv_proxyds() indicates that it has acquired
	* these attributes. nfsrv_proxyds() will return an error if the
	* server is not a pNFS one.
	*/
	gotattr = 0;
	if (vp->v_type == VREG && nfsrv_devidcnt > 0 && (attrbitp == NULL \|\|
	(nd->nd_flag & ND_NFSV4) == 0 \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_CHANGE) \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_SIZE) \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_TIMEACCESS) \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_TIMEMODIFY) \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_SPACEUSED))) {
	error = nfsrv_proxyds(vp, 0, 0, nd->nd_cred, p,
	NFSPROC_GETATTR, NULL, NULL, NULL, &na, NULL, NULL, 0,
	NULL);
	if (error == 0)
	gotattr = 1;
	}

	error = VOP_GETATTR(vp, &nvap->na_vattr, nd->nd_cred);
	if (lockedit != 0)
	NFSVOPUNLOCK(vp);

	/*
	* If we got the Change, Size and Modify Time from the DS,
	* replace them.
	*/
	if (gotattr != 0) {
	nvap->na_atime = na.na_atime;
	nvap->na_mtime = na.na_mtime;
	nvap->na_filerev = na.na_filerev;
	nvap->na_size = na.na_size;
	nvap->na_bytes = na.na_bytes;
	}
	NFSD_DEBUG(4, "nfsvno_getattr: gotattr=%d err=%d chg=%ju\n", gotattr,
	error, (uintmax_t)na.na_filerev);

	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Get a file handle for a vnode.
	*/
	int
	nfsvno_getfh(struct vnode vp, fhandle_t fhp, struct thread *p)
	{
	int error;

	NFSBZERO((caddr_t)fhp, sizeof(fhandle_t));
	fhp->fh_fsid = vp->v_mount->mnt_stat.f_fsid;
	error = VOP_VPTOFH(vp, &fhp->fh_fid);

	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Perform access checking for vnodes obtained from file handles that would
	* refer to files already opened by a Unix client. You cannot just use
	* vn_writechk() and VOP_ACCESSX() for two reasons.
	* 1 - You must check for exported rdonly as well as MNT_RDONLY for the write
	* case.
	* 2 - The owner is to be given access irrespective of mode bits for some
	* operations, so that processes that chmod after opening a file don't
	* break.
	*/
	int
	nfsvno_accchk(struct vnode vp, accmode_t accmode, struct ucred cred,
	struct nfsexstuff exp, struct thread p, int override, int vpislocked,
	u_int32_t *supportedtypep)
	{
	struct vattr vattr;
	int error = 0, getret = 0;

	if (vpislocked == 0) {
	if (NFSVOPLOCK(vp, LK_SHARED) != 0) {
	error = EPERM;
	goto out;
	}
	}
	if (accmode & VWRITE) {
	/* Just vn_writechk() changed to check rdonly */
	/*
	* Disallow write attempts on read-only file systems;
	* unless the file is a socket or a block or character
	* device resident on the file system.
	*/
	if (NFSVNO_EXRDONLY(exp) \|\|
	(vp->v_mount->mnt_flag & MNT_RDONLY)) {
	switch (vp->v_type) {
	case VREG:
	case VDIR:
	case VLNK:
	error = EROFS;
	default:
	break;
	}
	}
	/*
	* If there's shared text associated with
	* the inode, try to free it up once. If
	* we fail, we can't allow writing.
	*/
	if (VOP_IS_TEXT(vp) && error == 0)
	error = ETXTBSY;
	}
	if (error != 0) {
	if (vpislocked == 0)
	NFSVOPUNLOCK(vp);
	goto out;
	}

	/*
	* Should the override still be applied when ACLs are enabled?
	*/
	error = VOP_ACCESSX(vp, accmode, cred, p);
	if (error != 0 && (accmode & (VDELETE \| VDELETE_CHILD))) {
	/*
	* Try again with VEXPLICIT_DENY, to see if the test for
	* deletion is supported.
	*/
	error = VOP_ACCESSX(vp, accmode \| VEXPLICIT_DENY, cred, p);
	if (error == 0) {
	if (vp->v_type == VDIR) {
	accmode &= ~(VDELETE \| VDELETE_CHILD);
	accmode \|= VWRITE;
	error = VOP_ACCESSX(vp, accmode, cred, p);
	} else if (supportedtypep != NULL) {
	*supportedtypep &= ~NFSACCESS_DELETE;
	}
	}
	}

	/*
	* Allow certain operations for the owner (reads and writes
	* on files that are already open).
	*/
	if (override != NFSACCCHK_NOOVERRIDE &&
	(error == EPERM \|\| error == EACCES)) {
	if (cred->cr_uid == 0 && (override & NFSACCCHK_ALLOWROOT))
	error = 0;
	else if (override & NFSACCCHK_ALLOWOWNER) {
	getret = VOP_GETATTR(vp, &vattr, cred);
	if (getret == 0 && cred->cr_uid == vattr.va_uid)
	error = 0;
	}
	}
	if (vpislocked == 0)
	NFSVOPUNLOCK(vp);

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Set attribute(s) vnop.
	*/
	int
	nfsvno_setattr(struct vnode vp, struct nfsvattr nvap, struct ucred *cred,
	struct thread p, struct nfsexstuff exp)
	{
	u_quad_t savsize = 0;
	int error, savedit;
	+ time_t savbtime;

	/*
	* If this is an exported file system and a pNFS service is running,
	* don't VOP_SETATTR() of size for the MDS file system.
	*/
	savedit = 0;
	error = 0;
	if (vp->v_type == VREG && (vp->v_mount->mnt_flag & MNT_EXPORTED) != 0 &&
	nfsrv_devidcnt != 0 && nvap->na_vattr.va_size != VNOVAL &&
	nvap->na_vattr.va_size > 0) {
	savsize = nvap->na_vattr.va_size;
	nvap->na_vattr.va_size = VNOVAL;
	if (nvap->na_vattr.va_uid != (uid_t)VNOVAL \|\|
	nvap->na_vattr.va_gid != (gid_t)VNOVAL \|\|
	nvap->na_vattr.va_mode != (mode_t)VNOVAL \|\|
	nvap->na_vattr.va_atime.tv_sec != VNOVAL \|\|
	nvap->na_vattr.va_mtime.tv_sec != VNOVAL)
	savedit = 1;
	else
	savedit = 2;
	}
	if (savedit != 2)
	error = VOP_SETATTR(vp, &nvap->na_vattr, cred);
	if (savedit != 0)
	nvap->na_vattr.va_size = savsize;
	if (error == 0 && (nvap->na_vattr.va_uid != (uid_t)VNOVAL \|\|
	nvap->na_vattr.va_gid != (gid_t)VNOVAL \|\|
	nvap->na_vattr.va_size != VNOVAL \|\|
	nvap->na_vattr.va_mode != (mode_t)VNOVAL \|\|
	nvap->na_vattr.va_atime.tv_sec != VNOVAL \|\|
	nvap->na_vattr.va_mtime.tv_sec != VNOVAL)) {
	+ /* Never modify birthtime on a DS file. */
	+ savbtime = nvap->na_vattr.va_birthtime.tv_sec;
	+ nvap->na_vattr.va_birthtime.tv_sec = VNOVAL;
	/* For a pNFS server, set the attributes on the DS file. */
	error = nfsrv_proxyds(vp, 0, 0, cred, p, NFSPROC_SETATTR,
	NULL, NULL, NULL, nvap, NULL, NULL, 0, NULL);
	+ nvap->na_vattr.va_birthtime.tv_sec = savbtime;
	if (error == ENOENT)
	error = 0;
	}
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Set up nameidata for a lookup() call and do it.
	*/
	int
	nfsvno_namei(struct nfsrv_descript nd, struct nameidata ndp,
	struct vnode dp, int islocked, struct nfsexstuff exp, struct thread *p,
	struct vnode **retdirp)
	{
	struct componentname *cnp = &ndp->ni_cnd;
	int i;
	struct iovec aiov;
	struct uio auio;
	int lockleaf = (cnp->cn_flags & LOCKLEAF) != 0, linklen;
	int error = 0;
	char *cp;

	*retdirp = NULL;
	cnp->cn_nameptr = cnp->cn_pnbuf;
	ndp->ni_lcf = 0;
	/*
	* Extract and set starting directory.
	*/
	if (dp->v_type != VDIR) {
	if (islocked)
	vput(dp);
	else
	vrele(dp);
	nfsvno_relpathbuf(ndp);
	error = ENOTDIR;
	goto out1;
	}
	if (islocked)
	NFSVOPUNLOCK(dp);
	VREF(dp);
	*retdirp = dp;
	if (NFSVNO_EXRDONLY(exp))
	cnp->cn_flags \|= RDONLY;
	ndp->ni_segflg = UIO_SYSSPACE;

	if (nd->nd_flag & ND_PUBLOOKUP) {
	ndp->ni_loopcnt = 0;
	if (cnp->cn_pnbuf[0] == '/') {
	vrele(dp);
	/*
	* Check for degenerate pathnames here, since lookup()
	* panics on them.
	*/
	for (i = 1; i < ndp->ni_pathlen; i++)
	if (cnp->cn_pnbuf[i] != '/')
	break;
	if (i == ndp->ni_pathlen) {
	error = NFSERR_ACCES;
	goto out;
	}
	dp = rootvnode;
	VREF(dp);
	}
	} else if ((nfsrv_enable_crossmntpt == 0 && NFSVNO_EXPORTED(exp)) \|\|
	(nd->nd_flag & ND_NFSV4) == 0) {
	/*
	* Only cross mount points for NFSv4 when doing a
	* mount while traversing the file system above
	* the mount point, unless nfsrv_enable_crossmntpt is set.
	*/
	cnp->cn_flags \|= NOCROSSMOUNT;
	}

	/*
	* Initialize for scan, set ni_startdir and bump ref on dp again
	* because lookup() will dereference ni_startdir.
	*/

	cnp->cn_thread = p;
	ndp->ni_startdir = dp;
	ndp->ni_rootdir = rootvnode;
	ndp->ni_topdir = NULL;

	if (!lockleaf)
	cnp->cn_flags \|= LOCKLEAF;
	for (;;) {
	cnp->cn_nameptr = cnp->cn_pnbuf;
	/*
	* Call lookup() to do the real work. If an error occurs,
	* ndp->ni_vp and ni_dvp are left uninitialized or NULL and
	* we do not have to dereference anything before returning.
	* In either case ni_startdir will be dereferenced and NULLed
	* out.
	*/
	error = lookup(ndp);
	if (error)
	break;

	/*
	* Check for encountering a symbolic link. Trivial
	* termination occurs if no symlink encountered.
	*/
	if ((cnp->cn_flags & ISSYMLINK) == 0) {
	if ((cnp->cn_flags & (SAVENAME \| SAVESTART)) == 0)
	nfsvno_relpathbuf(ndp);
	if (ndp->ni_vp && !lockleaf)
	NFSVOPUNLOCK(ndp->ni_vp);
	break;
	}

	/*
	* Validate symlink
	*/
	if ((cnp->cn_flags & LOCKPARENT) && ndp->ni_pathlen == 1)
	NFSVOPUNLOCK(ndp->ni_dvp);
	if (!(nd->nd_flag & ND_PUBLOOKUP)) {
	error = EINVAL;
	goto badlink2;
	}

	if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
	error = ELOOP;
	goto badlink2;
	}
	if (ndp->ni_pathlen > 1)
	cp = uma_zalloc(namei_zone, M_WAITOK);
	else
	cp = cnp->cn_pnbuf;
	aiov.iov_base = cp;
	aiov.iov_len = MAXPATHLEN;
	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_offset = 0;
	auio.uio_rw = UIO_READ;
	auio.uio_segflg = UIO_SYSSPACE;
	auio.uio_td = NULL;
	auio.uio_resid = MAXPATHLEN;
	error = VOP_READLINK(ndp->ni_vp, &auio, cnp->cn_cred);
	if (error) {
	badlink1:
	if (ndp->ni_pathlen > 1)
	uma_zfree(namei_zone, cp);
	badlink2:
	vrele(ndp->ni_dvp);
	vput(ndp->ni_vp);
	break;
	}
	linklen = MAXPATHLEN - auio.uio_resid;
	if (linklen == 0) {
	error = ENOENT;
	goto badlink1;
	}
	if (linklen + ndp->ni_pathlen >= MAXPATHLEN) {
	error = ENAMETOOLONG;
	goto badlink1;
	}

	/*
	* Adjust or replace path
	*/
	if (ndp->ni_pathlen > 1) {
	NFSBCOPY(ndp->ni_next, cp + linklen, ndp->ni_pathlen);
	uma_zfree(namei_zone, cnp->cn_pnbuf);
	cnp->cn_pnbuf = cp;
	} else
	cnp->cn_pnbuf[linklen] = '\0';
	ndp->ni_pathlen += linklen;

	/*
	* Cleanup refs for next loop and check if root directory
	* should replace current directory. Normally ni_dvp
	* becomes the new base directory and is cleaned up when
	* we loop. Explicitly null pointers after invalidation
	* to clarify operation.
	*/
	vput(ndp->ni_vp);
	ndp->ni_vp = NULL;

	if (cnp->cn_pnbuf[0] == '/') {
	vrele(ndp->ni_dvp);
	ndp->ni_dvp = ndp->ni_rootdir;
	VREF(ndp->ni_dvp);
	}
	ndp->ni_startdir = ndp->ni_dvp;
	ndp->ni_dvp = NULL;
	}
	if (!lockleaf)
	cnp->cn_flags &= ~LOCKLEAF;

	out:
	if (error) {
	nfsvno_relpathbuf(ndp);
	ndp->ni_vp = NULL;
	ndp->ni_dvp = NULL;
	ndp->ni_startdir = NULL;
	} else if ((ndp->ni_cnd.cn_flags & (WANTPARENT\|LOCKPARENT)) == 0) {
	ndp->ni_dvp = NULL;
	}

	out1:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Set up a pathname buffer and return a pointer to it and, optionally
	* set a hash pointer.
	*/
	void
	nfsvno_setpathbuf(struct nameidata ndp, char bufpp, u_long *hashpp)
	{
	struct componentname *cnp = &ndp->ni_cnd;

	cnp->cn_flags \|= (NOMACCHECK \| HASBUF);
	cnp->cn_pnbuf = uma_zalloc(namei_zone, M_WAITOK);
	if (hashpp != NULL)
	*hashpp = NULL;
	*bufpp = cnp->cn_pnbuf;
	}

	/*
	* Release the above path buffer, if not released by nfsvno_namei().
	*/
	void
	nfsvno_relpathbuf(struct nameidata *ndp)
	{

	if ((ndp->ni_cnd.cn_flags & HASBUF) == 0)
	panic("nfsrelpath");
	uma_zfree(namei_zone, ndp->ni_cnd.cn_pnbuf);
	ndp->ni_cnd.cn_flags &= ~HASBUF;
	}

	/*
	* Readlink vnode op into an mbuf list.
	*/
	int
	nfsvno_readlink(struct vnode vp, struct ucred cred, struct thread *p,
	struct mbuf mpp, struct mbuf mpendp, int *lenp)
	{
	struct iovec *iv;
	struct uio io, *uiop = &io;
	struct mbuf mp, mp3;
	int len, tlen, error = 0;

	len = NFS_MAXPATHLEN;
	uiop->uio_iovcnt = nfsrv_createiovec(len, &mp3, &mp, &iv);
	uiop->uio_iov = iv;
	uiop->uio_offset = 0;
	uiop->uio_resid = len;
	uiop->uio_rw = UIO_READ;
	uiop->uio_segflg = UIO_SYSSPACE;
	uiop->uio_td = NULL;
	error = VOP_READLINK(vp, uiop, cred);
	free(iv, M_TEMP);
	if (error) {
	m_freem(mp3);
	*lenp = 0;
	goto out;
	}
	if (uiop->uio_resid > 0) {
	len -= uiop->uio_resid;
	tlen = NFSM_RNDUP(len);
	if (tlen == 0) {
	m_freem(mp3);
	mp3 = mp = NULL;
	} else if (tlen != NFS_MAXPATHLEN \|\| tlen != len)
	mp = nfsrv_adj(mp3, NFS_MAXPATHLEN - tlen,
	tlen - len);
	}
	*lenp = len;
	*mpp = mp3;
	*mpendp = mp;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Create an mbuf chain and an associated iovec that can be used to Read
	* or Getextattr of data.
	* Upon success, return pointers to the first and last mbufs in the chain
	* plus the malloc'd iovec and its iovlen.
	*/
	static int
	nfsrv_createiovec(int len, struct mbuf mpp, struct mbuf mpendp,
	struct iovec **ivp)
	{
	struct mbuf m, m2 = NULL, *m3;
	struct iovec *iv;
	int i, left, siz;

	left = len;
	m3 = NULL;
	/*
	* Generate the mbuf list with the uio_iov ref. to it.
	*/
	i = 0;
	while (left > 0) {
	NFSMGET(m);
	MCLGET(m, M_WAITOK);
	m->m_len = 0;
	siz = min(M_TRAILINGSPACE(m), left);
	left -= siz;
	i++;
	if (m3)
	m2->m_next = m;
	else
	m3 = m;
	m2 = m;
	}
	ivp = iv = malloc(i sizeof (struct iovec), M_TEMP, M_WAITOK);
	m = m3;
	left = len;
	i = 0;
	while (left > 0) {
	if (m == NULL)
	panic("nfsvno_read iov");
	siz = min(M_TRAILINGSPACE(m), left);
	if (siz > 0) {
	iv->iov_base = mtod(m, caddr_t) + m->m_len;
	iv->iov_len = siz;
	m->m_len += siz;
	left -= siz;
	iv++;
	i++;
	}
	m = m->m_next;
	}
	*mpp = m3;
	*mpendp = m2;
	return (i);
	}

	/*
	* Read vnode op call into mbuf list.
	*/
	int
	nfsvno_read(struct vnode vp, off_t off, int cnt, struct ucred cred,
	struct thread p, struct mbuf mpp, struct mbuf *mpendp)
	{
	struct mbuf *m;
	struct iovec *iv;
	int error = 0, len, tlen, ioflag = 0;
	struct mbuf *m3;
	struct uio io, *uiop = &io;
	struct nfsheur *nh;

	/*
	* Attempt to read from a DS file. A return of ENOENT implies
	* there is no DS file to read.
	*/
	error = nfsrv_proxyds(vp, off, cnt, cred, p, NFSPROC_READDS, mpp,
	NULL, mpendp, NULL, NULL, NULL, 0, NULL);
	if (error != ENOENT)
	return (error);

	len = NFSM_RNDUP(cnt);
	uiop->uio_iovcnt = nfsrv_createiovec(len, &m3, &m, &iv);
	uiop->uio_iov = iv;
	uiop->uio_offset = off;
	uiop->uio_resid = len;
	uiop->uio_rw = UIO_READ;
	uiop->uio_segflg = UIO_SYSSPACE;
	uiop->uio_td = NULL;
	nh = nfsrv_sequential_heuristic(uiop, vp);
	ioflag \|= nh->nh_seqcount << IO_SEQSHIFT;
	/* XXX KDM make this more systematic? */
	nfsstatsv1.srvbytes[NFSV4OP_READ] += uiop->uio_resid;
	error = VOP_READ(vp, uiop, IO_NODELOCKED \| ioflag, cred);
	free(iv, M_TEMP);
	if (error) {
	m_freem(m3);
	*mpp = NULL;
	goto out;
	}
	nh->nh_nextoff = uiop->uio_offset;
	tlen = len - uiop->uio_resid;
	cnt = cnt < tlen ? cnt : tlen;
	tlen = NFSM_RNDUP(cnt);
	if (tlen == 0) {
	m_freem(m3);
	m3 = m = NULL;
	} else if (len != tlen \|\| tlen != cnt)
	m = nfsrv_adj(m3, len - tlen, tlen - cnt);
	*mpp = m3;
	*mpendp = m;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Create the iovec for the mbuf chain passed in as an argument.
	* The "cp" argument is where the data starts within the first mbuf in
	* the chain. It returns the iovec and the iovcnt.
	*/
	static int
	nfsrv_createiovecw(int retlen, struct mbuf m, char cp, struct iovec **ivpp,
	int *iovcntp)
	{
	struct mbuf *mp;
	struct iovec *ivp;
	int cnt, i, len;

	/*
	* Loop through the mbuf chain, counting how many mbufs are a
	* part of this write operation, so the iovec size is known.
	*/
	cnt = 0;
	len = retlen;
	mp = m;
	i = mtod(mp, caddr_t) + mp->m_len - cp;
	while (len > 0) {
	if (i > 0) {
	len -= i;
	cnt++;
	}
	mp = mp->m_next;
	if (!mp) {
	if (len > 0)
	return (EBADRPC);
	} else
	i = mp->m_len;
	}

	/* Now, create the iovec. */
	mp = m;
	ivpp = ivp = malloc(cnt sizeof (struct iovec), M_TEMP,
	M_WAITOK);
	*iovcntp = cnt;
	i = mtod(mp, caddr_t) + mp->m_len - cp;
	len = retlen;
	while (len > 0) {
	if (mp == NULL)
	panic("nfsvno_write");
	if (i > 0) {
	i = min(i, len);
	ivp->iov_base = cp;
	ivp->iov_len = i;
	ivp++;
	len -= i;
	}
	mp = mp->m_next;
	if (mp) {
	i = mp->m_len;
	cp = mtod(mp, caddr_t);
	}
	}
	return (0);
	}

	/*
	* Write vnode op from an mbuf list.
	*/
	int
	nfsvno_write(struct vnode vp, off_t off, int retlen, int stable,
	struct mbuf mp, char cp, struct ucred cred, struct thread p)
	{
	struct iovec *iv;
	int cnt, ioflags, error;
	struct uio io, *uiop = &io;
	struct nfsheur *nh;

	/*
	* Attempt to write to a DS file. A return of ENOENT implies
	* there is no DS file to write.
	*/
	error = nfsrv_proxyds(vp, off, retlen, cred, p, NFSPROC_WRITEDS,
	&mp, cp, NULL, NULL, NULL, NULL, 0, NULL);
	if (error != ENOENT) {
	*stable = NFSWRITE_FILESYNC;
	return (error);
	}


	if (*stable == NFSWRITE_UNSTABLE)
	ioflags = IO_NODELOCKED;
	else
	ioflags = (IO_SYNC \| IO_NODELOCKED);
	error = nfsrv_createiovecw(retlen, mp, cp, &iv, &cnt);
	if (error != 0)
	return (error);
	uiop->uio_iov = iv;
	uiop->uio_iovcnt = cnt;
	uiop->uio_resid = retlen;
	uiop->uio_rw = UIO_WRITE;
	uiop->uio_segflg = UIO_SYSSPACE;
	NFSUIOPROC(uiop, p);
	uiop->uio_offset = off;
	nh = nfsrv_sequential_heuristic(uiop, vp);
	ioflags \|= nh->nh_seqcount << IO_SEQSHIFT;
	/* XXX KDM make this more systematic? */
	nfsstatsv1.srvbytes[NFSV4OP_WRITE] += uiop->uio_resid;
	error = VOP_WRITE(vp, uiop, ioflags, cred);
	if (error == 0)
	nh->nh_nextoff = uiop->uio_offset;
	free(iv, M_TEMP);

	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Common code for creating a regular file (plus special files for V2).
	*/
	int
	nfsvno_createsub(struct nfsrv_descript nd, struct nameidata ndp,
	struct vnode *vpp, struct nfsvattr nvap, int *exclusive_flagp,
	int32_t cverf, NFSDEV_T rdev, struct nfsexstuff exp)
	{
	u_quad_t tempsize;
	int error;
	struct thread *p = curthread;

	error = nd->nd_repstat;
	if (!error && ndp->ni_vp == NULL) {
	if (nvap->na_type == VREG \|\| nvap->na_type == VSOCK) {
	vrele(ndp->ni_startdir);
	error = VOP_CREATE(ndp->ni_dvp,
	&ndp->ni_vp, &ndp->ni_cnd, &nvap->na_vattr);
	/* For a pNFS server, create the data file on a DS. */
	if (error == 0 && nvap->na_type == VREG) {
	/*
	* Create a data file on a DS for a pNFS server.
	* This function just returns if not
	* running a pNFS DS or the creation fails.
	*/
	nfsrv_pnfscreate(ndp->ni_vp, &nvap->na_vattr,
	nd->nd_cred, p);
	}
	vput(ndp->ni_dvp);
	nfsvno_relpathbuf(ndp);
	if (!error) {
	if (*exclusive_flagp) {
	*exclusive_flagp = 0;
	NFSVNO_ATTRINIT(nvap);
	nvap->na_atime.tv_sec = cverf[0];
	nvap->na_atime.tv_nsec = cverf[1];
	error = VOP_SETATTR(ndp->ni_vp,
	&nvap->na_vattr, nd->nd_cred);
	if (error != 0) {
	vput(ndp->ni_vp);
	ndp->ni_vp = NULL;
	error = NFSERR_NOTSUPP;
	}
	}
	}
	/*
	* NFS V2 Only. nfsrvd_mknod() does this for V3.
	* (This implies, just get out on an error.)
	*/
	} else if (nvap->na_type == VCHR \|\| nvap->na_type == VBLK \|\|
	nvap->na_type == VFIFO) {
	if (nvap->na_type == VCHR && rdev == 0xffffffff)
	nvap->na_type = VFIFO;
	if (nvap->na_type != VFIFO &&
	(error = priv_check_cred(nd->nd_cred, PRIV_VFS_MKNOD_DEV))) {
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	vput(ndp->ni_dvp);
	goto out;
	}
	nvap->na_rdev = rdev;
	error = VOP_MKNOD(ndp->ni_dvp, &ndp->ni_vp,
	&ndp->ni_cnd, &nvap->na_vattr);
	vput(ndp->ni_dvp);
	nfsvno_relpathbuf(ndp);
	vrele(ndp->ni_startdir);
	if (error)
	goto out;
	} else {
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	vput(ndp->ni_dvp);
	error = ENXIO;
	goto out;
	}
	*vpp = ndp->ni_vp;
	} else {
	/*
	* Handle cases where error is already set and/or
	* the file exists.
	* 1 - clean up the lookup
	* 2 - iff !error and na_size set, truncate it
	*/
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	*vpp = ndp->ni_vp;
	if (ndp->ni_dvp == *vpp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	if (!error && nvap->na_size != VNOVAL) {
	error = nfsvno_accchk(*vpp, VWRITE,
	nd->nd_cred, exp, p, NFSACCCHK_NOOVERRIDE,
	NFSACCCHK_VPISLOCKED, NULL);
	if (!error) {
	tempsize = nvap->na_size;
	NFSVNO_ATTRINIT(nvap);
	nvap->na_size = tempsize;
	error = VOP_SETATTR(*vpp,
	&nvap->na_vattr, nd->nd_cred);
	}
	}
	if (error)
	vput(*vpp);
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Do a mknod vnode op.
	*/
	int
	nfsvno_mknod(struct nameidata ndp, struct nfsvattr nvap, struct ucred *cred,
	struct thread *p)
	{
	int error = 0;
	enum vtype vtyp;

	vtyp = nvap->na_type;
	/*
	* Iff doesn't exist, create it.
	*/
	if (ndp->ni_vp) {
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	vput(ndp->ni_dvp);
	vrele(ndp->ni_vp);
	error = EEXIST;
	goto out;
	}
	if (vtyp != VCHR && vtyp != VBLK && vtyp != VSOCK && vtyp != VFIFO) {
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	vput(ndp->ni_dvp);
	error = NFSERR_BADTYPE;
	goto out;
	}
	if (vtyp == VSOCK) {
	vrele(ndp->ni_startdir);
	error = VOP_CREATE(ndp->ni_dvp, &ndp->ni_vp,
	&ndp->ni_cnd, &nvap->na_vattr);
	vput(ndp->ni_dvp);
	nfsvno_relpathbuf(ndp);
	} else {
	if (nvap->na_type != VFIFO &&
	(error = priv_check_cred(cred, PRIV_VFS_MKNOD_DEV))) {
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	vput(ndp->ni_dvp);
	goto out;
	}
	error = VOP_MKNOD(ndp->ni_dvp, &ndp->ni_vp,
	&ndp->ni_cnd, &nvap->na_vattr);
	vput(ndp->ni_dvp);
	nfsvno_relpathbuf(ndp);
	vrele(ndp->ni_startdir);
	/*
	* Since VOP_MKNOD returns the ni_vp, I can't
	* see any reason to do the lookup.
	*/
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Mkdir vnode op.
	*/
	int
	nfsvno_mkdir(struct nameidata ndp, struct nfsvattr nvap, uid_t saved_uid,
	struct ucred cred, struct thread p, struct nfsexstuff *exp)
	{
	int error = 0;

	if (ndp->ni_vp != NULL) {
	if (ndp->ni_dvp == ndp->ni_vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	vrele(ndp->ni_vp);
	nfsvno_relpathbuf(ndp);
	error = EEXIST;
	goto out;
	}
	error = VOP_MKDIR(ndp->ni_dvp, &ndp->ni_vp, &ndp->ni_cnd,
	&nvap->na_vattr);
	vput(ndp->ni_dvp);
	nfsvno_relpathbuf(ndp);

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* symlink vnode op.
	*/
	int
	nfsvno_symlink(struct nameidata ndp, struct nfsvattr nvap, char *pathcp,
	int pathlen, int not_v2, uid_t saved_uid, struct ucred cred, struct thread p,
	struct nfsexstuff *exp)
	{
	int error = 0;

	if (ndp->ni_vp) {
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	if (ndp->ni_dvp == ndp->ni_vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	vrele(ndp->ni_vp);
	error = EEXIST;
	goto out;
	}

	error = VOP_SYMLINK(ndp->ni_dvp, &ndp->ni_vp, &ndp->ni_cnd,
	&nvap->na_vattr, pathcp);
	vput(ndp->ni_dvp);
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	/*
	* Although FreeBSD still had the lookup code in
	* it for 7/current, there doesn't seem to be any
	* point, since VOP_SYMLINK() returns the ni_vp.
	* Just vput it for v2.
	*/
	if (!not_v2 && !error)
	vput(ndp->ni_vp);

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Parse symbolic link arguments.
	* This function has an ugly side effect. It will malloc() an area for
	* the symlink and set iov_base to point to it, only if it succeeds.
	* So, if it returns with uiop->uio_iov->iov_base != NULL, that must
	* be FREE'd later.
	*/
	int
	nfsvno_getsymlink(struct nfsrv_descript nd, struct nfsvattr nvap,
	struct thread p, char pathcpp, int lenp)
	{
	u_int32_t *tl;
	char *pathcp = NULL;
	int error = 0, len;
	struct nfsv2_sattr *sp;

	*pathcpp = NULL;
	*lenp = 0;
	if ((nd->nd_flag & ND_NFSV3) &&
	(error = nfsrv_sattr(nd, NULL, nvap, NULL, NULL, p)))
	goto nfsmout;
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	len = fxdr_unsigned(int, *tl);
	if (len > NFS_MAXPATHLEN \|\| len <= 0) {
	error = EBADRPC;
	goto nfsmout;
	}
	pathcp = malloc(len + 1, M_TEMP, M_WAITOK);
	error = nfsrv_mtostr(nd, pathcp, len);
	if (error)
	goto nfsmout;
	if (nd->nd_flag & ND_NFSV2) {
	NFSM_DISSECT(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
	nvap->na_mode = fxdr_unsigned(u_int16_t, sp->sa_mode);
	}
	*pathcpp = pathcp;
	*lenp = len;
	NFSEXITCODE2(0, nd);
	return (0);
	nfsmout:
	if (pathcp)
	free(pathcp, M_TEMP);
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Remove a non-directory object.
	*/
	int
	nfsvno_removesub(struct nameidata ndp, int is_v4, struct ucred cred,
	struct thread p, struct nfsexstuff exp)
	{
	struct vnode vp, dsdvp[NFSDEV_MAXMIRRORS];
	int error = 0, mirrorcnt;
	char fname[PNFS_FILENAME_LEN + 1];
	fhandle_t fh;

	vp = ndp->ni_vp;
	dsdvp[0] = NULL;
	if (vp->v_type == VDIR)
	error = NFSERR_ISDIR;
	else if (is_v4)
	error = nfsrv_checkremove(vp, 1, NULL, (nfsquad_t)((u_quad_t)0),
	p);
	if (error == 0)
	nfsrv_pnfsremovesetup(vp, p, dsdvp, &mirrorcnt, fname, &fh);
	if (!error)
	error = VOP_REMOVE(ndp->ni_dvp, vp, &ndp->ni_cnd);
	if (error == 0 && dsdvp[0] != NULL)
	nfsrv_pnfsremove(dsdvp, mirrorcnt, fname, &fh, p);
	if (ndp->ni_dvp == vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	vput(vp);
	if ((ndp->ni_cnd.cn_flags & SAVENAME) != 0)
	nfsvno_relpathbuf(ndp);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Remove a directory.
	*/
	int
	nfsvno_rmdirsub(struct nameidata ndp, int is_v4, struct ucred cred,
	struct thread p, struct nfsexstuff exp)
	{
	struct vnode *vp;
	int error = 0;

	vp = ndp->ni_vp;
	if (vp->v_type != VDIR) {
	error = ENOTDIR;
	goto out;
	}
	/*
	* No rmdir "." please.
	*/
	if (ndp->ni_dvp == vp) {
	error = EINVAL;
	goto out;
	}
	/*
	* The root of a mounted filesystem cannot be deleted.
	*/
	if (vp->v_vflag & VV_ROOT)
	error = EBUSY;
	out:
	if (!error)
	error = VOP_RMDIR(ndp->ni_dvp, vp, &ndp->ni_cnd);
	if (ndp->ni_dvp == vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	vput(vp);
	if ((ndp->ni_cnd.cn_flags & SAVENAME) != 0)
	nfsvno_relpathbuf(ndp);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Rename vnode op.
	*/
	int
	nfsvno_rename(struct nameidata fromndp, struct nameidata tondp,
	u_int32_t ndstat, u_int32_t ndflag, struct ucred cred, struct thread p)
	{
	struct vnode fvp, tvp, tdvp, dsdvp[NFSDEV_MAXMIRRORS];
	int error = 0, mirrorcnt;
	char fname[PNFS_FILENAME_LEN + 1];
	fhandle_t fh;

	dsdvp[0] = NULL;
	fvp = fromndp->ni_vp;
	if (ndstat) {
	vrele(fromndp->ni_dvp);
	vrele(fvp);
	error = ndstat;
	goto out1;
	}
	tdvp = tondp->ni_dvp;
	tvp = tondp->ni_vp;
	if (tvp != NULL) {
	if (fvp->v_type == VDIR && tvp->v_type != VDIR) {
	error = (ndflag & ND_NFSV2) ? EISDIR : EEXIST;
	goto out;
	} else if (fvp->v_type != VDIR && tvp->v_type == VDIR) {
	error = (ndflag & ND_NFSV2) ? ENOTDIR : EEXIST;
	goto out;
	}
	if (tvp->v_type == VDIR && tvp->v_mountedhere) {
	error = (ndflag & ND_NFSV2) ? ENOTEMPTY : EXDEV;
	goto out;
	}

	/*
	* A rename to '.' or '..' results in a prematurely
	* unlocked vnode on FreeBSD5, so I'm just going to fail that
	* here.
	*/
	if ((tondp->ni_cnd.cn_namelen == 1 &&
	tondp->ni_cnd.cn_nameptr[0] == '.') \|\|
	(tondp->ni_cnd.cn_namelen == 2 &&
	tondp->ni_cnd.cn_nameptr[0] == '.' &&
	tondp->ni_cnd.cn_nameptr[1] == '.')) {
	error = EINVAL;
	goto out;
	}
	}
	if (fvp->v_type == VDIR && fvp->v_mountedhere) {
	error = (ndflag & ND_NFSV2) ? ENOTEMPTY : EXDEV;
	goto out;
	}
	if (fvp->v_mount != tdvp->v_mount) {
	error = (ndflag & ND_NFSV2) ? ENOTEMPTY : EXDEV;
	goto out;
	}
	if (fvp == tdvp) {
	error = (ndflag & ND_NFSV2) ? ENOTEMPTY : EINVAL;
	goto out;
	}
	if (fvp == tvp) {
	/*
	* If source and destination are the same, there is nothing to
	* do. Set error to -1 to indicate this.
	*/
	error = -1;
	goto out;
	}
	if (ndflag & ND_NFSV4) {
	if (NFSVOPLOCK(fvp, LK_EXCLUSIVE) == 0) {
	error = nfsrv_checkremove(fvp, 0, NULL,
	(nfsquad_t)((u_quad_t)0), p);
	NFSVOPUNLOCK(fvp);
	} else
	error = EPERM;
	if (tvp && !error)
	error = nfsrv_checkremove(tvp, 1, NULL,
	(nfsquad_t)((u_quad_t)0), p);
	} else {
	/*
	* For NFSv2 and NFSv3, try to get rid of the delegation, so
	* that the NFSv4 client won't be confused by the rename.
	* Since nfsd_recalldelegation() can only be called on an
	* unlocked vnode at this point and fvp is the file that will
	* still exist after the rename, just do fvp.
	*/
	nfsd_recalldelegation(fvp, p);
	}
	if (error == 0 && tvp != NULL) {
	nfsrv_pnfsremovesetup(tvp, p, dsdvp, &mirrorcnt, fname, &fh);
	NFSD_DEBUG(4, "nfsvno_rename: pnfsremovesetup"
	" dsdvp=%p\n", dsdvp[0]);
	}
	out:
	if (!error) {
	error = VOP_RENAME(fromndp->ni_dvp, fromndp->ni_vp,
	&fromndp->ni_cnd, tondp->ni_dvp, tondp->ni_vp,
	&tondp->ni_cnd);
	} else {
	if (tdvp == tvp)
	vrele(tdvp);
	else
	vput(tdvp);
	if (tvp)
	vput(tvp);
	vrele(fromndp->ni_dvp);
	vrele(fvp);
	if (error == -1)
	error = 0;
	}

	/*
	* If dsdvp[0] != NULL, it was set up by nfsrv_pnfsremovesetup() and
	* if the rename succeeded, the DS file for the tvp needs to be
	* removed.
	*/
	if (error == 0 && dsdvp[0] != NULL) {
	nfsrv_pnfsremove(dsdvp, mirrorcnt, fname, &fh, p);
	NFSD_DEBUG(4, "nfsvno_rename: pnfsremove\n");
	}

	vrele(tondp->ni_startdir);
	nfsvno_relpathbuf(tondp);
	out1:
	vrele(fromndp->ni_startdir);
	nfsvno_relpathbuf(fromndp);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Link vnode op.
	*/
	int
	nfsvno_link(struct nameidata ndp, struct vnode vp, struct ucred *cred,
	struct thread p, struct nfsexstuff exp)
	{
	struct vnode *xp;
	int error = 0;

	xp = ndp->ni_vp;
	if (xp != NULL) {
	error = EEXIST;
	} else {
	xp = ndp->ni_dvp;
	if (vp->v_mount != xp->v_mount)
	error = EXDEV;
	}
	if (!error) {
	NFSVOPLOCK(vp, LK_EXCLUSIVE \| LK_RETRY);
	if (!VN_IS_DOOMED(vp))
	error = VOP_LINK(ndp->ni_dvp, vp, &ndp->ni_cnd);
	else
	error = EPERM;
	if (ndp->ni_dvp == vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	NFSVOPUNLOCK(vp);
	} else {
	if (ndp->ni_dvp == ndp->ni_vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	if (ndp->ni_vp)
	vrele(ndp->ni_vp);
	}
	nfsvno_relpathbuf(ndp);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Do the fsync() appropriate for the commit.
	*/
	int
	nfsvno_fsync(struct vnode vp, u_int64_t off, int cnt, struct ucred cred,
	struct thread *td)
	{
	int error = 0;

	/*
	* RFC 1813 3.3.21: if count is 0, a flush from offset to the end of
	* file is done. At this time VOP_FSYNC does not accept offset and
	* byte count parameters so call VOP_FSYNC the whole file for now.
	* The same is true for NFSv4: RFC 3530 Sec. 14.2.3.
	* File systems that do not use the buffer cache (as indicated
	* by MNTK_USES_BCACHE not being set) must use VOP_FSYNC().
	*/
	if (cnt == 0 \|\| cnt > MAX_COMMIT_COUNT \|\|
	(vp->v_mount->mnt_kern_flag & MNTK_USES_BCACHE) == 0) {
	/*
	* Give up and do the whole thing
	*/
	if (vp->v_object && vm_object_mightbedirty(vp->v_object)) {
	VM_OBJECT_WLOCK(vp->v_object);
	vm_object_page_clean(vp->v_object, 0, 0, OBJPC_SYNC);
	VM_OBJECT_WUNLOCK(vp->v_object);
	}
	error = VOP_FSYNC(vp, MNT_WAIT, td);
	} else {
	/*
	* Locate and synchronously write any buffers that fall
	* into the requested range. Note: we are assuming that
	* f_iosize is a power of 2.
	*/
	int iosize = vp->v_mount->mnt_stat.f_iosize;
	int iomask = iosize - 1;
	struct bufobj *bo;
	daddr_t lblkno;

	/*
	* Align to iosize boundary, super-align to page boundary.
	*/
	if (off & iomask) {
	cnt += off & iomask;
	off &= ~(u_quad_t)iomask;
	}
	if (off & PAGE_MASK) {
	cnt += off & PAGE_MASK;
	off &= ~(u_quad_t)PAGE_MASK;
	}
	lblkno = off / iosize;

	if (vp->v_object && vm_object_mightbedirty(vp->v_object)) {
	VM_OBJECT_WLOCK(vp->v_object);
	vm_object_page_clean(vp->v_object, off, off + cnt,
	OBJPC_SYNC);
	VM_OBJECT_WUNLOCK(vp->v_object);
	}

	bo = &vp->v_bufobj;
	BO_LOCK(bo);
	while (cnt > 0) {
	struct buf *bp;

	/*
	* If we have a buffer and it is marked B_DELWRI we
	* have to lock and write it. Otherwise the prior
	* write is assumed to have already been committed.
	*
	* gbincore() can return invalid buffers now so we
	* have to check that bit as well (though B_DELWRI
	* should not be set if B_INVAL is set there could be
	* a race here since we haven't locked the buffer).
	*/
	if ((bp = gbincore(&vp->v_bufobj, lblkno)) != NULL) {
	if (BUF_LOCK(bp, LK_EXCLUSIVE \| LK_SLEEPFAIL \|
	LK_INTERLOCK, BO_LOCKPTR(bo)) == ENOLCK) {
	BO_LOCK(bo);
	continue; /* retry */
	}
	if ((bp->b_flags & (B_DELWRI\|B_INVAL)) ==
	B_DELWRI) {
	bremfree(bp);
	bp->b_flags &= ~B_ASYNC;
	bwrite(bp);
	++nfs_commit_miss;
	} else
	BUF_UNLOCK(bp);
	BO_LOCK(bo);
	}
	++nfs_commit_blks;
	if (cnt < iosize)
	break;
	cnt -= iosize;
	++lblkno;
	}
	BO_UNLOCK(bo);
	}
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Statfs vnode op.
	*/
	int
	nfsvno_statfs(struct vnode vp, struct statfs sf)
	{
	struct statfs *tsf;
	int error;

	tsf = NULL;
	if (nfsrv_devidcnt > 0) {
	/* For a pNFS service, get the DS numbers. */
	tsf = malloc(sizeof(*tsf), M_TEMP, M_WAITOK \| M_ZERO);
	error = nfsrv_pnfsstatfs(tsf, vp->v_mount);
	if (error != 0) {
	free(tsf, M_TEMP);
	tsf = NULL;
	}
	}
	error = VFS_STATFS(vp->v_mount, sf);
	if (error == 0) {
	if (tsf != NULL) {
	sf->f_blocks = tsf->f_blocks;
	sf->f_bavail = tsf->f_bavail;
	sf->f_bfree = tsf->f_bfree;
	sf->f_bsize = tsf->f_bsize;
	}
	/*
	* Since NFS handles these values as unsigned on the
	* wire, there is no way to represent negative values,
	* so set them to 0. Without this, they will appear
	* to be very large positive values for clients like
	* Solaris10.
	*/
	if (sf->f_bavail < 0)
	sf->f_bavail = 0;
	if (sf->f_ffree < 0)
	sf->f_ffree = 0;
	}
	free(tsf, M_TEMP);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Do the vnode op stuff for Open. Similar to nfsvno_createsub(), but
	* must handle nfsrv_opencheck() calls after any other access checks.
	*/
	void
	nfsvno_open(struct nfsrv_descript nd, struct nameidata ndp,
	nfsquad_t clientid, nfsv4stateid_t stateidp, struct nfsstate stp,
	int exclusive_flagp, struct nfsvattr nvap, int32_t *cverf, int create,
	NFSACL_T aclp, nfsattrbit_t attrbitp, struct ucred *cred,
	struct nfsexstuff exp, struct vnode *vpp)
	{
	struct vnode *vp = NULL;
	u_quad_t tempsize;
	struct nfsexstuff nes;
	struct thread *p = curthread;

	if (ndp->ni_vp == NULL)
	nd->nd_repstat = nfsrv_opencheck(clientid,
	stateidp, stp, NULL, nd, p, nd->nd_repstat);
	if (!nd->nd_repstat) {
	if (ndp->ni_vp == NULL) {
	vrele(ndp->ni_startdir);
	nd->nd_repstat = VOP_CREATE(ndp->ni_dvp,
	&ndp->ni_vp, &ndp->ni_cnd, &nvap->na_vattr);
	/* For a pNFS server, create the data file on a DS. */
	if (nd->nd_repstat == 0) {
	/*
	* Create a data file on a DS for a pNFS server.
	* This function just returns if not
	* running a pNFS DS or the creation fails.
	*/
	nfsrv_pnfscreate(ndp->ni_vp, &nvap->na_vattr,
	cred, p);
	}
	vput(ndp->ni_dvp);
	nfsvno_relpathbuf(ndp);
	if (!nd->nd_repstat) {
	if (*exclusive_flagp) {
	*exclusive_flagp = 0;
	NFSVNO_ATTRINIT(nvap);
	nvap->na_atime.tv_sec = cverf[0];
	nvap->na_atime.tv_nsec = cverf[1];
	nd->nd_repstat = VOP_SETATTR(ndp->ni_vp,
	&nvap->na_vattr, cred);
	if (nd->nd_repstat != 0) {
	vput(ndp->ni_vp);
	ndp->ni_vp = NULL;
	nd->nd_repstat = NFSERR_NOTSUPP;
	} else
	NFSSETBIT_ATTRBIT(attrbitp,
	NFSATTRBIT_TIMEACCESS);
	} else {
	nfsrv_fixattr(nd, ndp->ni_vp, nvap,
	aclp, p, attrbitp, exp);
	}
	}
	vp = ndp->ni_vp;
	} else {
	if (ndp->ni_startdir)
	vrele(ndp->ni_startdir);
	nfsvno_relpathbuf(ndp);
	vp = ndp->ni_vp;
	if (create == NFSV4OPEN_CREATE) {
	if (ndp->ni_dvp == vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	}
	if (NFSVNO_ISSETSIZE(nvap) && vp->v_type == VREG) {
	if (ndp->ni_cnd.cn_flags & RDONLY)
	NFSVNO_SETEXRDONLY(&nes);
	else
	NFSVNO_EXINIT(&nes);
	nd->nd_repstat = nfsvno_accchk(vp,
	VWRITE, cred, &nes, p,
	NFSACCCHK_NOOVERRIDE,
	NFSACCCHK_VPISLOCKED, NULL);
	nd->nd_repstat = nfsrv_opencheck(clientid,
	stateidp, stp, vp, nd, p, nd->nd_repstat);
	if (!nd->nd_repstat) {
	tempsize = nvap->na_size;
	NFSVNO_ATTRINIT(nvap);
	nvap->na_size = tempsize;
	nd->nd_repstat = VOP_SETATTR(vp,
	&nvap->na_vattr, cred);
	}
	} else if (vp->v_type == VREG) {
	nd->nd_repstat = nfsrv_opencheck(clientid,
	stateidp, stp, vp, nd, p, nd->nd_repstat);
	}
	}
	} else {
	if (ndp->ni_cnd.cn_flags & HASBUF)
	nfsvno_relpathbuf(ndp);
	if (ndp->ni_startdir && create == NFSV4OPEN_CREATE) {
	vrele(ndp->ni_startdir);
	if (ndp->ni_dvp == ndp->ni_vp)
	vrele(ndp->ni_dvp);
	else
	vput(ndp->ni_dvp);
	if (ndp->ni_vp)
	vput(ndp->ni_vp);
	}
	}
	*vpp = vp;

	NFSEXITCODE2(0, nd);
	}

	/*
	* Updates the file rev and sets the mtime and ctime
	* to the current clock time, returning the va_filerev and va_Xtime
	* values.
	* Return ESTALE to indicate the vnode is VIRF_DOOMED.
	*/
	int
	nfsvno_updfilerev(struct vnode vp, struct nfsvattr nvap,
	struct nfsrv_descript nd, struct thread p)
	{
	struct vattr va;

	VATTR_NULL(&va);
	vfs_timestamp(&va.va_mtime);
	if (NFSVOPISLOCKED(vp) != LK_EXCLUSIVE) {
	NFSVOPLOCK(vp, LK_UPGRADE \| LK_RETRY);
	if (VN_IS_DOOMED(vp))
	return (ESTALE);
	}
	(void) VOP_SETATTR(vp, &va, nd->nd_cred);
	(void) nfsvno_getattr(vp, nvap, nd, p, 1, NULL);
	return (0);
	}

	/*
	* Glue routine to nfsv4_fillattr().
	*/
	int
	nfsvno_fillattr(struct nfsrv_descript nd, struct mount mp, struct vnode *vp,
	struct nfsvattr nvap, fhandle_t fhp, int rderror, nfsattrbit_t *attrbitp,
	struct ucred cred, struct thread p, int isdgram, int reterr,
	int supports_nfsv4acls, int at_root, uint64_t mounted_on_fileno)
	{
	struct statfs *sf;
	int error;

	sf = NULL;
	if (nfsrv_devidcnt > 0 &&
	(NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_SPACEAVAIL) \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_SPACEFREE) \|\|
	NFSISSET_ATTRBIT(attrbitp, NFSATTRBIT_SPACETOTAL))) {
	sf = malloc(sizeof(*sf), M_TEMP, M_WAITOK \| M_ZERO);
	error = nfsrv_pnfsstatfs(sf, mp);
	if (error != 0) {
	free(sf, M_TEMP);
	sf = NULL;
	}
	}
	error = nfsv4_fillattr(nd, mp, vp, NULL, &nvap->na_vattr, fhp, rderror,
	attrbitp, cred, p, isdgram, reterr, supports_nfsv4acls, at_root,
	mounted_on_fileno, sf);
	free(sf, M_TEMP);
	NFSEXITCODE2(0, nd);
	return (error);
	}

	/* Since the Readdir vnode ops vary, put the entire functions in here. */
	/*
	* nfs readdir service
	* - mallocs what it thinks is enough to read
	* count rounded up to a multiple of DIRBLKSIZ <= NFS_MAXREADDIR
	* - calls VOP_READDIR()
	* - loops around building the reply
	* if the output generated exceeds count break out of loop
	* The NFSM_CLGET macro is used here so that the reply will be packed
	* tightly in mbuf clusters.
	* - it trims out records with d_fileno == 0
	* this doesn't matter for Unix clients, but they might confuse clients
	* for other os'.
	* - it trims out records with d_type == DT_WHT
	* these cannot be seen through NFS (unless we extend the protocol)
	* The alternate call nfsrvd_readdirplus() does lookups as well.
	* PS: The NFS protocol spec. does not clarify what the "count" byte
	* argument is a count of.. just name strings and file id's or the
	* entire reply rpc or ...
	* I tried just file name and id sizes and it confused the Sun client,
	* so I am using the full rpc size now. The "paranoia.." comment refers
	* to including the status longwords that are not a part of the dir.
	* "entry" structures, but are in the rpc.
	*/
	int
	nfsrvd_readdir(struct nfsrv_descript *nd, int isdgram,
	struct vnode vp, struct nfsexstuff exp)
	{
	struct dirent *dp;
	u_int32_t *tl;
	int dirlen;
	char cpos, cend, *rbuf;
	struct nfsvattr at;
	int nlen, error = 0, getret = 1;
	int siz, cnt, fullsiz, eofflag, ncookies;
	u_int64_t off, toff, verf __unused;
	u_long cookies = NULL, cookiep;
	struct uio io;
	struct iovec iv;
	int is_ufs;
	struct thread *p = curthread;

	if (nd->nd_repstat) {
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}
	if (nd->nd_flag & ND_NFSV2) {
	NFSM_DISSECT(tl, u_int32_t , 2 NFSX_UNSIGNED);
	off = fxdr_unsigned(u_quad_t, *tl++);
	} else {
	NFSM_DISSECT(tl, u_int32_t , 5 NFSX_UNSIGNED);
	off = fxdr_hyper(tl);
	tl += 2;
	verf = fxdr_hyper(tl);
	tl += 2;
	}
	toff = off;
	cnt = fxdr_unsigned(int, *tl);
	if (cnt > NFS_SRVMAXDATA(nd) \|\| cnt < 0)
	cnt = NFS_SRVMAXDATA(nd);
	siz = ((cnt + DIRBLKSIZ - 1) & ~(DIRBLKSIZ - 1));
	fullsiz = siz;
	if (nd->nd_flag & ND_NFSV3) {
	nd->nd_repstat = getret = nfsvno_getattr(vp, &at, nd, p, 1,
	NULL);
	#if 0
	/*
	* va_filerev is not sufficient as a cookie verifier,
	* since it is not supposed to change when entries are
	* removed/added unless that offset cookies returned to
	* the client are no longer valid.
	*/
	if (!nd->nd_repstat && toff && verf != at.na_filerev)
	nd->nd_repstat = NFSERR_BAD_COOKIE;
	#endif
	}
	if (!nd->nd_repstat && vp->v_type != VDIR)
	nd->nd_repstat = NFSERR_NOTDIR;
	if (nd->nd_repstat == 0 && cnt == 0) {
	if (nd->nd_flag & ND_NFSV2)
	/* NFSv2 does not have NFSERR_TOOSMALL */
	nd->nd_repstat = EPERM;
	else
	nd->nd_repstat = NFSERR_TOOSMALL;
	}
	if (!nd->nd_repstat)
	nd->nd_repstat = nfsvno_accchk(vp, VEXEC,
	nd->nd_cred, exp, p, NFSACCCHK_NOOVERRIDE,
	NFSACCCHK_VPISLOCKED, NULL);
	if (nd->nd_repstat) {
	vput(vp);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}
	is_ufs = strcmp(vp->v_mount->mnt_vfc->vfc_name, "ufs") == 0;
	rbuf = malloc(siz, M_TEMP, M_WAITOK);
	again:
	eofflag = 0;
	if (cookies) {
	free(cookies, M_TEMP);
	cookies = NULL;
	}

	iv.iov_base = rbuf;
	iv.iov_len = siz;
	io.uio_iov = &iv;
	io.uio_iovcnt = 1;
	io.uio_offset = (off_t)off;
	io.uio_resid = siz;
	io.uio_segflg = UIO_SYSSPACE;
	io.uio_rw = UIO_READ;
	io.uio_td = NULL;
	nd->nd_repstat = VOP_READDIR(vp, &io, nd->nd_cred, &eofflag, &ncookies,
	&cookies);
	off = (u_int64_t)io.uio_offset;
	if (io.uio_resid)
	siz -= io.uio_resid;

	if (!cookies && !nd->nd_repstat)
	nd->nd_repstat = NFSERR_PERM;
	if (nd->nd_flag & ND_NFSV3) {
	getret = nfsvno_getattr(vp, &at, nd, p, 1, NULL);
	if (!nd->nd_repstat)
	nd->nd_repstat = getret;
	}

	/*
	* Handles the failed cases. nd->nd_repstat == 0 past here.
	*/
	if (nd->nd_repstat) {
	vput(vp);
	free(rbuf, M_TEMP);
	if (cookies)
	free(cookies, M_TEMP);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}
	/*
	* If nothing read, return eof
	* rpc reply
	*/
	if (siz == 0) {
	vput(vp);
	if (nd->nd_flag & ND_NFSV2) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	} else {
	nfsrv_postopattr(nd, getret, &at);
	NFSM_BUILD(tl, u_int32_t , 4 NFSX_UNSIGNED);
	txdr_hyper(at.na_filerev, tl);
	tl += 2;
	}
	*tl++ = newnfs_false;
	*tl = newnfs_true;
	free(rbuf, M_TEMP);
	free(cookies, M_TEMP);
	goto out;
	}

	/*
	* Check for degenerate cases of nothing useful read.
	* If so go try again
	*/
	cpos = rbuf;
	cend = rbuf + siz;
	dp = (struct dirent *)cpos;
	cookiep = cookies;

	/*
	* For some reason FreeBSD's ufs_readdir() chooses to back the
	* directory offset up to a block boundary, so it is necessary to
	* skip over the records that precede the requested offset. This
	* requires the assumption that file offset cookies monotonically
	* increase.
	*/
	while (cpos < cend && ncookies > 0 &&
	(dp->d_fileno == 0 \|\| dp->d_type == DT_WHT \|\|
	(is_ufs == 1 && ((u_quad_t)(*cookiep)) <= toff))) {
	cpos += dp->d_reclen;
	dp = (struct dirent *)cpos;
	cookiep++;
	ncookies--;
	}
	if (cpos >= cend \|\| ncookies == 0) {
	siz = fullsiz;
	toff = off;
	goto again;
	}
	vput(vp);

	+ /*
	+ * If cnt > MCLBYTES and the reply will not be saved, use
	+ * ext_pgs mbufs for TLS.
	+ * For NFSv4.0, we do not know for sure if the reply will
	+ * be saved, so do not use ext_pgs mbufs for NFSv4.0.
	+ */
	+ if (cnt > MCLBYTES && siz > MCLBYTES &&
	+ (nd->nd_flag & (ND_TLS \| ND_EXTPG \| ND_SAVEREPLY)) == ND_TLS &&
	+ (nd->nd_flag & (ND_NFSV4 \| ND_NFSV41)) != ND_NFSV4)
	+ nd->nd_flag \|= ND_EXTPG;
	+
	/*
	* dirlen is the size of the reply, including all XDR and must
	* not exceed cnt. For NFSv2, RFC1094 didn't clearly indicate
	* if the XDR should be included in "count", but to be safe, we do.
	* (Include the two booleans at the end of the reply in dirlen now.)
	*/
	if (nd->nd_flag & ND_NFSV3) {
	nfsrv_postopattr(nd, getret, &at);
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	txdr_hyper(at.na_filerev, tl);
	dirlen = NFSX_V3POSTOPATTR + NFSX_VERF + 2 * NFSX_UNSIGNED;
	} else {
	dirlen = 2 * NFSX_UNSIGNED;
	}

	/* Loop through the records and build reply */
	while (cpos < cend && ncookies > 0) {
	nlen = dp->d_namlen;
	if (dp->d_fileno != 0 && dp->d_type != DT_WHT &&
	nlen <= NFS_MAXNAMLEN) {
	if (nd->nd_flag & ND_NFSV3)
	dirlen += (6*NFSX_UNSIGNED + NFSM_RNDUP(nlen));
	else
	dirlen += (4*NFSX_UNSIGNED + NFSM_RNDUP(nlen));
	if (dirlen > cnt) {
	eofflag = 0;
	break;
	}

	/*
	* Build the directory record xdr from
	* the dirent entry.
	*/
	if (nd->nd_flag & ND_NFSV3) {
	NFSM_BUILD(tl, u_int32_t , 3 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	*tl++ = 0;
	} else {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	}
	*tl = txdr_unsigned(dp->d_fileno);
	(void) nfsm_strtom(nd, dp->d_name, nlen);
	if (nd->nd_flag & ND_NFSV3) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = 0;
	} else
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	tl = txdr_unsigned(cookiep);
	}
	cpos += dp->d_reclen;
	dp = (struct dirent *)cpos;
	cookiep++;
	ncookies--;
	}
	if (cpos < cend)
	eofflag = 0;
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = newnfs_false;
	if (eofflag)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	free(rbuf, M_TEMP);
	free(cookies, M_TEMP);

	out:
	NFSEXITCODE2(0, nd);
	return (0);
	nfsmout:
	vput(vp);
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Readdirplus for V3 and Readdir for V4.
	*/
	int
	nfsrvd_readdirplus(struct nfsrv_descript *nd, int isdgram,
	struct vnode vp, struct nfsexstuff exp)
	{
	struct dirent *dp;
	u_int32_t *tl;
	int dirlen;
	char cpos, cend, *rbuf;
	struct vnode *nvp;
	fhandle_t nfh;
	struct nfsvattr nva, at, *nvap = &nva;
	struct mbuf mb0, mb1;
	struct nfsreferral *refp;
	int nlen, r, error = 0, getret = 1, usevget = 1;
	int siz, cnt, fullsiz, eofflag, ncookies, entrycnt;
	caddr_t bpos0, bpos1;
	u_int64_t off, toff, verf;
	u_long cookies = NULL, cookiep;
	nfsattrbit_t attrbits, rderrbits, savbits;
	struct uio io;
	struct iovec iv;
	struct componentname cn;
	int at_root, is_ufs, is_zfs, needs_unbusy, supports_nfsv4acls;
	struct mount mp, new_mp;
	uint64_t mounted_on_fileno;
	struct thread *p = curthread;
	+ int bextpg0, bextpg1, bextpgsiz0, bextpgsiz1;

	if (nd->nd_repstat) {
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}
	NFSM_DISSECT(tl, u_int32_t , 6 NFSX_UNSIGNED);
	off = fxdr_hyper(tl);
	toff = off;
	tl += 2;
	verf = fxdr_hyper(tl);
	tl += 2;
	siz = fxdr_unsigned(int, *tl++);
	cnt = fxdr_unsigned(int, *tl);

	/*
	* Use the server's maximum data transfer size as the upper bound
	* on reply datalen.
	*/
	if (cnt > NFS_SRVMAXDATA(nd) \|\| cnt < 0)
	cnt = NFS_SRVMAXDATA(nd);

	/*
	* siz is a "hint" of how much directory information (name, fileid,
	* cookie) should be in the reply. At least one client "hints" 0,
	* so I set it to cnt for that case. I also round it up to the
	* next multiple of DIRBLKSIZ.
	* Since the size of a Readdirplus directory entry reply will always
	* be greater than a directory entry returned by VOP_READDIR(), it
	* does not make sense to read more than NFS_SRVMAXDATA() via
	* VOP_READDIR().
	*/
	if (siz <= 0)
	siz = cnt;
	else if (siz > NFS_SRVMAXDATA(nd))
	siz = NFS_SRVMAXDATA(nd);
	siz = ((siz + DIRBLKSIZ - 1) & ~(DIRBLKSIZ - 1));

	if (nd->nd_flag & ND_NFSV4) {
	error = nfsrv_getattrbits(nd, &attrbits, NULL, NULL);
	if (error)
	goto nfsmout;
	NFSSET_ATTRBIT(&savbits, &attrbits);
	NFSCLRNOTFILLABLE_ATTRBIT(&attrbits, nd);
	NFSZERO_ATTRBIT(&rderrbits);
	NFSSETBIT_ATTRBIT(&rderrbits, NFSATTRBIT_RDATTRERROR);
	} else {
	NFSZERO_ATTRBIT(&attrbits);
	}
	fullsiz = siz;
	nd->nd_repstat = getret = nfsvno_getattr(vp, &at, nd, p, 1, NULL);
	#if 0
	if (!nd->nd_repstat) {
	if (off && verf != at.na_filerev) {
	/*
	* va_filerev is not sufficient as a cookie verifier,
	* since it is not supposed to change when entries are
	* removed/added unless that offset cookies returned to
	* the client are no longer valid.
	*/
	if (nd->nd_flag & ND_NFSV4) {
	nd->nd_repstat = NFSERR_NOTSAME;
	} else {
	nd->nd_repstat = NFSERR_BAD_COOKIE;
	}
	}
	}
	#endif
	if (!nd->nd_repstat && vp->v_type != VDIR)
	nd->nd_repstat = NFSERR_NOTDIR;
	if (!nd->nd_repstat && cnt == 0)
	nd->nd_repstat = NFSERR_TOOSMALL;
	if (!nd->nd_repstat)
	nd->nd_repstat = nfsvno_accchk(vp, VEXEC,
	nd->nd_cred, exp, p, NFSACCCHK_NOOVERRIDE,
	NFSACCCHK_VPISLOCKED, NULL);
	if (nd->nd_repstat) {
	vput(vp);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}
	is_ufs = strcmp(vp->v_mount->mnt_vfc->vfc_name, "ufs") == 0;
	is_zfs = strcmp(vp->v_mount->mnt_vfc->vfc_name, "zfs") == 0;

	rbuf = malloc(siz, M_TEMP, M_WAITOK);
	again:
	eofflag = 0;
	if (cookies) {
	free(cookies, M_TEMP);
	cookies = NULL;
	}

	iv.iov_base = rbuf;
	iv.iov_len = siz;
	io.uio_iov = &iv;
	io.uio_iovcnt = 1;
	io.uio_offset = (off_t)off;
	io.uio_resid = siz;
	io.uio_segflg = UIO_SYSSPACE;
	io.uio_rw = UIO_READ;
	io.uio_td = NULL;
	nd->nd_repstat = VOP_READDIR(vp, &io, nd->nd_cred, &eofflag, &ncookies,
	&cookies);
	off = (u_int64_t)io.uio_offset;
	if (io.uio_resid)
	siz -= io.uio_resid;

	getret = nfsvno_getattr(vp, &at, nd, p, 1, NULL);

	if (!cookies && !nd->nd_repstat)
	nd->nd_repstat = NFSERR_PERM;
	if (!nd->nd_repstat)
	nd->nd_repstat = getret;
	if (nd->nd_repstat) {
	vput(vp);
	if (cookies)
	free(cookies, M_TEMP);
	free(rbuf, M_TEMP);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}
	/*
	* If nothing read, return eof
	* rpc reply
	*/
	if (siz == 0) {
	vput(vp);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	NFSM_BUILD(tl, u_int32_t , 4 NFSX_UNSIGNED);
	txdr_hyper(at.na_filerev, tl);
	tl += 2;
	*tl++ = newnfs_false;
	*tl = newnfs_true;
	free(cookies, M_TEMP);
	free(rbuf, M_TEMP);
	goto out;
	}

	/*
	* Check for degenerate cases of nothing useful read.
	* If so go try again
	*/
	cpos = rbuf;
	cend = rbuf + siz;
	dp = (struct dirent *)cpos;
	cookiep = cookies;

	/*
	* For some reason FreeBSD's ufs_readdir() chooses to back the
	* directory offset up to a block boundary, so it is necessary to
	* skip over the records that precede the requested offset. This
	* requires the assumption that file offset cookies monotonically
	* increase.
	*/
	while (cpos < cend && ncookies > 0 &&
	(dp->d_fileno == 0 \|\| dp->d_type == DT_WHT \|\|
	(is_ufs == 1 && ((u_quad_t)(*cookiep)) <= toff) \|\|
	((nd->nd_flag & ND_NFSV4) &&
	((dp->d_namlen == 1 && dp->d_name[0] == '.') \|\|
	(dp->d_namlen==2 && dp->d_name[0]=='.' && dp->d_name[1]=='.'))))) {
	cpos += dp->d_reclen;
	dp = (struct dirent *)cpos;
	cookiep++;
	ncookies--;
	}
	if (cpos >= cend \|\| ncookies == 0) {
	siz = fullsiz;
	toff = off;
	goto again;
	}

	/*
	* Busy the file system so that the mount point won't go away
	* and, as such, VFS_VGET() can be used safely.
	*/
	mp = vp->v_mount;
	vfs_ref(mp);
	NFSVOPUNLOCK(vp);
	nd->nd_repstat = vfs_busy(mp, 0);
	vfs_rel(mp);
	if (nd->nd_repstat != 0) {
	vrele(vp);
	free(cookies, M_TEMP);
	free(rbuf, M_TEMP);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	goto out;
	}

	/*
	* Check to see if entries in this directory can be safely acquired
	* via VFS_VGET() or if a switch to VOP_LOOKUP() is required.
	* ZFS snapshot directories need VOP_LOOKUP(), so that any
	* automount of the snapshot directory that is required will
	* be done.
	* This needs to be done here for NFSv4, since NFSv4 never does
	* a VFS_VGET() for "." or "..".
	*/
	if (is_zfs == 1) {
	r = VFS_VGET(mp, at.na_fileid, LK_SHARED, &nvp);
	if (r == EOPNOTSUPP) {
	usevget = 0;
	cn.cn_nameiop = LOOKUP;
	cn.cn_lkflags = LK_SHARED \| LK_RETRY;
	cn.cn_cred = nd->nd_cred;
	cn.cn_thread = p;
	} else if (r == 0)
	vput(nvp);
	}

	+ /*
	+ * If the reply is likely to exceed MCLBYTES and the reply will
	+ * not be saved, use ext_pgs mbufs for TLS.
	+ * It is difficult to predict how large each entry will be and
	+ * how many entries have been read, so just assume the directory
	+ * entries grow by a factor of 4 when attributes are included.
	+ * For NFSv4.0, we do not know for sure if the reply will
	+ * be saved, so do not use ext_pgs mbufs for NFSv4.0.
	+ */
	+ if (cnt > MCLBYTES && siz > MCLBYTES / 4 &&
	+ (nd->nd_flag & (ND_TLS \| ND_EXTPG \| ND_SAVEREPLY)) == ND_TLS &&
	+ (nd->nd_flag & (ND_NFSV4 \| ND_NFSV41)) != ND_NFSV4)
	+ nd->nd_flag \|= ND_EXTPG;
	+
	/*
	* Save this position, in case there is an error before one entry
	* is created.
	*/
	mb0 = nd->nd_mb;
	bpos0 = nd->nd_bpos;
	+ bextpg0 = nd->nd_bextpg;
	+ bextpgsiz0 = nd->nd_bextpgsiz;

	/*
	* Fill in the first part of the reply.
	* dirlen is the reply length in bytes and cannot exceed cnt.
	* (Include the two booleans at the end of the reply in dirlen now,
	* so we recognize when we have exceeded cnt.)
	*/
	if (nd->nd_flag & ND_NFSV3) {
	dirlen = NFSX_V3POSTOPATTR + NFSX_VERF + 2 * NFSX_UNSIGNED;
	nfsrv_postopattr(nd, getret, &at);
	} else {
	dirlen = NFSX_VERF + 2 * NFSX_UNSIGNED;
	}
	NFSM_BUILD(tl, u_int32_t *, NFSX_VERF);
	txdr_hyper(at.na_filerev, tl);

	/*
	* Save this position, in case there is an empty reply needed.
	*/
	mb1 = nd->nd_mb;
	bpos1 = nd->nd_bpos;
	+ bextpg1 = nd->nd_bextpg;
	+ bextpgsiz1 = nd->nd_bextpgsiz;

	/* Loop through the records and build reply */
	entrycnt = 0;
	while (cpos < cend && ncookies > 0 && dirlen < cnt) {
	nlen = dp->d_namlen;
	if (dp->d_fileno != 0 && dp->d_type != DT_WHT &&
	nlen <= NFS_MAXNAMLEN &&
	((nd->nd_flag & ND_NFSV3) \|\| nlen > 2 \|\|
	(nlen==2 && (dp->d_name[0]!='.' \|\| dp->d_name[1]!='.'))
	\|\| (nlen == 1 && dp->d_name[0] != '.'))) {
	/*
	* Save the current position in the reply, in case
	* this entry exceeds cnt.
	*/
	mb1 = nd->nd_mb;
	bpos1 = nd->nd_bpos;
	+ bextpg1 = nd->nd_bextpg;
	+ bextpgsiz1 = nd->nd_bextpgsiz;

	/*
	* For readdir_and_lookup get the vnode using
	* the file number.
	*/
	nvp = NULL;
	refp = NULL;
	r = 0;
	at_root = 0;
	needs_unbusy = 0;
	new_mp = mp;
	mounted_on_fileno = (uint64_t)dp->d_fileno;
	if ((nd->nd_flag & ND_NFSV3) \|\|
	NFSNONZERO_ATTRBIT(&savbits)) {
	if (nd->nd_flag & ND_NFSV4)
	refp = nfsv4root_getreferral(NULL,
	vp, dp->d_fileno);
	if (refp == NULL) {
	if (usevget)
	r = VFS_VGET(mp, dp->d_fileno,
	LK_SHARED, &nvp);
	else
	r = EOPNOTSUPP;
	if (r == EOPNOTSUPP) {
	if (usevget) {
	usevget = 0;
	cn.cn_nameiop = LOOKUP;
	cn.cn_lkflags =
	LK_SHARED \|
	LK_RETRY;
	cn.cn_cred =
	nd->nd_cred;
	cn.cn_thread = p;
	}
	cn.cn_nameptr = dp->d_name;
	cn.cn_namelen = nlen;
	cn.cn_flags = ISLASTCN \|
	NOFOLLOW \| LOCKLEAF;
	if (nlen == 2 &&
	dp->d_name[0] == '.' &&
	dp->d_name[1] == '.')
	cn.cn_flags \|=
	ISDOTDOT;
	if (NFSVOPLOCK(vp, LK_SHARED)
	!= 0) {
	nd->nd_repstat = EPERM;
	break;
	}
	if ((vp->v_vflag & VV_ROOT) != 0
	&& (cn.cn_flags & ISDOTDOT)
	!= 0) {
	vref(vp);
	nvp = vp;
	r = 0;
	} else {
	r = VOP_LOOKUP(vp, &nvp,
	&cn);
	if (vp != nvp)
	NFSVOPUNLOCK(vp);
	}
	}

	/*
	* For NFSv4, check to see if nvp is
	* a mount point and get the mount
	* point vnode, as required.
	*/
	if (r == 0 &&
	nfsrv_enable_crossmntpt != 0 &&
	(nd->nd_flag & ND_NFSV4) != 0 &&
	nvp->v_type == VDIR &&
	nvp->v_mountedhere != NULL) {
	new_mp = nvp->v_mountedhere;
	r = vfs_busy(new_mp, 0);
	vput(nvp);
	nvp = NULL;
	if (r == 0) {
	r = VFS_ROOT(new_mp,
	LK_SHARED, &nvp);
	needs_unbusy = 1;
	if (r == 0)
	at_root = 1;
	}
	}
	}

	/*
	* If we failed to look up the entry, then it
	* has become invalid, most likely removed.
	*/
	if (r != 0) {
	if (needs_unbusy)
	vfs_unbusy(new_mp);
	goto invalid;
	}
	KASSERT(refp != NULL \|\| nvp != NULL,
	("%s: undetected lookup error", __func__));

	if (refp == NULL &&
	((nd->nd_flag & ND_NFSV3) \|\|
	NFSNONZERO_ATTRBIT(&attrbits))) {
	r = nfsvno_getfh(nvp, &nfh, p);
	if (!r)
	r = nfsvno_getattr(nvp, nvap, nd, p,
	1, &attrbits);
	if (r == 0 && is_zfs == 1 &&
	nfsrv_enable_crossmntpt != 0 &&
	(nd->nd_flag & ND_NFSV4) != 0 &&
	nvp->v_type == VDIR &&
	vp->v_mount != nvp->v_mount) {
	/*
	* For a ZFS snapshot, there is a
	* pseudo mount that does not set
	* v_mountedhere, so it needs to
	* be detected via a different
	* mount structure.
	*/
	at_root = 1;
	if (new_mp == mp)
	new_mp = nvp->v_mount;
	}
	}

	/*
	* If we failed to get attributes of the entry,
	* then just skip it for NFSv3 (the traditional
	* behavior in the old NFS server).
	* For NFSv4 the behavior is controlled by
	* RDATTRERROR: we either ignore the error or
	* fail the request.
	* Note that RDATTRERROR is never set for NFSv3.
	*/
	if (r != 0) {
	if (!NFSISSET_ATTRBIT(&attrbits,
	NFSATTRBIT_RDATTRERROR)) {
	vput(nvp);
	if (needs_unbusy != 0)
	vfs_unbusy(new_mp);
	if ((nd->nd_flag & ND_NFSV3))
	goto invalid;
	nd->nd_repstat = r;
	break;
	}
	}
	}

	/*
	* Build the directory record xdr
	*/
	if (nd->nd_flag & ND_NFSV3) {
	NFSM_BUILD(tl, u_int32_t , 3 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	*tl++ = 0;
	*tl = txdr_unsigned(dp->d_fileno);
	dirlen += nfsm_strtom(nd, dp->d_name, nlen);
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = 0;
	tl = txdr_unsigned(cookiep);
	nfsrv_postopattr(nd, 0, nvap);
	dirlen += nfsm_fhtom(nd,(u_int8_t *)&nfh,0,1);
	dirlen += (5*NFSX_UNSIGNED+NFSX_V3POSTOPATTR);
	if (nvp != NULL)
	vput(nvp);
	} else {
	NFSM_BUILD(tl, u_int32_t , 3 NFSX_UNSIGNED);
	*tl++ = newnfs_true;
	*tl++ = 0;
	tl = txdr_unsigned(cookiep);
	dirlen += nfsm_strtom(nd, dp->d_name, nlen);
	if (nvp != NULL) {
	supports_nfsv4acls =
	nfs_supportsnfsv4acls(nvp);
	NFSVOPUNLOCK(nvp);
	} else
	supports_nfsv4acls = 0;
	if (refp != NULL) {
	dirlen += nfsrv_putreferralattr(nd,
	&savbits, refp, 0,
	&nd->nd_repstat);
	if (nd->nd_repstat) {
	if (nvp != NULL)
	vrele(nvp);
	if (needs_unbusy != 0)
	vfs_unbusy(new_mp);
	break;
	}
	} else if (r) {
	dirlen += nfsvno_fillattr(nd, new_mp,
	nvp, nvap, &nfh, r, &rderrbits,
	nd->nd_cred, p, isdgram, 0,
	supports_nfsv4acls, at_root,
	mounted_on_fileno);
	} else {
	dirlen += nfsvno_fillattr(nd, new_mp,
	nvp, nvap, &nfh, r, &attrbits,
	nd->nd_cred, p, isdgram, 0,
	supports_nfsv4acls, at_root,
	mounted_on_fileno);
	}
	if (nvp != NULL)
	vrele(nvp);
	dirlen += (3 * NFSX_UNSIGNED);
	}
	if (needs_unbusy != 0)
	vfs_unbusy(new_mp);
	if (dirlen <= cnt)
	entrycnt++;
	}
	invalid:
	cpos += dp->d_reclen;
	dp = (struct dirent *)cpos;
	cookiep++;
	ncookies--;
	}
	vrele(vp);
	vfs_unbusy(mp);

	/*
	* If dirlen > cnt, we must strip off the last entry. If that
	* results in an empty reply, report NFSERR_TOOSMALL.
	*/
	if (dirlen > cnt \|\| nd->nd_repstat) {
	if (!nd->nd_repstat && entrycnt == 0)
	nd->nd_repstat = NFSERR_TOOSMALL;
	if (nd->nd_repstat) {
	- newnfs_trimtrailing(nd, mb0, bpos0);
	+ nfsm_trimtrailing(nd, mb0, bpos0, bextpg0, bextpgsiz0);
	if (nd->nd_flag & ND_NFSV3)
	nfsrv_postopattr(nd, getret, &at);
	} else
	- newnfs_trimtrailing(nd, mb1, bpos1);
	+ nfsm_trimtrailing(nd, mb1, bpos1, bextpg1, bextpgsiz1);
	eofflag = 0;
	} else if (cpos < cend)
	eofflag = 0;
	if (!nd->nd_repstat) {
	NFSM_BUILD(tl, u_int32_t , 2 NFSX_UNSIGNED);
	*tl++ = newnfs_false;
	if (eofflag)
	*tl = newnfs_true;
	else
	*tl = newnfs_false;
	}
	free(cookies, M_TEMP);
	free(rbuf, M_TEMP);

	out:
	NFSEXITCODE2(0, nd);
	return (0);
	nfsmout:
	vput(vp);
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Get the settable attributes out of the mbuf list.
	* (Return 0 or EBADRPC)
	*/
	int
	nfsrv_sattr(struct nfsrv_descript nd, vnode_t vp, struct nfsvattr nvap,
	nfsattrbit_t attrbitp, NFSACL_T aclp, struct thread *p)
	{
	u_int32_t *tl;
	struct nfsv2_sattr *sp;
	int error = 0, toclient = 0;

	switch (nd->nd_flag & (ND_NFSV2 \| ND_NFSV3 \| ND_NFSV4)) {
	case ND_NFSV2:
	NFSM_DISSECT(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
	/*
	* Some old clients didn't fill in the high order 16bits.
	* --> check the low order 2 bytes for 0xffff
	*/
	if ((fxdr_unsigned(int, sp->sa_mode) & 0xffff) != 0xffff)
	nvap->na_mode = nfstov_mode(sp->sa_mode);
	if (sp->sa_uid != newnfs_xdrneg1)
	nvap->na_uid = fxdr_unsigned(uid_t, sp->sa_uid);
	if (sp->sa_gid != newnfs_xdrneg1)
	nvap->na_gid = fxdr_unsigned(gid_t, sp->sa_gid);
	if (sp->sa_size != newnfs_xdrneg1)
	nvap->na_size = fxdr_unsigned(u_quad_t, sp->sa_size);
	if (sp->sa_atime.nfsv2_sec != newnfs_xdrneg1) {
	#ifdef notyet
	fxdr_nfsv2time(&sp->sa_atime, &nvap->na_atime);
	#else
	nvap->na_atime.tv_sec =
	fxdr_unsigned(u_int32_t,sp->sa_atime.nfsv2_sec);
	nvap->na_atime.tv_nsec = 0;
	#endif
	}
	if (sp->sa_mtime.nfsv2_sec != newnfs_xdrneg1)
	fxdr_nfsv2time(&sp->sa_mtime, &nvap->na_mtime);
	break;
	case ND_NFSV3:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (*tl == newnfs_true) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	nvap->na_mode = nfstov_mode(*tl);
	}
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (*tl == newnfs_true) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	nvap->na_uid = fxdr_unsigned(uid_t, *tl);
	}
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (*tl == newnfs_true) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	nvap->na_gid = fxdr_unsigned(gid_t, *tl);
	}
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (*tl == newnfs_true) {
	NFSM_DISSECT(tl, u_int32_t , 2 NFSX_UNSIGNED);
	nvap->na_size = fxdr_hyper(tl);
	}
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	switch (fxdr_unsigned(int, *tl)) {
	case NFSV3SATTRTIME_TOCLIENT:
	NFSM_DISSECT(tl, u_int32_t , 2 NFSX_UNSIGNED);
	fxdr_nfsv3time(tl, &nvap->na_atime);
	toclient = 1;
	break;
	case NFSV3SATTRTIME_TOSERVER:
	vfs_timestamp(&nvap->na_atime);
	nvap->na_vaflags \|= VA_UTIMES_NULL;
	break;
	}
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	switch (fxdr_unsigned(int, *tl)) {
	case NFSV3SATTRTIME_TOCLIENT:
	NFSM_DISSECT(tl, u_int32_t , 2 NFSX_UNSIGNED);
	fxdr_nfsv3time(tl, &nvap->na_mtime);
	nvap->na_vaflags &= ~VA_UTIMES_NULL;
	break;
	case NFSV3SATTRTIME_TOSERVER:
	vfs_timestamp(&nvap->na_mtime);
	if (!toclient)
	nvap->na_vaflags \|= VA_UTIMES_NULL;
	break;
	}
	break;
	case ND_NFSV4:
	error = nfsv4_sattr(nd, vp, nvap, attrbitp, aclp, p);
	}
	nfsmout:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Handle the setable attributes for V4.
	* Returns NFSERR_BADXDR if it can't be parsed, 0 otherwise.
	*/
	int
	nfsv4_sattr(struct nfsrv_descript nd, vnode_t vp, struct nfsvattr nvap,
	nfsattrbit_t attrbitp, NFSACL_T aclp, struct thread *p)
	{
	u_int32_t *tl;
	int attrsum = 0;
	int i, j;
	int error, attrsize, bitpos, aclsize, aceerr, retnotsup = 0;
	int moderet, toclient = 0;
	u_char *cp, namestr[NFSV4_SMALLSTR + 1];
	uid_t uid;
	gid_t gid;
	u_short mode, mask; /* Same type as va_mode. */
	struct vattr va;

	error = nfsrv_getattrbits(nd, attrbitp, NULL, &retnotsup);
	if (error)
	goto nfsmout;
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsize = fxdr_unsigned(int, *tl);

	/*
	* Loop around getting the setable attributes. If an unsupported
	* one is found, set nd_repstat == NFSERR_ATTRNOTSUPP and return.
	*/
	if (retnotsup) {
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	bitpos = NFSATTRBIT_MAX;
	} else {
	bitpos = 0;
	}
	moderet = 0;
	for (; bitpos < NFSATTRBIT_MAX; bitpos++) {
	if (attrsum > attrsize) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	if (NFSISSET_ATTRBIT(attrbitp, bitpos))
	switch (bitpos) {
	case NFSATTRBIT_SIZE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_HYPER);
	if (vp != NULL && vp->v_type != VREG) {
	error = (vp->v_type == VDIR) ? NFSERR_ISDIR :
	NFSERR_INVAL;
	goto nfsmout;
	}
	nvap->na_size = fxdr_hyper(tl);
	attrsum += NFSX_HYPER;
	break;
	case NFSATTRBIT_ACL:
	error = nfsrv_dissectacl(nd, aclp, &aceerr, &aclsize,
	p);
	if (error)
	goto nfsmout;
	if (aceerr && !nd->nd_repstat)
	nd->nd_repstat = aceerr;
	attrsum += aclsize;
	break;
	case NFSATTRBIT_ARCHIVE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (!nd->nd_repstat)
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_HIDDEN:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (!nd->nd_repstat)
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_MIMETYPE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	i = fxdr_unsigned(int, *tl);
	error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
	if (error)
	goto nfsmout;
	if (!nd->nd_repstat)
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(i));
	break;
	case NFSATTRBIT_MODE:
	moderet = NFSERR_INVAL; /* Can't do MODESETMASKED. */
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	nvap->na_mode = nfstov_mode(*tl);
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_OWNER:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	j = fxdr_unsigned(int, *tl);
	if (j < 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	if (j > NFSV4_SMALLSTR)
	cp = malloc(j + 1, M_NFSSTRING, M_WAITOK);
	else
	cp = namestr;
	error = nfsrv_mtostr(nd, cp, j);
	if (error) {
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	goto nfsmout;
	}
	if (!nd->nd_repstat) {
	nd->nd_repstat = nfsv4_strtouid(nd, cp, j,
	&uid);
	if (!nd->nd_repstat)
	nvap->na_uid = uid;
	}
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(j));
	break;
	case NFSATTRBIT_OWNERGROUP:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	j = fxdr_unsigned(int, *tl);
	if (j < 0) {
	error = NFSERR_BADXDR;
	goto nfsmout;
	}
	if (j > NFSV4_SMALLSTR)
	cp = malloc(j + 1, M_NFSSTRING, M_WAITOK);
	else
	cp = namestr;
	error = nfsrv_mtostr(nd, cp, j);
	if (error) {
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	goto nfsmout;
	}
	if (!nd->nd_repstat) {
	nd->nd_repstat = nfsv4_strtogid(nd, cp, j,
	&gid);
	if (!nd->nd_repstat)
	nvap->na_gid = gid;
	}
	if (j > NFSV4_SMALLSTR)
	free(cp, M_NFSSTRING);
	attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(j));
	break;
	case NFSATTRBIT_SYSTEM:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	if (!nd->nd_repstat)
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_UNSIGNED;
	break;
	case NFSATTRBIT_TIMEACCESSSET:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsum += NFSX_UNSIGNED;
	if (fxdr_unsigned(int, *tl)==NFSV4SATTRTIME_TOCLIENT) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	fxdr_nfsv4time(tl, &nvap->na_atime);
	toclient = 1;
	attrsum += NFSX_V4TIME;
	} else {
	vfs_timestamp(&nvap->na_atime);
	nvap->na_vaflags \|= VA_UTIMES_NULL;
	}
	break;
	case NFSATTRBIT_TIMEBACKUP:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	if (!nd->nd_repstat)
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMECREATE:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	- if (!nd->nd_repstat)
	- nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	+ fxdr_nfsv4time(tl, &nvap->na_btime);
	attrsum += NFSX_V4TIME;
	break;
	case NFSATTRBIT_TIMEMODIFYSET:
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	attrsum += NFSX_UNSIGNED;
	if (fxdr_unsigned(int, *tl)==NFSV4SATTRTIME_TOCLIENT) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_V4TIME);
	fxdr_nfsv4time(tl, &nvap->na_mtime);
	nvap->na_vaflags &= ~VA_UTIMES_NULL;
	attrsum += NFSX_V4TIME;
	} else {
	vfs_timestamp(&nvap->na_mtime);
	if (!toclient)
	nvap->na_vaflags \|= VA_UTIMES_NULL;
	}
	break;
	case NFSATTRBIT_MODESETMASKED:
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED);
	mode = fxdr_unsigned(u_short, *tl++);
	mask = fxdr_unsigned(u_short, *tl);
	/*
	* vp == NULL implies an Open/Create operation.
	* This attribute can only be used for Setattr and
	* only for NFSv4.1 or higher.
	* If moderet != 0, a mode attribute has also been
	* specified and this attribute cannot be done in the
	* same Setattr operation.
	*/
	if ((nd->nd_flag & ND_NFSV41) == 0)
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	else if ((mode & ~07777) != 0 \|\| (mask & ~07777) != 0 \|\|
	vp == NULL)
	nd->nd_repstat = NFSERR_INVAL;
	else if (moderet == 0)
	moderet = VOP_GETATTR(vp, &va, nd->nd_cred);
	if (moderet == 0)
	nvap->na_mode = (mode & mask) \|
	(va.va_mode & ~mask);
	else
	nd->nd_repstat = moderet;
	attrsum += 2 * NFSX_UNSIGNED;
	break;
	default:
	nd->nd_repstat = NFSERR_ATTRNOTSUPP;
	/*
	* set bitpos so we drop out of the loop.
	*/
	bitpos = NFSATTRBIT_MAX;
	break;
	}
	}

	/*
	* some clients pad the attrlist, so we need to skip over the
	* padding.
	*/
	if (attrsum > attrsize) {
	error = NFSERR_BADXDR;
	} else {
	attrsize = NFSM_RNDUP(attrsize);
	if (attrsum < attrsize)
	error = nfsm_advance(nd, attrsize - attrsum, -1);
	}
	nfsmout:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Check/setup export credentials.
	*/
	int
	nfsd_excred(struct nfsrv_descript nd, struct nfsexstuff exp,
	struct ucred *credanon)
	{
	int error = 0;

	/*
	* Check/setup credentials.
	*/
	if (nd->nd_flag & ND_GSS)
	exp->nes_exflag &= ~MNT_EXPORTANON;

	/*
	* Check to see if the operation is allowed for this security flavor.
	* RFC2623 suggests that the NFSv3 Fsinfo RPC be allowed to
	* AUTH_NONE or AUTH_SYS for file systems requiring RPCSEC_GSS.
	* Also, allow Secinfo, so that it can acquire the correct flavor(s).
	*/
	if (nfsvno_testexp(nd, exp) &&
	nd->nd_procnum != NFSV4OP_SECINFO &&
	nd->nd_procnum != NFSPROC_FSINFO) {
	if (nd->nd_flag & ND_NFSV4)
	error = NFSERR_WRONGSEC;
	else
	error = (NFSERR_AUTHERR \| AUTH_TOOWEAK);
	goto out;
	}

	/*
	* Check to see if the file system is exported V4 only.
	*/
	if (NFSVNO_EXV4ONLY(exp) && !(nd->nd_flag & ND_NFSV4)) {
	error = NFSERR_PROGNOTV4;
	goto out;
	}

	/*
	* Now, map the user credentials.
	* (Note that ND_AUTHNONE will only be set for an NFSv3
	* Fsinfo RPC. If set for anything else, this code might need
	* to change.)
	*/
	if (NFSVNO_EXPORTED(exp)) {
	if (((nd->nd_flag & ND_GSS) == 0 && nd->nd_cred->cr_uid == 0) \|\|
	NFSVNO_EXPORTANON(exp) \|\|
	(nd->nd_flag & ND_AUTHNONE) != 0) {
	nd->nd_cred->cr_uid = credanon->cr_uid;
	nd->nd_cred->cr_gid = credanon->cr_gid;
	crsetgroups(nd->nd_cred, credanon->cr_ngroups,
	credanon->cr_groups);
	} else if ((nd->nd_flag & ND_GSS) == 0) {
	/*
	* If using AUTH_SYS, call nfsrv_getgrpscred() to see
	* if there is a replacement credential with a group
	* list set up by "nfsuserd -manage-gids".
	* If there is no replacement, nfsrv_getgrpscred()
	* simply returns its argument.
	*/
	nd->nd_cred = nfsrv_getgrpscred(nd->nd_cred);
	}
	}

	out:
	NFSEXITCODE2(error, nd);
	return (error);
	}

	/*
	* Check exports.
	*/
	int
	nfsvno_checkexp(struct mount mp, struct sockaddr nam, struct nfsexstuff *exp,
	struct ucred **credp)
	{
	int error;

	error = VFS_CHECKEXP(mp, nam, &exp->nes_exflag, credp,
	&exp->nes_numsecflavor, exp->nes_secflavors);
	if (error) {
	if (nfs_rootfhset) {
	exp->nes_exflag = 0;
	exp->nes_numsecflavor = 0;
	error = 0;
	}
	} else if (exp->nes_numsecflavor < 1 \|\| exp->nes_numsecflavor >
	MAXSECFLAVORS) {
	printf("nfsvno_checkexp: numsecflavors out of range\n");
	exp->nes_numsecflavor = 0;
	error = EACCES;
	}
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Get a vnode for a file handle and export stuff.
	*/
	int
	nfsvno_fhtovp(struct mount mp, fhandle_t fhp, struct sockaddr *nam,
	int lktype, struct vnode *vpp, struct nfsexstuff exp,
	struct ucred **credp)
	{
	int error;

	*credp = NULL;
	exp->nes_numsecflavor = 0;
	error = VFS_FHTOVP(mp, &fhp->fh_fid, lktype, vpp);
	if (error != 0)
	/* Make sure the server replies ESTALE to the client. */
	error = ESTALE;
	if (nam && !error) {
	error = VFS_CHECKEXP(mp, nam, &exp->nes_exflag, credp,
	&exp->nes_numsecflavor, exp->nes_secflavors);
	if (error) {
	if (nfs_rootfhset) {
	exp->nes_exflag = 0;
	exp->nes_numsecflavor = 0;
	error = 0;
	} else {
	vput(*vpp);
	}
	} else if (exp->nes_numsecflavor < 1 \|\| exp->nes_numsecflavor >
	MAXSECFLAVORS) {
	printf("nfsvno_fhtovp: numsecflavors out of range\n");
	exp->nes_numsecflavor = 0;
	error = EACCES;
	vput(*vpp);
	}
	}
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* nfsd_fhtovp() - convert a fh to a vnode ptr
	* - look up fsid in mount list (if not found ret error)
	* - get vp and export rights by calling nfsvno_fhtovp()
	* - if cred->cr_uid == 0 or MNT_EXPORTANON set it to credanon
	* for AUTH_SYS
	* - if mpp != NULL, return the mount point so that it can
	* be used for vn_finished_write() by the caller
	*/
	void
	nfsd_fhtovp(struct nfsrv_descript nd, struct nfsrvfh nfp, int lktype,
	struct vnode *vpp, struct nfsexstuff exp,
	struct mount **mpp, int startwrite)
	{
	struct mount *mp;
	struct ucred *credanon;
	fhandle_t *fhp;

	fhp = (fhandle_t *)nfp->nfsrvfh_data;
	/*
	* Check for the special case of the nfsv4root_fh.
	*/
	mp = vfs_busyfs(&fhp->fh_fsid);
	if (mpp != NULL)
	*mpp = mp;
	if (mp == NULL) {
	*vpp = NULL;
	nd->nd_repstat = ESTALE;
	goto out;
	}

	if (startwrite) {
	vn_start_write(NULL, mpp, V_WAIT);
	if (lktype == LK_SHARED && !(MNT_SHARED_WRITES(mp)))
	lktype = LK_EXCLUSIVE;
	}
	nd->nd_repstat = nfsvno_fhtovp(mp, fhp, nd->nd_nam, lktype, vpp, exp,
	&credanon);
	vfs_unbusy(mp);

	/*
	* For NFSv4 without a pseudo root fs, unexported file handles
	* can be returned, so that Lookup works everywhere.
	*/
	if (!nd->nd_repstat && exp->nes_exflag == 0 &&
	!(nd->nd_flag & ND_NFSV4)) {
	vput(*vpp);
	nd->nd_repstat = EACCES;
	}

	/*
	* Personally, I've never seen any point in requiring a
	* reserved port#, since only in the rare case where the
	* clients are all boxes with secure system privileges,
	* does it provide any enhanced security, but... some people
	* believe it to be useful and keep putting this code back in.
	* (There is also some "security checker" out there that
	* complains if the nfs server doesn't enforce this.)
	* However, note the following:
	* RFC3530 (NFSv4) specifies that a reserved port# not be
	* required.
	* RFC2623 recommends that, if a reserved port# is checked for,
	* that there be a way to turn that off--> ifdef'd.
	*/
	#ifdef NFS_REQRSVPORT
	if (!nd->nd_repstat) {
	struct sockaddr_in *saddr;
	struct sockaddr_in6 *saddr6;

	saddr = NFSSOCKADDR(nd->nd_nam, struct sockaddr_in *);
	saddr6 = NFSSOCKADDR(nd->nd_nam, struct sockaddr_in6 *);
	if (!(nd->nd_flag & ND_NFSV4) &&
	((saddr->sin_family == AF_INET &&
	ntohs(saddr->sin_port) >= IPPORT_RESERVED) \|\|
	(saddr6->sin6_family == AF_INET6 &&
	ntohs(saddr6->sin6_port) >= IPPORT_RESERVED))) {
	vput(*vpp);
	nd->nd_repstat = (NFSERR_AUTHERR \| AUTH_TOOWEAK);
	}
	}
	#endif /* NFS_REQRSVPORT */

	/*
	* Check/setup credentials.
	*/
	if (!nd->nd_repstat) {
	nd->nd_saveduid = nd->nd_cred->cr_uid;
	nd->nd_repstat = nfsd_excred(nd, exp, credanon);
	if (nd->nd_repstat)
	vput(*vpp);
	}
	if (credanon != NULL)
	crfree(credanon);
	if (nd->nd_repstat) {
	if (startwrite)
	vn_finished_write(mp);
	*vpp = NULL;
	if (mpp != NULL)
	*mpp = NULL;
	}

	out:
	NFSEXITCODE2(0, nd);
	}

	/*
	* glue for fp.
	*/
	static int
	fp_getfvp(struct thread p, int fd, struct file fpp, struct vnode *vpp)
	{
	struct filedesc *fdp;
	struct file *fp;
	int error = 0;

	fdp = p->td_proc->p_fd;
	if (fd < 0 \|\| fd >= fdp->fd_nfiles \|\|
	(fp = fdp->fd_ofiles[fd].fde_file) == NULL) {
	error = EBADF;
	goto out;
	}
	*fpp = fp;

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Called from nfssvc() to update the exports list. Just call
	* vfs_export(). This has to be done, since the v4 root fake fs isn't
	* in the mount list.
	*/
	int
	nfsrv_v4rootexport(void argp, struct ucred cred, struct thread *p)
	{
	struct nfsex_args nfsexargp = (struct nfsex_args )argp;
	int error = 0;
	struct nameidata nd;
	fhandle_t fh;

	error = vfs_export(&nfsv4root_mnt, &nfsexargp->export);
	if ((nfsexargp->export.ex_flags & MNT_DELEXPORT) != 0)
	nfs_rootfhset = 0;
	else if (error == 0) {
	if (nfsexargp->fspec == NULL) {
	error = EPERM;
	goto out;
	}
	/*
	* If fspec != NULL, this is the v4root path.
	*/
	NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE,
	nfsexargp->fspec, p);
	if ((error = namei(&nd)) != 0)
	goto out;
	error = nfsvno_getfh(nd.ni_vp, &fh, p);
	vrele(nd.ni_vp);
	if (!error) {
	nfs_rootfh.nfsrvfh_len = NFSX_MYFH;
	NFSBCOPY((caddr_t)&fh,
	nfs_rootfh.nfsrvfh_data,
	sizeof (fhandle_t));
	nfs_rootfhset = 1;
	}
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* This function needs to test to see if the system is near its limit
	* for memory allocation via malloc() or mget() and return True iff
	* either of these resources are near their limit.
	* XXX (For now, this is just a stub.)
	*/
	int nfsrv_testmalloclimit = 0;
	int
	nfsrv_mallocmget_limit(void)
	{
	static int printmesg = 0;
	static int testval = 1;

	if (nfsrv_testmalloclimit && (testval++ % 1000) == 0) {
	if ((printmesg++ % 100) == 0)
	printf("nfsd: malloc/mget near limit\n");
	return (1);
	}
	return (0);
	}

	/*
	* BSD specific initialization of a mount point.
	*/
	void
	nfsd_mntinit(void)
	{
	static int inited = 0;

	if (inited)
	return;
	inited = 1;
	nfsv4root_mnt.mnt_flag = (MNT_RDONLY \| MNT_EXPORTED);
	TAILQ_INIT(&nfsv4root_mnt.mnt_nvnodelist);
	TAILQ_INIT(&nfsv4root_mnt.mnt_lazyvnodelist);
	nfsv4root_mnt.mnt_export = NULL;
	TAILQ_INIT(&nfsv4root_opt);
	TAILQ_INIT(&nfsv4root_newopt);
	nfsv4root_mnt.mnt_opt = &nfsv4root_opt;
	nfsv4root_mnt.mnt_optnew = &nfsv4root_newopt;
	nfsv4root_mnt.mnt_nvnodelistsize = 0;
	nfsv4root_mnt.mnt_lazyvnodelistsize = 0;
	}

	/*
	* Get a vnode for a file handle, without checking exports, etc.
	*/
	struct vnode *
	nfsvno_getvp(fhandle_t *fhp)
	{
	struct mount *mp;
	struct vnode *vp;
	int error;

	mp = vfs_busyfs(&fhp->fh_fsid);
	if (mp == NULL)
	return (NULL);
	error = VFS_FHTOVP(mp, &fhp->fh_fid, LK_EXCLUSIVE, &vp);
	vfs_unbusy(mp);
	if (error)
	return (NULL);
	return (vp);
	}

	/*
	* Do a local VOP_ADVLOCK().
	*/
	int
	nfsvno_advlock(struct vnode *vp, int ftype, u_int64_t first,
	u_int64_t end, struct thread *td)
	{
	int error = 0;
	struct flock fl;
	u_int64_t tlen;

	if (nfsrv_dolocallocks == 0)
	goto out;
	ASSERT_VOP_UNLOCKED(vp, "nfsvno_advlock: vp locked");

	fl.l_whence = SEEK_SET;
	fl.l_type = ftype;
	fl.l_start = (off_t)first;
	if (end == NFS64BITSSET) {
	fl.l_len = 0;
	} else {
	tlen = end - first;
	fl.l_len = (off_t)tlen;
	}
	/*
	* For FreeBSD8, the l_pid and l_sysid must be set to the same
	* values for all calls, so that all locks will be held by the
	* nfsd server. (The nfsd server handles conflicts between the
	* various clients.)
	* Since an NFSv4 lockowner is a ClientID plus an array of up to 1024
	* bytes, so it can't be put in l_sysid.
	*/
	if (nfsv4_sysid == 0)
	nfsv4_sysid = nlm_acquire_next_sysid();
	fl.l_pid = (pid_t)0;
	fl.l_sysid = (int)nfsv4_sysid;

	if (ftype == F_UNLCK)
	error = VOP_ADVLOCK(vp, (caddr_t)td->td_proc, F_UNLCK, &fl,
	(F_POSIX \| F_REMOTE));
	else
	error = VOP_ADVLOCK(vp, (caddr_t)td->td_proc, F_SETLK, &fl,
	(F_POSIX \| F_REMOTE));

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Check the nfsv4 root exports.
	*/
	int
	nfsvno_v4rootexport(struct nfsrv_descript *nd)
	{
	struct ucred *credanon;
	int error = 0, numsecflavor, secflavors[MAXSECFLAVORS], i;
	uint64_t exflags;

	error = vfs_stdcheckexp(&nfsv4root_mnt, nd->nd_nam, &exflags,
	&credanon, &numsecflavor, secflavors);
	if (error) {
	error = NFSERR_PROGUNAVAIL;
	goto out;
	}
	if (credanon != NULL)
	crfree(credanon);
	for (i = 0; i < numsecflavor; i++) {
	if (secflavors[i] == AUTH_SYS)
	nd->nd_flag \|= ND_EXAUTHSYS;
	else if (secflavors[i] == RPCSEC_GSS_KRB5)
	nd->nd_flag \|= ND_EXGSS;
	else if (secflavors[i] == RPCSEC_GSS_KRB5I)
	nd->nd_flag \|= ND_EXGSSINTEGRITY;
	else if (secflavors[i] == RPCSEC_GSS_KRB5P)
	nd->nd_flag \|= ND_EXGSSPRIVACY;
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Nfs server pseudo system call for the nfsd's
	*/
	/*
	* MPSAFE
	*/
	static int
	nfssvc_nfsd(struct thread td, struct nfssvc_args uap)
	{
	struct file *fp;
	struct nfsd_addsock_args sockarg;
	struct nfsd_nfsd_args nfsdarg;
	struct nfsd_nfsd_oargs onfsdarg;
	struct nfsd_pnfsd_args pnfsdarg;
	struct vnode vp, nvp, *curdvp;
	struct pnfsdsfile *pf;
	struct nfsdevice ds, fds;
	cap_rights_t rights;
	int buflen, error, ret;
	char buf, cp, cp2, cp3;
	char fname[PNFS_FILENAME_LEN + 1];

	if (uap->flag & NFSSVC_NFSDADDSOCK) {
	error = copyin(uap->argp, (caddr_t)&sockarg, sizeof (sockarg));
	if (error)
	goto out;
	/*
	* Since we don't know what rights might be required,
	* pretend that we need them all. It is better to be too
	* careful than too reckless.
	*/
	error = fget(td, sockarg.sock,
	cap_rights_init(&rights, CAP_SOCK_SERVER), &fp);
	if (error != 0)
	goto out;
	if (fp->f_type != DTYPE_SOCKET) {
	fdrop(fp, td);
	error = EPERM;
	goto out;
	}
	error = nfsrvd_addsock(fp);
	fdrop(fp, td);
	} else if (uap->flag & NFSSVC_NFSDNFSD) {
	if (uap->argp == NULL) {
	error = EINVAL;
	goto out;
	}
	if ((uap->flag & NFSSVC_NEWSTRUCT) == 0) {
	error = copyin(uap->argp, &onfsdarg, sizeof(onfsdarg));
	if (error == 0) {
	nfsdarg.principal = onfsdarg.principal;
	nfsdarg.minthreads = onfsdarg.minthreads;
	nfsdarg.maxthreads = onfsdarg.maxthreads;
	nfsdarg.version = 1;
	nfsdarg.addr = NULL;
	nfsdarg.addrlen = 0;
	nfsdarg.dnshost = NULL;
	nfsdarg.dnshostlen = 0;
	nfsdarg.dspath = NULL;
	nfsdarg.dspathlen = 0;
	nfsdarg.mdspath = NULL;
	nfsdarg.mdspathlen = 0;
	nfsdarg.mirrorcnt = 1;
	}
	} else
	error = copyin(uap->argp, &nfsdarg, sizeof(nfsdarg));
	if (error)
	goto out;
	if (nfsdarg.addrlen > 0 && nfsdarg.addrlen < 10000 &&
	nfsdarg.dnshostlen > 0 && nfsdarg.dnshostlen < 10000 &&
	nfsdarg.dspathlen > 0 && nfsdarg.dspathlen < 10000 &&
	nfsdarg.mdspathlen > 0 && nfsdarg.mdspathlen < 10000 &&
	nfsdarg.mirrorcnt >= 1 &&
	nfsdarg.mirrorcnt <= NFSDEV_MAXMIRRORS &&
	nfsdarg.addr != NULL && nfsdarg.dnshost != NULL &&
	nfsdarg.dspath != NULL && nfsdarg.mdspath != NULL) {
	NFSD_DEBUG(1, "addrlen=%d dspathlen=%d dnslen=%d"
	" mdspathlen=%d mirrorcnt=%d\n", nfsdarg.addrlen,
	nfsdarg.dspathlen, nfsdarg.dnshostlen,
	nfsdarg.mdspathlen, nfsdarg.mirrorcnt);
	cp = malloc(nfsdarg.addrlen + 1, M_TEMP, M_WAITOK);
	error = copyin(nfsdarg.addr, cp, nfsdarg.addrlen);
	if (error != 0) {
	free(cp, M_TEMP);
	goto out;
	}
	cp[nfsdarg.addrlen] = '\0'; /* Ensure nul term. */
	nfsdarg.addr = cp;
	cp = malloc(nfsdarg.dnshostlen + 1, M_TEMP, M_WAITOK);
	error = copyin(nfsdarg.dnshost, cp, nfsdarg.dnshostlen);
	if (error != 0) {
	free(nfsdarg.addr, M_TEMP);
	free(cp, M_TEMP);
	goto out;
	}
	cp[nfsdarg.dnshostlen] = '\0'; /* Ensure nul term. */
	nfsdarg.dnshost = cp;
	cp = malloc(nfsdarg.dspathlen + 1, M_TEMP, M_WAITOK);
	error = copyin(nfsdarg.dspath, cp, nfsdarg.dspathlen);
	if (error != 0) {
	free(nfsdarg.addr, M_TEMP);
	free(nfsdarg.dnshost, M_TEMP);
	free(cp, M_TEMP);
	goto out;
	}
	cp[nfsdarg.dspathlen] = '\0'; /* Ensure nul term. */
	nfsdarg.dspath = cp;
	cp = malloc(nfsdarg.mdspathlen + 1, M_TEMP, M_WAITOK);
	error = copyin(nfsdarg.mdspath, cp, nfsdarg.mdspathlen);
	if (error != 0) {
	free(nfsdarg.addr, M_TEMP);
	free(nfsdarg.dnshost, M_TEMP);
	free(nfsdarg.dspath, M_TEMP);
	free(cp, M_TEMP);
	goto out;
	}
	cp[nfsdarg.mdspathlen] = '\0'; /* Ensure nul term. */
	nfsdarg.mdspath = cp;
	} else {
	nfsdarg.addr = NULL;
	nfsdarg.addrlen = 0;
	nfsdarg.dnshost = NULL;
	nfsdarg.dnshostlen = 0;
	nfsdarg.dspath = NULL;
	nfsdarg.dspathlen = 0;
	nfsdarg.mdspath = NULL;
	nfsdarg.mdspathlen = 0;
	nfsdarg.mirrorcnt = 1;
	}
	error = nfsrvd_nfsd(td, &nfsdarg);
	free(nfsdarg.addr, M_TEMP);
	free(nfsdarg.dnshost, M_TEMP);
	free(nfsdarg.dspath, M_TEMP);
	free(nfsdarg.mdspath, M_TEMP);
	} else if (uap->flag & NFSSVC_PNFSDS) {
	error = copyin(uap->argp, &pnfsdarg, sizeof(pnfsdarg));
	if (error == 0 && (pnfsdarg.op == PNFSDOP_DELDSSERVER \|\|
	pnfsdarg.op == PNFSDOP_FORCEDELDS)) {
	cp = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK);
	error = copyinstr(pnfsdarg.dspath, cp, PATH_MAX + 1,
	NULL);
	if (error == 0)
	error = nfsrv_deldsserver(pnfsdarg.op, cp, td);
	free(cp, M_TEMP);
	} else if (error == 0 && pnfsdarg.op == PNFSDOP_COPYMR) {
	cp = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK);
	buflen = sizeof(pf) NFSDEV_MAXMIRRORS;
	buf = malloc(buflen, M_TEMP, M_WAITOK);
	error = copyinstr(pnfsdarg.mdspath, cp, PATH_MAX + 1,
	NULL);
	NFSD_DEBUG(4, "pnfsdcopymr cp mdspath=%d\n", error);
	if (error == 0 && pnfsdarg.dspath != NULL) {
	cp2 = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK);
	error = copyinstr(pnfsdarg.dspath, cp2,
	PATH_MAX + 1, NULL);
	NFSD_DEBUG(4, "pnfsdcopymr cp dspath=%d\n",
	error);
	} else
	cp2 = NULL;
	if (error == 0 && pnfsdarg.curdspath != NULL) {
	cp3 = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK);
	error = copyinstr(pnfsdarg.curdspath, cp3,
	PATH_MAX + 1, NULL);
	NFSD_DEBUG(4, "pnfsdcopymr cp curdspath=%d\n",
	error);
	} else
	cp3 = NULL;
	curdvp = NULL;
	fds = NULL;
	if (error == 0)
	error = nfsrv_mdscopymr(cp, cp2, cp3, buf,
	&buflen, fname, td, &vp, &nvp, &pf, &ds,
	&fds);
	NFSD_DEBUG(4, "nfsrv_mdscopymr=%d\n", error);
	if (error == 0) {
	if (pf->dsf_dir >= nfsrv_dsdirsize) {
	printf("copymr: dsdir out of range\n");
	pf->dsf_dir = 0;
	}
	NFSD_DEBUG(4, "copymr: buflen=%d\n", buflen);
	error = nfsrv_copymr(vp, nvp,
	ds->nfsdev_dsdir[pf->dsf_dir], ds, pf,
	(struct pnfsdsfile *)buf,
	buflen / sizeof(*pf), td->td_ucred, td);
	vput(vp);
	vput(nvp);
	if (fds != NULL && error == 0) {
	curdvp = fds->nfsdev_dsdir[pf->dsf_dir];
	ret = vn_lock(curdvp, LK_EXCLUSIVE);
	if (ret == 0) {
	nfsrv_dsremove(curdvp, fname,
	td->td_ucred, td);
	NFSVOPUNLOCK(curdvp);
	}
	}
	NFSD_DEBUG(4, "nfsrv_copymr=%d\n", error);
	}
	free(cp, M_TEMP);
	free(cp2, M_TEMP);
	free(cp3, M_TEMP);
	free(buf, M_TEMP);
	}
	} else {
	error = nfssvc_srvcall(td, uap, td->td_ucred);
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	static int
	nfssvc_srvcall(struct thread p, struct nfssvc_args uap, struct ucred *cred)
	{
	struct nfsex_args export;
	struct nfsex_oldargs oexp;
	struct file *fp = NULL;
	int stablefd, i, len;
	struct nfsd_clid adminrevoke;
	struct nfsd_dumplist dumplist;
	struct nfsd_dumpclients *dumpclients;
	struct nfsd_dumplocklist dumplocklist;
	struct nfsd_dumplocks *dumplocks;
	struct nameidata nd;
	vnode_t vp;
	int error = EINVAL, igotlock;
	struct proc *procp;
	gid_t *grps;
	static int suspend_nfsd = 0;

	if (uap->flag & NFSSVC_PUBLICFH) {
	NFSBZERO((caddr_t)&nfs_pubfh.nfsrvfh_data,
	sizeof (fhandle_t));
	error = copyin(uap->argp,
	&nfs_pubfh.nfsrvfh_data, sizeof (fhandle_t));
	if (!error)
	nfs_pubfhset = 1;
	} else if ((uap->flag & (NFSSVC_V4ROOTEXPORT \| NFSSVC_NEWSTRUCT)) ==
	(NFSSVC_V4ROOTEXPORT \| NFSSVC_NEWSTRUCT)) {
	error = copyin(uap->argp,(caddr_t)&export,
	sizeof (struct nfsex_args));
	if (!error) {
	grps = NULL;
	if (export.export.ex_ngroups > NGROUPS_MAX \|\|
	export.export.ex_ngroups < 0)
	error = EINVAL;
	else if (export.export.ex_ngroups > 0) {
	grps = malloc(export.export.ex_ngroups *
	sizeof(gid_t), M_TEMP, M_WAITOK);
	error = copyin(export.export.ex_groups, grps,
	export.export.ex_ngroups * sizeof(gid_t));
	export.export.ex_groups = grps;
	} else
	export.export.ex_groups = NULL;
	if (!error)
	error = nfsrv_v4rootexport(&export, cred, p);
	free(grps, M_TEMP);
	}
	} else if ((uap->flag & (NFSSVC_V4ROOTEXPORT \| NFSSVC_NEWSTRUCT)) ==
	NFSSVC_V4ROOTEXPORT) {
	error = copyin(uap->argp,(caddr_t)&oexp,
	sizeof (struct nfsex_oldargs));
	if (!error) {
	memset(&export.export, 0, sizeof(export.export));
	export.export.ex_flags = (uint64_t)oexp.export.ex_flags;
	export.export.ex_root = oexp.export.ex_root;
	export.export.ex_uid = oexp.export.ex_anon.cr_uid;
	export.export.ex_ngroups =
	oexp.export.ex_anon.cr_ngroups;
	export.export.ex_groups = NULL;
	if (export.export.ex_ngroups > XU_NGROUPS \|\|
	export.export.ex_ngroups < 0)
	error = EINVAL;
	else if (export.export.ex_ngroups > 0) {
	export.export.ex_groups = malloc(
	export.export.ex_ngroups * sizeof(gid_t),
	M_TEMP, M_WAITOK);
	for (i = 0; i < export.export.ex_ngroups; i++)
	export.export.ex_groups[i] =
	oexp.export.ex_anon.cr_groups[i];
	}
	export.export.ex_addr = oexp.export.ex_addr;
	export.export.ex_addrlen = oexp.export.ex_addrlen;
	export.export.ex_mask = oexp.export.ex_mask;
	export.export.ex_masklen = oexp.export.ex_masklen;
	export.export.ex_indexfile = oexp.export.ex_indexfile;
	export.export.ex_numsecflavors =
	oexp.export.ex_numsecflavors;
	if (export.export.ex_numsecflavors >= MAXSECFLAVORS \|\|
	export.export.ex_numsecflavors < 0)
	error = EINVAL;
	else {
	for (i = 0; i < export.export.ex_numsecflavors;
	i++)
	export.export.ex_secflavors[i] =
	oexp.export.ex_secflavors[i];
	}
	export.fspec = oexp.fspec;
	if (error == 0)
	error = nfsrv_v4rootexport(&export, cred, p);
	free(export.export.ex_groups, M_TEMP);
	}
	} else if (uap->flag & NFSSVC_NOPUBLICFH) {
	nfs_pubfhset = 0;
	error = 0;
	} else if (uap->flag & NFSSVC_STABLERESTART) {
	error = copyin(uap->argp, (caddr_t)&stablefd,
	sizeof (int));
	if (!error)
	error = fp_getfvp(p, stablefd, &fp, &vp);
	if (!error && (NFSFPFLAG(fp) & (FREAD \| FWRITE)) != (FREAD \| FWRITE))
	error = EBADF;
	if (!error && newnfs_numnfsd != 0)
	error = EPERM;
	if (!error) {
	nfsrv_stablefirst.nsf_fp = fp;
	nfsrv_setupstable(p);
	}
	} else if (uap->flag & NFSSVC_ADMINREVOKE) {
	error = copyin(uap->argp, (caddr_t)&adminrevoke,
	sizeof (struct nfsd_clid));
	if (!error)
	error = nfsrv_adminrevoke(&adminrevoke, p);
	} else if (uap->flag & NFSSVC_DUMPCLIENTS) {
	error = copyin(uap->argp, (caddr_t)&dumplist,
	sizeof (struct nfsd_dumplist));
	if (!error && (dumplist.ndl_size < 1 \|\|
	dumplist.ndl_size > NFSRV_MAXDUMPLIST))
	error = EPERM;
	if (!error) {
	len = sizeof (struct nfsd_dumpclients) * dumplist.ndl_size;
	dumpclients = malloc(len, M_TEMP, M_WAITOK \| M_ZERO);
	nfsrv_dumpclients(dumpclients, dumplist.ndl_size);
	error = copyout(dumpclients, dumplist.ndl_list, len);
	free(dumpclients, M_TEMP);
	}
	} else if (uap->flag & NFSSVC_DUMPLOCKS) {
	error = copyin(uap->argp, (caddr_t)&dumplocklist,
	sizeof (struct nfsd_dumplocklist));
	if (!error && (dumplocklist.ndllck_size < 1 \|\|
	dumplocklist.ndllck_size > NFSRV_MAXDUMPLIST))
	error = EPERM;
	if (!error)
	error = nfsrv_lookupfilename(&nd,
	dumplocklist.ndllck_fname, p);
	if (!error) {
	len = sizeof (struct nfsd_dumplocks) *
	dumplocklist.ndllck_size;
	dumplocks = malloc(len, M_TEMP, M_WAITOK \| M_ZERO);
	nfsrv_dumplocks(nd.ni_vp, dumplocks,
	dumplocklist.ndllck_size, p);
	vput(nd.ni_vp);
	error = copyout(dumplocks, dumplocklist.ndllck_list,
	len);
	free(dumplocks, M_TEMP);
	}
	} else if (uap->flag & NFSSVC_BACKUPSTABLE) {
	procp = p->td_proc;
	PROC_LOCK(procp);
	nfsd_master_pid = procp->p_pid;
	bcopy(procp->p_comm, nfsd_master_comm, MAXCOMLEN + 1);
	nfsd_master_start = procp->p_stats->p_start;
	nfsd_master_proc = procp;
	PROC_UNLOCK(procp);
	} else if ((uap->flag & NFSSVC_SUSPENDNFSD) != 0) {
	NFSLOCKV4ROOTMUTEX();
	if (suspend_nfsd == 0) {
	/* Lock out all nfsd threads */
	do {
	igotlock = nfsv4_lock(&nfsd_suspend_lock, 1,
	NULL, NFSV4ROOTLOCKMUTEXPTR, NULL);
	} while (igotlock == 0 && suspend_nfsd == 0);
	suspend_nfsd = 1;
	}
	NFSUNLOCKV4ROOTMUTEX();
	error = 0;
	} else if ((uap->flag & NFSSVC_RESUMENFSD) != 0) {
	NFSLOCKV4ROOTMUTEX();
	if (suspend_nfsd != 0) {
	nfsv4_unlock(&nfsd_suspend_lock, 0);
	suspend_nfsd = 0;
	}
	NFSUNLOCKV4ROOTMUTEX();
	error = 0;
	}

	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Check exports.
	* Returns 0 if ok, 1 otherwise.
	*/
	int
	nfsvno_testexp(struct nfsrv_descript nd, struct nfsexstuff exp)
	{
	int i;

	/*
	* This seems odd, but allow the case where the security flavor
	* list is empty. This happens when NFSv4 is traversing non-exported
	* file systems. Exported file systems should always have a non-empty
	* security flavor list.
	*/
	if (exp->nes_numsecflavor == 0)
	return (0);

	for (i = 0; i < exp->nes_numsecflavor; i++) {
	/*
	* The tests for privacy and integrity must be first,
	* since ND_GSS is set for everything but AUTH_SYS.
	*/
	if (exp->nes_secflavors[i] == RPCSEC_GSS_KRB5P &&
	(nd->nd_flag & ND_GSSPRIVACY))
	return (0);
	if (exp->nes_secflavors[i] == RPCSEC_GSS_KRB5I &&
	(nd->nd_flag & ND_GSSINTEGRITY))
	return (0);
	if (exp->nes_secflavors[i] == RPCSEC_GSS_KRB5 &&
	(nd->nd_flag & ND_GSS))
	return (0);
	if (exp->nes_secflavors[i] == AUTH_SYS &&
	(nd->nd_flag & ND_GSS) == 0)
	return (0);
	}
	return (1);
	}

	/*
	* Calculate a hash value for the fid in a file handle.
	*/
	uint32_t
	nfsrv_hashfh(fhandle_t *fhp)
	{
	uint32_t hashval;

	hashval = hash32_buf(&fhp->fh_fid, sizeof(struct fid), 0);
	return (hashval);
	}

	/*
	* Calculate a hash value for the sessionid.
	*/
	uint32_t
	nfsrv_hashsessionid(uint8_t *sessionid)
	{
	uint32_t hashval;

	hashval = hash32_buf(sessionid, NFSX_V4SESSIONID, 0);
	return (hashval);
	}

	/*
	* Signal the userland master nfsd to backup the stable restart file.
	*/
	void
	nfsrv_backupstable(void)
	{
	struct proc *procp;

	if (nfsd_master_proc != NULL) {
	procp = pfind(nfsd_master_pid);
	/* Try to make sure it is the correct process. */
	if (procp == nfsd_master_proc &&
	procp->p_stats->p_start.tv_sec ==
	nfsd_master_start.tv_sec &&
	procp->p_stats->p_start.tv_usec ==
	nfsd_master_start.tv_usec &&
	strcmp(procp->p_comm, nfsd_master_comm) == 0)
	kern_psignal(procp, SIGUSR2);
	else
	nfsd_master_proc = NULL;

	if (procp != NULL)
	PROC_UNLOCK(procp);
	}
	}

	/*
	* Create a DS data file for nfsrv_pnfscreate(). Called for each mirror.
	* The arguments are in a structure, so that they can be passed through
	* taskqueue for a kernel process to execute this function.
	*/
	struct nfsrvdscreate {
	int done;
	int inprog;
	struct task tsk;
	struct ucred *tcred;
	struct vnode *dvp;
	NFSPROC_T *p;
	struct pnfsdsfile *pf;
	int err;
	fhandle_t fh;
	struct vattr va;
	struct vattr createva;
	};

	int
	nfsrv_dscreate(struct vnode dvp, struct vattr vap, struct vattr *nvap,
	fhandle_t fhp, struct pnfsdsfile pf, struct pnfsdsattr *dsa,
	char fnamep, struct ucred tcred, NFSPROC_T p, struct vnode *nvpp)
	{
	struct vnode *nvp;
	struct nameidata named;
	struct vattr va;
	char *bufp;
	u_long *hashp;
	struct nfsnode *np;
	struct nfsmount *nmp;
	int error;

	NFSNAMEICNDSET(&named.ni_cnd, tcred, CREATE,
	LOCKPARENT \| LOCKLEAF \| SAVESTART \| NOCACHE);
	nfsvno_setpathbuf(&named, &bufp, &hashp);
	named.ni_cnd.cn_lkflags = LK_EXCLUSIVE;
	named.ni_cnd.cn_thread = p;
	named.ni_cnd.cn_nameptr = bufp;
	if (fnamep != NULL) {
	strlcpy(bufp, fnamep, PNFS_FILENAME_LEN + 1);
	named.ni_cnd.cn_namelen = strlen(bufp);
	} else
	named.ni_cnd.cn_namelen = nfsrv_putfhname(fhp, bufp);
	NFSD_DEBUG(4, "nfsrv_dscreate: dvp=%p fname=%s\n", dvp, bufp);

	/* Create the date file in the DS mount. */
	error = NFSVOPLOCK(dvp, LK_EXCLUSIVE);
	if (error == 0) {
	error = VOP_CREATE(dvp, &nvp, &named.ni_cnd, vap);
	NFSVOPUNLOCK(dvp);
	if (error == 0) {
	/* Set the ownership of the file. */
	error = VOP_SETATTR(nvp, nvap, tcred);
	NFSD_DEBUG(4, "nfsrv_dscreate:"
	" setattr-uid=%d\n", error);
	if (error != 0)
	vput(nvp);
	}
	if (error != 0)
	printf("pNFS: pnfscreate failed=%d\n", error);
	} else
	printf("pNFS: pnfscreate vnlock=%d\n", error);
	if (error == 0) {
	np = VTONFS(nvp);
	nmp = VFSTONFS(nvp->v_mount);
	if (strcmp(nvp->v_mount->mnt_vfc->vfc_name, "nfs")
	!= 0 \|\| nmp->nm_nam->sa_len > sizeof(
	struct sockaddr_in6) \|\|
	np->n_fhp->nfh_len != NFSX_MYFH) {
	printf("Bad DS file: fstype=%s salen=%d"
	" fhlen=%d\n",
	nvp->v_mount->mnt_vfc->vfc_name,
	nmp->nm_nam->sa_len, np->n_fhp->nfh_len);
	error = ENOENT;
	}

	/* Set extattrs for the DS on the MDS file. */
	if (error == 0) {
	if (dsa != NULL) {
	error = VOP_GETATTR(nvp, &va, tcred);
	if (error == 0) {
	dsa->dsa_filerev = va.va_filerev;
	dsa->dsa_size = va.va_size;
	dsa->dsa_atime = va.va_atime;
	dsa->dsa_mtime = va.va_mtime;
	dsa->dsa_bytes = va.va_bytes;
	}
	}
	if (error == 0) {
	NFSBCOPY(np->n_fhp->nfh_fh, &pf->dsf_fh,
	NFSX_MYFH);
	NFSBCOPY(nmp->nm_nam, &pf->dsf_sin,
	nmp->nm_nam->sa_len);
	NFSBCOPY(named.ni_cnd.cn_nameptr,
	pf->dsf_filename,
	sizeof(pf->dsf_filename));
	}
	} else
	printf("pNFS: pnfscreate can't get DS"
	" attr=%d\n", error);
	if (nvpp != NULL && error == 0)
	*nvpp = nvp;
	else
	vput(nvp);
	}
	nfsvno_relpathbuf(&named);
	return (error);
	}

	/*
	* Start up the thread that will execute nfsrv_dscreate().
	*/
	static void
	start_dscreate(void *arg, int pending)
	{
	struct nfsrvdscreate *dsc;

	dsc = (struct nfsrvdscreate *)arg;
	dsc->err = nfsrv_dscreate(dsc->dvp, &dsc->createva, &dsc->va, &dsc->fh,
	dsc->pf, NULL, NULL, dsc->tcred, dsc->p, NULL);
	dsc->done = 1;
	NFSD_DEBUG(4, "start_dscreate: err=%d\n", dsc->err);
	}

	/*
	* Create a pNFS data file on the Data Server(s).
	*/
	static void
	nfsrv_pnfscreate(struct vnode vp, struct vattr vap, struct ucred *cred,
	NFSPROC_T *p)
	{
	struct nfsrvdscreate dsc, tdsc = NULL;
	struct nfsdevice ds, tds, *fds;
	struct mount *mp;
	struct pnfsdsfile pf, tpf;
	struct pnfsdsattr dsattr;
	struct vattr va;
	struct vnode *dvp[NFSDEV_MAXMIRRORS];
	struct nfsmount *nmp;
	fhandle_t fh;
	uid_t vauid;
	gid_t vagid;
	u_short vamode;
	struct ucred *tcred;
	int dsdir[NFSDEV_MAXMIRRORS], error, i, mirrorcnt, ret;
	int failpos, timo;

	/* Get a DS server directory in a round-robin order. */
	mirrorcnt = 1;
	mp = vp->v_mount;
	ds = fds = NULL;
	NFSDDSLOCK();
	/*
	* Search for the first entry that handles this MDS fs, but use the
	* first entry for all MDS fs's otherwise.
	*/
	TAILQ_FOREACH(tds, &nfsrv_devidhead, nfsdev_list) {
	if (tds->nfsdev_nmp != NULL) {
	if (tds->nfsdev_mdsisset == 0 && ds == NULL)
	ds = tds;
	else if (tds->nfsdev_mdsisset != 0 && fsidcmp(
	&mp->mnt_stat.f_fsid, &tds->nfsdev_mdsfsid) == 0) {
	ds = fds = tds;
	break;
	}
	}
	}
	if (ds == NULL) {
	NFSDDSUNLOCK();
	NFSD_DEBUG(4, "nfsrv_pnfscreate: no srv\n");
	return;
	}
	i = dsdir[0] = ds->nfsdev_nextdir;
	ds->nfsdev_nextdir = (ds->nfsdev_nextdir + 1) % nfsrv_dsdirsize;
	dvp[0] = ds->nfsdev_dsdir[i];
	tds = TAILQ_NEXT(ds, nfsdev_list);
	if (nfsrv_maxpnfsmirror > 1 && tds != NULL) {
	TAILQ_FOREACH_FROM(tds, &nfsrv_devidhead, nfsdev_list) {
	if (tds->nfsdev_nmp != NULL &&
	((tds->nfsdev_mdsisset == 0 && fds == NULL) \|\|
	(tds->nfsdev_mdsisset != 0 && fds != NULL &&
	fsidcmp(&mp->mnt_stat.f_fsid,
	&tds->nfsdev_mdsfsid) == 0))) {
	dsdir[mirrorcnt] = i;
	dvp[mirrorcnt] = tds->nfsdev_dsdir[i];
	mirrorcnt++;
	if (mirrorcnt >= nfsrv_maxpnfsmirror)
	break;
	}
	}
	}
	/* Put at end of list to implement round-robin usage. */
	TAILQ_REMOVE(&nfsrv_devidhead, ds, nfsdev_list);
	TAILQ_INSERT_TAIL(&nfsrv_devidhead, ds, nfsdev_list);
	NFSDDSUNLOCK();
	dsc = NULL;
	if (mirrorcnt > 1)
	tdsc = dsc = malloc(sizeof(dsc) (mirrorcnt - 1), M_TEMP,
	M_WAITOK \| M_ZERO);
	tpf = pf = malloc(sizeof(pf) nfsrv_maxpnfsmirror, M_TEMP, M_WAITOK \|
	M_ZERO);

	error = nfsvno_getfh(vp, &fh, p);
	if (error == 0)
	error = VOP_GETATTR(vp, &va, cred);
	if (error == 0) {
	/* Set the attributes for "vp" to Setattr the DS vp. */
	vauid = va.va_uid;
	vagid = va.va_gid;
	vamode = va.va_mode;
	VATTR_NULL(&va);
	va.va_uid = vauid;
	va.va_gid = vagid;
	va.va_mode = vamode;
	va.va_size = 0;
	} else
	printf("pNFS: pnfscreate getfh+attr=%d\n", error);

	NFSD_DEBUG(4, "nfsrv_pnfscreate: cruid=%d crgid=%d\n", cred->cr_uid,
	cred->cr_gid);
	/* Make data file name based on FH. */
	tcred = newnfs_getcred();

	/*
	* Create the file on each DS mirror, using kernel process(es) for the
	* additional mirrors.
	*/
	failpos = -1;
	for (i = 0; i < mirrorcnt - 1 && error == 0; i++, tpf++, tdsc++) {
	tpf->dsf_dir = dsdir[i];
	tdsc->tcred = tcred;
	tdsc->p = p;
	tdsc->pf = tpf;
	tdsc->createva = *vap;
	NFSBCOPY(&fh, &tdsc->fh, sizeof(fh));
	tdsc->va = va;
	tdsc->dvp = dvp[i];
	tdsc->done = 0;
	tdsc->inprog = 0;
	tdsc->err = 0;
	ret = EIO;
	if (nfs_pnfsiothreads != 0) {
	ret = nfs_pnfsio(start_dscreate, tdsc);
	NFSD_DEBUG(4, "nfsrv_pnfscreate: nfs_pnfsio=%d\n", ret);
	}
	if (ret != 0) {
	ret = nfsrv_dscreate(dvp[i], vap, &va, &fh, tpf, NULL,
	NULL, tcred, p, NULL);
	if (ret != 0) {
	KASSERT(error == 0, ("nfsrv_dscreate err=%d",
	error));
	if (failpos == -1 && nfsds_failerr(ret))
	failpos = i;
	else
	error = ret;
	}
	}
	}
	if (error == 0) {
	tpf->dsf_dir = dsdir[mirrorcnt - 1];
	error = nfsrv_dscreate(dvp[mirrorcnt - 1], vap, &va, &fh, tpf,
	&dsattr, NULL, tcred, p, NULL);
	if (failpos == -1 && mirrorcnt > 1 && nfsds_failerr(error)) {
	failpos = mirrorcnt - 1;
	error = 0;
	}
	}
	timo = hz / 50; /* Wait for 20msec. */
	if (timo < 1)
	timo = 1;
	/* Wait for kernel task(s) to complete. */
	for (tdsc = dsc, i = 0; i < mirrorcnt - 1; i++, tdsc++) {
	while (tdsc->inprog != 0 && tdsc->done == 0)
	tsleep(&tdsc->tsk, PVFS, "srvdcr", timo);
	if (tdsc->err != 0) {
	if (failpos == -1 && nfsds_failerr(tdsc->err))
	failpos = i;
	else if (error == 0)
	error = tdsc->err;
	}
	}

	/*
	* If failpos has been set, that mirror has failed, so it needs
	* to be disabled.
	*/
	if (failpos >= 0) {
	nmp = VFSTONFS(dvp[failpos]->v_mount);
	NFSLOCKMNT(nmp);
	if ((nmp->nm_privflag & (NFSMNTP_FORCEDISM \|
	NFSMNTP_CANCELRPCS)) == 0) {
	nmp->nm_privflag \|= NFSMNTP_CANCELRPCS;
	NFSUNLOCKMNT(nmp);
	ds = nfsrv_deldsnmp(PNFSDOP_DELDSSERVER, nmp, p);
	NFSD_DEBUG(4, "dscreatfail fail=%d ds=%p\n", failpos,
	ds);
	if (ds != NULL)
	nfsrv_killrpcs(nmp);
	NFSLOCKMNT(nmp);
	nmp->nm_privflag &= ~NFSMNTP_CANCELRPCS;
	wakeup(nmp);
	}
	NFSUNLOCKMNT(nmp);
	}

	NFSFREECRED(tcred);
	if (error == 0) {
	ASSERT_VOP_ELOCKED(vp, "nfsrv_pnfscreate vp");

	NFSD_DEBUG(4, "nfsrv_pnfscreate: mirrorcnt=%d maxmirror=%d\n",
	mirrorcnt, nfsrv_maxpnfsmirror);
	/*
	* For all mirrors that couldn't be created, fill in the
	* *pf structure, but with an IP address == 0.0.0.0.
	*/
	tpf = pf + mirrorcnt;
	for (i = mirrorcnt; i < nfsrv_maxpnfsmirror; i++, tpf++) {
	tpf = pf;
	tpf->dsf_sin.sin_family = AF_INET;
	tpf->dsf_sin.sin_len = sizeof(struct sockaddr_in);
	tpf->dsf_sin.sin_addr.s_addr = 0;
	tpf->dsf_sin.sin_port = 0;
	}

	error = vn_extattr_set(vp, IO_NODELOCKED,
	EXTATTR_NAMESPACE_SYSTEM, "pnfsd.dsfile",
	sizeof(pf) nfsrv_maxpnfsmirror, (char *)pf, p);
	if (error == 0)
	error = vn_extattr_set(vp, IO_NODELOCKED,
	EXTATTR_NAMESPACE_SYSTEM, "pnfsd.dsattr",
	sizeof(dsattr), (char *)&dsattr, p);
	if (error != 0)
	printf("pNFS: pnfscreate setextattr=%d\n",
	error);
	} else
	printf("pNFS: pnfscreate=%d\n", error);
	free(pf, M_TEMP);
	free(dsc, M_TEMP);
	}

	/*
	* Get the information needed to remove the pNFS Data Server file from the
	* Metadata file. Upon success, ddvp is set non-NULL to the locked
	* DS directory vnode. The caller must unlock *ddvp when done with it.
	*/
	static void
	nfsrv_pnfsremovesetup(struct vnode vp, NFSPROC_T p, struct vnode **dvpp,
	int mirrorcntp, char fname, fhandle_t *fhp)
	{
	struct vattr va;
	struct ucred *tcred;
	char *buf;
	int buflen, error;

	dvpp[0] = NULL;
	/* If not an exported regular file or not a pNFS server, just return. */
	if (vp->v_type != VREG \|\| (vp->v_mount->mnt_flag & MNT_EXPORTED) == 0 \|\|
	nfsrv_devidcnt == 0)
	return;

	/* Check to see if this is the last hard link. */
	tcred = newnfs_getcred();
	error = VOP_GETATTR(vp, &va, tcred);
	NFSFREECRED(tcred);
	if (error != 0) {
	printf("pNFS: nfsrv_pnfsremovesetup getattr=%d\n", error);
	return;
	}
	if (va.va_nlink > 1)
	return;

	error = nfsvno_getfh(vp, fhp, p);
	if (error != 0) {
	printf("pNFS: nfsrv_pnfsremovesetup getfh=%d\n", error);
	return;
	}

	buflen = 1024;
	buf = malloc(buflen, M_TEMP, M_WAITOK);
	/* Get the directory vnode for the DS mount and the file handle. */
	error = nfsrv_dsgetsockmnt(vp, 0, buf, &buflen, mirrorcntp, p, dvpp,
	NULL, NULL, fname, NULL, NULL, NULL, NULL, NULL);
	free(buf, M_TEMP);
	if (error != 0)
	printf("pNFS: nfsrv_pnfsremovesetup getsockmnt=%d\n", error);
	}

	/*
	* Remove a DS data file for nfsrv_pnfsremove(). Called for each mirror.
	* The arguments are in a structure, so that they can be passed through
	* taskqueue for a kernel process to execute this function.
	*/
	struct nfsrvdsremove {
	int done;
	int inprog;
	struct task tsk;
	struct ucred *tcred;
	struct vnode *dvp;
	NFSPROC_T *p;
	int err;
	char fname[PNFS_FILENAME_LEN + 1];
	};

	static int
	nfsrv_dsremove(struct vnode dvp, char fname, struct ucred *tcred,
	NFSPROC_T *p)
	{
	struct nameidata named;
	struct vnode *nvp;
	char *bufp;
	u_long *hashp;
	int error;

	error = NFSVOPLOCK(dvp, LK_EXCLUSIVE);
	if (error != 0)
	return (error);
	named.ni_cnd.cn_nameiop = DELETE;
	named.ni_cnd.cn_lkflags = LK_EXCLUSIVE \| LK_RETRY;
	named.ni_cnd.cn_cred = tcred;
	named.ni_cnd.cn_thread = p;
	named.ni_cnd.cn_flags = ISLASTCN \| LOCKPARENT \| LOCKLEAF \| SAVENAME;
	nfsvno_setpathbuf(&named, &bufp, &hashp);
	named.ni_cnd.cn_nameptr = bufp;
	named.ni_cnd.cn_namelen = strlen(fname);
	strlcpy(bufp, fname, NAME_MAX);
	NFSD_DEBUG(4, "nfsrv_pnfsremove: filename=%s\n", bufp);
	error = VOP_LOOKUP(dvp, &nvp, &named.ni_cnd);
	NFSD_DEBUG(4, "nfsrv_pnfsremove: aft LOOKUP=%d\n", error);
	if (error == 0) {
	error = VOP_REMOVE(dvp, nvp, &named.ni_cnd);
	vput(nvp);
	}
	NFSVOPUNLOCK(dvp);
	nfsvno_relpathbuf(&named);
	if (error != 0)
	printf("pNFS: nfsrv_pnfsremove failed=%d\n", error);
	return (error);
	}

	/*
	* Start up the thread that will execute nfsrv_dsremove().
	*/
	static void
	start_dsremove(void *arg, int pending)
	{
	struct nfsrvdsremove *dsrm;

	dsrm = (struct nfsrvdsremove *)arg;
	dsrm->err = nfsrv_dsremove(dsrm->dvp, dsrm->fname, dsrm->tcred,
	dsrm->p);
	dsrm->done = 1;
	NFSD_DEBUG(4, "start_dsremove: err=%d\n", dsrm->err);
	}

	/*
	* Remove a pNFS data file from a Data Server.
	* nfsrv_pnfsremovesetup() must have been called before the MDS file was
	* removed to set up the dvp and fill in the FH.
	*/
	static void
	nfsrv_pnfsremove(struct vnode *dvp, int mirrorcnt, char fname, fhandle_t *fhp,
	NFSPROC_T *p)
	{
	struct ucred *tcred;
	struct nfsrvdsremove dsrm, tdsrm;
	struct nfsdevice *ds;
	struct nfsmount *nmp;
	int failpos, i, ret, timo;

	tcred = newnfs_getcred();
	dsrm = NULL;
	if (mirrorcnt > 1)
	dsrm = malloc(sizeof(dsrm) mirrorcnt - 1, M_TEMP, M_WAITOK);
	/*
	* Remove the file on each DS mirror, using kernel process(es) for the
	* additional mirrors.
	*/
	failpos = -1;
	for (tdsrm = dsrm, i = 0; i < mirrorcnt - 1; i++, tdsrm++) {
	tdsrm->tcred = tcred;
	tdsrm->p = p;
	tdsrm->dvp = dvp[i];
	strlcpy(tdsrm->fname, fname, PNFS_FILENAME_LEN + 1);
	tdsrm->inprog = 0;
	tdsrm->done = 0;
	tdsrm->err = 0;
	ret = EIO;
	if (nfs_pnfsiothreads != 0) {
	ret = nfs_pnfsio(start_dsremove, tdsrm);
	NFSD_DEBUG(4, "nfsrv_pnfsremove: nfs_pnfsio=%d\n", ret);
	}
	if (ret != 0) {
	ret = nfsrv_dsremove(dvp[i], fname, tcred, p);
	if (failpos == -1 && nfsds_failerr(ret))
	failpos = i;
	}
	}
	ret = nfsrv_dsremove(dvp[mirrorcnt - 1], fname, tcred, p);
	if (failpos == -1 && mirrorcnt > 1 && nfsds_failerr(ret))
	failpos = mirrorcnt - 1;
	timo = hz / 50; /* Wait for 20msec. */
	if (timo < 1)
	timo = 1;
	/* Wait for kernel task(s) to complete. */
	for (tdsrm = dsrm, i = 0; i < mirrorcnt - 1; i++, tdsrm++) {
	while (tdsrm->inprog != 0 && tdsrm->done == 0)
	tsleep(&tdsrm->tsk, PVFS, "srvdsrm", timo);
	if (failpos == -1 && nfsds_failerr(tdsrm->err))
	failpos = i;
	}

	/*
	* If failpos has been set, that mirror has failed, so it needs
	* to be disabled.
	*/
	if (failpos >= 0) {
	nmp = VFSTONFS(dvp[failpos]->v_mount);
	NFSLOCKMNT(nmp);
	if ((nmp->nm_privflag & (NFSMNTP_FORCEDISM \|
	NFSMNTP_CANCELRPCS)) == 0) {
	nmp->nm_privflag \|= NFSMNTP_CANCELRPCS;
	NFSUNLOCKMNT(nmp);
	ds = nfsrv_deldsnmp(PNFSDOP_DELDSSERVER, nmp, p);
	NFSD_DEBUG(4, "dsremovefail fail=%d ds=%p\n", failpos,
	ds);
	if (ds != NULL)
	nfsrv_killrpcs(nmp);
	NFSLOCKMNT(nmp);
	nmp->nm_privflag &= ~NFSMNTP_CANCELRPCS;
	wakeup(nmp);
	}
	NFSUNLOCKMNT(nmp);
	}

	/* Get rid all layouts for the file. */
	nfsrv_freefilelayouts(fhp);

	NFSFREECRED(tcred);
	free(dsrm, M_TEMP);
	}

	/*
	* Generate a file name based on the file handle and put it in *bufp.
	* Return the number of bytes generated.
	*/
	static int
	nfsrv_putfhname(fhandle_t fhp, char bufp)
	{
	int i;
	uint8_t *cp;
	const uint8_t *hexdigits = "0123456789abcdef";

	cp = (uint8_t *)fhp;
	for (i = 0; i < sizeof(*fhp); i++) {
	bufp[2 * i] = hexdigits[(*cp >> 4) & 0xf];
	bufp[2 * i + 1] = hexdigits[*cp++ & 0xf];
	}
	bufp[2 * i] = '\0';
	return (2 * i);
	}

	/*
	* Update the Metadata file's attributes from the DS file when a Read/Write
	* layout is returned.
	* Basically just call nfsrv_proxyds() with procedure == NFSPROC_LAYOUTRETURN
	* so that it does a nfsrv_getattrdsrpc() and nfsrv_setextattr() on the DS file.
	*/
	int
	nfsrv_updatemdsattr(struct vnode vp, struct nfsvattr nap, NFSPROC_T *p)
	{
	struct ucred *tcred;
	int error;

	/* Do this as root so that it won't fail with EACCES. */
	tcred = newnfs_getcred();
	error = nfsrv_proxyds(vp, 0, 0, tcred, p, NFSPROC_LAYOUTRETURN,
	NULL, NULL, NULL, nap, NULL, NULL, 0, NULL);
	NFSFREECRED(tcred);
	return (error);
	}

	/*
	* Set the NFSv4 ACL on the DS file to the same ACL as the MDS file.
	*/
	static int
	nfsrv_dssetacl(struct vnode vp, struct acl aclp, struct ucred *cred,
	NFSPROC_T *p)
	{
	int error;

	error = nfsrv_proxyds(vp, 0, 0, cred, p, NFSPROC_SETACL,
	NULL, NULL, NULL, NULL, aclp, NULL, 0, NULL);
	return (error);
	}

	static int
	nfsrv_proxyds(struct vnode vp, off_t off, int cnt, struct ucred cred,
	struct thread p, int ioproc, struct mbuf mpp, char cp,
	struct mbuf *mpp2, struct nfsvattr nap, struct acl *aclp,
	off_t offp, int content, bool eofp)
	{
	struct nfsmount nmp[NFSDEV_MAXMIRRORS], failnmp;
	fhandle_t fh[NFSDEV_MAXMIRRORS];
	struct vnode *dvp[NFSDEV_MAXMIRRORS];
	struct nfsdevice *ds;
	struct pnfsdsattr dsattr;
	struct opnfsdsattr odsattr;
	char *buf;
	int buflen, error, failpos, i, mirrorcnt, origmircnt, trycnt;

	NFSD_DEBUG(4, "in nfsrv_proxyds\n");
	/*
	* If not a regular file, not exported or not a pNFS server,
	* just return ENOENT.
	*/
	if (vp->v_type != VREG \|\| (vp->v_mount->mnt_flag & MNT_EXPORTED) == 0 \|\|
	nfsrv_devidcnt == 0)
	return (ENOENT);

	buflen = 1024;
	buf = malloc(buflen, M_TEMP, M_WAITOK);
	error = 0;

	/*
	* For Getattr, get the Change attribute (va_filerev) and size (va_size)
	* from the MetaData file's extended attribute.
	*/
	if (ioproc == NFSPROC_GETATTR) {
	error = vn_extattr_get(vp, IO_NODELOCKED,
	EXTATTR_NAMESPACE_SYSTEM, "pnfsd.dsattr", &buflen, buf,
	p);
	if (error == 0) {
	if (buflen == sizeof(odsattr)) {
	NFSBCOPY(buf, &odsattr, buflen);
	nap->na_filerev = odsattr.dsa_filerev;
	nap->na_size = odsattr.dsa_size;
	nap->na_atime = odsattr.dsa_atime;
	nap->na_mtime = odsattr.dsa_mtime;
	/*
	* Fake na_bytes by rounding up na_size.
	* Since we don't know the block size, just
	* use BLKDEV_IOSIZE.
	*/
	nap->na_bytes = (odsattr.dsa_size +
	BLKDEV_IOSIZE - 1) & ~(BLKDEV_IOSIZE - 1);
	} else if (buflen == sizeof(dsattr)) {
	NFSBCOPY(buf, &dsattr, buflen);
	nap->na_filerev = dsattr.dsa_filerev;
	nap->na_size = dsattr.dsa_size;
	nap->na_atime = dsattr.dsa_atime;
	nap->na_mtime = dsattr.dsa_mtime;
	nap->na_bytes = dsattr.dsa_bytes;
	} else
	error = ENXIO;
	}
	if (error == 0) {
	/*
	* If nfsrv_pnfsgetdsattr is 0 or nfsrv_checkdsattr()
	* returns 0, just return now. nfsrv_checkdsattr()
	* returns 0 if there is no Read/Write layout
	* plus either an Open/Write_access or Write
	* delegation issued to a client for the file.
	*/
	if (nfsrv_pnfsgetdsattr == 0 \|\|
	nfsrv_checkdsattr(vp, p) == 0) {
	free(buf, M_TEMP);
	return (error);
	}
	}

	/*
	* Clear ENOATTR so the code below will attempt to do a
	* nfsrv_getattrdsrpc() to get the attributes and (re)create
	* the extended attribute.
	*/
	if (error == ENOATTR)
	error = 0;
	}

	origmircnt = -1;
	trycnt = 0;
	tryagain:
	if (error == 0) {
	buflen = 1024;
	if (ioproc == NFSPROC_READDS && NFSVOPISLOCKED(vp) ==
	LK_EXCLUSIVE)
	printf("nfsrv_proxyds: Readds vp exclusively locked\n");
	error = nfsrv_dsgetsockmnt(vp, LK_SHARED, buf, &buflen,
	&mirrorcnt, p, dvp, fh, NULL, NULL, NULL, NULL, NULL,
	NULL, NULL);
	if (error == 0) {
	for (i = 0; i < mirrorcnt; i++)
	nmp[i] = VFSTONFS(dvp[i]->v_mount);
	} else
	printf("pNFS: proxy getextattr sockaddr=%d\n", error);
	} else
	printf("pNFS: nfsrv_dsgetsockmnt=%d\n", error);
	if (error == 0) {
	failpos = -1;
	if (origmircnt == -1)
	origmircnt = mirrorcnt;
	/*
	* If failpos is set to a mirror#, then that mirror has
	* failed and will be disabled. For Read, Getattr and Seek, the
	* function only tries one mirror, so if that mirror has
	* failed, it will need to be retried. As such, increment
	* tryitagain for these cases.
	* For Write, Setattr and Setacl, the function tries all
	* mirrors and will not return an error for the case where
	* one mirror has failed. For these cases, the functioning
	* mirror(s) will have been modified, so a retry isn't
	* necessary. These functions will set failpos for the
	* failed mirror#.
	*/
	if (ioproc == NFSPROC_READDS) {
	error = nfsrv_readdsrpc(fh, off, cnt, cred, p, nmp[0],
	mpp, mpp2);
	if (nfsds_failerr(error) && mirrorcnt > 1) {
	/*
	* Setting failpos will cause the mirror
	* to be disabled and then a retry of this
	* read is required.
	*/
	failpos = 0;
	error = 0;
	trycnt++;
	}
	} else if (ioproc == NFSPROC_WRITEDS)
	error = nfsrv_writedsrpc(fh, off, cnt, cred, p, vp,
	&nmp[0], mirrorcnt, mpp, cp, &failpos);
	else if (ioproc == NFSPROC_SETATTR)
	error = nfsrv_setattrdsrpc(fh, cred, p, vp, &nmp[0],
	mirrorcnt, nap, &failpos);
	else if (ioproc == NFSPROC_SETACL)
	error = nfsrv_setacldsrpc(fh, cred, p, vp, &nmp[0],
	mirrorcnt, aclp, &failpos);
	else if (ioproc == NFSPROC_SEEKDS) {
	error = nfsrv_seekdsrpc(fh, offp, content, eofp, cred,
	p, nmp[0]);
	if (nfsds_failerr(error) && mirrorcnt > 1) {
	/*
	* Setting failpos will cause the mirror
	* to be disabled and then a retry of this
	* read is required.
	*/
	failpos = 0;
	error = 0;
	trycnt++;
	}
	} else if (ioproc == NFSPROC_ALLOCATE)
	error = nfsrv_allocatedsrpc(fh, off, *offp, cred, p, vp,
	&nmp[0], mirrorcnt, &failpos);
	else {
	error = nfsrv_getattrdsrpc(&fh[mirrorcnt - 1], cred, p,
	vp, nmp[mirrorcnt - 1], nap);
	if (nfsds_failerr(error) && mirrorcnt > 1) {
	/*
	* Setting failpos will cause the mirror
	* to be disabled and then a retry of this
	* getattr is required.
	*/
	failpos = mirrorcnt - 1;
	error = 0;
	trycnt++;
	}
	}
	ds = NULL;
	if (failpos >= 0) {
	failnmp = nmp[failpos];
	NFSLOCKMNT(failnmp);
	if ((failnmp->nm_privflag & (NFSMNTP_FORCEDISM \|
	NFSMNTP_CANCELRPCS)) == 0) {
	failnmp->nm_privflag \|= NFSMNTP_CANCELRPCS;
	NFSUNLOCKMNT(failnmp);
	ds = nfsrv_deldsnmp(PNFSDOP_DELDSSERVER,
	failnmp, p);
	NFSD_DEBUG(4, "dsldsnmp fail=%d ds=%p\n",
	failpos, ds);
	if (ds != NULL)
	nfsrv_killrpcs(failnmp);
	NFSLOCKMNT(failnmp);
	failnmp->nm_privflag &= ~NFSMNTP_CANCELRPCS;
	wakeup(failnmp);
	}
	NFSUNLOCKMNT(failnmp);
	}
	for (i = 0; i < mirrorcnt; i++)
	NFSVOPUNLOCK(dvp[i]);
	NFSD_DEBUG(4, "nfsrv_proxyds: aft RPC=%d trya=%d\n", error,
	trycnt);
	/* Try the Read/Getattr again if a mirror was deleted. */
	if (ds != NULL && trycnt > 0 && trycnt < origmircnt)
	goto tryagain;
	} else {
	/* Return ENOENT for any Extended Attribute error. */
	error = ENOENT;
	}
	free(buf, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_proxyds: error=%d\n", error);
	return (error);
	}

	/*
	* Get the DS mount point, fh and directory from the "pnfsd.dsfile" extended
	* attribute.
	* newnmpp - If it points to a non-NULL nmp, that is the destination and needs
	* to be checked. If it points to a NULL nmp, then it returns
	* a suitable destination.
	* curnmp - If non-NULL, it is the source mount for the copy.
	*/
	int
	nfsrv_dsgetsockmnt(struct vnode vp, int lktype, char buf, int *buflenp,
	int mirrorcntp, NFSPROC_T p, struct vnode *dvpp, fhandle_t fhp,
	char devid, char fnamep, struct vnode nvpp, struct nfsmount newnmpp,
	struct nfsmount curnmp, int ippos, int *dsdirp)
	{
	struct vnode dvp, nvp = NULL, **tdvpp;
	struct mount *mp;
	struct nfsmount nmp, newnmp;
	struct sockaddr *sad;
	struct sockaddr_in *sin;
	struct nfsdevice ds, tds, *fndds;
	struct pnfsdsfile *pf;
	uint32_t dsdir;
	int error, fhiszero, fnd, gotone, i, mirrorcnt;

	ASSERT_VOP_LOCKED(vp, "nfsrv_dsgetsockmnt vp");
	*mirrorcntp = 1;
	tdvpp = dvpp;
	if (nvpp != NULL)
	*nvpp = NULL;
	if (dvpp != NULL)
	*dvpp = NULL;
	if (ippos != NULL)
	*ippos = -1;
	if (newnmpp != NULL)
	newnmp = *newnmpp;
	else
	newnmp = NULL;
	mp = vp->v_mount;
	error = vn_extattr_get(vp, IO_NODELOCKED, EXTATTR_NAMESPACE_SYSTEM,
	"pnfsd.dsfile", buflenp, buf, p);
	mirrorcnt = buflenp / sizeof(pf);
	if (error == 0 && (mirrorcnt < 1 \|\| mirrorcnt > NFSDEV_MAXMIRRORS \|\|
	buflenp != sizeof(pf) * mirrorcnt))
	error = ENOATTR;

	pf = (struct pnfsdsfile *)buf;
	/* If curnmp != NULL, check for a match in the mirror list. */
	if (curnmp != NULL && error == 0) {
	fnd = 0;
	for (i = 0; i < mirrorcnt; i++, pf++) {
	sad = (struct sockaddr *)&pf->dsf_sin;
	if (nfsaddr2_match(sad, curnmp->nm_nam)) {
	if (ippos != NULL)
	*ippos = i;
	fnd = 1;
	break;
	}
	}
	if (fnd == 0)
	error = ENXIO;
	}

	gotone = 0;
	pf = (struct pnfsdsfile *)buf;
	NFSD_DEBUG(4, "nfsrv_dsgetsockmnt: mirrorcnt=%d err=%d\n", mirrorcnt,
	error);
	for (i = 0; i < mirrorcnt && error == 0; i++, pf++) {
	fhiszero = 0;
	sad = (struct sockaddr *)&pf->dsf_sin;
	sin = &pf->dsf_sin;
	dsdir = pf->dsf_dir;
	if (dsdir >= nfsrv_dsdirsize) {
	printf("nfsrv_dsgetsockmnt: dsdir=%d\n", dsdir);
	error = ENOATTR;
	} else if (nvpp != NULL && newnmp != NULL &&
	nfsaddr2_match(sad, newnmp->nm_nam))
	error = EEXIST;
	if (error == 0) {
	if (ippos != NULL && curnmp == NULL &&
	sad->sa_family == AF_INET &&
	sin->sin_addr.s_addr == 0)
	*ippos = i;
	if (NFSBCMP(&zerofh, &pf->dsf_fh, sizeof(zerofh)) == 0)
	fhiszero = 1;
	/* Use the socket address to find the mount point. */
	fndds = NULL;
	NFSDDSLOCK();
	/* Find a match for the IP address. */
	TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) {
	if (ds->nfsdev_nmp != NULL) {
	dvp = ds->nfsdev_dvp;
	nmp = VFSTONFS(dvp->v_mount);
	if (nmp != ds->nfsdev_nmp)
	printf("different2 nmp %p %p\n",
	nmp, ds->nfsdev_nmp);
	if (nfsaddr2_match(sad, nmp->nm_nam)) {
	fndds = ds;
	break;
	}
	}
	}
	if (fndds != NULL && newnmpp != NULL &&
	newnmp == NULL) {
	/* Search for a place to make a mirror copy. */
	TAILQ_FOREACH(tds, &nfsrv_devidhead,
	nfsdev_list) {
	if (tds->nfsdev_nmp != NULL &&
	fndds != tds &&
	((tds->nfsdev_mdsisset == 0 &&
	fndds->nfsdev_mdsisset == 0) \|\|
	(tds->nfsdev_mdsisset != 0 &&
	fndds->nfsdev_mdsisset != 0 &&
	fsidcmp(&tds->nfsdev_mdsfsid,
	&mp->mnt_stat.f_fsid) == 0))) {
	*newnmpp = tds->nfsdev_nmp;
	break;
	}
	}
	if (tds != NULL) {
	/*
	* Move this entry to the end of the
	* list, so it won't be selected as
	* easily the next time.
	*/
	TAILQ_REMOVE(&nfsrv_devidhead, tds,
	nfsdev_list);
	TAILQ_INSERT_TAIL(&nfsrv_devidhead, tds,
	nfsdev_list);
	}
	}
	NFSDDSUNLOCK();
	if (fndds != NULL) {
	dvp = fndds->nfsdev_dsdir[dsdir];
	if (lktype != 0 \|\| fhiszero != 0 \|\|
	(nvpp != NULL && *nvpp == NULL)) {
	if (fhiszero != 0)
	error = vn_lock(dvp,
	LK_EXCLUSIVE);
	else if (lktype != 0)
	error = vn_lock(dvp, lktype);
	else
	error = vn_lock(dvp, LK_SHARED);
	/*
	* If the file handle is all 0's, try to
	* do a Lookup against the DS to acquire
	* it.
	* If dvpp == NULL or the Lookup fails,
	* unlock dvp after the call.
	*/
	if (error == 0 && (fhiszero != 0 \|\|
	(nvpp != NULL && *nvpp == NULL))) {
	error = nfsrv_pnfslookupds(vp,
	dvp, pf, &nvp, p);
	if (error == 0) {
	if (fhiszero != 0)
	nfsrv_pnfssetfh(
	vp, pf,
	devid,
	fnamep,
	nvp, p);
	if (nvpp != NULL &&
	*nvpp == NULL) {
	*nvpp = nvp;
	*dsdirp = dsdir;
	} else
	vput(nvp);
	}
	if (error != 0 \|\| lktype == 0)
	NFSVOPUNLOCK(dvp);
	}
	}
	if (error == 0) {
	gotone++;
	NFSD_DEBUG(4, "gotone=%d\n", gotone);
	if (devid != NULL) {
	NFSBCOPY(fndds->nfsdev_deviceid,
	devid, NFSX_V4DEVICEID);
	devid += NFSX_V4DEVICEID;
	}
	if (dvpp != NULL)
	*tdvpp++ = dvp;
	if (fhp != NULL)
	NFSBCOPY(&pf->dsf_fh, fhp++,
	NFSX_MYFH);
	if (fnamep != NULL && gotone == 1)
	strlcpy(fnamep,
	pf->dsf_filename,
	sizeof(pf->dsf_filename));
	} else
	NFSD_DEBUG(4, "nfsrv_dsgetsockmnt "
	"err=%d\n", error);
	}
	}
	}
	if (error == 0 && gotone == 0)
	error = ENOENT;

	NFSD_DEBUG(4, "eo nfsrv_dsgetsockmnt: gotone=%d err=%d\n", gotone,
	error);
	if (error == 0)
	*mirrorcntp = gotone;
	else {
	if (gotone > 0 && dvpp != NULL) {
	/*
	* If the error didn't occur on the first one and
	* dvpp != NULL, the one(s) prior to the failure will
	* have locked dvp's that need to be unlocked.
	*/
	for (i = 0; i < gotone; i++) {
	NFSVOPUNLOCK(*dvpp);
	*dvpp++ = NULL;
	}
	}
	/*
	* If it found the vnode to be copied from before a failure,
	* it needs to be vput()'d.
	*/
	if (nvpp != NULL && *nvpp != NULL) {
	vput(*nvpp);
	*nvpp = NULL;
	}
	}
	return (error);
	}

	/*
	* Set the extended attribute for the Change attribute.
	*/
	static int
	nfsrv_setextattr(struct vnode vp, struct nfsvattr nap, NFSPROC_T *p)
	{
	struct pnfsdsattr dsattr;
	int error;

	ASSERT_VOP_ELOCKED(vp, "nfsrv_setextattr vp");
	dsattr.dsa_filerev = nap->na_filerev;
	dsattr.dsa_size = nap->na_size;
	dsattr.dsa_atime = nap->na_atime;
	dsattr.dsa_mtime = nap->na_mtime;
	dsattr.dsa_bytes = nap->na_bytes;
	error = vn_extattr_set(vp, IO_NODELOCKED, EXTATTR_NAMESPACE_SYSTEM,
	"pnfsd.dsattr", sizeof(dsattr), (char *)&dsattr, p);
	if (error != 0)
	printf("pNFS: setextattr=%d\n", error);
	return (error);
	}

	static int
	nfsrv_readdsrpc(fhandle_t fhp, off_t off, int len, struct ucred cred,
	NFSPROC_T p, struct nfsmount nmp, struct mbuf mpp, struct mbuf mpendp)
	{
	uint32_t *tl;
	struct nfsrv_descript *nd;
	nfsv4stateid_t st;
	struct mbuf m, m2;
	int error = 0, retlen, tlen, trimlen;

	NFSD_DEBUG(4, "in nfsrv_readdsrpc\n");
	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	*mpp = NULL;
	/*
	* Use a stateid where other is an alternating 01010 pattern and
	* seqid is 0xffffffff. This value is not defined as special by
	* the RFC and is used by the FreeBSD NFS server to indicate an
	* MDS->DS proxy operation.
	*/
	st.other[0] = 0x55555555;
	st.other[1] = 0x55555555;
	st.other[2] = 0x55555555;
	st.seqid = 0xffffffff;
	nfscl_reqstart(nd, NFSPROC_READDS, nmp, (u_int8_t )fhp, sizeof(fhp),
	NULL, NULL, 0, 0, false);
	nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID);
	NFSM_BUILD(tl, uint32_t , NFSX_UNSIGNED 3);
	txdr_hyper(off, tl);
	*(tl + 2) = txdr_unsigned(len);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred,
	NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	if (nd->nd_repstat == 0) {
	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
	NFSM_STRSIZ(retlen, len);
	if (retlen > 0) {
	/* Trim off the pre-data XDR from the mbuf chain. */
	m = nd->nd_mrep;
	while (m != NULL && m != nd->nd_md) {
	if (m->m_next == nd->nd_md) {
	m->m_next = NULL;
	m_freem(nd->nd_mrep);
	nd->nd_mrep = m = nd->nd_md;
	} else
	m = m->m_next;
	}
	if (m == NULL) {
	printf("nfsrv_readdsrpc: busted mbuf list\n");
	error = ENOENT;
	goto nfsmout;
	}

	/*
	* Now, adjust first mbuf so that any XDR before the
	* read data is skipped over.
	*/
	trimlen = nd->nd_dpos - mtod(m, char *);
	if (trimlen > 0) {
	m->m_len -= trimlen;
	NFSM_DATAP(m, trimlen);
	}

	/*
	* Truncate the mbuf chain at retlen bytes of data,
	* plus XDR padding that brings the length up to a
	* multiple of 4.
	*/
	tlen = NFSM_RNDUP(retlen);
	do {
	if (m->m_len >= tlen) {
	m->m_len = tlen;
	tlen = 0;
	m2 = m->m_next;
	m->m_next = NULL;
	m_freem(m2);
	break;
	}
	tlen -= m->m_len;
	m = m->m_next;
	} while (m != NULL);
	if (tlen > 0) {
	printf("nfsrv_readdsrpc: busted mbuf list\n");
	error = ENOENT;
	goto nfsmout;
	}
	*mpp = nd->nd_mrep;
	*mpendp = m;
	nd->nd_mrep = NULL;
	}
	} else
	error = nd->nd_repstat;
	nfsmout:
	/* If nd->nd_mrep is already NULL, this is a no-op. */
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_readdsrpc error=%d\n", error);
	return (error);
	}

	/*
	* Do a write RPC on a DS data file, using this structure for the arguments,
	* so that this function can be executed by a separate kernel process.
	*/
	struct nfsrvwritedsdorpc {
	int done;
	int inprog;
	struct task tsk;
	fhandle_t fh;
	off_t off;
	int len;
	struct nfsmount *nmp;
	struct ucred *cred;
	NFSPROC_T *p;
	struct mbuf *m;
	int err;
	};

	static int
	nfsrv_writedsdorpc(struct nfsmount nmp, fhandle_t fhp, off_t off, int len,
	struct nfsvattr nap, struct mbuf m, struct ucred cred, NFSPROC_T p)
	{
	uint32_t *tl;
	struct nfsrv_descript *nd;
	nfsattrbit_t attrbits;
	nfsv4stateid_t st;
	int commit, error, retlen;

	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	nfscl_reqstart(nd, NFSPROC_WRITE, nmp, (u_int8_t *)fhp,
	sizeof(fhandle_t), NULL, NULL, 0, 0, false);

	/*
	* Use a stateid where other is an alternating 01010 pattern and
	* seqid is 0xffffffff. This value is not defined as special by
	* the RFC and is used by the FreeBSD NFS server to indicate an
	* MDS->DS proxy operation.
	*/
	st.other[0] = 0x55555555;
	st.other[1] = 0x55555555;
	st.other[2] = 0x55555555;
	st.seqid = 0xffffffff;
	nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID);
	NFSM_BUILD(tl, u_int32_t , NFSX_HYPER + 2 NFSX_UNSIGNED);
	txdr_hyper(off, tl);
	tl += 2;
	/*
	* Do all writes FileSync, since the server doesn't hold onto dirty
	* buffers. Since clients should be accessing the DS servers directly
	* using the pNFS layouts, this just needs to work correctly as a
	* fallback.
	*/
	*tl++ = txdr_unsigned(NFSWRITE_FILESYNC);
	*tl = txdr_unsigned(len);
	NFSD_DEBUG(4, "nfsrv_writedsdorpc: len=%d\n", len);

	/* Put data in mbuf chain. */
	nd->nd_mb->m_next = m;

	/* Set nd_mb and nd_bpos to end of data. */
	while (m->m_next != NULL)
	m = m->m_next;
	nd->nd_mb = m;
	nfsm_set(nd, m->m_len);
	NFSD_DEBUG(4, "nfsrv_writedsdorpc: lastmb len=%d\n", m->m_len);

	/* Do a Getattr for the attributes that change upon writing. */
	NFSZERO_ATTRBIT(&attrbits);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SIZE);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_CHANGE);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEACCESS);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEMODIFY);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SPACEUSED);
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4OP_GETATTR);
	(void) nfsrv_putattrbit(nd, &attrbits);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p,
	cred, NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	NFSD_DEBUG(4, "nfsrv_writedsdorpc: aft writerpc=%d\n", nd->nd_repstat);
	/* Get rid of weak cache consistency data for now. */
	if ((nd->nd_flag & (ND_NOMOREDATA \| ND_NFSV4 \| ND_V4WCCATTR)) ==
	(ND_NFSV4 \| ND_V4WCCATTR)) {
	error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL,
	NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL);
	NFSD_DEBUG(4, "nfsrv_writedsdorpc: wcc attr=%d\n", error);
	if (error != 0)
	goto nfsmout;
	/*
	* Get rid of Op# and status for next op.
	*/
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED);
	if (*++tl != 0)
	nd->nd_flag \|= ND_NOMOREDATA;
	}
	if (nd->nd_repstat == 0) {
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED + NFSX_VERF);
	retlen = fxdr_unsigned(int, *tl++);
	commit = fxdr_unsigned(int, *tl);
	if (commit != NFSWRITE_FILESYNC)
	error = NFSERR_IO;
	NFSD_DEBUG(4, "nfsrv_writedsdorpc:retlen=%d commit=%d err=%d\n",
	retlen, commit, error);
	} else
	error = nd->nd_repstat;
	/* We have no use for the Write Verifier since we use FileSync. */

	/*
	* Get the Change, Size, Access Time and Modify Time attributes and set
	* on the Metadata file, so its attributes will be what the file's
	* would be if it had been written.
	*/
	if (error == 0) {
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED);
	error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL,
	NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL);
	}
	NFSD_DEBUG(4, "nfsrv_writedsdorpc: aft loadattr=%d\n", error);
	nfsmout:
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_writedsdorpc error=%d\n", error);
	return (error);
	}

	/*
	* Start up the thread that will execute nfsrv_writedsdorpc().
	*/
	static void
	start_writedsdorpc(void *arg, int pending)
	{
	struct nfsrvwritedsdorpc *drpc;

	drpc = (struct nfsrvwritedsdorpc *)arg;
	drpc->err = nfsrv_writedsdorpc(drpc->nmp, &drpc->fh, drpc->off,
	drpc->len, NULL, drpc->m, drpc->cred, drpc->p);
	drpc->done = 1;
	NFSD_DEBUG(4, "start_writedsdorpc: err=%d\n", drpc->err);
	}

	static int
	nfsrv_writedsrpc(fhandle_t fhp, off_t off, int len, struct ucred cred,
	NFSPROC_T p, struct vnode vp, struct nfsmount **nmpp, int mirrorcnt,
	struct mbuf *mpp, char cp, int *failposp)
	{
	struct nfsrvwritedsdorpc drpc, tdrpc = NULL;
	struct nfsvattr na;
	struct mbuf *m;
	int error, i, offs, ret, timo;

	NFSD_DEBUG(4, "in nfsrv_writedsrpc\n");
	KASSERT(*mpp != NULL, ("nfsrv_writedsrpc: NULL mbuf chain"));
	drpc = NULL;
	if (mirrorcnt > 1)
	tdrpc = drpc = malloc(sizeof(drpc) (mirrorcnt - 1), M_TEMP,
	M_WAITOK);

	/* Calculate offset in mbuf chain that data starts. */
	offs = cp - mtod(mpp, char );
	NFSD_DEBUG(4, "nfsrv_writedsrpc: mcopy offs=%d len=%d\n", offs, len);

	/*
	* Do the write RPC for every DS, using a separate kernel process
	* for every DS except the last one.
	*/
	error = 0;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	tdrpc->done = 0;
	NFSBCOPY(fhp, &tdrpc->fh, sizeof(*fhp));
	tdrpc->off = off;
	tdrpc->len = len;
	tdrpc->nmp = *nmpp;
	tdrpc->cred = cred;
	tdrpc->p = p;
	tdrpc->inprog = 0;
	tdrpc->err = 0;
	tdrpc->m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK);
	ret = EIO;
	if (nfs_pnfsiothreads != 0) {
	ret = nfs_pnfsio(start_writedsdorpc, tdrpc);
	NFSD_DEBUG(4, "nfsrv_writedsrpc: nfs_pnfsio=%d\n",
	ret);
	}
	if (ret != 0) {
	ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, NULL,
	tdrpc->m, cred, p);
	if (nfsds_failerr(ret) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && ret != 0)
	error = ret;
	}
	nmpp++;
	fhp++;
	}
	m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK);
	ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, &na, m, cred, p);
	if (nfsds_failerr(ret) && *failposp == -1 && mirrorcnt > 1)
	*failposp = mirrorcnt - 1;
	else if (error == 0 && ret != 0)
	error = ret;
	if (error == 0)
	error = nfsrv_setextattr(vp, &na, p);
	NFSD_DEBUG(4, "nfsrv_writedsrpc: aft setextat=%d\n", error);
	tdrpc = drpc;
	timo = hz / 50; /* Wait for 20msec. */
	if (timo < 1)
	timo = 1;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	/* Wait for RPCs on separate threads to complete. */
	while (tdrpc->inprog != 0 && tdrpc->done == 0)
	tsleep(&tdrpc->tsk, PVFS, "srvwrds", timo);
	if (nfsds_failerr(tdrpc->err) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && tdrpc->err != 0)
	error = tdrpc->err;
	}
	free(drpc, M_TEMP);
	return (error);
	}

	/*
	* Do a allocate RPC on a DS data file, using this structure for the arguments,
	* so that this function can be executed by a separate kernel process.
	*/
	struct nfsrvallocatedsdorpc {
	int done;
	int inprog;
	struct task tsk;
	fhandle_t fh;
	off_t off;
	off_t len;
	struct nfsmount *nmp;
	struct ucred *cred;
	NFSPROC_T *p;
	int err;
	};

	static int
	nfsrv_allocatedsdorpc(struct nfsmount nmp, fhandle_t fhp, off_t off,
	off_t len, struct nfsvattr nap, struct ucred cred, NFSPROC_T *p)
	{
	uint32_t *tl;
	struct nfsrv_descript *nd;
	nfsattrbit_t attrbits;
	nfsv4stateid_t st;
	int error;

	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	nfscl_reqstart(nd, NFSPROC_ALLOCATE, nmp, (u_int8_t *)fhp,
	sizeof(fhandle_t), NULL, NULL, 0, 0, false);

	/*
	* Use a stateid where other is an alternating 01010 pattern and
	* seqid is 0xffffffff. This value is not defined as special by
	* the RFC and is used by the FreeBSD NFS server to indicate an
	* MDS->DS proxy operation.
	*/
	st.other[0] = 0x55555555;
	st.other[1] = 0x55555555;
	st.other[2] = 0x55555555;
	st.seqid = 0xffffffff;
	nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID);
	NFSM_BUILD(tl, uint32_t , 2 NFSX_HYPER + NFSX_UNSIGNED);
	txdr_hyper(off, tl); tl += 2;
	txdr_hyper(len, tl); tl += 2;
	NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: len=%jd\n", (intmax_t)len);

	*tl = txdr_unsigned(NFSV4OP_GETATTR);
	NFSGETATTR_ATTRBIT(&attrbits);
	nfsrv_putattrbit(nd, &attrbits);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p,
	cred, NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: aft allocaterpc=%d\n",
	nd->nd_repstat);
	if (nd->nd_repstat == 0) {
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED);
	error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0, NULL, NULL,
	NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL);
	} else
	error = nd->nd_repstat;
	NFSD_DEBUG(4, "nfsrv_allocatedsdorpc: aft loadattr=%d\n", error);
	nfsmout:
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_allocatedsdorpc error=%d\n", error);
	return (error);
	}

	/*
	* Start up the thread that will execute nfsrv_allocatedsdorpc().
	*/
	static void
	start_allocatedsdorpc(void *arg, int pending)
	{
	struct nfsrvallocatedsdorpc *drpc;

	drpc = (struct nfsrvallocatedsdorpc *)arg;
	drpc->err = nfsrv_allocatedsdorpc(drpc->nmp, &drpc->fh, drpc->off,
	drpc->len, NULL, drpc->cred, drpc->p);
	drpc->done = 1;
	NFSD_DEBUG(4, "start_allocatedsdorpc: err=%d\n", drpc->err);
	}

	static int
	nfsrv_allocatedsrpc(fhandle_t fhp, off_t off, off_t len, struct ucred cred,
	NFSPROC_T p, struct vnode vp, struct nfsmount **nmpp, int mirrorcnt,
	int *failposp)
	{
	struct nfsrvallocatedsdorpc drpc, tdrpc = NULL;
	struct nfsvattr na;
	int error, i, ret, timo;

	NFSD_DEBUG(4, "in nfsrv_allocatedsrpc\n");
	drpc = NULL;
	if (mirrorcnt > 1)
	tdrpc = drpc = malloc(sizeof(drpc) (mirrorcnt - 1), M_TEMP,
	M_WAITOK);

	/*
	* Do the allocate RPC for every DS, using a separate kernel process
	* for every DS except the last one.
	*/
	error = 0;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	tdrpc->done = 0;
	NFSBCOPY(fhp, &tdrpc->fh, sizeof(*fhp));
	tdrpc->off = off;
	tdrpc->len = len;
	tdrpc->nmp = *nmpp;
	tdrpc->cred = cred;
	tdrpc->p = p;
	tdrpc->inprog = 0;
	tdrpc->err = 0;
	ret = EIO;
	if (nfs_pnfsiothreads != 0) {
	ret = nfs_pnfsio(start_allocatedsdorpc, tdrpc);
	NFSD_DEBUG(4, "nfsrv_allocatedsrpc: nfs_pnfsio=%d\n",
	ret);
	}
	if (ret != 0) {
	ret = nfsrv_allocatedsdorpc(*nmpp, fhp, off, len, NULL,
	cred, p);
	if (nfsds_failerr(ret) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && ret != 0)
	error = ret;
	}
	nmpp++;
	fhp++;
	}
	ret = nfsrv_allocatedsdorpc(*nmpp, fhp, off, len, &na, cred, p);
	if (nfsds_failerr(ret) && *failposp == -1 && mirrorcnt > 1)
	*failposp = mirrorcnt - 1;
	else if (error == 0 && ret != 0)
	error = ret;
	if (error == 0)
	error = nfsrv_setextattr(vp, &na, p);
	NFSD_DEBUG(4, "nfsrv_allocatedsrpc: aft setextat=%d\n", error);
	tdrpc = drpc;
	timo = hz / 50; /* Wait for 20msec. */
	if (timo < 1)
	timo = 1;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	/* Wait for RPCs on separate threads to complete. */
	while (tdrpc->inprog != 0 && tdrpc->done == 0)
	tsleep(&tdrpc->tsk, PVFS, "srvalds", timo);
	if (nfsds_failerr(tdrpc->err) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && tdrpc->err != 0)
	error = tdrpc->err;
	}
	free(drpc, M_TEMP);
	return (error);
	}

	static int
	nfsrv_setattrdsdorpc(fhandle_t fhp, struct ucred cred, NFSPROC_T *p,
	struct vnode vp, struct nfsmount nmp, struct nfsvattr *nap,
	struct nfsvattr *dsnap)
	{
	uint32_t *tl;
	struct nfsrv_descript *nd;
	nfsv4stateid_t st;
	nfsattrbit_t attrbits;
	int error;

	NFSD_DEBUG(4, "in nfsrv_setattrdsdorpc\n");
	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	/*
	* Use a stateid where other is an alternating 01010 pattern and
	* seqid is 0xffffffff. This value is not defined as special by
	* the RFC and is used by the FreeBSD NFS server to indicate an
	* MDS->DS proxy operation.
	*/
	st.other[0] = 0x55555555;
	st.other[1] = 0x55555555;
	st.other[2] = 0x55555555;
	st.seqid = 0xffffffff;
	nfscl_reqstart(nd, NFSPROC_SETATTR, nmp, (u_int8_t )fhp, sizeof(fhp),
	NULL, NULL, 0, 0, false);
	nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID);
	nfscl_fillsattr(nd, &nap->na_vattr, vp, NFSSATTR_FULL, 0);

	/* Do a Getattr for the attributes that change due to writing. */
	NFSZERO_ATTRBIT(&attrbits);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SIZE);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_CHANGE);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEACCESS);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEMODIFY);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SPACEUSED);
	NFSM_BUILD(tl, u_int32_t *, NFSX_UNSIGNED);
	*tl = txdr_unsigned(NFSV4OP_GETATTR);
	(void) nfsrv_putattrbit(nd, &attrbits);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred,
	NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: aft setattrrpc=%d\n",
	nd->nd_repstat);
	/* Get rid of weak cache consistency data for now. */
	if ((nd->nd_flag & (ND_NOMOREDATA \| ND_NFSV4 \| ND_V4WCCATTR)) ==
	(ND_NFSV4 \| ND_V4WCCATTR)) {
	error = nfsv4_loadattr(nd, NULL, dsnap, NULL, NULL, 0, NULL,
	NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL);
	NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: wcc attr=%d\n", error);
	if (error != 0)
	goto nfsmout;
	/*
	* Get rid of Op# and status for next op.
	*/
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED);
	if (*++tl != 0)
	nd->nd_flag \|= ND_NOMOREDATA;
	}
	error = nfsrv_getattrbits(nd, &attrbits, NULL, NULL);
	if (error != 0)
	goto nfsmout;
	if (nd->nd_repstat != 0)
	error = nd->nd_repstat;
	/*
	* Get the Change, Size, Access Time and Modify Time attributes and set
	* on the Metadata file, so its attributes will be what the file's
	* would be if it had been written.
	*/
	if (error == 0) {
	NFSM_DISSECT(tl, uint32_t , 2 NFSX_UNSIGNED);
	error = nfsv4_loadattr(nd, NULL, dsnap, NULL, NULL, 0, NULL,
	NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL);
	}
	NFSD_DEBUG(4, "nfsrv_setattrdsdorpc: aft setattr loadattr=%d\n", error);
	nfsmout:
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_setattrdsdorpc error=%d\n", error);
	return (error);
	}

	struct nfsrvsetattrdsdorpc {
	int done;
	int inprog;
	struct task tsk;
	fhandle_t fh;
	struct nfsmount *nmp;
	struct vnode *vp;
	struct ucred *cred;
	NFSPROC_T *p;
	struct nfsvattr na;
	struct nfsvattr dsna;
	int err;
	};

	/*
	* Start up the thread that will execute nfsrv_setattrdsdorpc().
	*/
	static void
	start_setattrdsdorpc(void *arg, int pending)
	{
	struct nfsrvsetattrdsdorpc *drpc;

	drpc = (struct nfsrvsetattrdsdorpc *)arg;
	drpc->err = nfsrv_setattrdsdorpc(&drpc->fh, drpc->cred, drpc->p,
	drpc->vp, drpc->nmp, &drpc->na, &drpc->dsna);
	drpc->done = 1;
	}

	static int
	nfsrv_setattrdsrpc(fhandle_t fhp, struct ucred cred, NFSPROC_T *p,
	struct vnode vp, struct nfsmount *nmpp, int mirrorcnt,
	struct nfsvattr nap, int failposp)
	{
	struct nfsrvsetattrdsdorpc drpc, tdrpc = NULL;
	struct nfsvattr na;
	int error, i, ret, timo;

	NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n");
	drpc = NULL;
	if (mirrorcnt > 1)
	tdrpc = drpc = malloc(sizeof(drpc) (mirrorcnt - 1), M_TEMP,
	M_WAITOK);

	/*
	* Do the setattr RPC for every DS, using a separate kernel process
	* for every DS except the last one.
	*/
	error = 0;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	tdrpc->done = 0;
	tdrpc->inprog = 0;
	NFSBCOPY(fhp, &tdrpc->fh, sizeof(*fhp));
	tdrpc->nmp = *nmpp;
	tdrpc->vp = vp;
	tdrpc->cred = cred;
	tdrpc->p = p;
	tdrpc->na = *nap;
	tdrpc->err = 0;
	ret = EIO;
	if (nfs_pnfsiothreads != 0) {
	ret = nfs_pnfsio(start_setattrdsdorpc, tdrpc);
	NFSD_DEBUG(4, "nfsrv_setattrdsrpc: nfs_pnfsio=%d\n",
	ret);
	}
	if (ret != 0) {
	ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap,
	&na);
	if (nfsds_failerr(ret) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && ret != 0)
	error = ret;
	}
	nmpp++;
	fhp++;
	}
	ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap, &na);
	if (nfsds_failerr(ret) && *failposp == -1 && mirrorcnt > 1)
	*failposp = mirrorcnt - 1;
	else if (error == 0 && ret != 0)
	error = ret;
	if (error == 0)
	error = nfsrv_setextattr(vp, &na, p);
	NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error);
	tdrpc = drpc;
	timo = hz / 50; /* Wait for 20msec. */
	if (timo < 1)
	timo = 1;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	/* Wait for RPCs on separate threads to complete. */
	while (tdrpc->inprog != 0 && tdrpc->done == 0)
	tsleep(&tdrpc->tsk, PVFS, "srvsads", timo);
	if (nfsds_failerr(tdrpc->err) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && tdrpc->err != 0)
	error = tdrpc->err;
	}
	free(drpc, M_TEMP);
	return (error);
	}

	/*
	* Do a Setattr of an NFSv4 ACL on the DS file.
	*/
	static int
	nfsrv_setacldsdorpc(fhandle_t fhp, struct ucred cred, NFSPROC_T *p,
	struct vnode vp, struct nfsmount nmp, struct acl *aclp)
	{
	struct nfsrv_descript *nd;
	nfsv4stateid_t st;
	nfsattrbit_t attrbits;
	int error;

	NFSD_DEBUG(4, "in nfsrv_setacldsdorpc\n");
	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	/*
	* Use a stateid where other is an alternating 01010 pattern and
	* seqid is 0xffffffff. This value is not defined as special by
	* the RFC and is used by the FreeBSD NFS server to indicate an
	* MDS->DS proxy operation.
	*/
	st.other[0] = 0x55555555;
	st.other[1] = 0x55555555;
	st.other[2] = 0x55555555;
	st.seqid = 0xffffffff;
	nfscl_reqstart(nd, NFSPROC_SETACL, nmp, (u_int8_t )fhp, sizeof(fhp),
	NULL, NULL, 0, 0, false);
	nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID);
	NFSZERO_ATTRBIT(&attrbits);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_ACL);
	/*
	* The "vp" argument to nfsv4_fillattr() is only used for vnode_type(),
	* so passing in the metadata "vp" will be ok, since it is of
	* the same type (VREG).
	*/
	nfsv4_fillattr(nd, NULL, vp, aclp, NULL, NULL, 0, &attrbits, NULL,
	NULL, 0, 0, 0, 0, 0, NULL);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred,
	NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	NFSD_DEBUG(4, "nfsrv_setacldsdorpc: aft setaclrpc=%d\n",
	nd->nd_repstat);
	error = nd->nd_repstat;
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	return (error);
	}

	struct nfsrvsetacldsdorpc {
	int done;
	int inprog;
	struct task tsk;
	fhandle_t fh;
	struct nfsmount *nmp;
	struct vnode *vp;
	struct ucred *cred;
	NFSPROC_T *p;
	struct acl *aclp;
	int err;
	};

	/*
	* Start up the thread that will execute nfsrv_setacldsdorpc().
	*/
	static void
	start_setacldsdorpc(void *arg, int pending)
	{
	struct nfsrvsetacldsdorpc *drpc;

	drpc = (struct nfsrvsetacldsdorpc *)arg;
	drpc->err = nfsrv_setacldsdorpc(&drpc->fh, drpc->cred, drpc->p,
	drpc->vp, drpc->nmp, drpc->aclp);
	drpc->done = 1;
	}

	static int
	nfsrv_setacldsrpc(fhandle_t fhp, struct ucred cred, NFSPROC_T *p,
	struct vnode vp, struct nfsmount nmpp, int mirrorcnt, struct acl aclp,
	int *failposp)
	{
	struct nfsrvsetacldsdorpc drpc, tdrpc = NULL;
	int error, i, ret, timo;

	NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n");
	drpc = NULL;
	if (mirrorcnt > 1)
	tdrpc = drpc = malloc(sizeof(drpc) (mirrorcnt - 1), M_TEMP,
	M_WAITOK);

	/*
	* Do the setattr RPC for every DS, using a separate kernel process
	* for every DS except the last one.
	*/
	error = 0;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	tdrpc->done = 0;
	tdrpc->inprog = 0;
	NFSBCOPY(fhp, &tdrpc->fh, sizeof(*fhp));
	tdrpc->nmp = *nmpp;
	tdrpc->vp = vp;
	tdrpc->cred = cred;
	tdrpc->p = p;
	tdrpc->aclp = aclp;
	tdrpc->err = 0;
	ret = EIO;
	if (nfs_pnfsiothreads != 0) {
	ret = nfs_pnfsio(start_setacldsdorpc, tdrpc);
	NFSD_DEBUG(4, "nfsrv_setacldsrpc: nfs_pnfsio=%d\n",
	ret);
	}
	if (ret != 0) {
	ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp,
	aclp);
	if (nfsds_failerr(ret) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && ret != 0)
	error = ret;
	}
	nmpp++;
	fhp++;
	}
	ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp, aclp);
	if (nfsds_failerr(ret) && *failposp == -1 && mirrorcnt > 1)
	*failposp = mirrorcnt - 1;
	else if (error == 0 && ret != 0)
	error = ret;
	NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setextat=%d\n", error);
	tdrpc = drpc;
	timo = hz / 50; /* Wait for 20msec. */
	if (timo < 1)
	timo = 1;
	for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
	/* Wait for RPCs on separate threads to complete. */
	while (tdrpc->inprog != 0 && tdrpc->done == 0)
	tsleep(&tdrpc->tsk, PVFS, "srvacds", timo);
	if (nfsds_failerr(tdrpc->err) && *failposp == -1)
	*failposp = i;
	else if (error == 0 && tdrpc->err != 0)
	error = tdrpc->err;
	}
	free(drpc, M_TEMP);
	return (error);
	}

	/*
	* Getattr call to the DS for the attributes that change due to writing.
	*/
	static int
	nfsrv_getattrdsrpc(fhandle_t fhp, struct ucred cred, NFSPROC_T *p,
	struct vnode vp, struct nfsmount nmp, struct nfsvattr *nap)
	{
	struct nfsrv_descript *nd;
	int error;
	nfsattrbit_t attrbits;

	NFSD_DEBUG(4, "in nfsrv_getattrdsrpc\n");
	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	nfscl_reqstart(nd, NFSPROC_GETATTR, nmp, (u_int8_t *)fhp,
	sizeof(fhandle_t), NULL, NULL, 0, 0, false);
	NFSZERO_ATTRBIT(&attrbits);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SIZE);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_CHANGE);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEACCESS);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_TIMEMODIFY);
	NFSSETBIT_ATTRBIT(&attrbits, NFSATTRBIT_SPACEUSED);
	(void) nfsrv_putattrbit(nd, &attrbits);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred,
	NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	NFSD_DEBUG(4, "nfsrv_getattrdsrpc: aft getattrrpc=%d\n",
	nd->nd_repstat);
	if (nd->nd_repstat == 0) {
	error = nfsv4_loadattr(nd, NULL, nap, NULL, NULL, 0,
	NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL,
	NULL, NULL);
	/*
	* We can only save the updated values in the extended
	* attribute if the vp is exclusively locked.
	* This should happen when any of the following operations
	* occur on the vnode:
	* Close, Delegreturn, LayoutCommit, LayoutReturn
	* As such, the updated extended attribute should get saved
	* before nfsrv_checkdsattr() returns 0 and allows the cached
	* attributes to be returned without calling this function.
	*/
	if (error == 0 && VOP_ISLOCKED(vp) == LK_EXCLUSIVE) {
	error = nfsrv_setextattr(vp, nap, p);
	NFSD_DEBUG(4, "nfsrv_getattrdsrpc: aft setextat=%d\n",
	error);
	}
	} else
	error = nd->nd_repstat;
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_getattrdsrpc error=%d\n", error);
	return (error);
	}

	/*
	* Seek call to a DS.
	*/
	static int
	nfsrv_seekdsrpc(fhandle_t fhp, off_t offp, int content, bool *eofp,
	struct ucred cred, NFSPROC_T p, struct nfsmount *nmp)
	{
	uint32_t *tl;
	struct nfsrv_descript *nd;
	nfsv4stateid_t st;
	int error;

	NFSD_DEBUG(4, "in nfsrv_seekdsrpc\n");
	/*
	* Use a stateid where other is an alternating 01010 pattern and
	* seqid is 0xffffffff. This value is not defined as special by
	* the RFC and is used by the FreeBSD NFS server to indicate an
	* MDS->DS proxy operation.
	*/
	st.other[0] = 0x55555555;
	st.other[1] = 0x55555555;
	st.other[2] = 0x55555555;
	st.seqid = 0xffffffff;
	nd = malloc(sizeof(*nd), M_TEMP, M_WAITOK \| M_ZERO);
	nfscl_reqstart(nd, NFSPROC_SEEKDS, nmp, (u_int8_t *)fhp,
	sizeof(fhandle_t), NULL, NULL, 0, 0, false);
	nfsm_stateidtom(nd, &st, NFSSTATEID_PUTSTATEID);
	NFSM_BUILD(tl, uint32_t *, NFSX_HYPER + NFSX_UNSIGNED);
	txdr_hyper(*offp, tl); tl += 2;
	*tl = txdr_unsigned(content);
	error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred,
	NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL);
	if (error != 0) {
	free(nd, M_TEMP);
	return (error);
	}
	NFSD_DEBUG(4, "nfsrv_seekdsrpc: aft seekrpc=%d\n", nd->nd_repstat);
	if (nd->nd_repstat == 0) {
	NFSM_DISSECT(tl, uint32_t *, NFSX_UNSIGNED + NFSX_HYPER);
	if (*tl++ == newnfs_true)
	*eofp = true;
	else
	*eofp = false;
	*offp = fxdr_hyper(tl);
	} else
	error = nd->nd_repstat;
	nfsmout:
	m_freem(nd->nd_mrep);
	free(nd, M_TEMP);
	NFSD_DEBUG(4, "nfsrv_seekdsrpc error=%d\n", error);
	return (error);
	}

	/*
	* Get the device id and file handle for a DS file.
	*/
	int
	nfsrv_dsgetdevandfh(struct vnode vp, NFSPROC_T p, int *mirrorcntp,
	fhandle_t fhp, char devid)
	{
	int buflen, error;
	char *buf;

	buflen = 1024;
	buf = malloc(buflen, M_TEMP, M_WAITOK);
	error = nfsrv_dsgetsockmnt(vp, 0, buf, &buflen, mirrorcntp, p, NULL,
	fhp, devid, NULL, NULL, NULL, NULL, NULL, NULL);
	free(buf, M_TEMP);
	return (error);
	}

	/*
	* Do a Lookup against the DS for the filename.
	*/
	static int
	nfsrv_pnfslookupds(struct vnode vp, struct vnode dvp, struct pnfsdsfile *pf,
	struct vnode *nvpp, NFSPROC_T p)
	{
	struct nameidata named;
	struct ucred *tcred;
	char *bufp;
	u_long *hashp;
	struct vnode *nvp;
	int error;

	tcred = newnfs_getcred();
	named.ni_cnd.cn_nameiop = LOOKUP;
	named.ni_cnd.cn_lkflags = LK_SHARED \| LK_RETRY;
	named.ni_cnd.cn_cred = tcred;
	named.ni_cnd.cn_thread = p;
	named.ni_cnd.cn_flags = ISLASTCN \| LOCKPARENT \| LOCKLEAF \| SAVENAME;
	nfsvno_setpathbuf(&named, &bufp, &hashp);
	named.ni_cnd.cn_nameptr = bufp;
	named.ni_cnd.cn_namelen = strlen(pf->dsf_filename);
	strlcpy(bufp, pf->dsf_filename, NAME_MAX);
	NFSD_DEBUG(4, "nfsrv_pnfslookupds: filename=%s\n", bufp);
	error = VOP_LOOKUP(dvp, &nvp, &named.ni_cnd);
	NFSD_DEBUG(4, "nfsrv_pnfslookupds: aft LOOKUP=%d\n", error);
	NFSFREECRED(tcred);
	nfsvno_relpathbuf(&named);
	if (error == 0)
	*nvpp = nvp;
	NFSD_DEBUG(4, "eo nfsrv_pnfslookupds=%d\n", error);
	return (error);
	}

	/*
	* Set the file handle to the correct one.
	*/
	static void
	nfsrv_pnfssetfh(struct vnode vp, struct pnfsdsfile pf, char *devid,
	char fnamep, struct vnode nvp, NFSPROC_T *p)
	{
	struct nfsnode *np;
	int ret = 0;

	np = VTONFS(nvp);
	NFSBCOPY(np->n_fhp->nfh_fh, &pf->dsf_fh, NFSX_MYFH);
	/*
	* We can only do a vn_set_extattr() if the vnode is exclusively
	* locked and vn_start_write() has been done. If devid != NULL or
	* fnamep != NULL or the vnode is shared locked, vn_start_write()
	* may not have been done.
	* If not done now, it will be done on a future call.
	*/
	if (devid == NULL && fnamep == NULL && NFSVOPISLOCKED(vp) ==
	LK_EXCLUSIVE)
	ret = vn_extattr_set(vp, IO_NODELOCKED,
	EXTATTR_NAMESPACE_SYSTEM, "pnfsd.dsfile", sizeof(*pf),
	(char *)pf, p);
	NFSD_DEBUG(4, "eo nfsrv_pnfssetfh=%d\n", ret);
	}

	/*
	* Cause RPCs waiting on "nmp" to fail. This is called for a DS mount point
	* when the DS has failed.
	*/
	void
	nfsrv_killrpcs(struct nfsmount *nmp)
	{

	/*
	* Call newnfs_nmcancelreqs() to cause
	* any RPCs in progress on the mount point to
	* fail.
	* This will cause any process waiting for an
	* RPC to complete while holding a vnode lock
	* on the mounted-on vnode (such as "df" or
	* a non-forced "umount") to fail.
	* This will unlock the mounted-on vnode so
	* a forced dismount can succeed.
	* The NFSMNTP_CANCELRPCS flag should be set when this function is
	* called.
	*/
	newnfs_nmcancelreqs(nmp);
	}

	/*
	* Sum up the statfs info for each of the DSs, so that the client will
	* receive the total for all DSs.
	*/
	static int
	nfsrv_pnfsstatfs(struct statfs sf, struct mount mp)
	{
	struct statfs *tsf;
	struct nfsdevice *ds;
	struct vnode dvpp, tdvpp, *dvp;
	uint64_t tot;
	int cnt, error = 0, i;

	if (nfsrv_devidcnt <= 0)
	return (ENXIO);
	dvpp = mallocarray(nfsrv_devidcnt, sizeof(*dvpp), M_TEMP, M_WAITOK);
	tsf = malloc(sizeof(*tsf), M_TEMP, M_WAITOK);

	/* Get an array of the dvps for the DSs. */
	tdvpp = dvpp;
	i = 0;
	NFSDDSLOCK();
	/* First, search for matches for same file system. */
	TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) {
	if (ds->nfsdev_nmp != NULL && ds->nfsdev_mdsisset != 0 &&
	fsidcmp(&ds->nfsdev_mdsfsid, &mp->mnt_stat.f_fsid) == 0) {
	if (++i > nfsrv_devidcnt)
	break;
	*tdvpp++ = ds->nfsdev_dvp;
	}
	}
	/*
	* If no matches for same file system, total all servers not assigned
	* to a file system.
	*/
	if (i == 0) {
	TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) {
	if (ds->nfsdev_nmp != NULL &&
	ds->nfsdev_mdsisset == 0) {
	if (++i > nfsrv_devidcnt)
	break;
	*tdvpp++ = ds->nfsdev_dvp;
	}
	}
	}
	NFSDDSUNLOCK();
	cnt = i;

	/* Do a VFS_STATFS() for each of the DSs and sum them up. */
	tdvpp = dvpp;
	for (i = 0; i < cnt && error == 0; i++) {
	dvp = *tdvpp++;
	error = VFS_STATFS(dvp->v_mount, tsf);
	if (error == 0) {
	if (sf->f_bsize == 0) {
	if (tsf->f_bsize > 0)
	sf->f_bsize = tsf->f_bsize;
	else
	sf->f_bsize = 8192;
	}
	if (tsf->f_blocks > 0) {
	if (sf->f_bsize != tsf->f_bsize) {
	tot = tsf->f_blocks * tsf->f_bsize;
	sf->f_blocks += (tot / sf->f_bsize);
	} else
	sf->f_blocks += tsf->f_blocks;
	}
	if (tsf->f_bfree > 0) {
	if (sf->f_bsize != tsf->f_bsize) {
	tot = tsf->f_bfree * tsf->f_bsize;
	sf->f_bfree += (tot / sf->f_bsize);
	} else
	sf->f_bfree += tsf->f_bfree;
	}
	if (tsf->f_bavail > 0) {
	if (sf->f_bsize != tsf->f_bsize) {
	tot = tsf->f_bavail * tsf->f_bsize;
	sf->f_bavail += (tot / sf->f_bsize);
	} else
	sf->f_bavail += tsf->f_bavail;
	}
	}
	}
	free(tsf, M_TEMP);
	free(dvpp, M_TEMP);
	return (error);
	}

	/*
	* Set an NFSv4 acl.
	*/
	int
	nfsrv_setacl(struct vnode vp, NFSACL_T aclp, struct ucred cred, NFSPROC_T p)
	{
	int error;

	if (nfsrv_useacl == 0 \|\| nfs_supportsnfsv4acls(vp) == 0) {
	error = NFSERR_ATTRNOTSUPP;
	goto out;
	}
	/*
	* With NFSv4 ACLs, chmod(2) may need to add additional entries.
	* Make sure it has enough room for that - splitting every entry
	* into two and appending "canonical six" entries at the end.
	* Cribbed out of kern/vfs_acl.c - Rick M.
	*/
	if (aclp->acl_cnt > (ACL_MAX_ENTRIES - 6) / 2) {
	error = NFSERR_ATTRNOTSUPP;
	goto out;
	}
	error = VOP_SETACL(vp, ACL_TYPE_NFS4, aclp, cred, p);
	if (error == 0) {
	error = nfsrv_dssetacl(vp, aclp, cred, p);
	if (error == ENOENT)
	error = 0;
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Seek vnode op call (actually it is a VOP_IOCTL()).
	* This function is called with the vnode locked, but unlocks and vrele()s
	* the vp before returning.
	*/
	int
	nfsvno_seek(struct nfsrv_descript nd, struct vnode vp, u_long cmd,
	off_t offp, int content, bool eofp, struct ucred cred, NFSPROC_T p)
	{
	struct nfsvattr at;
	int error, ret;

	ASSERT_VOP_LOCKED(vp, "nfsvno_seek vp");
	/*
	* Attempt to seek on a DS file. A return of ENOENT implies
	* there is no DS file to seek on.
	*/
	error = nfsrv_proxyds(vp, 0, 0, cred, p, NFSPROC_SEEKDS, NULL,
	NULL, NULL, NULL, NULL, offp, content, eofp);
	if (error != ENOENT) {
	vput(vp);
	return (error);
	}

	/*
	* Do the VOP_IOCTL() call. For the case where *offp == file_size,
	* VOP_IOCTL() will return ENXIO. However, the correct reply for
	* NFSv4.2 is *eofp == true and error == 0 for this case.
	*/
	NFSVOPUNLOCK(vp);
	error = VOP_IOCTL(vp, cmd, offp, 0, cred, p);
	*eofp = false;
	if (error == ENXIO \|\| (error == 0 && cmd == FIOSEEKHOLE)) {
	/* Handle the cases where we might be at EOF. */
	ret = nfsvno_getattr(vp, &at, nd, p, 0, NULL);
	if (ret == 0 && *offp == at.na_size) {
	*eofp = true;
	error = 0;
	}
	if (ret != 0 && error == 0)
	error = ret;
	}
	vrele(vp);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Allocate vnode op call.
	*/
	int
	nfsvno_allocate(struct vnode vp, off_t off, off_t len, struct ucred cred,
	NFSPROC_T *p)
	{
	int error, trycnt;

	ASSERT_VOP_ELOCKED(vp, "nfsvno_allocate vp");
	/*
	* Attempt to allocate on a DS file. A return of ENOENT implies
	* there is no DS file to allocate on.
	*/
	error = nfsrv_proxyds(vp, off, 0, cred, p, NFSPROC_ALLOCATE, NULL,
	NULL, NULL, NULL, NULL, &len, 0, NULL);
	if (error != ENOENT)
	return (error);
	error = 0;

	/*
	* Do the actual VOP_ALLOCATE(), looping a reasonable number of
	* times to achieve completion.
	*/
	trycnt = 0;
	while (error == 0 && len > 0 && trycnt++ < 20)
	error = VOP_ALLOCATE(vp, &off, &len);
	if (error == 0 && len > 0)
	error = NFSERR_IO;
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Get Extended Atribute vnode op into an mbuf list.
	*/
	int
	nfsvno_getxattr(struct vnode vp, char name, uint32_t maxresp,
	struct ucred cred, struct thread p, struct mbuf **mpp,
	struct mbuf *mpendp, int lenp)
	{
	struct iovec *iv;
	struct uio io, *uiop = &io;
	struct mbuf m, m2;
	int alen, error, len, tlen;
	size_t siz;

	/* First, find out the size of the extended attribute. */
	error = VOP_GETEXTATTR(vp, EXTATTR_NAMESPACE_USER, name, NULL,
	&siz, cred, p);
	if (error != 0)
	return (NFSERR_NOXATTR);
	if (siz > maxresp - NFS_MAXXDR)
	return (NFSERR_XATTR2BIG);
	len = siz;
	tlen = NFSM_RNDUP(len);
	if (tlen > 0) {
	uiop->uio_iovcnt = nfsrv_createiovec(tlen, &m, &m2, &iv);
	uiop->uio_iov = iv;
	} else {
	uiop->uio_iovcnt = 0;
	uiop->uio_iov = iv = NULL;
	m = m2 = NULL;
	}
	uiop->uio_offset = 0;
	uiop->uio_resid = tlen;
	uiop->uio_rw = UIO_READ;
	uiop->uio_segflg = UIO_SYSSPACE;
	uiop->uio_td = p;
	#ifdef MAC
	error = mac_vnode_check_getextattr(cred, vp, EXTATTR_NAMESPACE_USER,
	name);
	if (error != 0)
	goto out;
	#endif

	if (tlen > 0)
	error = VOP_GETEXTATTR(vp, EXTATTR_NAMESPACE_USER, name, uiop,
	NULL, cred, p);
	if (error != 0)
	goto out;
	if (uiop->uio_resid > 0) {
	alen = tlen;
	len = tlen - uiop->uio_resid;
	tlen = NFSM_RNDUP(len);
	if (alen != tlen)
	printf("nfsvno_getxattr: weird size read\n");
	if (tlen == 0) {
	m_freem(m);
	m = m2 = NULL;
	} else if (alen != tlen \|\| tlen != len)
	m2 = nfsrv_adj(m, alen - tlen, tlen - len);
	}
	*lenp = len;
	*mpp = m;
	*mpendp = m2;

	out:
	if (error != 0) {
	if (m != NULL)
	m_freem(m);
	*lenp = 0;
	}
	free(iv, M_TEMP);
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Set Extended attribute vnode op from an mbuf list.
	*/
	int
	nfsvno_setxattr(struct vnode vp, char name, int len, struct mbuf *m,
	char cp, struct ucred cred, struct thread *p)
	{
	struct iovec *iv;
	struct uio uio, *uiop = &uio;
	int cnt, error;

	error = 0;
	#ifdef MAC
	error = mac_vnode_check_setextattr(cred, vp, EXTATTR_NAMESPACE_USER,
	name);
	#endif
	if (error != 0)
	goto out;

	uiop->uio_rw = UIO_WRITE;
	uiop->uio_segflg = UIO_SYSSPACE;
	uiop->uio_td = p;
	uiop->uio_offset = 0;
	uiop->uio_resid = len;
	if (len > 0) {
	error = nfsrv_createiovecw(len, m, cp, &iv, &cnt);
	uiop->uio_iov = iv;
	uiop->uio_iovcnt = cnt;
	} else {
	uiop->uio_iov = iv = NULL;
	uiop->uio_iovcnt = 0;
	}
	if (error == 0) {
	error = VOP_SETEXTATTR(vp, EXTATTR_NAMESPACE_USER, name, uiop,
	cred, p);
	free(iv, M_TEMP);
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* Remove Extended attribute vnode op.
	*/
	int
	nfsvno_rmxattr(struct nfsrv_descript nd, struct vnode vp, char *name,
	struct ucred cred, struct thread p)
	{
	int error;

	/*
	* Get rid of any delegations. I am not sure why this is required,
	* but RFC-8276 says so.
	*/
	error = nfsrv_checkremove(vp, 0, nd, nd->nd_clientid, p);
	if (error != 0)
	goto out;
	#ifdef MAC
	error = mac_vnode_check_deleteextattr(cred, vp, EXTATTR_NAMESPACE_USER,
	name);
	if (error != 0)
	goto out;
	#endif

	error = VOP_DELETEEXTATTR(vp, EXTATTR_NAMESPACE_USER, name, cred, p);
	if (error == EOPNOTSUPP)
	error = VOP_SETEXTATTR(vp, EXTATTR_NAMESPACE_USER, name, NULL,
	cred, p);
	out:
	NFSEXITCODE(error);
	return (error);
	}

	/*
	* List Extended Atribute vnode op into an mbuf list.
	*/
	int
	nfsvno_listxattr(struct vnode vp, uint64_t cookie, struct ucred cred,
	struct thread p, u_char bufp, uint32_t lenp, bool *eofp)
	{
	struct iovec iv;
	struct uio io;
	int error;
	size_t siz;

	*bufp = NULL;
	/* First, find out the size of the extended attribute. */
	error = VOP_LISTEXTATTR(vp, EXTATTR_NAMESPACE_USER, NULL, &siz, cred,
	p);
	if (error != 0)
	return (NFSERR_NOXATTR);
	if (siz <= cookie) {
	*lenp = 0;
	*eofp = true;
	goto out;
	}
	if (siz > cookie + *lenp) {
	siz = cookie + *lenp;
	*eofp = false;
	} else
	*eofp = true;
	/* Just choose a sanity limit of 10Mbytes for malloc(M_TEMP). */
	if (siz > 10 * 1024 * 1024) {
	error = NFSERR_XATTR2BIG;
	goto out;
	}
	*bufp = malloc(siz, M_TEMP, M_WAITOK);
	iv.iov_base = *bufp;
	iv.iov_len = siz;
	io.uio_iovcnt = 1;
	io.uio_iov = &iv;
	io.uio_offset = 0;
	io.uio_resid = siz;
	io.uio_rw = UIO_READ;
	io.uio_segflg = UIO_SYSSPACE;
	io.uio_td = p;
	#ifdef MAC
	error = mac_vnode_check_listextattr(cred, vp, EXTATTR_NAMESPACE_USER);
	if (error != 0)
	goto out;
	#endif

	error = VOP_LISTEXTATTR(vp, EXTATTR_NAMESPACE_USER, &io, NULL, cred,
	p);
	if (error != 0)
	goto out;
	if (io.uio_resid > 0)
	siz -= io.uio_resid;
	*lenp = siz;

	out:
	if (error != 0) {
	free(*bufp, M_TEMP);
	*bufp = NULL;
	}
	NFSEXITCODE(error);
	return (error);
	}

	+/*
	+ * Trim trailing data off the mbuf list being built.
	+ */
	+static void
	+nfsm_trimtrailing(struct nfsrv_descript nd, struct mbuf mb, char *bpos,
	+ int bextpg, int bextpgsiz)
	+{
	+ vm_page_t pg;
	+ int fullpgsiz, i;
	+
	+ if (mb->m_next != NULL) {
	+ m_freem(mb->m_next);
	+ mb->m_next = NULL;
	+ }
	+ if ((mb->m_flags & M_EXTPG) != 0) {
	+ /* First, get rid of any pages after this position. */
	+ for (i = mb->m_epg_npgs - 1; i > bextpg; i--) {
	+ pg = PHYS_TO_VM_PAGE(mb->m_epg_pa[i]);
	+ vm_page_unwire_noq(pg);
	+ vm_page_free(pg);
	+ }
	+ mb->m_epg_npgs = bextpg + 1;
	+ if (bextpg == 0)
	+ fullpgsiz = PAGE_SIZE - mb->m_epg_1st_off;
	+ else
	+ fullpgsiz = PAGE_SIZE;
	+ mb->m_epg_last_len = fullpgsiz - bextpgsiz;
	+ mb->m_len = m_epg_pagelen(mb, 0, mb->m_epg_1st_off);
	+ for (i = 1; i < mb->m_epg_npgs; i++)
	+ mb->m_len += m_epg_pagelen(mb, i, 0);
	+ nd->nd_bextpgsiz = bextpgsiz;
	+ nd->nd_bextpg = bextpg;
	+ } else
	+ mb->m_len = bpos - mtod(mb, char *);
	+ nd->nd_mb = mb;
	+ nd->nd_bpos = bpos;
	+}
	+
	extern int (nfsd_call_nfsd)(struct thread , struct nfssvc_args *);

	/*
	* Called once to initialize data structures...
	*/
	static int
	nfsd_modevent(module_t mod, int type, void *data)
	{
	int error = 0, i;
	static int loaded = 0;

	switch (type) {
	case MOD_LOAD:
	if (loaded)
	goto out;
	newnfs_portinit();
	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
	mtx_init(&nfsrchash_table[i].mtx, "nfsrtc", NULL,
	MTX_DEF);
	mtx_init(&nfsrcahash_table[i].mtx, "nfsrtca", NULL,
	MTX_DEF);
	}
	mtx_init(&nfsrc_udpmtx, "nfsuc", NULL, MTX_DEF);
	mtx_init(&nfs_v4root_mutex, "nfs4rt", NULL, MTX_DEF);
	mtx_init(&nfsv4root_mnt.mnt_mtx, "nfs4mnt", NULL, MTX_DEF);
	mtx_init(&nfsrv_dontlistlock_mtx, "nfs4dnl", NULL, MTX_DEF);
	mtx_init(&nfsrv_recalllock_mtx, "nfs4rec", NULL, MTX_DEF);
	lockinit(&nfsv4root_mnt.mnt_explock, PVFS, "explock", 0, 0);
	nfsrvd_initcache();
	nfsd_init();
	NFSD_LOCK();
	nfsrvd_init(0);
	NFSD_UNLOCK();
	nfsd_mntinit();
	#ifdef VV_DISABLEDELEG
	vn_deleg_ops.vndeleg_recall = nfsd_recalldelegation;
	vn_deleg_ops.vndeleg_disable = nfsd_disabledelegation;
	#endif
	nfsd_call_servertimer = nfsrv_servertimer;
	nfsd_call_nfsd = nfssvc_nfsd;
	loaded = 1;
	break;

	case MOD_UNLOAD:
	if (newnfs_numnfsd != 0) {
	error = EBUSY;
	break;
	}

	#ifdef VV_DISABLEDELEG
	vn_deleg_ops.vndeleg_recall = NULL;
	vn_deleg_ops.vndeleg_disable = NULL;
	#endif
	nfsd_call_servertimer = NULL;
	nfsd_call_nfsd = NULL;

	/* Clean out all NFSv4 state. */
	nfsrv_throwawayallstate(curthread);

	/* Clean the NFS server reply cache */
	nfsrvd_cleancache();

	/* Free up the krpc server pool. */
	if (nfsrvd_pool != NULL)
	svcpool_destroy(nfsrvd_pool);

	/* and get rid of the locks */
	for (i = 0; i < NFSRVCACHE_HASHSIZE; i++) {
	mtx_destroy(&nfsrchash_table[i].mtx);
	mtx_destroy(&nfsrcahash_table[i].mtx);
	}
	mtx_destroy(&nfsrc_udpmtx);
	mtx_destroy(&nfs_v4root_mutex);
	mtx_destroy(&nfsv4root_mnt.mnt_mtx);
	mtx_destroy(&nfsrv_dontlistlock_mtx);
	mtx_destroy(&nfsrv_recalllock_mtx);
	for (i = 0; i < nfsrv_sessionhashsize; i++)
	mtx_destroy(&nfssessionhash[i].mtx);
	if (nfslayouthash != NULL) {
	for (i = 0; i < nfsrv_layouthashsize; i++)
	mtx_destroy(&nfslayouthash[i].mtx);
	free(nfslayouthash, M_NFSDSESSION);
	}
	lockdestroy(&nfsv4root_mnt.mnt_explock);
	free(nfsclienthash, M_NFSDCLIENT);
	free(nfslockhash, M_NFSDLOCKFILE);
	free(nfssessionhash, M_NFSDSESSION);
	loaded = 0;
	break;
	default:
	error = EOPNOTSUPP;
	break;
	}

	out:
	NFSEXITCODE(error);
	return (error);
	}
	static moduledata_t nfsd_mod = {
	"nfsd",
	nfsd_modevent,
	NULL,
	};
	DECLARE_MODULE(nfsd, nfsd_mod, SI_SUB_VFS, SI_ORDER_ANY);

	/* So that loader and kldload(2) can find us, wherever we are.. */
	MODULE_VERSION(nfsd, 1);
	MODULE_DEPEND(nfsd, nfscommon, 1, 1, 1);
	MODULE_DEPEND(nfsd, nfslockd, 1, 1, 1);
	MODULE_DEPEND(nfsd, krpc, 1, 1, 1);
	MODULE_DEPEND(nfsd, nfssvc, 1, 1, 1);

	diff --git a/sys/geom/eli/g_eli_integrity.c b/sys/geom/eli/g_eli_integrity.c
	index e4f31046c45b..d20b753256dd 100644
	--- a/sys/geom/eli/g_eli_integrity.c
	+++ b/sys/geom/eli/g_eli_integrity.c
	@@ -1,556 +1,556 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/linker.h>
	#include <sys/module.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/bio.h>
	#include <sys/sysctl.h>
	#include <sys/malloc.h>
	#include <sys/kthread.h>
	#include <sys/proc.h>
	#include <sys/sched.h>
	#include <sys/smp.h>
	#include <sys/vnode.h>

	#include <vm/uma.h>

	#include <geom/geom.h>
	#include <geom/geom_dbg.h>
	#include <geom/eli/g_eli.h>
	#include <geom/eli/pkcs5v2.h>

	/*
	* The data layout description when integrity verification is configured.
	*
	* One of the most important assumption here is that authenticated data and its
	* HMAC has to be stored in the same place (namely in the same sector) to make
	* it work reliable.
	* The problem is that file systems work only with sectors that are multiple of
	* 512 bytes and a power of two number.
	* My idea to implement it is as follows.
	* Let's store HMAC in sector. This is a must. This leaves us 480 bytes for
	* data. We can't use that directly (ie. we can't create provider with 480 bytes
	* sector size). We need another sector from where we take only 32 bytes of data
	* and we store HMAC of this data as well. This takes two sectors from the
	* original provider at the input and leaves us one sector of authenticated data
	* at the output. Not very efficient, but you got the idea.
	* Now, let's assume, we want to create provider with 4096 bytes sector.
	* To output 4096 bytes of authenticated data we need 8x480 plus 1x256, so we
	* need nine 512-bytes sectors at the input to get one 4096-bytes sector at the
	* output. That's better. With 4096 bytes sector we can use 89% of size of the
	* original provider. I find it as an acceptable cost.
	* The reliability comes from the fact, that every HMAC stored inside the sector
	* is calculated only for the data in the same sector, so its impossible to
	* write new data and leave old HMAC or vice versa.
	*
	* And here is the picture:
	*
	* da0: +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+
	* \|32b \|480b\| \|32b \|480b\| \|32b \|480b\| \|32b \|480b\| \|32b \|480b\| \|32b \|480b\| \|32b \|480b\| \|32b \|480b\| \|32b \|256b \|
	* \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data\| \|HMAC\|Data \|
	* +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+
	* \|512 bytes\| \|512 bytes\| \|512 bytes\| \|512 bytes\| \|512 bytes\| \|512 bytes\| \|512 bytes\| \|512 bytes\| \|288 bytes \|
	* +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ \|224 unused\|
	* +----------+
	* da0.eli: +----+----+----+----+----+----+----+----+----+
	* \|480b\|480b\|480b\|480b\|480b\|480b\|480b\|480b\|256b\|
	* +----+----+----+----+----+----+----+----+----+
	* \| 4096 bytes \|
	* +--------------------------------------------+
	*
	* PS. You can use any sector size with geli(8). My example is using 4kB,
	* because it's most efficient. For 8kB sectors you need 2 extra sectors,
	* so the cost is the same as for 4kB sectors.
	*/

	/*
	* Code paths:
	* BIO_READ:
	* g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> g_eli_auth_read_done -> g_io_deliver
	* BIO_WRITE:
	* g_eli_start -> g_eli_auth_run -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
	*/

	MALLOC_DECLARE(M_ELI);

	/*
	* Here we generate key for HMAC. Every sector has its own HMAC key, so it is
	* not possible to copy sectors.
	* We cannot depend on fact, that every sector has its own IV, because different
	* IV doesn't change HMAC, when we use encrypt-then-authenticate method.
	*/
	static void
	g_eli_auth_keygen(struct g_eli_softc sc, off_t offset, u_char key)
	{
	SHA256_CTX ctx;

	/* Copy precalculated SHA256 context. */
	bcopy(&sc->sc_akeyctx, &ctx, sizeof(ctx));
	SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
	SHA256_Final(key, &ctx);
	}

	/*
	* The function is called after we read and decrypt data.
	*
	* g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> G_ELI_AUTH_READ_DONE -> g_io_deliver
	*/
	static int
	g_eli_auth_read_done(struct cryptop *crp)
	{
	struct g_eli_softc *sc;
	struct bio *bp;

	if (crp->crp_etype == EAGAIN) {
	if (g_eli_crypto_rerun(crp) == 0)
	return (0);
	}
	bp = (struct bio *)crp->crp_opaque;
	bp->bio_inbed++;
	sc = bp->bio_to->geom->softc;
	if (crp->crp_etype == 0) {
	bp->bio_completed += crp->crp_payload_length;
	G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%d completed=%jd).",
	bp->bio_inbed, bp->bio_children, crp->crp_payload_length, (intmax_t)bp->bio_completed);
	} else {
	u_int nsec, decr_secsize, encr_secsize, rel_sec;
	int *errorp;

	/* Sectorsize of decrypted provider eg. 4096. */
	decr_secsize = bp->bio_to->sectorsize;
	/* The real sectorsize of encrypted provider, eg. 512. */
	encr_secsize =
	LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
	/* Number of sectors from decrypted provider, eg. 2. */
	nsec = bp->bio_length / decr_secsize;
	/* Number of sectors from encrypted provider, eg. 18. */
	nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
	/* Which relative sector this request decrypted. */
	rel_sec = ((crp->crp_buf.cb_buf + crp->crp_payload_start) -
	(char *)bp->bio_driver2) / encr_secsize;

	errorp = (int )((char )bp->bio_driver2 + encr_secsize * nsec +
	sizeof(int) * rel_sec);
	*errorp = crp->crp_etype;
	G_ELI_DEBUG(1,
	"Crypto READ request failed (%d/%d) error=%d.",
	bp->bio_inbed, bp->bio_children, crp->crp_etype);
	if (bp->bio_error == 0 \|\| bp->bio_error == EINTEGRITY)
	bp->bio_error = crp->crp_etype == EBADMSG ?
	EINTEGRITY : crp->crp_etype;
	}
	if (crp->crp_cipher_key != NULL)
	g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
	crypto_freereq(crp);
	/*
	* Do we have all sectors already?
	*/
	if (bp->bio_inbed < bp->bio_children)
	return (0);

	if (bp->bio_error == 0) {
	u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
	u_char srcdata, dstdata;

	/* Sectorsize of decrypted provider eg. 4096. */
	decr_secsize = bp->bio_to->sectorsize;
	/* The real sectorsize of encrypted provider, eg. 512. */
	encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
	/* Number of data bytes in one encrypted sector, eg. 480. */
	data_secsize = sc->sc_data_per_sector;
	/* Number of sectors from decrypted provider, eg. 2. */
	nsec = bp->bio_length / decr_secsize;
	/* Number of sectors from encrypted provider, eg. 18. */
	nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
	/* Last sector number in every big sector, eg. 9. */
	lsec = sc->sc_bytes_per_sector / encr_secsize;

	srcdata = bp->bio_driver2;
	dstdata = bp->bio_data;

	for (i = 1; i <= nsec; i++) {
	data_secsize = sc->sc_data_per_sector;
	if ((i % lsec) == 0)
	data_secsize = decr_secsize % data_secsize;
	bcopy(srcdata + sc->sc_alen, dstdata, data_secsize);
	srcdata += encr_secsize;
	dstdata += data_secsize;
	}
	} else if (bp->bio_error == EINTEGRITY) {
	u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
	int *errorp;
	off_t coroff, corsize, dstoff;

	/* Sectorsize of decrypted provider eg. 4096. */
	decr_secsize = bp->bio_to->sectorsize;
	/* The real sectorsize of encrypted provider, eg. 512. */
	encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
	/* Number of data bytes in one encrypted sector, eg. 480. */
	data_secsize = sc->sc_data_per_sector;
	/* Number of sectors from decrypted provider, eg. 2. */
	nsec = bp->bio_length / decr_secsize;
	/* Number of sectors from encrypted provider, eg. 18. */
	nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
	/* Last sector number in every big sector, eg. 9. */
	lsec = sc->sc_bytes_per_sector / encr_secsize;

	errorp = (int )((char )bp->bio_driver2 + encr_secsize * nsec);
	coroff = -1;
	corsize = 0;
	dstoff = bp->bio_offset;

	for (i = 1; i <= nsec; i++) {
	data_secsize = sc->sc_data_per_sector;
	if ((i % lsec) == 0)
	data_secsize = decr_secsize % data_secsize;
	if (errorp[i - 1] == EBADMSG) {
	/*
	* Corruption detected, remember the offset if
	* this is the first corrupted sector and
	* increase size.
	*/
	if (coroff == -1)
	coroff = dstoff;
	corsize += data_secsize;
	} else {
	/*
	* No corruption, good.
	* Report previous corruption if there was one.
	*/
	if (coroff != -1) {
	G_ELI_DEBUG(0, "%s: Failed to authenticate %jd "
	"bytes of data at offset %jd.",
	sc->sc_name, (intmax_t)corsize,
	(intmax_t)coroff);
	coroff = -1;
	corsize = 0;
	}
	}
	dstoff += data_secsize;
	}
	/* Report previous corruption if there was one. */
	if (coroff != -1) {
	G_ELI_DEBUG(0, "%s: Failed to authenticate %jd "
	"bytes of data at offset %jd.",
	sc->sc_name, (intmax_t)corsize, (intmax_t)coroff);
	}
	}
	free(bp->bio_driver2, M_ELI);
	bp->bio_driver2 = NULL;
	if (bp->bio_error != 0) {
	if (bp->bio_error != EINTEGRITY) {
	G_ELI_LOGREQ(0, bp,
	"Crypto READ request failed (error=%d).",
	bp->bio_error);
	}
	bp->bio_completed = 0;
	}
	/*
	* Read is finished, send it up.
	*/
	g_io_deliver(bp, bp->bio_error);
	atomic_subtract_int(&sc->sc_inflight, 1);
	return (0);
	}

	/*
	* The function is called after data encryption.
	*
	* g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
	*/
	static int
	g_eli_auth_write_done(struct cryptop *crp)
	{
	struct g_eli_softc *sc;
	struct g_consumer *cp;
	struct bio bp, cbp, *cbp2;
	u_int nsec;

	if (crp->crp_etype == EAGAIN) {
	if (g_eli_crypto_rerun(crp) == 0)
	return (0);
	}
	bp = (struct bio *)crp->crp_opaque;
	bp->bio_inbed++;
	if (crp->crp_etype == 0) {
	G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
	bp->bio_inbed, bp->bio_children);
	} else {
	G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
	bp->bio_inbed, bp->bio_children, crp->crp_etype);
	if (bp->bio_error == 0)
	bp->bio_error = crp->crp_etype;
	}
	sc = bp->bio_to->geom->softc;
	if (crp->crp_cipher_key != NULL)
	g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
	crypto_freereq(crp);
	/*
	* All sectors are already encrypted?
	*/
	if (bp->bio_inbed < bp->bio_children)
	return (0);
	if (bp->bio_error != 0) {
	G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
	bp->bio_error);
	free(bp->bio_driver2, M_ELI);
	bp->bio_driver2 = NULL;
	cbp = bp->bio_driver1;
	bp->bio_driver1 = NULL;
	g_destroy_bio(cbp);
	g_io_deliver(bp, bp->bio_error);
	atomic_subtract_int(&sc->sc_inflight, 1);
	return (0);
	}
	cp = LIST_FIRST(&sc->sc_geom->consumer);
	cbp = bp->bio_driver1;
	bp->bio_driver1 = NULL;
	cbp->bio_to = cp->provider;
	cbp->bio_done = g_eli_write_done;

	/* Number of sectors from decrypted provider, eg. 1. */
	nsec = bp->bio_length / bp->bio_to->sectorsize;
	/* Number of sectors from encrypted provider, eg. 9. */
	nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;

	cbp->bio_length = cp->provider->sectorsize * nsec;
	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
	cbp->bio_data = bp->bio_driver2;

	/*
	* We write more than what is requested, so we have to be ready to write
	* more than MAXPHYS.
	*/
	cbp2 = NULL;
	if (cbp->bio_length > MAXPHYS) {
	cbp2 = g_duplicate_bio(bp);
	cbp2->bio_length = cbp->bio_length - MAXPHYS;
	cbp2->bio_data = cbp->bio_data + MAXPHYS;
	cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
	cbp2->bio_to = cp->provider;
	cbp2->bio_done = g_eli_write_done;
	cbp->bio_length = MAXPHYS;
	}
	/*
	* Send encrypted data to the provider.
	*/
	G_ELI_LOGREQ(2, cbp, "Sending request.");
	bp->bio_inbed = 0;
	bp->bio_children = (cbp2 != NULL ? 2 : 1);
	g_io_request(cbp, cp);
	if (cbp2 != NULL) {
	G_ELI_LOGREQ(2, cbp2, "Sending request.");
	g_io_request(cbp2, cp);
	}
	return (0);
	}

	void
	g_eli_auth_read(struct g_eli_softc sc, struct bio bp)
	{
	struct g_consumer *cp;
	struct bio cbp, cbp2;
	size_t size;
	off_t nsec;

	bp->bio_pflags = 0;

	cp = LIST_FIRST(&sc->sc_geom->consumer);
	cbp = bp->bio_driver1;
	bp->bio_driver1 = NULL;
	cbp->bio_to = cp->provider;
	cbp->bio_done = g_eli_read_done;

	/* Number of sectors from decrypted provider, eg. 1. */
	nsec = bp->bio_length / bp->bio_to->sectorsize;
	/* Number of sectors from encrypted provider, eg. 9. */
	nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;

	cbp->bio_length = cp->provider->sectorsize * nsec;
	size = cbp->bio_length;
	size += sizeof(int) * nsec;
	size += G_ELI_AUTH_SECKEYLEN * nsec;
	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
	bp->bio_driver2 = malloc(size, M_ELI, M_WAITOK);
	cbp->bio_data = bp->bio_driver2;

	/* Clear the error array. */
	memset((char *)bp->bio_driver2 + cbp->bio_length, 0,
	sizeof(int) * nsec);

	/*
	* We read more than what is requested, so we have to be ready to read
	* more than MAXPHYS.
	*/
	cbp2 = NULL;
	if (cbp->bio_length > MAXPHYS) {
	cbp2 = g_duplicate_bio(bp);
	cbp2->bio_length = cbp->bio_length - MAXPHYS;
	cbp2->bio_data = cbp->bio_data + MAXPHYS;
	cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
	cbp2->bio_to = cp->provider;
	cbp2->bio_done = g_eli_read_done;
	cbp->bio_length = MAXPHYS;
	}
	/*
	* Read encrypted data from provider.
	*/
	G_ELI_LOGREQ(2, cbp, "Sending request.");
	g_io_request(cbp, cp);
	if (cbp2 != NULL) {
	G_ELI_LOGREQ(2, cbp2, "Sending request.");
	g_io_request(cbp2, cp);
	}
	}

	/*
	* This is the main function responsible for cryptography (ie. communication
	* with crypto(9) subsystem).
	*
	* BIO_READ:
	* g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> G_ELI_AUTH_RUN -> g_eli_auth_read_done -> g_io_deliver
	* BIO_WRITE:
	* g_eli_start -> G_ELI_AUTH_RUN -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
	*/
	void
	g_eli_auth_run(struct g_eli_worker wr, struct bio bp)
	{
	struct g_eli_softc *sc;
	struct cryptop *crp;
	u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
	off_t dstoff;
	u_char p, data, authkey, plaindata;
	int error;

	G_ELI_LOGREQ(3, bp, "%s", __func__);

	bp->bio_pflags = wr->w_number;
	sc = wr->w_softc;
	/* Sectorsize of decrypted provider eg. 4096. */
	decr_secsize = bp->bio_to->sectorsize;
	/* The real sectorsize of encrypted provider, eg. 512. */
	encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
	/* Number of data bytes in one encrypted sector, eg. 480. */
	data_secsize = sc->sc_data_per_sector;
	/* Number of sectors from decrypted provider, eg. 2. */
	nsec = bp->bio_length / decr_secsize;
	/* Number of sectors from encrypted provider, eg. 18. */
	nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
	/* Last sector number in every big sector, eg. 9. */
	lsec = sc->sc_bytes_per_sector / encr_secsize;
	/* Destination offset, used for IV generation. */
	dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;

	plaindata = bp->bio_data;
	if (bp->bio_cmd == BIO_READ) {
	data = bp->bio_driver2;
	p = data + encr_secsize * nsec;
	p += sizeof(int) * nsec;
	} else {
	size_t size;

	size = encr_secsize * nsec;
	size += G_ELI_AUTH_SECKEYLEN * nsec;
	size += sizeof(uintptr_t); /* Space for alignment. */
	data = malloc(size, M_ELI, M_WAITOK);
	bp->bio_driver2 = data;
	p = data + encr_secsize * nsec;
	}
	bp->bio_inbed = 0;
	bp->bio_children = nsec;

	#if defined(__mips_n64) \|\| defined(__mips_o64)
	p = (char *)roundup((uintptr_t)p, sizeof(uintptr_t));
	#endif

	for (i = 1; i <= nsec; i++, dstoff += encr_secsize) {
	crp = crypto_getreq(wr->w_sid, M_WAITOK);
	authkey = (u_char *)p; p += G_ELI_AUTH_SECKEYLEN;

	data_secsize = sc->sc_data_per_sector;
	if ((i % lsec) == 0) {
	data_secsize = decr_secsize % data_secsize;
	/*
	* Last encrypted sector of each decrypted sector is
	* only partially filled.
	*/
	if (bp->bio_cmd == BIO_WRITE)
	memset(data + sc->sc_alen + data_secsize, 0,
	encr_secsize - sc->sc_alen - data_secsize);
	}

	if (bp->bio_cmd == BIO_WRITE) {
	bcopy(plaindata, data + sc->sc_alen, data_secsize);
	plaindata += data_secsize;
	}

	crypto_use_buf(crp, data, sc->sc_alen + data_secsize);
	crp->crp_opaque = (void *)bp;
	data += encr_secsize;
	crp->crp_flags = CRYPTO_F_CBIFSYNC;
	if (g_eli_batch)
	crp->crp_flags \|= CRYPTO_F_BATCH;
	if (bp->bio_cmd == BIO_WRITE) {
	crp->crp_callback = g_eli_auth_write_done;
	crp->crp_op = CRYPTO_OP_ENCRYPT \|
	CRYPTO_OP_COMPUTE_DIGEST;
	} else {
	crp->crp_callback = g_eli_auth_read_done;
	crp->crp_op = CRYPTO_OP_DECRYPT \|
	CRYPTO_OP_VERIFY_DIGEST;
	}

	crp->crp_digest_start = 0;
	crp->crp_payload_start = sc->sc_alen;
	crp->crp_payload_length = data_secsize;
	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) {
	crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
	encr_secsize);
	}
	if (g_eli_ivlen(sc->sc_ealgo) != 0) {
	crp->crp_flags \|= CRYPTO_F_IV_SEPARATE;
	g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
	- sizeof(crp->crp_iv));
	+ sizeof(crp->crp_iv));
	}

	g_eli_auth_keygen(sc, dstoff, authkey);
	crp->crp_auth_key = authkey;

	error = crypto_dispatch(crp);
	KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
	error));
	}
	}
	diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c
	index 4a3e91948ebb..7bd8eb29cc28 100644
	--- a/sys/geom/eli/g_eli_privacy.c
	+++ b/sys/geom/eli/g_eli_privacy.c
	@@ -1,298 +1,298 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/linker.h>
	#include <sys/module.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/bio.h>
	#include <sys/sysctl.h>
	#include <sys/malloc.h>
	#include <sys/kthread.h>
	#include <sys/proc.h>
	#include <sys/sched.h>
	#include <sys/smp.h>
	#include <sys/vnode.h>

	#include <vm/uma.h>

	#include <geom/geom.h>
	#include <geom/geom_dbg.h>
	#include <geom/eli/g_eli.h>
	#include <geom/eli/pkcs5v2.h>

	/*
	* Code paths:
	* BIO_READ:
	* g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
	* BIO_WRITE:
	* g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
	*/

	MALLOC_DECLARE(M_ELI);

	/*
	* The function is called after we read and decrypt data.
	*
	* g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
	*/
	static int
	g_eli_crypto_read_done(struct cryptop *crp)
	{
	struct g_eli_softc *sc;
	struct bio *bp;

	if (crp->crp_etype == EAGAIN) {
	if (g_eli_crypto_rerun(crp) == 0)
	return (0);
	}
	bp = (struct bio *)crp->crp_opaque;
	bp->bio_inbed++;
	if (crp->crp_etype == 0) {
	G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
	bp->bio_inbed, bp->bio_children);
	bp->bio_completed += crp->crp_payload_length;
	} else {
	G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
	bp->bio_inbed, bp->bio_children, crp->crp_etype);
	if (bp->bio_error == 0)
	bp->bio_error = crp->crp_etype;
	}
	sc = bp->bio_to->geom->softc;
	if (sc != NULL && crp->crp_cipher_key != NULL)
	g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
	crypto_freereq(crp);
	/*
	* Do we have all sectors already?
	*/
	if (bp->bio_inbed < bp->bio_children)
	return (0);
	free(bp->bio_driver2, M_ELI);
	bp->bio_driver2 = NULL;
	if (bp->bio_error != 0) {
	G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
	bp->bio_error);
	bp->bio_completed = 0;
	}
	/*
	* Read is finished, send it up.
	*/
	g_io_deliver(bp, bp->bio_error);
	if (sc != NULL)
	atomic_subtract_int(&sc->sc_inflight, 1);
	return (0);
	}

	/*
	* The function is called after data encryption.
	*
	* g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
	*/
	static int
	g_eli_crypto_write_done(struct cryptop *crp)
	{
	struct g_eli_softc *sc;
	struct g_geom *gp;
	struct g_consumer *cp;
	struct bio bp, cbp;

	if (crp->crp_etype == EAGAIN) {
	if (g_eli_crypto_rerun(crp) == 0)
	return (0);
	}
	bp = (struct bio *)crp->crp_opaque;
	bp->bio_inbed++;
	if (crp->crp_etype == 0) {
	G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
	bp->bio_inbed, bp->bio_children);
	} else {
	G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
	bp->bio_inbed, bp->bio_children, crp->crp_etype);
	if (bp->bio_error == 0)
	bp->bio_error = crp->crp_etype;
	}
	gp = bp->bio_to->geom;
	sc = gp->softc;
	if (crp->crp_cipher_key != NULL)
	g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
	crypto_freereq(crp);
	/*
	* All sectors are already encrypted?
	*/
	if (bp->bio_inbed < bp->bio_children)
	return (0);
	bp->bio_inbed = 0;
	bp->bio_children = 1;
	cbp = bp->bio_driver1;
	bp->bio_driver1 = NULL;
	if (bp->bio_error != 0) {
	G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
	bp->bio_error);
	free(bp->bio_driver2, M_ELI);
	bp->bio_driver2 = NULL;
	g_destroy_bio(cbp);
	g_io_deliver(bp, bp->bio_error);
	atomic_subtract_int(&sc->sc_inflight, 1);
	return (0);
	}
	cbp->bio_data = bp->bio_driver2;
	cbp->bio_done = g_eli_write_done;
	cp = LIST_FIRST(&gp->consumer);
	cbp->bio_to = cp->provider;
	G_ELI_LOGREQ(2, cbp, "Sending request.");
	/*
	* Send encrypted data to the provider.
	*/
	g_io_request(cbp, cp);
	return (0);
	}

	/*
	* The function is called to read encrypted data.
	*
	* g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
	*/
	void
	g_eli_crypto_read(struct g_eli_softc sc, struct bio bp, boolean_t fromworker)
	{
	struct g_consumer *cp;
	struct bio *cbp;

	if (!fromworker) {
	/*
	* We are not called from the worker thread, so check if
	* device is suspended.
	*/
	mtx_lock(&sc->sc_queue_mtx);
	if (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
	/*
	* If device is suspended, we place the request onto
	* the queue, so it can be handled after resume.
	*/
	G_ELI_DEBUG(0, "device suspended, move onto queue");
	bioq_insert_tail(&sc->sc_queue, bp);
	mtx_unlock(&sc->sc_queue_mtx);
	wakeup(sc);
	return;
	}
	atomic_add_int(&sc->sc_inflight, 1);
	mtx_unlock(&sc->sc_queue_mtx);
	}
	bp->bio_pflags = 0;
	bp->bio_driver2 = NULL;
	cbp = bp->bio_driver1;
	cbp->bio_done = g_eli_read_done;
	cp = LIST_FIRST(&sc->sc_geom->consumer);
	cbp->bio_to = cp->provider;
	G_ELI_LOGREQ(2, cbp, "Sending request.");
	/*
	* Read encrypted data from provider.
	*/
	g_io_request(cbp, cp);
	}

	/*
	* This is the main function responsible for cryptography (ie. communication
	* with crypto(9) subsystem).
	*
	* BIO_READ:
	* g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver
	* BIO_WRITE:
	* g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
	*/
	void
	g_eli_crypto_run(struct g_eli_worker wr, struct bio bp)
	{
	struct g_eli_softc *sc;
	struct cryptop *crp;
	u_int i, nsec, secsize;
	off_t dstoff;
	u_char *data;
	int error;

	G_ELI_LOGREQ(3, bp, "%s", __func__);

	bp->bio_pflags = wr->w_number;
	sc = wr->w_softc;
	secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
	nsec = bp->bio_length / secsize;

	bp->bio_inbed = 0;
	bp->bio_children = nsec;

	/*
	* If we write the data we cannot destroy current bio_data content,
	* so we need to allocate more memory for encrypted data.
	*/
	if (bp->bio_cmd == BIO_WRITE) {
	data = malloc(bp->bio_length, M_ELI, M_WAITOK);
	bp->bio_driver2 = data;
	bcopy(bp->bio_data, data, bp->bio_length);
	} else
	data = bp->bio_data;

	for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
	crp = crypto_getreq(wr->w_sid, M_WAITOK);

	crypto_use_buf(crp, data, secsize);
	crp->crp_opaque = (void *)bp;
	data += secsize;
	if (bp->bio_cmd == BIO_WRITE) {
	crp->crp_op = CRYPTO_OP_ENCRYPT;
	crp->crp_callback = g_eli_crypto_write_done;
	} else /* if (bp->bio_cmd == BIO_READ) */ {
	crp->crp_op = CRYPTO_OP_DECRYPT;
	crp->crp_callback = g_eli_crypto_read_done;
	}
	crp->crp_flags = CRYPTO_F_CBIFSYNC;
	if (g_eli_batch)
	crp->crp_flags \|= CRYPTO_F_BATCH;

	crp->crp_payload_start = 0;
	crp->crp_payload_length = secsize;
	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) {
	crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
	secsize);
	}
	if (g_eli_ivlen(sc->sc_ealgo) != 0) {
	crp->crp_flags \|= CRYPTO_F_IV_SEPARATE;
	g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
	- sizeof(crp->crp_iv));
	+ sizeof(crp->crp_iv));
	}

	error = crypto_dispatch(crp);
	KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
	error));
	}
	}
	diff --git a/sys/geom/geom.h b/sys/geom/geom.h
	index 46ec01e6c94a..25cf0131355d 100644
	--- a/sys/geom/geom.h
	+++ b/sys/geom/geom.h
	@@ -1,440 +1,440 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 2002 Poul-Henning Kamp
	* Copyright (c) 2002 Networks Associates Technology, Inc.
	* All rights reserved.
	*
	* This software was developed for the FreeBSD Project by Poul-Henning Kamp
	* and NAI Labs, the Security Research Division of Network Associates, Inc.
	* under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
	* DARPA CHATS research program.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. The names of the authors may not be used to endorse or promote
	* products derived from this software without specific prior written
	* permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef _GEOM_GEOM_H_
	#define _GEOM_GEOM_H_

	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/sx.h>
	#include <sys/queue.h>
	#include <sys/ioccom.h>
	#include <sys/conf.h>
	#include <sys/module.h>

	struct g_class;
	struct g_geom;
	struct g_consumer;
	struct g_provider;
	struct g_stat;
	struct thread;
	struct bio;
	struct sbuf;
	struct gctl_req;
	struct g_configargs;
	struct disk_zone_args;

	typedef int g_config_t (struct g_configargs *ca);
	typedef void g_ctl_req_t (struct gctl_req , struct g_class cp, char const *verb);
	typedef int g_ctl_create_geom_t (struct gctl_req , struct g_class cp, struct g_provider *pp);
	typedef int g_ctl_destroy_geom_t (struct gctl_req , struct g_class cp, struct g_geom *gp);
	typedef int g_ctl_config_geom_t (struct gctl_req , struct g_geom gp, const char *verb);
	typedef void g_init_t (struct g_class *mp);
	typedef void g_fini_t (struct g_class *mp);
	typedef struct g_geom * g_taste_t (struct g_class , struct g_provider , int flags);
	typedef int g_ioctl_t(struct g_provider pp, u_long cmd, void data, int fflag, struct thread *td);
	#define G_TF_NORMAL 0
	#define G_TF_INSIST 1
	#define G_TF_TRANSPARENT 2
	typedef int g_access_t (struct g_provider *, int, int, int);
	/* XXX: not sure about the thread arg */
	typedef void g_orphan_t (struct g_consumer *);

	typedef void g_start_t (struct bio *);
	typedef void g_spoiled_t (struct g_consumer *);
	typedef void g_attrchanged_t (struct g_consumer , const char attr);
	typedef void g_provgone_t (struct g_provider *);
	typedef void g_dumpconf_t (struct sbuf , const char indent, struct g_geom *,
	struct g_consumer , struct g_provider );
	typedef void g_resize_t(struct g_consumer *cp);

	/*
	* The g_class structure describes a transformation class. In other words
	* all BSD disklabel handlers share one g_class, all MBR handlers share
	* one common g_class and so on.
	* Certain operations are instantiated on the class, most notably the
	* taste and config_geom functions.
	*/
	struct g_class {
	const char *name;
	u_int version;
	u_int spare0;
	g_taste_t *taste;
	g_config_t *config;
	g_ctl_req_t *ctlreq;
	g_init_t *init;
	g_fini_t *fini;
	g_ctl_destroy_geom_t *destroy_geom;
	/*
	* Default values for geom methods
	*/
	g_start_t *start;
	g_spoiled_t *spoiled;
	g_attrchanged_t *attrchanged;
	g_dumpconf_t *dumpconf;
	g_access_t *access;
	g_orphan_t *orphan;
	g_ioctl_t *ioctl;
	g_provgone_t *providergone;
	g_resize_t *resize;
	void *spare1;
	void *spare2;
	/*
	* The remaining elements are private
	*/
	LIST_ENTRY(g_class) class;
	LIST_HEAD(,g_geom) geom;
	};

	#define G_VERSION_00 0x19950323
	#define G_VERSION_01 0x20041207 /* add fflag to g_ioctl_t */
	#define G_VERSION G_VERSION_01

	/*
	* The g_geom is an instance of a g_class.
	*/
	struct g_geom {
	char *name;
	struct g_class *class;
	LIST_ENTRY(g_geom) geom;
	LIST_HEAD(,g_consumer) consumer;
	LIST_HEAD(,g_provider) provider;
	TAILQ_ENTRY(g_geom) geoms; /* XXX: better name */
	int rank;
	g_start_t *start;
	g_spoiled_t *spoiled;
	g_attrchanged_t *attrchanged;
	g_dumpconf_t *dumpconf;
	g_access_t *access;
	g_orphan_t *orphan;
	g_ioctl_t *ioctl;
	g_provgone_t *providergone;
	g_resize_t *resize;
	void *spare0;
	void *spare1;
	void *softc;
	unsigned flags;
	#define G_GEOM_WITHER 0x01
	#define G_GEOM_VOLATILE_BIO 0x02
	#define G_GEOM_IN_ACCESS 0x04
	#define G_GEOM_ACCESS_WAIT 0x08
	};

	/*
	* The g_bioq is a queue of struct bio's.
	* XXX: possibly collection point for statistics.
	* XXX: should (possibly) be collapsed with sys/bio.h::bio_queue_head.
	*/
	struct g_bioq {
	TAILQ_HEAD(, bio) bio_queue;
	struct mtx bio_queue_lock;
	int bio_queue_length;
	};

	/*
	* A g_consumer is an attachment point for a g_provider. One g_consumer
	* can only be attached to one g_provider, but multiple g_consumers
	* can be attached to one g_provider.
	*/

	struct g_consumer {
	struct g_geom *geom;
	LIST_ENTRY(g_consumer) consumer;
	struct g_provider *provider;
	LIST_ENTRY(g_consumer) consumers; /* XXX: better name */
	int acr, acw, ace;
	int flags;
	#define G_CF_SPOILED 0x1
	#define G_CF_ORPHAN 0x4
	#define G_CF_DIRECT_SEND 0x10
	#define G_CF_DIRECT_RECEIVE 0x20
	struct devstat *stat;
	u_int nstart, nend;

	/* Two fields for the implementing class to use */
	void *private;
	u_int index;
	};

	/*
	* The g_geom_alias is a list node for aliases for the provider name for device
	* node creation.
	*/
	struct g_geom_alias {
	LIST_ENTRY(g_geom_alias) ga_next;
	const char *ga_alias;
	};

	/*
	* A g_provider is a "logical disk".
	*/
	struct g_provider {
	char *name;
	LIST_ENTRY(g_provider) provider;
	struct g_geom *geom;
	LIST_HEAD(,g_consumer) consumers;
	int acr, acw, ace;
	int error;
	TAILQ_ENTRY(g_provider) orphan;
	off_t mediasize;
	u_int sectorsize;
	off_t stripesize;
	off_t stripeoffset;
	struct devstat *stat;
	u_int spare1;
	u_int spare2;
	u_int flags;
	#define G_PF_WITHER 0x2
	#define G_PF_ORPHAN 0x4
	#define G_PF_ACCEPT_UNMAPPED 0x8
	#define G_PF_DIRECT_SEND 0x10
	#define G_PF_DIRECT_RECEIVE 0x20
	LIST_HEAD(,g_geom_alias) aliases;

	/* Two fields for the implementing class to use */
	void *private;
	u_int index;
	};

	/* BIO_GETATTR("GEOM::setstate") argument values. */
	#define G_STATE_FAILED 0
	#define G_STATE_REBUILD 1
	#define G_STATE_RESYNC 2
	#define G_STATE_ACTIVE 3

	/* geom_dev.c */
	struct cdev;
	void g_dev_print(void);
	void g_dev_physpath_changed(void);
	struct g_provider g_dev_getprovider(struct cdev dev);

	/* geom_dump.c */
	void (g_trace)(int level, const char *, ...) __printflike(2, 3);
	#define G_T_TOPOLOGY 0x01
	#define G_T_BIO 0x02
	#define G_T_ACCESS 0x04
	extern int g_debugflags;
	#define G_F_FOOTSHOOTING 0x10
	#define G_F_DISKIOCTL 0x40
	#define G_F_CTLDUMP 0x80
	#define g_trace(level, fmt, ...) do { \
	if (__predict_false(g_debugflags & (level))) \
	(g_trace)(level, fmt, ## __VA_ARGS__); \
	} while (0)

	/* geom_event.c */
	typedef void g_event_t(void *, int flag);
	#define EV_CANCEL 1
	int g_post_event(g_event_t func, void arg, int flag, ...);
	int g_waitfor_event(g_event_t func, void arg, int flag, ...);
	void g_cancel_event(void *ref);
	int g_attr_changed(struct g_provider pp, const char attr, int flag);
	int g_media_changed(struct g_provider *pp, int flag);
	int g_media_gone(struct g_provider *pp, int flag);
	void g_orphan_provider(struct g_provider *pp, int error);
	void g_waitidlelock(void);

	/* geom_subr.c */
	int g_access(struct g_consumer *cp, int nread, int nwrite, int nexcl);
	int g_attach(struct g_consumer cp, struct g_provider pp);
	int g_compare_names(const char namea, const char nameb);
	void g_destroy_consumer(struct g_consumer *cp);
	void g_destroy_geom(struct g_geom *pp);
	void g_destroy_provider(struct g_provider *pp);
	void g_detach(struct g_consumer *cp);
	void g_error_provider(struct g_provider *pp, int error);
	struct g_provider g_provider_by_name(char const arg);
	int g_getattr__(const char attr, struct g_consumer cp, void *var, int len);
	#define g_getattr(a, c, v) g_getattr__((a), (c), (v), sizeof *(v))
	int g_handleattr(struct bio bp, const char attribute, const void *val,
	int len);
	int g_handleattr_int(struct bio bp, const char attribute, int val);
	int g_handleattr_off_t(struct bio bp, const char attribute, off_t val);
	int g_handleattr_uint16_t(struct bio bp, const char attribute, uint16_t val);
	int g_handleattr_str(struct bio bp, const char attribute, const char *str);
	struct g_consumer * g_new_consumer(struct g_geom *gp);
	struct g_geom * g_new_geomf(struct g_class mp, const char fmt, ...)
	__printflike(2, 3);
	struct g_provider * g_new_providerf(struct g_geom gp, const char fmt, ...)
	__printflike(2, 3);
	void g_provider_add_alias(struct g_provider pp, const char fmt, ...)
	__printflike(2, 3);
	void g_resize_provider(struct g_provider *pp, off_t size);
	int g_retaste(struct g_class *mp);
	void g_spoil(struct g_provider pp, struct g_consumer cp);
	int g_std_access(struct g_provider *pp, int dr, int dw, int de);
	void g_std_done(struct bio *bp);
	void g_std_spoiled(struct g_consumer *cp);
	void g_wither_geom(struct g_geom *gp, int error);
	void g_wither_geom_close(struct g_geom *gp, int error);
	void g_wither_provider(struct g_provider *pp, int error);

	#if defined(DIAGNOSTIC) \|\| defined(DDB)
	int g_valid_obj(void const *ptr);
	#endif
	#ifdef DIAGNOSTIC
	#define G_VALID_CLASS(foo) \
	KASSERT(g_valid_obj(foo) == 1, ("%p is not a g_class", foo))
	#define G_VALID_GEOM(foo) \
	KASSERT(g_valid_obj(foo) == 2, ("%p is not a g_geom", foo))
	#define G_VALID_CONSUMER(foo) \
	KASSERT(g_valid_obj(foo) == 3, ("%p is not a g_consumer", foo))
	#define G_VALID_PROVIDER(foo) \
	KASSERT(g_valid_obj(foo) == 4, ("%p is not a g_provider", foo))
	#else
	#define G_VALID_CLASS(foo) do { } while (0)
	#define G_VALID_GEOM(foo) do { } while (0)
	#define G_VALID_CONSUMER(foo) do { } while (0)
	#define G_VALID_PROVIDER(foo) do { } while (0)
	#endif

	int g_modevent(module_t, int, void *);

	/* geom_io.c */
	struct bio * g_clone_bio(struct bio *);
	struct bio * g_duplicate_bio(struct bio *);
	void g_destroy_bio(struct bio *);
	void g_io_deliver(struct bio *bp, int error);
	int g_io_getattr(const char attr, struct g_consumer cp, int len, void ptr);
	int g_io_zonecmd(struct disk_zone_args zone_args, struct g_consumer cp);
	int g_io_flush(struct g_consumer *cp);
	int g_io_speedup(size_t shortage, u_int flags, size_t resid, struct g_consumer cp);
	void g_io_request(struct bio bp, struct g_consumer cp);
	struct bio *g_new_bio(void);
	struct bio *g_alloc_bio(void);
	void g_reset_bio(struct bio *);
	void * g_read_data(struct g_consumer cp, off_t offset, off_t length, int error);
	int g_write_data(struct g_consumer cp, off_t offset, void ptr, off_t length);
	int g_delete_data(struct g_consumer *cp, off_t offset, off_t length);
	void g_format_bio(struct sbuf , const struct bio bp);
	void g_print_bio(const char prefix, const struct bio bp, const char *fmtsuffix, ...) __printflike(3, 4);
	int g_use_g_read_data(void , off_t, void *, int);
	int g_use_g_write_data(void , off_t, void , int);

	/* geom_kern.c / geom_kernsim.c */

	#ifdef _KERNEL

	extern struct sx topology_lock;

	struct g_kerneldump {
	off_t offset;
	off_t length;
	struct dumperinfo di;
	};

	MALLOC_DECLARE(M_GEOM);

	static __inline void *
	g_malloc(int size, int flags)
	{
	void *p;

	p = malloc(size, M_GEOM, flags);
	return (p);
	}

	static __inline void
	g_free(void *ptr)
	{

	#ifdef DIAGNOSTIC
	if (sx_xlocked(&topology_lock)) {
	KASSERT(g_valid_obj(ptr) == 0,
	("g_free(%p) of live object, type %d", ptr,
	g_valid_obj(ptr)));
	}
	#endif
	free(ptr, M_GEOM);
	}

	#define g_topology_lock() \
	do { \
	sx_xlock(&topology_lock); \
	} while (0)

	#define g_topology_try_lock() sx_try_xlock(&topology_lock)

	#define g_topology_unlock() \
	do { \
	sx_xunlock(&topology_lock); \
	} while (0)

	#define g_topology_locked() sx_xlocked(&topology_lock)

	#define g_topology_assert() \
	do { \
	sx_assert(&topology_lock, SX_XLOCKED); \
	} while (0)

	#define g_topology_assert_not() \
	do { \
	sx_assert(&topology_lock, SX_UNLOCKED); \
	} while (0)

	#define g_topology_sleep(chan, timo) \
	sx_sleep(chan, &topology_lock, 0, "gtopol", timo)

	#define DECLARE_GEOM_CLASS(class, name) \
	static moduledata_t name##_mod = { \
	#name, g_modevent, &class \
	}; \
	DECLARE_MODULE(name, name##_mod, SI_SUB_DRIVERS, SI_ORDER_SECOND);

	int g_is_geom_thread(struct thread *td);

	#ifndef _PATH_DEV
	#define _PATH_DEV "/dev/"
	#endif

	#endif /* _KERNEL */

	/* geom_ctl.c */
	int gctl_set_param(struct gctl_req req, const char param, void const *ptr, int len);
	void gctl_set_param_err(struct gctl_req req, const char param, void const *ptr, int len);
	void gctl_get_param(struct gctl_req req, const char param, int len);
	char const gctl_get_asciiparam(struct gctl_req req, const char *param);
	void gctl_get_paraml(struct gctl_req req, const char *param, int len);
	void gctl_get_paraml_opt(struct gctl_req req, const char *param, int len);
	int gctl_error(struct gctl_req req, const char fmt, ...) __printflike(2, 3);
	struct g_class gctl_get_class(struct gctl_req req, char const *arg);
	-struct g_geom gctl_get_geom(struct gctl_req req, struct g_class mpr, char const arg);
	+struct g_geom gctl_get_geom(struct gctl_req req, struct g_class mp, char const arg);
	struct g_provider gctl_get_provider(struct gctl_req req, char const *arg);

	#endif /* _GEOM_GEOM_H_ */
	diff --git a/sys/geom/geom_ctl.c b/sys/geom/geom_ctl.c
	index 8b9af1aa54ab..7dbe012a3320 100644
	--- a/sys/geom/geom_ctl.c
	+++ b/sys/geom/geom_ctl.c
	@@ -1,535 +1,530 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 2002 Poul-Henning Kamp
	* Copyright (c) 2002 Networks Associates Technology, Inc.
	* All rights reserved.
	*
	* This software was developed for the FreeBSD Project by Poul-Henning Kamp
	* and NAI Labs, the Security Research Division of Network Associates, Inc.
	* under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
	* DARPA CHATS research program.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. The names of the authors may not be used to endorse or promote
	* products derived from this software without specific prior written
	* permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_geom.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/sysctl.h>
	#include <sys/bio.h>
	#include <sys/conf.h>
	#include <sys/disk.h>
	#include <sys/malloc.h>
	#include <sys/sysctl.h>
	#include <sys/sbuf.h>

	#include <sys/lock.h>
	#include <sys/mutex.h>

	#include <vm/vm.h>
	#include <vm/vm_extern.h>

	#include <geom/geom.h>
	#include <geom/geom_int.h>
	#define GCTL_TABLE 1
	#include <geom/geom_ctl.h>

	#include <machine/stdarg.h>

	static d_ioctl_t g_ctl_ioctl;

	static struct cdevsw g_ctl_cdevsw = {
	.d_version = D_VERSION,
	.d_flags = 0,
	.d_ioctl = g_ctl_ioctl,
	.d_name = "g_ctl",
	};

	void
	g_ctl_init(void)
	{

	make_dev_credf(MAKEDEV_ETERNAL, &g_ctl_cdevsw, 0, NULL,
	UID_ROOT, GID_OPERATOR, 0640, PATH_GEOM_CTL);
	KASSERT(GCTL_PARAM_RD == VM_PROT_READ,
	("GCTL_PARAM_RD != VM_PROT_READ"));
	KASSERT(GCTL_PARAM_WR == VM_PROT_WRITE,
	("GCTL_PARAM_WR != VM_PROT_WRITE"));
	}

	/*
	* Report an error back to the user in ascii format. Return nerror
	* or EINVAL if nerror isn't specified.
	*/
	int
	gctl_error(struct gctl_req req, const char fmt, ...)
	{
	va_list ap;

	if (req == NULL)
	return (EINVAL);

	/* We only record the first error */
	if (sbuf_done(req->serror)) {
	if (!req->nerror)
	req->nerror = EEXIST;
	return (req->nerror);
	}
	if (!req->nerror)
	req->nerror = EINVAL;

	va_start(ap, fmt);
	sbuf_vprintf(req->serror, fmt, ap);
	va_end(ap);
	sbuf_finish(req->serror);
	if (g_debugflags & G_F_CTLDUMP)
	printf("gctl %p error \"%s\"\n", req, sbuf_data(req->serror));
	return (req->nerror);
	}

	/*
	* Allocate space and copyin() something.
	* XXX: this should really be a standard function in the kernel.
	*/
	static void *
	geom_alloc_copyin(struct gctl_req req, void uaddr, size_t len)
	{
	void *ptr;

	ptr = g_malloc(len, M_WAITOK);
	req->nerror = copyin(uaddr, ptr, len);
	if (!req->nerror)
	return (ptr);
	g_free(ptr);
	return (NULL);
	}

	static void
	gctl_copyin(struct gctl_req *req)
	{
	struct gctl_req_arg *ap;
	char *p;
	u_int i;

	if (req->narg > GEOM_CTL_ARG_MAX) {
	gctl_error(req, "too many arguments");
	req->arg = NULL;
	return;
	}

	ap = geom_alloc_copyin(req, req->arg, req->narg * sizeof(*ap));
	if (ap == NULL) {
	gctl_error(req, "bad control request");
	req->arg = NULL;
	return;
	}

	/* Nothing have been copyin()'ed yet */
	for (i = 0; i < req->narg; i++) {
	ap[i].flag &= ~(GCTL_PARAM_NAMEKERNEL\|GCTL_PARAM_VALUEKERNEL);
	ap[i].flag &= ~GCTL_PARAM_CHANGED;
	ap[i].kvalue = NULL;
	}

	for (i = 0; i < req->narg; i++) {
	if (ap[i].nlen < 1 \|\| ap[i].nlen > SPECNAMELEN) {
	gctl_error(req,
	"wrong param name length %d: %d", i, ap[i].nlen);
	break;
	}
	p = geom_alloc_copyin(req, ap[i].name, ap[i].nlen);
	if (p == NULL)
	break;
	if (p[ap[i].nlen - 1] != '\0') {
	gctl_error(req, "unterminated param name");
	g_free(p);
	break;
	}
	ap[i].name = p;
	ap[i].flag \|= GCTL_PARAM_NAMEKERNEL;
	if (ap[i].len <= 0) {
	gctl_error(req, "negative param length");
	break;
	}
	p = geom_alloc_copyin(req, ap[i].value, ap[i].len);
	if (p == NULL)
	break;
	if ((ap[i].flag & GCTL_PARAM_ASCII) &&
	p[ap[i].len - 1] != '\0') {
	gctl_error(req, "unterminated param value");
	g_free(p);
	break;
	}
	ap[i].kvalue = p;
	ap[i].flag \|= GCTL_PARAM_VALUEKERNEL;
	}
	req->arg = ap;
	return;
	}

	static void
	gctl_copyout(struct gctl_req *req)
	{
	int error, i;
	struct gctl_req_arg *ap;

	if (req->nerror)
	return;
	error = 0;
	ap = req->arg;
	for (i = 0; i < req->narg; i++, ap++) {
	if (!(ap->flag & GCTL_PARAM_CHANGED))
	continue;
	error = copyout(ap->kvalue, ap->value, ap->len);
	if (!error)
	continue;
	req->nerror = error;
	return;
	}
	return;
	}

	static void
	gctl_free(struct gctl_req *req)
	{
	u_int i;

	sbuf_delete(req->serror);
	if (req->arg == NULL)
	return;
	for (i = 0; i < req->narg; i++) {
	if (req->arg[i].flag & GCTL_PARAM_NAMEKERNEL)
	g_free(req->arg[i].name);
	if ((req->arg[i].flag & GCTL_PARAM_VALUEKERNEL) &&
	req->arg[i].len > 0)
	g_free(req->arg[i].kvalue);
	}
	g_free(req->arg);
	}

	static void
	gctl_dump(struct gctl_req *req)
	{
	struct gctl_req_arg *ap;
	u_int i;
	int j;

	printf("Dump of gctl request at %p:\n", req);
	if (req->nerror > 0) {
	printf(" nerror:\t%d\n", req->nerror);
	if (sbuf_len(req->serror) > 0)
	printf(" error:\t\"%s\"\n", sbuf_data(req->serror));
	}
	if (req->arg == NULL)
	return;
	for (i = 0; i < req->narg; i++) {
	ap = &req->arg[i];
	if (!(ap->flag & GCTL_PARAM_NAMEKERNEL))
	printf(" param:\t%d@%p", ap->nlen, ap->name);
	else
	printf(" param:\t\"%s\"", ap->name);
	printf(" [%s%s%d] = ",
	ap->flag & GCTL_PARAM_RD ? "R" : "",
	ap->flag & GCTL_PARAM_WR ? "W" : "",
	ap->len);
	if (!(ap->flag & GCTL_PARAM_VALUEKERNEL)) {
	printf(" =@ %p", ap->value);
	} else if (ap->flag & GCTL_PARAM_ASCII) {
	printf("\"%s\"", (char *)ap->kvalue);
	} else if (ap->len > 0) {
	for (j = 0; j < ap->len && j < 512; j++)
	printf(" %02x", ((u_char *)ap->kvalue)[j]);
	} else {
	printf(" = %p", ap->kvalue);
	}
	printf("\n");
	}
	}

	int
	gctl_set_param(struct gctl_req req, const char param, void const *ptr,
	int len)
	{
	u_int i;
	struct gctl_req_arg *ap;

	for (i = 0; i < req->narg; i++) {
	ap = &req->arg[i];
	if (strcmp(param, ap->name))
	continue;
	if (!(ap->flag & GCTL_PARAM_WR))
	return (EPERM);
	ap->flag \|= GCTL_PARAM_CHANGED;
	if (ap->len < len) {
	bcopy(ptr, ap->kvalue, ap->len);
	return (ENOSPC);
	}
	bcopy(ptr, ap->kvalue, len);
	return (0);
	}
	return (EINVAL);
	}

	void
	gctl_set_param_err(struct gctl_req req, const char param, void const *ptr,
	int len)
	{

	switch (gctl_set_param(req, param, ptr, len)) {
	case EPERM:
	gctl_error(req, "No write access %s argument", param);
	break;
	case ENOSPC:
	gctl_error(req, "Wrong length %s argument", param);
	break;
	case EINVAL:
	gctl_error(req, "Missing %s argument", param);
	break;
	}
	}

	void *
	gctl_get_param(struct gctl_req req, const char param, int *len)
	{
	u_int i;
	void *p;
	struct gctl_req_arg *ap;

	for (i = 0; i < req->narg; i++) {
	ap = &req->arg[i];
	if (strcmp(param, ap->name))
	continue;
	if (!(ap->flag & GCTL_PARAM_RD))
	continue;
	p = ap->kvalue;
	if (len != NULL)
	*len = ap->len;
	return (p);
	}
	return (NULL);
	}

	char const *
	gctl_get_asciiparam(struct gctl_req req, const char param)
	{
	u_int i;
	char const *p;
	struct gctl_req_arg *ap;

	for (i = 0; i < req->narg; i++) {
	ap = &req->arg[i];
	if (strcmp(param, ap->name))
	continue;
	if (!(ap->flag & GCTL_PARAM_RD))
	continue;
	p = ap->kvalue;
	if (ap->len < 1) {
	gctl_error(req, "No length argument (%s)", param);
	return (NULL);
	}
	if (p[ap->len - 1] != '\0') {
	gctl_error(req, "Unterminated argument (%s)", param);
	return (NULL);
	}
	return (p);
	}
	return (NULL);
	}

	void *
	gctl_get_paraml_opt(struct gctl_req req, const char param, int len)
	{
	int i;
	void *p;

	p = gctl_get_param(req, param, &i);
	if (i != len) {
	p = NULL;
	gctl_error(req, "Wrong length %s argument", param);
	}
	return (p);
	}

	void *
	gctl_get_paraml(struct gctl_req req, const char param, int len)
	{
	void *p;

	p = gctl_get_paraml_opt(req, param, len);
	if (p == NULL)
	gctl_error(req, "Missing %s argument", param);
	return (p);
	}

	struct g_class *
	gctl_get_class(struct gctl_req req, char const arg)
	{
	char const *p;
	struct g_class *cp;

	p = gctl_get_asciiparam(req, arg);
	if (p == NULL) {
	gctl_error(req, "Missing %s argument", arg);
	return (NULL);
	}
	LIST_FOREACH(cp, &g_classes, class) {
	if (!strcmp(p, cp->name))
	return (cp);
	}
	gctl_error(req, "Class not found: \"%s\"", p);
	return (NULL);
	}

	struct g_geom *
	-gctl_get_geom(struct gctl_req req, struct g_class mpr, char const *arg)
	+gctl_get_geom(struct gctl_req req, struct g_class mp, char const *arg)
	{
	char const *p;
	- struct g_class *mp;
	struct g_geom *gp;

	+ MPASS(mp != NULL);
	p = gctl_get_asciiparam(req, arg);
	if (p == NULL) {
	gctl_error(req, "Missing %s argument", arg);
	return (NULL);
	}
	- LIST_FOREACH(mp, &g_classes, class) {
	- if (mpr != NULL && mpr != mp)
	- continue;
	- LIST_FOREACH(gp, &mp->geom, geom) {
	- if (!strcmp(p, gp->name))
	- return (gp);
	- }
	- }
	+ LIST_FOREACH(gp, &mp->geom, geom)
	+ if (!strcmp(p, gp->name))
	+ return (gp);
	gctl_error(req, "Geom not found: \"%s\"", p);
	return (NULL);
	}

	struct g_provider *
	gctl_get_provider(struct gctl_req req, char const arg)
	{
	char const *p;
	struct g_provider *pp;

	p = gctl_get_asciiparam(req, arg);
	if (p == NULL) {
	gctl_error(req, "Missing '%s' argument", arg);
	return (NULL);
	}
	pp = g_provider_by_name(p);
	if (pp != NULL)
	return (pp);
	gctl_error(req, "Provider not found: \"%s\"", p);
	return (NULL);
	}

	static void
	g_ctl_req(void *arg, int flag __unused)
	{
	struct g_class *mp;
	struct gctl_req *req;
	char const *verb;

	g_topology_assert();
	req = arg;
	mp = gctl_get_class(req, "class");
	if (mp == NULL)
	return;
	if (mp->ctlreq == NULL) {
	gctl_error(req, "Class takes no requests");
	return;
	}
	verb = gctl_get_param(req, "verb", NULL);
	if (verb == NULL) {
	gctl_error(req, "Verb missing");
	return;
	}
	mp->ctlreq(req, mp, verb);
	g_topology_assert();
	}


	static int
	g_ctl_ioctl_ctl(struct cdev dev, u_long cmd, caddr_t data, int fflag, struct thread td)
	{
	struct gctl_req *req;
	int nerror;

	req = (void *)data;
	req->nerror = 0;
	/* It is an error if we cannot return an error text */
	if (req->lerror < 2)
	return (EINVAL);
	if (!useracc(req->error, req->lerror, VM_PROT_WRITE))
	return (EINVAL);

	req->serror = sbuf_new_auto();
	/* Check the version */
	if (req->version != GCTL_VERSION) {
	gctl_error(req, "kernel and libgeom version mismatch.");
	req->arg = NULL;
	} else {
	/* Get things on board */
	gctl_copyin(req);

	if (g_debugflags & G_F_CTLDUMP)
	gctl_dump(req);

	if (!req->nerror) {
	g_waitfor_event(g_ctl_req, req, M_WAITOK, NULL);
	gctl_copyout(req);
	}
	}
	if (sbuf_done(req->serror)) {
	copyout(sbuf_data(req->serror), req->error,
	imin(req->lerror, sbuf_len(req->serror) + 1));
	}

	nerror = req->nerror;
	gctl_free(req);
	return (nerror);
	}

	static int
	g_ctl_ioctl(struct cdev dev, u_long cmd, caddr_t data, int fflag, struct thread td)
	{
	int error;

	switch(cmd) {
	case GEOM_CTL:
	error = g_ctl_ioctl_ctl(dev, cmd, data, fflag, td);
	break;
	default:
	error = ENOIOCTL;
	break;
	}
	return (error);

	}
	diff --git a/sys/geom/geom_map.c b/sys/geom/geom_map.c
	index fcdba38c8055..e5f1cb649421 100644
	--- a/sys/geom/geom_map.c
	+++ b/sys/geom/geom_map.c
	@@ -1,410 +1,397 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2010-2011 Aleksandr Rybalko <ray@dlink.ua>
	* based on geom_redboot.c
	* Copyright (c) 2009 Sam Leffler, Errno Consulting
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer,
	* without modification.
	* 2. Redistributions in binary form must reproduce at minimum a disclaimer
	* similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
	* redistribution must be conditioned upon including a substantially
	* similar Disclaimer requirement for further binary redistribution.
	*
	* NO WARRANTY
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
	* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
	* THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
	* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
	* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	* THE POSSIBILITY OF SUCH DAMAGES.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/errno.h>
	#include <sys/endian.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/fcntl.h>
	#include <sys/malloc.h>
	#include <sys/bio.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/sbuf.h>

	#include <geom/geom.h>
	#include <geom/geom_slice.h>

	#define MAP_CLASS_NAME "MAP"
	#define MAP_MAXSLICE 64
	#define MAP_MAX_MARKER_LEN 64

	struct g_map_softc {
	off_t offset[MAP_MAXSLICE]; /* offset in flash */
	off_t size[MAP_MAXSLICE]; /* image size in bytes */
	off_t entry[MAP_MAXSLICE];
	off_t dsize[MAP_MAXSLICE];
	uint8_t readonly[MAP_MAXSLICE];
	g_access_t *parent_access;
	};

	static int
	g_map_access(struct g_provider *pp, int dread, int dwrite, int dexcl)
	{
	struct g_geom *gp;
	struct g_slicer *gsp;
	struct g_map_softc *sc;

	gp = pp->geom;
	gsp = gp->softc;
	sc = gsp->softc;

	if (dwrite > 0 && sc->readonly[pp->index])
	return (EPERM);

	return (sc->parent_access(pp, dread, dwrite, dexcl));
	}

	static int
	g_map_start(struct bio *bp)
	{
	struct g_provider *pp;
	struct g_geom *gp;
	struct g_map_softc *sc;
	struct g_slicer *gsp;
	int idx;

	pp = bp->bio_to;
	idx = pp->index;
	gp = pp->geom;
	gsp = gp->softc;
	sc = gsp->softc;

	if (bp->bio_cmd == BIO_GETATTR) {
	if (g_handleattr_int(bp, MAP_CLASS_NAME "::entry",
	sc->entry[idx])) {
	return (1);
	}
	if (g_handleattr_int(bp, MAP_CLASS_NAME "::dsize",
	sc->dsize[idx])) {
	return (1);
	}
	}

	return (0);
	}

	static void
	g_map_dumpconf(struct sbuf sb, const char indent, struct g_geom *gp,
	struct g_consumer cp __unused, struct g_provider pp)
	{
	struct g_map_softc *sc;
	struct g_slicer *gsp;

	gsp = gp->softc;
	sc = gsp->softc;
	g_slice_dumpconf(sb, indent, gp, cp, pp);
	if (pp != NULL) {
	if (indent == NULL) {
	sbuf_printf(sb, " entry %jd", (intmax_t)sc->entry[pp->index]);
	sbuf_printf(sb, " dsize %jd", (intmax_t)sc->dsize[pp->index]);
	} else {
	sbuf_printf(sb, "%s<entry>%jd</entry>\n", indent,
	(intmax_t)sc->entry[pp->index]);
	sbuf_printf(sb, "%s<dsize>%jd</dsize>\n", indent,
	(intmax_t)sc->dsize[pp->index]);
	}
	}
	}

	static int
	find_marker(struct g_consumer cp, const char line, off_t *offset)
	{
	off_t search_start, search_offset, search_step;
	size_t sectorsize;
	uint8_t *buf;
	char *op, key[MAP_MAX_MARKER_LEN], search_key[MAP_MAX_MARKER_LEN];
	int ret, c;

	/* Try convert to numeric first */
	*offset = strtouq(line, &op, 0);
	if (*op == '\0')
	return (0);

	bzero(search_key, MAP_MAX_MARKER_LEN);
	sectorsize = cp->provider->sectorsize;

	#ifdef __LP64__
	ret = sscanf(line, "search:%li:%li:%63c",
	&search_start, &search_step, search_key);
	#else
	ret = sscanf(line, "search:%qi:%qi:%63c",
	&search_start, &search_step, search_key);
	#endif
	if (ret < 3)
	return (1);

	if (bootverbose) {
	printf("MAP: search %s for key \"%s\" from 0x%jx, step 0x%jx\n",
	cp->geom->name, search_key, (intmax_t)search_start, (intmax_t)search_step);
	}

	/* error if search_key is empty */
	if (strlen(search_key) < 1)
	return (1);

	/* sscanf successful, and we start marker search */
	for (search_offset = search_start;
	search_offset < cp->provider->mediasize;
	search_offset += search_step) {

	g_topology_unlock();
	buf = g_read_data(cp, rounddown(search_offset, sectorsize),
	roundup(strlen(search_key), sectorsize), NULL);
	g_topology_lock();

	/*
	* Don't bother doing the rest if buf==NULL; eg derefencing
	* to assemble 'key'.
	*/
	if (buf == NULL)
	continue;

	/* Wildcard, replace '.' with byte from data */
	/* TODO: add support wildcard escape '\.' */

	strncpy(key, search_key, MAP_MAX_MARKER_LEN);

	for (c = 0; c < MAP_MAX_MARKER_LEN && key[c]; c++) {
	if (key[c] == '.') {
	key[c] = ((char *)(buf +
	(search_offset % sectorsize)))[c];
	}
	}

	/* Assume buf != NULL here */
	if (memcmp(buf + search_offset % sectorsize,
	key, strlen(search_key)) == 0) {
	g_free(buf);
	/* Marker found, so return their offset */
	*offset = search_offset;
	return (0);
	}
	g_free(buf);
	}

	/* Marker not found */
	return (1);
	}

	static int
	g_map_parse_part(struct g_class mp, struct g_provider pp,
	struct g_consumer cp, struct g_geom gp, struct g_map_softc *sc, int i)
	{
	const char value, name;
	char *op;
	off_t start, end, offset, size, dsize;
	int readonly, ret;

	/* hint.map.0.at="cfid0" - bind to cfid0 media */
	if (resource_string_value("map", i, "at", &value) != 0)
	return (1);

	/* Check if this correct provider */
	if (strcmp(pp->name, value) != 0)
	return (1);

	/*
	* hint.map.0.name="uboot" - name of partition, will be available
	* as "/dev/map/uboot"
	*/
	if (resource_string_value("map", i, "name", &name) != 0) {
	if (bootverbose)
	printf("MAP: hint.map.%d has no name\n", i);
	return (1);
	}

	/*
	* hint.map.0.start="0x00010000" - partition start at 0x00010000
	* or hint.map.0.start="search:0x00010000:0x200:marker text" -
	* search for text "marker text", begin at 0x10000, step 0x200
	* until we found marker or end of media reached
	*/
	if (resource_string_value("map", i, "start", &value) != 0) {
	if (bootverbose)
	printf("MAP: \"%s\" has no start value\n", name);
	return (1);
	}
	if (find_marker(cp, value, &start) != 0) {
	if (bootverbose) {
	printf("MAP: \"%s\" can't parse/use start value\n",
	name);
	}
	return (1);
	}

	/* like "start" */
	if (resource_string_value("map", i, "end", &value) != 0) {
	if (bootverbose)
	printf("MAP: \"%s\" has no end value\n", name);
	return (1);
	}
	if (find_marker(cp, value, &end) != 0) {
	if (bootverbose) {
	printf("MAP: \"%s\" can't parse/use end value\n",
	name);
	}
	return (1);
	}

	/* variable readonly optional, disable write access */
	if (resource_int_value("map", i, "readonly", &readonly) != 0)
	readonly = 0;

	/* offset of partition data, from partition begin */
	if (resource_string_value("map", i, "offset", &value) == 0) {
	offset = strtouq(value, &op, 0);
	if (*op != '\0') {
	if (bootverbose) {
	printf("MAP: \"%s\" can't parse offset\n",
	name);
	}
	return (1);
	}
	} else {
	offset = 0;
	}

	/* partition data size */
	if (resource_string_value("map", i, "dsize", &value) == 0) {
	dsize = strtouq(value, &op, 0);
	if (*op != '\0') {
	if (bootverbose) {
	printf("MAP: \"%s\" can't parse dsize\n",
	name);
	}
	return (1);
	}
	} else {
	dsize = 0;
	}

	size = end - start;
	if (dsize == 0)
	dsize = size - offset;

	/* end is 0 or size is 0, No MAP - so next */
	if (end < start) {
	if (bootverbose) {
	printf("MAP: \"%s\", \"end\" less than "
	"\"start\"\n", name);
	}
	return (1);
	}

	if (offset + dsize > size) {
	if (bootverbose) {
	printf("MAP: \"%s\", \"dsize\" bigger than "
	"partition - offset\n", name);
	}
	return (1);
	}

	ret = g_slice_config(gp, i, G_SLICE_CONFIG_SET, start + offset,
	dsize, cp->provider->sectorsize, "map/%s", name);
	if (ret != 0) {
	if (bootverbose) {
	printf("MAP: g_slice_config returns %d for \"%s\"\n",
	ret, name);
	}
	return (1);
	}

	if (bootverbose) {
	printf("MAP: %s: %jxx%jx, data=%jxx%jx "
	"\"/dev/map/%s\"\n",
	cp->geom->name, (intmax_t)start, (intmax_t)size, (intmax_t)offset,
	(intmax_t)dsize, name);
	}

	sc->offset[i] = start;
	sc->size[i] = size;
	sc->entry[i] = offset;
	sc->dsize[i] = dsize;
	sc->readonly[i] = readonly ? 1 : 0;

	return (0);
	}

	static struct g_geom *
	g_map_taste(struct g_class mp, struct g_provider pp, int insist __unused)
	{
	struct g_map_softc *sc;
	struct g_consumer *cp;
	struct g_geom *gp;
	int i;

	g_trace(G_T_TOPOLOGY, "map_taste(%s,%s)", mp->name, pp->name);
	g_topology_assert();
	if (strcmp(pp->geom->class->name, MAP_CLASS_NAME) == 0)
	return (NULL);

	gp = g_slice_new(mp, MAP_MAXSLICE, pp, &cp, &sc, sizeof(*sc),
	g_map_start);
	if (gp == NULL)
	return (NULL);

	/* interpose our access method */
	sc->parent_access = gp->access;
	gp->access = g_map_access;

	for (i = 0; i < MAP_MAXSLICE; i++)
	g_map_parse_part(mp, pp, cp, gp, sc, i);


	g_access(cp, -1, 0, 0);
	if (LIST_EMPTY(&gp->provider)) {
	if (bootverbose)
	printf("MAP: No valid partition found at %s\n", pp->name);
	g_slice_spoiled(cp);
	return (NULL);
	}
	return (gp);
	}

	-static void
	-g_map_config(struct gctl_req req, struct g_class mp, const char *verb)
	-{
	- struct g_geom *gp;
	-
	- g_topology_assert();
	- gp = gctl_get_geom(req, mp, "geom");
	- if (gp == NULL)
	- return;
	- gctl_error(req, "Unknown verb");
	-}
	-
	static struct g_class g_map_class = {
	.name = MAP_CLASS_NAME,
	.version = G_VERSION,
	.taste = g_map_taste,
	.dumpconf = g_map_dumpconf,
	- .ctlreq = g_map_config,
	};
	DECLARE_GEOM_CLASS(g_map_class, g_map);
	MODULE_VERSION(geom_map, 0);
	diff --git a/sys/geom/geom_redboot.c b/sys/geom/geom_redboot.c
	index febee5bd8377..ffdb64d16274 100644
	--- a/sys/geom/geom_redboot.c
	+++ b/sys/geom/geom_redboot.c
	@@ -1,360 +1,347 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2009 Sam Leffler, Errno Consulting
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer,
	* without modification.
	* 2. Redistributions in binary form must reproduce at minimum a disclaimer
	* similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
	* redistribution must be conditioned upon including a substantially
	* similar Disclaimer requirement for further binary redistribution.
	*
	* NO WARRANTY
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
	* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
	* THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
	* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
	* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	* THE POSSIBILITY OF SUCH DAMAGES.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/errno.h>
	#include <sys/endian.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/fcntl.h>
	#include <sys/malloc.h>
	#include <sys/bio.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/bus.h>

	#include <sys/sbuf.h>
	#include <geom/geom.h>
	#include <geom/geom_slice.h>

	#define REDBOOT_CLASS_NAME "REDBOOT"

	struct fis_image_desc {
	uint8_t name[16]; /* null-terminated name */
	uint32_t offset; /* offset in flash */
	uint32_t addr; /* address in memory */
	uint32_t size; /* image size in bytes */
	uint32_t entry; /* offset in image for entry point */
	uint32_t dsize; /* data size in bytes */
	uint8_t pad[256-(16+7sizeof(uint32_t)+sizeof(void))];
	struct fis_image_desc next; / linked list (in memory) */
	uint32_t dsum; /* descriptor checksum */
	uint32_t fsum; /* checksum over image data */
	};

	#define FISDIR_NAME "FIS directory"
	#define REDBCFG_NAME "RedBoot config"
	#define REDBOOT_NAME "RedBoot"

	#define REDBOOT_MAXSLICE 64
	#define REDBOOT_MAXOFF \
	(REDBOOT_MAXSLICE*sizeof(struct fis_image_desc))

	struct g_redboot_softc {
	uint32_t entry[REDBOOT_MAXSLICE];
	uint32_t dsize[REDBOOT_MAXSLICE];
	uint8_t readonly[REDBOOT_MAXSLICE];
	g_access_t *parent_access;
	};

	static void
	g_redboot_print(int i, struct fis_image_desc *fd)
	{

	printf("[%2d] \"%-15.15s\" %08x:%08x", i, fd->name,
	fd->offset, fd->size);
	printf(" addr %08x entry %08x\n", fd->addr, fd->entry);
	printf(" dsize 0x%x dsum 0x%x fsum 0x%x\n", fd->dsize,
	fd->dsum, fd->fsum);
	}

	static int
	g_redboot_ioctl(struct g_provider pp, u_long cmd, void data, int fflag, struct thread *td)
	{
	return (ENOIOCTL);
	}

	static int
	g_redboot_access(struct g_provider *pp, int dread, int dwrite, int dexcl)
	{
	struct g_geom *gp = pp->geom;
	struct g_slicer *gsp = gp->softc;
	struct g_redboot_softc *sc = gsp->softc;

	if (dwrite > 0 && sc->readonly[pp->index])
	return (EPERM);
	return (sc->parent_access(pp, dread, dwrite, dexcl));
	}

	static int
	g_redboot_start(struct bio *bp)
	{
	struct g_provider *pp;
	struct g_geom *gp;
	struct g_redboot_softc *sc;
	struct g_slicer *gsp;
	int idx;

	pp = bp->bio_to;
	idx = pp->index;
	gp = pp->geom;
	gsp = gp->softc;
	sc = gsp->softc;
	if (bp->bio_cmd == BIO_GETATTR) {
	if (g_handleattr_int(bp, REDBOOT_CLASS_NAME "::entry",
	sc->entry[idx]))
	return (1);
	if (g_handleattr_int(bp, REDBOOT_CLASS_NAME "::dsize",
	sc->dsize[idx]))
	return (1);
	}

	return (0);
	}

	static void
	g_redboot_dumpconf(struct sbuf sb, const char indent, struct g_geom *gp,
	struct g_consumer cp __unused, struct g_provider pp)
	{
	struct g_redboot_softc *sc;
	struct g_slicer *gsp;

	gsp = gp->softc;
	sc = gsp->softc;
	g_slice_dumpconf(sb, indent, gp, cp, pp);
	if (pp != NULL) {
	if (indent == NULL) {
	sbuf_printf(sb, " entry %d", sc->entry[pp->index]);
	sbuf_printf(sb, " dsize %d", sc->dsize[pp->index]);
	} else {
	sbuf_printf(sb, "%s<entry>%d</entry>\n", indent,
	sc->entry[pp->index]);
	sbuf_printf(sb, "%s<dsize>%d</dsize>\n", indent,
	sc->dsize[pp->index]);
	}
	}
	}

	#include <sys/ctype.h>

	static int
	nameok(const char name[16])
	{
	int i;

	/* descriptor names are null-terminated printable ascii */
	for (i = 0; i < 15; i++)
	if (!isprint(name[i]))
	break;
	return (name[i] == '\0');
	}

	static struct fis_image_desc *
	parse_fis_directory(u_char *buf, size_t bufsize, off_t offset, uint32_t offmask)
	{
	#define match(a,b) (bcmp(a, b, sizeof(b)-1) == 0)
	struct fis_image_desc fd, efd;
	struct fis_image_desc fisdir, redbcfg;
	struct fis_image_desc head, *tail;
	int i;

	fd = (struct fis_image_desc *)buf;
	efd = fd + (bufsize / sizeof(struct fis_image_desc));
	#if 0
	/*
	* Find the start of the FIS table.
	*/
	while (fd < efd && fd->name[0] != 0xff)
	fd++;
	if (fd == efd)
	return (NULL);
	if (bootverbose)
	printf("RedBoot FIS table starts at 0x%jx\n",
	offset + fd - (struct fis_image_desc *) buf);
	#endif
	/*
	* Scan forward collecting entries in a list.
	*/
	fisdir = redbcfg = NULL;
	*(tail = &head) = NULL;
	for (i = 0; fd < efd; i++, fd++) {
	if (fd->name[0] == 0xff)
	continue;
	if (match(fd->name, FISDIR_NAME))
	fisdir = fd;
	else if (match(fd->name, REDBCFG_NAME))
	redbcfg = fd;
	if (nameok(fd->name)) {
	/*
	* NB: flash address includes platform mapping;
	* strip it so we have only a flash offset.
	*/
	fd->offset &= offmask;
	if (bootverbose)
	g_redboot_print(i, fd);
	*tail = fd;
	*(tail = &fd->next) = NULL;
	}
	}
	if (fisdir == NULL) {
	if (bootverbose)
	printf("No RedBoot FIS table located at %lu\n",
	(long) offset);
	return (NULL);
	}
	if (redbcfg != NULL &&
	fisdir->offset + fisdir->size == redbcfg->offset) {
	/*
	* Merged FIS/RedBoot config directory.
	*/
	if (bootverbose)
	printf("FIS/RedBoot merged at 0x%jx (not yet)\n",
	offset + fisdir->offset);
	/* XXX */
	}
	return head;
	#undef match
	}

	static struct g_geom *
	g_redboot_taste(struct g_class mp, struct g_provider pp, int insist)
	{
	struct g_geom *gp;
	struct g_consumer *cp;
	struct g_redboot_softc *sc;
	int error, sectorsize, i;
	struct fis_image_desc fd, head;
	uint32_t offmask;
	off_t blksize; /* NB: flash block size stored as stripesize */
	u_char *buf;
	off_t offset;
	const char *value;
	char *op;

	offset = 0;
	if (resource_string_value("redboot", 0, "fisoffset", &value) == 0) {
	offset = strtouq(value, &op, 0);
	if (*op != '\0') {
	offset = 0;
	}
	}

	g_trace(G_T_TOPOLOGY, "redboot_taste(%s,%s)", mp->name, pp->name);
	g_topology_assert();
	if (!strcmp(pp->geom->class->name, REDBOOT_CLASS_NAME))
	return (NULL);
	/* XXX only taste flash providers */
	if (strncmp(pp->name, "cfi", 3) &&
	strncmp(pp->name, "flash/spi", 9))
	return (NULL);
	gp = g_slice_new(mp, REDBOOT_MAXSLICE, pp, &cp, &sc, sizeof(*sc),
	g_redboot_start);
	if (gp == NULL)
	return (NULL);
	/* interpose our access method */
	sc->parent_access = gp->access;
	gp->access = g_redboot_access;

	sectorsize = cp->provider->sectorsize;
	blksize = cp->provider->stripesize;
	if (powerof2(cp->provider->mediasize))
	offmask = cp->provider->mediasize-1;
	else
	offmask = 0xffffffff; /* XXX */
	if (bootverbose)
	printf("%s: mediasize %ld secsize %d blksize %ju offmask 0x%x\n",
	__func__, (long) cp->provider->mediasize, sectorsize,
	(uintmax_t)blksize, offmask);
	if (sectorsize < sizeof(struct fis_image_desc) \|\|
	(sectorsize % sizeof(struct fis_image_desc)))
	return (NULL);
	g_topology_unlock();
	head = NULL;
	if(offset == 0)
	offset = cp->provider->mediasize - blksize;
	again:
	buf = g_read_data(cp, offset, blksize, NULL);
	if (buf != NULL)
	head = parse_fis_directory(buf, blksize, offset, offmask);
	if (head == NULL && offset != 0) {
	if (buf != NULL)
	g_free(buf);
	offset = 0; /* check the front */
	goto again;
	}
	g_topology_lock();
	if (head == NULL) {
	if (buf != NULL)
	g_free(buf);
	return NULL;
	}
	/*
	* Craft a slice for each entry.
	*/
	for (fd = head, i = 0; fd != NULL; fd = fd->next) {
	if (fd->name[0] == '\0')
	continue;
	error = g_slice_config(gp, i, G_SLICE_CONFIG_SET,
	fd->offset, fd->size, sectorsize, "redboot/%s", fd->name);
	if (error)
	printf("%s: g_slice_config returns %d for \"%s\"\n",
	__func__, error, fd->name);
	sc->entry[i] = fd->entry;
	sc->dsize[i] = fd->dsize;
	/* disallow writing hard-to-recover entries */
	sc->readonly[i] = (strcmp(fd->name, FISDIR_NAME) == 0) \|\|
	(strcmp(fd->name, REDBOOT_NAME) == 0);
	i++;
	}
	g_free(buf);
	g_access(cp, -1, 0, 0);
	if (LIST_EMPTY(&gp->provider)) {
	g_slice_spoiled(cp);
	return (NULL);
	}
	return (gp);
	}

	-static void
	-g_redboot_config(struct gctl_req req, struct g_class mp, const char *verb)
	-{
	- struct g_geom *gp;
	-
	- g_topology_assert();
	- gp = gctl_get_geom(req, mp, "geom");
	- if (gp == NULL)
	- return;
	- gctl_error(req, "Unknown verb");
	-}
	-
	static struct g_class g_redboot_class = {
	.name = REDBOOT_CLASS_NAME,
	.version = G_VERSION,
	.taste = g_redboot_taste,
	.dumpconf = g_redboot_dumpconf,
	- .ctlreq = g_redboot_config,
	.ioctl = g_redboot_ioctl,
	};
	DECLARE_GEOM_CLASS(g_redboot_class, g_redboot);
	MODULE_VERSION(geom_redboot, 0);
	diff --git a/sys/kern/kern_dump.c b/sys/kern/kern_dump.c
	index e5e6a5d1208c..79dc7766a964 100644
	--- a/sys/kern/kern_dump.c
	+++ b/sys/kern/kern_dump.c
	@@ -1,390 +1,386 @@
	/*-
	* Copyright (c) 2002 Marcel Moolenaar
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/conf.h>
	#include <sys/cons.h>
	#include <sys/kernel.h>
	#include <sys/proc.h>
	#include <sys/kerneldump.h>
	#include <sys/watchdog.h>
	#include <vm/vm.h>
	#include <vm/vm_param.h>
	#include <vm/vm_page.h>
	#include <vm/vm_phys.h>
	#include <vm/pmap.h>
	#include <machine/dump.h>
	#include <machine/elf.h>
	#include <machine/md_var.h>
	#include <machine/pcb.h>

	CTASSERT(sizeof(struct kerneldumpheader) == 512);

	#define MD_ALIGN(x) roundup2((off_t)(x), PAGE_SIZE)

	/* Handle buffered writes. */
	static size_t fragsz;

	struct dump_pa dump_map[DUMPSYS_MD_PA_NPAIRS];

	-#if !defined(__powerpc__) && !defined(__sparc__)
	+#if !defined(__powerpc__)
	void
	dumpsys_gen_pa_init(void)
	{
	int n, idx;

	bzero(dump_map, sizeof(dump_map));
	for (n = 0; n < nitems(dump_map); n++) {
	idx = n * 2;
	if (dump_avail[idx] == 0 && dump_avail[idx + 1] == 0)
	break;
	dump_map[n].pa_start = dump_avail[idx];
	dump_map[n].pa_size = dump_avail[idx + 1] - dump_avail[idx];
	}
	}
	#endif

	struct dump_pa *
	dumpsys_gen_pa_next(struct dump_pa *mdp)
	{

	if (mdp == NULL)
	return (&dump_map[0]);

	mdp++;
	if (mdp->pa_size == 0)
	mdp = NULL;
	return (mdp);
	}

	void
	dumpsys_gen_wbinv_all(void)
	{

	}

	void
	dumpsys_gen_unmap_chunk(vm_paddr_t pa __unused, size_t chunk __unused,
	void *va __unused)
	{

	}

	-#if !defined(__sparc__)
	int
	dumpsys_gen_write_aux_headers(struct dumperinfo *di)
	{

	return (0);
	}
	-#endif

	int
	dumpsys_buf_seek(struct dumperinfo *di, size_t sz)
	{
	static uint8_t buf[DEV_BSIZE];
	size_t nbytes;
	int error;

	bzero(buf, sizeof(buf));

	while (sz > 0) {
	nbytes = MIN(sz, sizeof(buf));

	error = dump_append(di, buf, 0, nbytes);
	if (error)
	return (error);
	sz -= nbytes;
	}

	return (0);
	}

	int
	dumpsys_buf_write(struct dumperinfo di, char ptr, size_t sz)
	{
	size_t len;
	int error;

	while (sz) {
	len = di->blocksize - fragsz;
	if (len > sz)
	len = sz;
	memcpy((char *)di->blockbuf + fragsz, ptr, len);
	fragsz += len;
	ptr += len;
	sz -= len;
	if (fragsz == di->blocksize) {
	error = dump_append(di, di->blockbuf, 0, di->blocksize);
	if (error)
	return (error);
	fragsz = 0;
	}
	}
	return (0);
	}

	int
	dumpsys_buf_flush(struct dumperinfo *di)
	{
	int error;

	if (fragsz == 0)
	return (0);

	error = dump_append(di, di->blockbuf, 0, di->blocksize);
	fragsz = 0;
	return (error);
	}

	CTASSERT(PAGE_SHIFT < 20);
	#define PG2MB(pgs) ((pgs + (1 << (20 - PAGE_SHIFT)) - 1) >> (20 - PAGE_SHIFT))

	int
	dumpsys_cb_dumpdata(struct dump_pa mdp, int seqnr, void arg)
	{
	struct dumperinfo di = (struct dumperinfo)arg;
	vm_paddr_t pa;
	void *va;
	uint64_t pgs;
	size_t counter, sz, chunk;
	int c, error;
	u_int maxdumppgs;

	error = 0; /* catch case in which chunk size is 0 */
	counter = 0; /* Update twiddle every 16MB */
	va = NULL;
	pgs = mdp->pa_size / PAGE_SIZE;
	pa = mdp->pa_start;
	maxdumppgs = min(di->maxiosize / PAGE_SIZE, MAXDUMPPGS);
	if (maxdumppgs == 0) /* seatbelt */
	maxdumppgs = 1;

	printf(" chunk %d: %juMB (%ju pages)", seqnr, (uintmax_t)PG2MB(pgs),
	(uintmax_t)pgs);

	dumpsys_wbinv_all();
	while (pgs) {
	chunk = pgs;
	if (chunk > maxdumppgs)
	chunk = maxdumppgs;
	sz = chunk << PAGE_SHIFT;
	counter += sz;
	if (counter >> 24) {
	printf(" %ju", (uintmax_t)PG2MB(pgs));
	counter &= (1 << 24) - 1;
	}

	dumpsys_map_chunk(pa, chunk, &va);
	wdog_kern_pat(WD_LASTVAL);

	error = dump_append(di, va, 0, sz);
	dumpsys_unmap_chunk(pa, chunk, va);
	if (error)
	break;
	pgs -= chunk;
	pa += sz;

	/* Check for user abort. */
	c = cncheckc();
	if (c == 0x03)
	return (ECANCELED);
	if (c != -1)
	printf(" (CTRL-C to abort) ");
	}
	printf(" ... %s\n", (error) ? "fail" : "ok");
	return (error);
	}

	int
	dumpsys_foreach_chunk(dumpsys_callback_t cb, void *arg)
	{
	struct dump_pa *mdp;
	int error, seqnr;

	seqnr = 0;
	mdp = dumpsys_pa_next(NULL);
	while (mdp != NULL) {
	error = (*cb)(mdp, seqnr++, arg);
	if (error)
	return (-error);
	mdp = dumpsys_pa_next(mdp);
	}
	return (seqnr);
	}

	-#if !defined(__sparc__)
	static off_t fileofs;

	static int
	cb_dumphdr(struct dump_pa mdp, int seqnr, void arg)
	{
	struct dumperinfo di = (struct dumperinfo)arg;
	Elf_Phdr phdr;
	uint64_t size;
	int error;

	size = mdp->pa_size;
	bzero(&phdr, sizeof(phdr));
	phdr.p_type = PT_LOAD;
	phdr.p_flags = PF_R; /* XXX */
	phdr.p_offset = fileofs;
	#ifdef __powerpc__
	phdr.p_vaddr = (do_minidump? mdp->pa_start : ~0L);
	phdr.p_paddr = (do_minidump? ~0L : mdp->pa_start);
	#else
	phdr.p_vaddr = mdp->pa_start;
	phdr.p_paddr = mdp->pa_start;
	#endif
	phdr.p_filesz = size;
	phdr.p_memsz = size;
	phdr.p_align = PAGE_SIZE;

	error = dumpsys_buf_write(di, (char*)&phdr, sizeof(phdr));
	fileofs += phdr.p_filesz;
	return (error);
	}

	static int
	cb_size(struct dump_pa mdp, int seqnr, void arg)
	{
	uint64_t *sz;

	sz = (uint64_t *)arg;
	*sz += (uint64_t)mdp->pa_size;
	return (0);
	}

	int
	dumpsys_generic(struct dumperinfo *di)
	{
	static struct kerneldumpheader kdh;
	Elf_Ehdr ehdr;
	uint64_t dumpsize;
	off_t hdrgap;
	size_t hdrsz;
	int error;

	#if !defined(__powerpc__) \|\| defined(__powerpc64__)
	if (do_minidump)
	return (minidumpsys(di));
	#endif

	bzero(&ehdr, sizeof(ehdr));
	ehdr.e_ident[EI_MAG0] = ELFMAG0;
	ehdr.e_ident[EI_MAG1] = ELFMAG1;
	ehdr.e_ident[EI_MAG2] = ELFMAG2;
	ehdr.e_ident[EI_MAG3] = ELFMAG3;
	ehdr.e_ident[EI_CLASS] = ELF_CLASS;
	#if BYTE_ORDER == LITTLE_ENDIAN
	ehdr.e_ident[EI_DATA] = ELFDATA2LSB;
	#else
	ehdr.e_ident[EI_DATA] = ELFDATA2MSB;
	#endif
	ehdr.e_ident[EI_VERSION] = EV_CURRENT;
	ehdr.e_ident[EI_OSABI] = ELFOSABI_STANDALONE; /* XXX big picture? */
	ehdr.e_type = ET_CORE;
	ehdr.e_machine = EM_VALUE;
	ehdr.e_phoff = sizeof(ehdr);
	ehdr.e_flags = 0;
	ehdr.e_ehsize = sizeof(ehdr);
	ehdr.e_phentsize = sizeof(Elf_Phdr);
	ehdr.e_shentsize = sizeof(Elf_Shdr);

	dumpsys_pa_init();

	/* Calculate dump size. */
	dumpsize = 0L;
	ehdr.e_phnum = dumpsys_foreach_chunk(cb_size, &dumpsize) +
	DUMPSYS_NUM_AUX_HDRS;
	hdrsz = ehdr.e_phoff + ehdr.e_phnum * ehdr.e_phentsize;
	fileofs = MD_ALIGN(hdrsz);
	dumpsize += fileofs;
	hdrgap = fileofs - roundup2((off_t)hdrsz, di->blocksize);

	dump_init_header(di, &kdh, KERNELDUMPMAGIC, KERNELDUMP_ARCH_VERSION,
	dumpsize);

	error = dump_start(di, &kdh);
	if (error != 0)
	goto fail;

	printf("Dumping %ju MB (%d chunks)\n", (uintmax_t)dumpsize >> 20,
	ehdr.e_phnum - DUMPSYS_NUM_AUX_HDRS);

	/* Dump ELF header */
	error = dumpsys_buf_write(di, (char*)&ehdr, sizeof(ehdr));
	if (error)
	goto fail;

	/* Dump program headers */
	error = dumpsys_foreach_chunk(cb_dumphdr, di);
	if (error < 0)
	goto fail;
	error = dumpsys_write_aux_headers(di);
	if (error < 0)
	goto fail;
	dumpsys_buf_flush(di);

	/*
	* All headers are written using blocked I/O, so we know the
	* current offset is (still) block aligned. Skip the alignement
	* in the file to have the segment contents aligned at page
	* boundary.
	*/
	error = dumpsys_buf_seek(di, (size_t)hdrgap);
	if (error)
	goto fail;

	/* Dump memory chunks. */
	error = dumpsys_foreach_chunk(dumpsys_cb_dumpdata, di);
	if (error < 0)
	goto fail;

	error = dump_finish(di, &kdh);
	if (error != 0)
	goto fail;

	printf("\nDump complete\n");
	return (0);

	fail:
	if (error < 0)
	error = -error;

	if (error == ECANCELED)
	printf("\nDump aborted\n");
	else if (error == E2BIG \|\| error == ENOSPC)
	printf("\nDump failed. Partition too small.\n");
	else
	printf("\n DUMP FAILED (ERROR %d) \n", error);
	return (error);
	}
	-#endif
	diff --git a/sys/kern/kern_priv.c b/sys/kern/kern_priv.c
	index a912336cbbca..710e9b90894f 100644
	--- a/sys/kern/kern_priv.c
	+++ b/sys/kern/kern_priv.c
	@@ -1,273 +1,288 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2006 nCircle Network Security, Inc.
	* Copyright (c) 2009 Robert N. M. Watson
	* All rights reserved.
	*
	* This software was developed by Robert N. M. Watson for the TrustedBSD
	* Project under contract to nCircle Network Security, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR, NCIRCLE NETWORK SECURITY,
	* INC., OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
	* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
	* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
	* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
	* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/jail.h>
	#include <sys/kernel.h>
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/sdt.h>
	#include <sys/sysctl.h>
	#include <sys/systm.h>

	#include <security/mac/mac_framework.h>

	/*
	* `suser_enabled' (which can be set by the security.bsd.suser_enabled
	* sysctl) determines whether the system 'super-user' policy is in effect. If
	* it is nonzero, an effective uid of 0 connotes special privilege,
	* overriding many mandatory and discretionary protections. If it is zero,
	* uid 0 is offered no special privilege in the kernel security policy.
	* Setting it to zero may seriously impact the functionality of many existing
	* userland programs, and should not be done without careful consideration of
	* the consequences.
	*/
	static int __read_mostly suser_enabled = 1;
	SYSCTL_INT(_security_bsd, OID_AUTO, suser_enabled, CTLFLAG_RWTUN,
	&suser_enabled, 0, "processes with uid 0 have privilege");

	static int unprivileged_mlock = 1;
	SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_mlock, CTLFLAG_RWTUN,
	&unprivileged_mlock, 0, "Allow non-root users to call mlock(2)");

	static int unprivileged_read_msgbuf = 1;
	SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_read_msgbuf,
	CTLFLAG_RW, &unprivileged_read_msgbuf, 0,
	"Unprivileged processes may read the kernel message buffer");

	SDT_PROVIDER_DEFINE(priv);
	SDT_PROBE_DEFINE1(priv, kernel, priv_check, priv__ok, "int");
	SDT_PROBE_DEFINE1(priv, kernel, priv_check, priv__err, "int");

	static __always_inline int
	priv_check_cred_pre(struct ucred *cred, int priv)
	{
	int error;

	#ifdef MAC
	error = mac_priv_check(cred, priv);
	#else
	error = 0;
	#endif
	return (error);
	}

	static __always_inline int
	priv_check_cred_post(struct ucred *cred, int priv, int error, bool handled)
	{

	if (__predict_true(handled))
	goto out;
	/*
	* Now check with MAC, if enabled, to see if a policy module grants
	* privilege.
	*/
	#ifdef MAC
	if (mac_priv_grant(cred, priv) == 0) {
	error = 0;
	goto out;
	}
	#endif

	/*
	* The default is deny, so if no policies have granted it, reject
	* with a privilege error here.
	*/
	error = EPERM;
	out:
	if (SDT_PROBES_ENABLED()) {
	if (error)
	SDT_PROBE1(priv, kernel, priv_check, priv__err, priv);
	else
	SDT_PROBE1(priv, kernel, priv_check, priv__ok, priv);
	}
	return (error);
	}

	/*
	* Check a credential for privilege. Lots of good reasons to deny privilege;
	* only a few to grant it.
	*/
	int
	priv_check_cred(struct ucred *cred, int priv)
	{
	int error;

	KASSERT(PRIV_VALID(priv), ("priv_check_cred: invalid privilege %d",
	priv));

	switch (priv) {
	case PRIV_VFS_GENERATION:
	return (priv_check_cred_vfs_generation(cred));
	}

	/*
	* We first evaluate policies that may deny the granting of
	* privilege unilaterally.
	*/
	error = priv_check_cred_pre(cred, priv);
	if (error)
	goto out;

	/*
	* Jail policy will restrict certain privileges that may otherwise be
	* be granted.
	*/
	error = prison_priv_check(cred, priv);
	if (error)
	goto out;

	if (unprivileged_mlock) {
	/*
	* Allow unprivileged users to call mlock(2)/munlock(2) and
	* mlockall(2)/munlockall(2).
	*/
	switch (priv) {
	case PRIV_VM_MLOCK:
	case PRIV_VM_MUNLOCK:
	error = 0;
	goto out;
	}
	}

	if (unprivileged_read_msgbuf) {
	/*
	* Allow an unprivileged user to read the kernel message
	* buffer.
	*/
	if (priv == PRIV_MSGBUF) {
	error = 0;
	goto out;
	}
	}

	/*
	* Having determined if privilege is restricted by various policies,
	* now determine if privilege is granted. At this point, any policy
	* may grant privilege. For now, we allow short-circuit boolean
	* evaluation, so may not call all policies. Perhaps we should.
	*
	* Superuser policy grants privilege based on the effective (or in
	* the case of specific privileges, real) uid being 0. We allow the
	* superuser policy to be globally disabled, although this is
	* currenty of limited utility.
	*/
	if (suser_enabled) {
	switch (priv) {
	case PRIV_MAXFILES:
	case PRIV_MAXPROC:
	case PRIV_PROC_LIMIT:
	if (cred->cr_ruid == 0) {
	error = 0;
	goto out;
	}
	break;
	case PRIV_VFS_READ_DIR:
	/*
	* Allow PRIV_VFS_READ_DIR for root if we're not in a
	* jail, otherwise deny unless a MAC policy grants it.
	*/
	if (jailed(cred))
	break;
	/* FALLTHROUGH */
	default:
	if (cred->cr_uid == 0) {
	error = 0;
	goto out;
	}
	break;
	}
	}

	/*
	* Writes to kernel/physical memory are a typical root-only operation,
	* but non-root users are expected to be able to read it (provided they
	* have permission to access /dev/[k]mem).
	*/
	if (priv == PRIV_KMEM_READ) {
	error = 0;
	goto out;
	}

	/*
	* Allow unprivileged process debugging on a per-jail basis.
	* Do this here instead of prison_priv_check(), so it can also
	* apply to prison0.
	*/
	if (priv == PRIV_DEBUG_UNPRIV) {
	if (prison_allow(cred, PR_ALLOW_UNPRIV_DEBUG)) {
	error = 0;
	goto out;
	}
	}

	return (priv_check_cred_post(cred, priv, error, false));
	out:
	return (priv_check_cred_post(cred, priv, error, true));
	}

	int
	priv_check(struct thread *td, int priv)
	{

	KASSERT(td == curthread, ("priv_check: td != curthread"));

	return (priv_check_cred(td->td_ucred, priv));
	}

	-int
	-priv_check_cred_vfs_generation(struct ucred *cred)
	+static int __noinline
	+priv_check_cred_vfs_generation_slow(struct ucred *cred)
	{
	int error;

	error = priv_check_cred_pre(cred, PRIV_VFS_GENERATION);
	if (error)
	goto out;

	if (jailed(cred)) {
	error = EPERM;
	goto out;
	}

	if (cred->cr_uid == 0 && suser_enabled) {
	error = 0;
	goto out;
	}

	return (priv_check_cred_post(cred, PRIV_VFS_GENERATION, error, false));
	out:
	return (priv_check_cred_post(cred, PRIV_VFS_GENERATION, error, true));

	}
	+
	+int
	+priv_check_cred_vfs_generation(struct ucred *cred)
	+{
	+ int error;
	+
	+ if (__predict_false(mac_priv_check_fp_flag \|\|
	+ mac_priv_grant_fp_flag \|\| SDT_PROBES_ENABLED()))
	+ return (priv_check_cred_vfs_generation_slow(cred));
	+
	+ error = EPERM;
	+ if (!jailed(cred) && cred->cr_uid == 0 && suser_enabled)
	+ error = 0;
	+ return (error);
	+}
	diff --git a/sys/kern/makesyscalls.sh b/sys/kern/makesyscalls.sh
	index 5a0fcfedfb1a..8cbd00a69fcd 100644
	--- a/sys/kern/makesyscalls.sh
	+++ b/sys/kern/makesyscalls.sh
	@@ -1,802 +1,805 @@
	#! /bin/sh -
	# @(#)makesyscalls.sh 8.1 (Berkeley) 6/10/93
	# $FreeBSD$

	set -e

	# name of compat options:
	compat=COMPAT_43
	compat4=COMPAT_FREEBSD4
	compat6=COMPAT_FREEBSD6
	compat7=COMPAT_FREEBSD7
	compat10=COMPAT_FREEBSD10
	compat11=COMPAT_FREEBSD11
	compat12=COMPAT_FREEBSD12

	# output files:
	sysnames="syscalls.c"
	sysproto="../sys/sysproto.h"
	sysproto_h=_SYS_SYSPROTO_H_
	syshdr="../sys/syscall.h"
	sysmk="../sys/syscall.mk"
	syssw="init_sysent.c"
	syscallprefix="SYS_"
	switchname="sysent"
	namesname="syscallnames"
	systrace="systrace_args.c"

	# tmp files:
	sysaue="sysent.aue.$$"
	sysdcl="sysent.dcl.$$"
	syscompat="sysent.compat.$$"
	syscompatdcl="sysent.compatdcl.$$"
	syscompat4="sysent.compat4.$$"
	syscompat4dcl="sysent.compat4dcl.$$"
	syscompat6="sysent.compat6.$$"
	syscompat6dcl="sysent.compat6dcl.$$"
	syscompat7="sysent.compat7.$$"
	syscompat7dcl="sysent.compat7dcl.$$"
	syscompat10="sysent.compat10.$$"
	syscompat10dcl="sysent.compat10dcl.$$"
	syscompat11="sysent.compat11.$$"
	syscompat11dcl="sysent.compat11dcl.$$"
	syscompat12="sysent.compat12.$$"
	syscompat12dcl="sysent.compat12dcl.$$"
	sysent="sysent.switch.$$"
	sysinc="sysinc.switch.$$"
	sysarg="sysarg.switch.$$"
	sysprotoend="sysprotoend.$$"
	systracetmp="systrace.$$"
	systraceret="systraceret.$$"
	capabilities_conf="capabilities.conf"

	trap "rm $sysaue $sysdcl $syscompat $syscompatdcl $syscompat4 $syscompat4dcl $syscompat6 $syscompat6dcl $syscompat7 $syscompat7dcl $syscompat10 $syscompat10dcl $syscompat11 $syscompat11dcl $syscompat12 $syscompat12dcl $sysent $sysinc $sysarg $sysprotoend $systracetmp $systraceret" 0

	touch $sysaue $sysdcl $syscompat $syscompatdcl $syscompat4 $syscompat4dcl $syscompat6 $syscompat6dcl $syscompat7 $syscompat7dcl $syscompat10 $syscompat10dcl $syscompat11 $syscompat11dcl $syscompat12 $syscompat12dcl $sysent $sysinc $sysarg $sysprotoend $systracetmp $systraceret

	case $# in
	0) echo "usage: $0 input-file <config-file>" 1>&2
	exit 1
	;;
	esac

	+1>&2 echo "$0: This script has been replaced by sys/tools/makesyscalls.lua and"
	+1>&2 echo "$0: will be removed before FreeBSD 13. See also: sys/conf/sysent.mk"
	+
	if [ -n "$2" ]; then
	. "$2"
	fi

	if [ -n "$capenabled" ]; then
	# do nothing
	elif [ -r $capabilities_conf ]; then
	capenabled=`egrep -v '^#\|^$' $capabilities_conf`
	capenabled=`echo $capenabled \| sed 's/ /,/g'`
	else
	capenabled=""
	fi

	sed -e '
	# FreeBSD ID, includes, comments, and blank lines
	/.*\$FreeBSD/b done_joining
	/^[#;]/b done_joining
	/^$/b done_joining

	# Join lines ending in backslash
	:joining
	/\\$/{a\

	N
	s/\\\n//
	b joining
	}

	# OBSOL, etc lines without function signatures
	/^[0-9][^{]*$/b done_joining

	# Join incomplete signatures. The { must appear on the first line
	# and the } must appear on the last line (modulo lines joined by
	# backslashes).
	/^[^}]*$/{a\

	N
	s/\n//
	b joining
	}
	:done_joining
	2,${
	/^#/!s/$[{}()*,]$/ \1 /g
	}
	' < $1 \| awk "
	BEGIN {
	sysaue = \"$sysaue\"
	sysdcl = \"$sysdcl\"
	sysproto = \"$sysproto\"
	sysprotoend = \"$sysprotoend\"
	sysproto_h = \"$sysproto_h\"
	syscompat = \"$syscompat\"
	syscompatdcl = \"$syscompatdcl\"
	syscompat4 = \"$syscompat4\"
	syscompat4dcl = \"$syscompat4dcl\"
	syscompat6 = \"$syscompat6\"
	syscompat6dcl = \"$syscompat6dcl\"
	syscompat7 = \"$syscompat7\"
	syscompat7dcl = \"$syscompat7dcl\"
	syscompat10 = \"$syscompat10\"
	syscompat10dcl = \"$syscompat10dcl\"
	syscompat11 = \"$syscompat11\"
	syscompat11dcl = \"$syscompat11dcl\"
	syscompat12 = \"$syscompat12\"
	syscompat12dcl = \"$syscompat12dcl\"
	sysent = \"$sysent\"
	syssw = \"$syssw\"
	sysinc = \"$sysinc\"
	sysarg = \"$sysarg\"
	sysnames = \"$sysnames\"
	syshdr = \"$syshdr\"
	sysmk = \"$sysmk\"
	systrace = \"$systrace\"
	systracetmp = \"$systracetmp\"
	systraceret = \"$systraceret\"
	compat = \"$compat\"
	compat4 = \"$compat4\"
	compat6 = \"$compat6\"
	compat7 = \"$compat7\"
	compat10 = \"$compat10\"
	compat11 = \"$compat11\"
	compat12 = \"$compat12\"
	syscallprefix = \"$syscallprefix\"
	switchname = \"$switchname\"
	namesname = \"$namesname\"
	infile = \"$1\"
	abi_func_prefix = \"$abi_func_prefix\"
	capenabled_string = \"$capenabled\"
	"'

	# Avoid a literal generated file tag here.
	generated = "@" "generated";

	split(capenabled_string, capenabled, ",");

	printf "\n/* The casts are bogus but will do for now. */\n" > sysent
	printf "struct sysent %s[] = {\n",switchname > sysent

	printf "/\n System call switch table.\n *\n" > syssw
	printf " * DO NOT EDIT-- this file is automatically " generated ".\n" > syssw
	printf " * $%s$\n", "FreeBSD" > syssw
	printf " */\n\n" > syssw

	printf "/\n System call prototypes.\n *\n" > sysarg
	printf " * DO NOT EDIT-- this file is automatically " generated ".\n" > sysarg
	printf " * $%s$\n", "FreeBSD" > sysarg
	printf " */\n\n" > sysarg
	printf "#ifndef %s\n", sysproto_h > sysarg
	printf "#define\t%s\n\n", sysproto_h > sysarg
	printf "#include <sys/signal.h>\n" > sysarg
	printf "#include <sys/acl.h>\n" > sysarg
	printf "#include <sys/cpuset.h>\n" > sysarg
	printf "#include <sys/domainset.h>\n" > sysarg
	printf "#include <sys/_ffcounter.h>\n" > sysarg
	printf "#include <sys/_semaphore.h>\n" > sysarg
	printf "#include <sys/ucontext.h>\n" > sysarg
	printf "#include <sys/wait.h>\n\n" > sysarg
	printf "#include <bsm/audit_kevents.h>\n\n" > sysarg
	printf "struct proc;\n\n" > sysarg
	printf "struct thread;\n\n" > sysarg
	printf "#define\tPAD_(t)\t(sizeof(register_t) <= sizeof(t) ? \\\n" > sysarg
	printf "\t\t0 : sizeof(register_t) - sizeof(t))\n\n" > sysarg
	printf "#if BYTE_ORDER == LITTLE_ENDIAN\n"> sysarg
	printf "#define\tPADL_(t)\t0\n" > sysarg
	printf "#define\tPADR_(t)\tPAD_(t)\n" > sysarg
	printf "#else\n" > sysarg
	printf "#define\tPADL_(t)\tPAD_(t)\n" > sysarg
	printf "#define\tPADR_(t)\t0\n" > sysarg
	printf "#endif\n\n" > sysarg

	printf "\n#ifdef %s\n\n", compat > syscompat
	printf "\n#ifdef %s\n\n", compat4 > syscompat4
	printf "\n#ifdef %s\n\n", compat6 > syscompat6
	printf "\n#ifdef %s\n\n", compat7 > syscompat7
	printf "\n#ifdef %s\n\n", compat10 > syscompat10
	printf "\n#ifdef %s\n\n", compat11 > syscompat11
	printf "\n#ifdef %s\n\n", compat12 > syscompat12

	printf "/\n System call names.\n *\n" > sysnames
	printf " * DO NOT EDIT-- this file is automatically " generated ".\n" > sysnames
	printf " * $%s$\n", "FreeBSD" > sysnames
	printf " */\n\n" > sysnames
	printf "const char *%s[] = {\n", namesname > sysnames

	printf "/\n System call numbers.\n *\n" > syshdr
	printf " * DO NOT EDIT-- this file is automatically " generated ".\n" > syshdr
	printf " * $%s$\n", "FreeBSD" > syshdr
	printf " */\n\n" > syshdr

	printf "# FreeBSD system call object files.\n" > sysmk
	printf "# DO NOT EDIT-- this file is automatically " generated ".\n" > sysmk
	printf "# $%s$\n", "FreeBSD" > sysmk
	printf "MIASM = " > sysmk

	printf "/\n System call argument to DTrace register array converstion.\n *\n" > systrace
	printf " * DO NOT EDIT-- this file is automatically " generated ".\n" > systrace
	printf " * $%s$\n", "FreeBSD" > systrace
	printf " * This file is part of the DTrace syscall provider.\n */\n\n" > systrace
	printf "static void\nsystrace_args(int sysnum, void params, uint64_t uarg, int *n_args)\n{\n" > systrace
	printf "\tint64_t iarg = (int64_t ) uarg;\n" > systrace
	printf "\tswitch (sysnum) {\n" > systrace

	printf "static void\nsystrace_entry_setargdesc(int sysnum, int ndx, char desc, size_t descsz)\n{\n\tconst char p = NULL;\n" > systracetmp
	printf "\tswitch (sysnum) {\n" > systracetmp

	printf "static void\nsystrace_return_setargdesc(int sysnum, int ndx, char desc, size_t descsz)\n{\n\tconst char p = NULL;\n" > systraceret
	printf "\tswitch (sysnum) {\n" > systraceret
	}
	NR == 1 {
	next
	}
	NF == 0 \|\| $1 ~ /^;/ {
	next
	}
	$1 ~ /^#[ ]*include/ {
	print > sysinc
	next
	}
	$1 ~ /^#[ ]*if/ {
	print > sysent
	print > sysdcl
	print > sysarg
	print > syscompat
	print > syscompat4
	print > syscompat6
	print > syscompat7
	print > syscompat10
	print > syscompat11
	print > syscompat12
	print > sysnames
	print > systrace
	print > systracetmp
	print > systraceret
	savesyscall = syscall
	next
	}
	$1 ~ /^#[ ]*else/ {
	print > sysent
	print > sysdcl
	print > sysarg
	print > syscompat
	print > syscompat4
	print > syscompat6
	print > syscompat7
	print > syscompat10
	print > syscompat11
	print > syscompat12
	print > sysnames
	print > systrace
	print > systracetmp
	print > systraceret
	syscall = savesyscall
	next
	}
	$1 ~ /^#/ {
	print > sysent
	print > sysdcl
	print > sysarg
	print > syscompat
	print > syscompat4
	print > syscompat6
	print > syscompat7
	print > syscompat10
	print > syscompat11
	print > syscompat12
	print > sysnames
	print > systrace
	print > systracetmp
	print > systraceret
	next
	}
	# Returns true if the type "name" is the first flag in the type field
	function type(name, flags, n) {
	n = split($3, flags, /\\|/)
	return (n > 0 && flags[1] == name)
	}
	# Returns true if the flag "name" is set in the type field
	function flag(name, flags, i, n) {
	n = split($3, flags, /\\|/)
	for (i = 1; i <= n; i++)
	if (flags[i] == name)
	return 1
	return 0
	}
	{
	n = split($1, syscall_range, /-/)
	if (n == 1) {
	syscall_range[2] = syscall_range[1]
	} else if (n == 2) {
	if (!type("UNIMPL")) {
	printf "%s: line %d: range permitted only with UNIMPL\n",
	infile, NR
	exit 1
	}
	} else {
	printf "%s: line %d: invalid syscall number or range %s\n",
	infile, NR, $1
	exit 1
	}
	}
	syscall != syscall_range[1] {
	printf "%s: line %d: syscall number out of sync at %d\n",
	infile, NR, syscall
	printf "line is:\n"
	print
	exit 1
	}
	function align_sysent_comment(column) {
	printf("\t") > sysent
	column = column + 8 - column % 8
	while (column < 56) {
	printf("\t") > sysent
	column = column + 8
	}
	}
	function parserr(was, wanted) {
	printf "%s: line %d: unexpected %s (expected %s)\n",
	infile, NR, was, wanted
	exit 1
	}
	function parseline() {
	f=4 # toss number, type, audit event
	ret_inc = 0
	argc= 0;
	argssize = "0"
	thr_flag = "SY_THR_STATIC"
	if (flag("NOTSTATIC")) {
	thr_flag = "SY_THR_ABSENT"
	}
	if ($NF != "}") {
	funcalias=$(NF-2)
	argalias=$(NF-1)
	rettype=$NF
	end=NF-3
	} else {
	funcalias=""
	argalias=""
	rettype="int"
	if ($(f+2) == "*") {
	ret_inc = 1
	}
	end=NF
	}
	if (flag("NODEF")) {
	auditev="AUE_NULL"
	funcname=$(4 + ret_inc)
	argssize = "AS(" $(6 + ret_inc) ")"
	return
	}
	if ($f != "{")
	parserr($f, "{")
	f++
	if ($end != "}")
	parserr($end, "}")
	end--
	if ($end != ";")
	parserr($end, ";")
	end--
	if ($end != ")")
	parserr($end, ")")
	end--

	syscallret=$f
	f++
	while (ret_inc > 0) {
	syscallret=syscallret " " $f
	f++
	ret_inc--
	}

	funcname=$f

	#
	# We now know the func name, so define a flags field for it.
	# Do this before any other processing as we may return early
	# from it.
	#
	for (cap in capenabled) {
	if (funcname == capenabled[cap] \|\|
	funcname == abi_func_prefix capenabled[cap]) {
	flags = "SYF_CAPENABLED";
	break;
	}
	}

	if (funcalias == "")
	funcalias = funcname
	if (argalias == "") {
	argalias = funcname "_args"
	if (flag("COMPAT"))
	argalias = "o" argalias
	if (flag("COMPAT4"))
	argalias = "freebsd4_" argalias
	if (flag("COMPAT6"))
	argalias = "freebsd6_" argalias
	if (flag("COMPAT7"))
	argalias = "freebsd7_" argalias
	if (flag("COMPAT10"))
	argalias = "freebsd10_" argalias
	if (flag("COMPAT11"))
	argalias = "freebsd11_" argalias
	if (flag("COMPAT12"))
	argalias = "freebsd12_" argalias
	}
	f++

	if ($f != "(")
	parserr($f, ")")
	f++

	if (f == end) {
	if ($f != "void")
	parserr($f, "argument definition")
	return
	}

	while (f <= end) {
	argc++
	argtype[argc]=""
	oldf=""
	while (f < end && $(f+1) != ",") {
	if (argtype[argc] != "" && oldf != "*")
	argtype[argc] = argtype[argc]" ";
	argtype[argc] = argtype[argc]$f;
	oldf = $f;
	f++
	}
	if (argtype[argc] == "")
	parserr($f, "argument definition")

	# The parser adds space around parens.
	# Remove it from annotations.
	gsub(/ \( /, "(", argtype[argc]);
	gsub(/ \)/, ")", argtype[argc]);

	#remove annotations
	gsub(/_In[^ ]*[_)] /, "", argtype[argc]);
	gsub(/_Out[^ ]*[_)] /, "", argtype[argc]);

	argname[argc]=$f;
	f += 2; # skip name, and any comma
	}
	if (argc != 0)
	argssize = "AS(" argalias ")"
	}
	{ comment = $4
	if (NF < 7)
	for (i = 5; i <= NF; i++)
	comment = comment " " $i
	}

	#
	# The AUE_ audit event identifier.
	#
	{
	auditev = $2;
	}

	#
	# The flags, if any.
	#
	{
	flags = "0";
	}

	type("STD") \|\| type("NODEF") \|\| type("NOARGS") \|\| type("NOPROTO") \
	\|\| type("NOSTD") {
	parseline()
	printf("\t/* %s */\n\tcase %d: {\n", funcname, syscall) > systrace
	printf("\t/* %s */\n\tcase %d:\n", funcname, syscall) > systracetmp
	printf("\t/* %s */\n\tcase %d:\n", funcname, syscall) > systraceret
	if (argc > 0) {
	printf("\t\tswitch(ndx) {\n") > systracetmp
	printf("\t\tstruct %s *p = params;\n", argalias) > systrace
	for (i = 1; i <= argc; i++) {
	arg = argtype[i]
	sub("__restrict$", "", arg)
	if (index(arg, "*") > 0)
	printf("\t\tcase %d:\n\t\t\tp = \"userland %s\";\n\t\t\tbreak;\n", i - 1, arg) > systracetmp
	else
	printf("\t\tcase %d:\n\t\t\tp = \"%s\";\n\t\t\tbreak;\n", i - 1, arg) > systracetmp
	if (index(arg, "*") > 0 \|\| arg == "caddr_t")
	printf("\t\tuarg[%d] = (intptr_t) p->%s; /* %s */\n", \
	i - 1, \
	argname[i], arg) > systrace
	else if (arg == "union l_semun")
	printf("\t\tuarg[%d] = p->%s.buf; /* %s */\n", \
	i - 1, \
	argname[i], arg) > systrace
	else if (substr(arg, 1, 1) == "u" \|\| arg == "size_t")
	printf("\t\tuarg[%d] = p->%s; /* %s */\n", \
	i - 1, \
	argname[i], arg) > systrace
	else
	printf("\t\tiarg[%d] = p->%s; /* %s */\n", \
	i - 1, \
	argname[i], arg) > systrace
	}
	printf("\t\tdefault:\n\t\t\tbreak;\n\t\t};\n") > systracetmp

	printf("\t\tif (ndx == 0 \|\| ndx == 1)\n") > systraceret
	printf("\t\t\tp = \"%s\";\n", syscallret) > systraceret
	printf("\t\tbreak;\n") > systraceret
	}
	printf("\t\t*n_args = %d;\n\t\tbreak;\n\t}\n", argc) > systrace
	printf("\t\tbreak;\n") > systracetmp
	if (!flag("NOARGS") && !flag("NOPROTO") && !flag("NODEF")) {
	if (argc != 0) {
	printf("struct %s {\n", argalias) > sysarg
	for (i = 1; i <= argc; i++)
	printf("\tchar %s_l_[PADL_(%s)]; " \
	"%s %s; char %s_r_[PADR_(%s)];\n",
	argname[i], argtype[i],
	argtype[i], argname[i],
	argname[i], argtype[i]) > sysarg
	printf("};\n") > sysarg
	} else
	printf("struct %s {\n\tregister_t dummy;\n};\n",
	argalias) > sysarg
	}
	if (!flag("NOPROTO") && !flag("NODEF")) {
	if (funcname == "nosys" \|\| funcname == "lkmnosys" \|\|
	funcname == "sysarch" \|\| funcname ~ /^freebsd/ \|\|
	funcname ~ /^linux/ \|\| funcname ~ /^cloudabi/) {
	printf("%s\t%s(struct thread , struct %s )",
	rettype, funcname, argalias) > sysdcl
	} else {
	printf("%s\tsys_%s(struct thread , struct %s )",
	rettype, funcname, argalias) > sysdcl
	}
	printf(";\n") > sysdcl
	printf("#define\t%sAUE_%s\t%s\n", syscallprefix,
	funcalias, auditev) > sysaue
	}
	printf("\t{ %s, (sy_call_t *)", argssize) > sysent
	column = 8 + 2 + length(argssize) + 15
	if (flag("NOSTD")) {
	printf("lkmressys, AUE_NULL, NULL, 0, 0, %s, SY_THR_ABSENT },", flags) > sysent
	column = column + length("lkmressys") + length("AUE_NULL") + 3
	} else {
	if (funcname == "nosys" \|\| funcname == "sysarch" \|\|
	funcname == "lkmnosys" \|\| funcname ~ /^freebsd/ \|\|
	funcname ~ /^linux/ \|\| funcname ~ /^cloudabi/) {
	printf("%s, %s, NULL, 0, 0, %s, %s },", funcname, auditev, flags, thr_flag) > sysent
	column = column + length(funcname) + length(auditev) + length(flags) + 3
	} else {
	printf("sys_%s, %s, NULL, 0, 0, %s, %s },", funcname, auditev, flags, thr_flag) > sysent
	column = column + length(funcname) + length(auditev) + length(flags) + 3 + 4
	}
	}
	align_sysent_comment(column)
	printf("/* %d = %s */\n", syscall, funcalias) > sysent
	printf("\t\"%s\",\t\t\t/* %d = %s */\n",
	funcalias, syscall, funcalias) > sysnames
	if (!flag("NODEF")) {
	printf("#define\t%s%s\t%d\n", syscallprefix,
	funcalias, syscall) > syshdr
	printf(" \\\n\t%s.o", funcalias) > sysmk
	}
	syscall++
	next
	}
	type("COMPAT") \|\| type("COMPAT4") \|\| type("COMPAT6") \|\| \
	type("COMPAT7") \|\| type("COMPAT10") \|\| type("COMPAT11") \|\| \
	type("COMPAT12") {
	if (flag("COMPAT")) {
	ncompat++
	out = syscompat
	outdcl = syscompatdcl
	wrap = "compat"
	prefix = "o"
	descr = "old"
	} else if (flag("COMPAT4")) {
	ncompat4++
	out = syscompat4
	outdcl = syscompat4dcl
	wrap = "compat4"
	prefix = "freebsd4_"
	descr = "freebsd4"
	} else if (flag("COMPAT6")) {
	ncompat6++
	out = syscompat6
	outdcl = syscompat6dcl
	wrap = "compat6"
	prefix = "freebsd6_"
	descr = "freebsd6"
	} else if (flag("COMPAT7")) {
	ncompat7++
	out = syscompat7
	outdcl = syscompat7dcl
	wrap = "compat7"
	prefix = "freebsd7_"
	descr = "freebsd7"
	} else if (flag("COMPAT10")) {
	ncompat10++
	out = syscompat10
	outdcl = syscompat10dcl
	wrap = "compat10"
	prefix = "freebsd10_"
	descr = "freebsd10"
	} else if (flag("COMPAT11")) {
	ncompat11++
	out = syscompat11
	outdcl = syscompat11dcl
	wrap = "compat11"
	prefix = "freebsd11_"
	descr = "freebsd11"
	} else if (flag("COMPAT12")) {
	ncompat12++
	out = syscompat12
	outdcl = syscompat12dcl
	wrap = "compat12"
	prefix = "freebsd12_"
	descr = "freebsd12"
	}
	parseline()
	if (argc != 0 && !flag("NOARGS") && !flag("NOPROTO") && \
	!flag("NODEF")) {
	printf("struct %s {\n", argalias) > out
	for (i = 1; i <= argc; i++)
	printf("\tchar %s_l_[PADL_(%s)]; %s %s; " \
	"char %s_r_[PADR_(%s)];\n",
	argname[i], argtype[i],
	argtype[i], argname[i],
	argname[i], argtype[i]) > out
	printf("};\n") > out
	}
	else if (!flag("NOARGS") && !flag("NOPROTO") && !flag("NODEF"))
	printf("struct %s {\n\tregister_t dummy;\n};\n",
	argalias) > sysarg
	if (!flag("NOPROTO") && !flag("NODEF")) {
	printf("%s\t%s%s(struct thread , struct %s );\n",
	rettype, prefix, funcname, argalias) > outdcl
	printf("#define\t%sAUE_%s%s\t%s\n", syscallprefix,
	prefix, funcname, auditev) > sysaue
	}
	if (flag("NOSTD")) {
	printf("\t{ %s, (sy_call_t *)%s, %s, NULL, 0, 0, 0, SY_THR_ABSENT },",
	"0", "lkmressys", "AUE_NULL") > sysent
	align_sysent_comment(8 + 2 + length("0") + 15 + \
	length("lkmressys") + length("AUE_NULL") + 3)
	} else {
	printf("\t{ %s(%s,%s), %s, NULL, 0, 0, %s, %s },",
	wrap, argssize, funcname, auditev, flags, thr_flag) > sysent
	align_sysent_comment(8 + 9 + length(argssize) + 1 + \
	length(funcname) + length(auditev) + \
	length(flags) + 4)
	}
	printf("/* %d = %s %s */\n", syscall, descr, funcalias) > sysent
	printf("\t\"%s.%s\",\t\t/* %d = %s %s */\n",
	wrap, funcalias, syscall, descr, funcalias) > sysnames
	# Do not provide freebsdN_* symbols in libc for < FreeBSD 7
	if (flag("COMPAT") \|\| flag("COMPAT4") \|\| flag("COMPAT6")) {
	printf("\t\t\t\t/* %d is %s %s */\n",
	syscall, descr, funcalias) > syshdr
	} else if (!flag("NODEF")) {
	printf("#define\t%s%s%s\t%d\n", syscallprefix,
	prefix, funcalias, syscall) > syshdr
	printf(" \\\n\t%s%s.o", prefix, funcalias) > sysmk
	}
	syscall++
	next
	}
	type("OBSOL") {
	printf("\t{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },") > sysent
	align_sysent_comment(34)
	printf("/* %d = obsolete %s */\n", syscall, comment) > sysent
	printf("\t\"obs_%s\",\t\t\t/* %d = obsolete %s */\n",
	$4, syscall, comment) > sysnames
	printf("\t\t\t\t/* %d is obsolete %s */\n",
	syscall, comment) > syshdr
	syscall++
	next
	}
	type("UNIMPL") {
	while (syscall <= syscall_range[2]) {
	printf("\t{ 0, (sy_call_t )nosys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },\t\t\t/ %d = %s */\n",
	syscall, comment) > sysent
	printf("\t\"#%d\",\t\t\t/* %d = %s */\n",
	syscall, syscall, comment) > sysnames
	syscall++
	}
	next
	}
	{
	printf "%s: line %d: unrecognized keyword %s\n", infile, NR, $3
	exit 1
	}
	END {
	printf "\n#define AS(name) (sizeof(struct name) / sizeof(register_t))\n" > sysinc

	if (ncompat != 0) {
	printf "\n#ifdef %s\n", compat > sysinc
	printf "#define compat(n, name) n, (sy_call_t *)__CONCAT(o,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}

	if (ncompat4 != 0) {
	printf "\n#ifdef %s\n", compat4 > sysinc
	printf "#define compat4(n, name) n, (sy_call_t *)__CONCAT(freebsd4_,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat4(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}

	if (ncompat6 != 0) {
	printf "\n#ifdef %s\n", compat6 > sysinc
	printf "#define compat6(n, name) n, (sy_call_t *)__CONCAT(freebsd6_,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat6(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}

	if (ncompat7 != 0) {
	printf "\n#ifdef %s\n", compat7 > sysinc
	printf "#define compat7(n, name) n, (sy_call_t *)__CONCAT(freebsd7_,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat7(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}
	if (ncompat10 != 0) {
	printf "\n#ifdef %s\n", compat10 > sysinc
	printf "#define compat10(n, name) n, (sy_call_t *)__CONCAT(freebsd10_,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat10(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}
	if (ncompat11 != 0) {
	printf "\n#ifdef %s\n", compat11 > sysinc
	printf "#define compat11(n, name) n, (sy_call_t *)__CONCAT(freebsd11_,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat11(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}
	if (ncompat12 != 0) {
	printf "\n#ifdef %s\n", compat12 > sysinc
	printf "#define compat12(n, name) n, (sy_call_t *)__CONCAT(freebsd12_,name)\n" > sysinc
	printf "#else\n" > sysinc
	printf "#define compat12(n, name) 0, (sy_call_t *)nosys\n" > sysinc
	printf "#endif\n" > sysinc
	}

	printf("\n#endif /* %s */\n\n", compat) > syscompatdcl
	printf("\n#endif /* %s */\n\n", compat4) > syscompat4dcl
	printf("\n#endif /* %s */\n\n", compat6) > syscompat6dcl
	printf("\n#endif /* %s */\n\n", compat7) > syscompat7dcl
	printf("\n#endif /* %s */\n\n", compat10) > syscompat10dcl
	printf("\n#endif /* %s */\n\n", compat11) > syscompat11dcl
	printf("\n#endif /* %s */\n\n", compat12) > syscompat12dcl

	printf("\n#undef PAD_\n") > sysprotoend
	printf("#undef PADL_\n") > sysprotoend
	printf("#undef PADR_\n") > sysprotoend
	printf("\n#endif /* !%s */\n", sysproto_h) > sysprotoend

	printf("\n") > sysmk
	printf("};\n") > sysent
	printf("};\n") > sysnames
	printf("#define\t%sMAXSYSCALL\t%d\n", syscallprefix, syscall) \
	> syshdr
	printf "\tdefault:\n\t\t*n_args = 0;\n\t\tbreak;\n\t};\n}\n" > systrace
	printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systracetmp
	printf "\tdefault:\n\t\tbreak;\n\t};\n\tif (p != NULL)\n\t\tstrlcpy(desc, p, descsz);\n}\n" > systraceret
	} '

	cat $sysinc $sysent >> $syssw
	cat $sysarg $sysdcl \
	$syscompat $syscompatdcl \
	$syscompat4 $syscompat4dcl \
	$syscompat6 $syscompat6dcl \
	$syscompat7 $syscompat7dcl \
	$syscompat10 $syscompat10dcl \
	$syscompat11 $syscompat11dcl \
	$syscompat12 $syscompat12dcl \
	$sysaue $sysprotoend > $sysproto
	cat $systracetmp >> $systrace
	cat $systraceret >> $systrace

	diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
	index 15a8b1439463..ac138679c850 100644
	--- a/sys/kern/uipc_socket.c
	+++ b/sys/kern/uipc_socket.c
	@@ -1,4393 +1,4398 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1982, 1986, 1988, 1990, 1993
	* The Regents of the University of California.
	* Copyright (c) 2004 The FreeBSD Foundation
	* Copyright (c) 2004-2008 Robert N. M. Watson
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
	*/

	/*
	* Comments on the socket life cycle:
	*
	* soalloc() sets of socket layer state for a socket, called only by
	* socreate() and sonewconn(). Socket layer private.
	*
	* sodealloc() tears down socket layer state for a socket, called only by
	* sofree() and sonewconn(). Socket layer private.
	*
	* pru_attach() associates protocol layer state with an allocated socket;
	* called only once, may fail, aborting socket allocation. This is called
	* from socreate() and sonewconn(). Socket layer private.
	*
	* pru_detach() disassociates protocol layer state from an attached socket,
	* and will be called exactly once for sockets in which pru_attach() has
	* been successfully called. If pru_attach() returned an error,
	* pru_detach() will not be called. Socket layer private.
	*
	* pru_abort() and pru_close() notify the protocol layer that the last
	* consumer of a socket is starting to tear down the socket, and that the
	* protocol should terminate the connection. Historically, pru_abort() also
	* detached protocol state from the socket state, but this is no longer the
	* case.
	*
	* socreate() creates a socket and attaches protocol state. This is a public
	* interface that may be used by socket layer consumers to create new
	* sockets.
	*
	* sonewconn() creates a socket and attaches protocol state. This is a
	* public interface that may be used by protocols to create new sockets when
	* a new connection is received and will be available for accept() on a
	* listen socket.
	*
	* soclose() destroys a socket after possibly waiting for it to disconnect.
	* This is a public interface that socket consumers should use to close and
	* release a socket when done with it.
	*
	* soabort() destroys a socket without waiting for it to disconnect (used
	* only for incoming connections that are already partially or fully
	* connected). This is used internally by the socket layer when clearing
	* listen socket queues (due to overflow or close on the listen socket), but
	* is also a public interface protocols may use to abort connections in
	* their incomplete listen queues should they no longer be required. Sockets
	* placed in completed connection listen queues should not be aborted for
	* reasons described in the comment above the soclose() implementation. This
	* is not a general purpose close routine, and except in the specific
	* circumstances described here, should not be used.
	*
	* sofree() will free a socket and its protocol state if all references on
	* the socket have been released, and is the public interface to attempt to
	* free a socket when a reference is removed. This is a socket layer private
	* interface.
	*
	* NOTE: In addition to socreate() and soclose(), which provide a single
	* socket reference to the consumer to be managed as required, there are two
	* calls to explicitly manage socket references, soref(), and sorele().
	* Currently, these are generally required only when transitioning a socket
	* from a listen queue to a file descriptor, in order to prevent garbage
	* collection of the socket at an untimely moment. For a number of reasons,
	* these interfaces are not preferred, and should be avoided.
	*
	* NOTE: With regard to VNETs the general rule is that callers do not set
	* curvnet. Exceptions to this rule include soabort(), sodisconnect(),
	* sofree() (and with that sorele(), sotryfree()), as well as sonewconn()
	* and sorflush(), which are usually called from a pre-set VNET context.
	* sopoll() currently does not need a VNET context to be set.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_kern_tls.h"
	#include "opt_sctp.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/fcntl.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/mac.h>
	#include <sys/malloc.h>
	#include <sys/mbuf.h>
	#include <sys/mutex.h>
	#include <sys/domain.h>
	#include <sys/file.h> /* for struct knote */
	#include <sys/hhook.h>
	#include <sys/kernel.h>
	#include <sys/khelp.h>
	#include <sys/ktls.h>
	#include <sys/event.h>
	#include <sys/eventhandler.h>
	#include <sys/poll.h>
	#include <sys/proc.h>
	#include <sys/protosw.h>
	#include <sys/sbuf.h>
	#include <sys/socket.h>
	#include <sys/socketvar.h>
	#include <sys/resourcevar.h>
	#include <net/route.h>
	#include <sys/signalvar.h>
	#include <sys/stat.h>
	#include <sys/sx.h>
	#include <sys/sysctl.h>
	#include <sys/taskqueue.h>
	#include <sys/uio.h>
	#include <sys/un.h>
	#include <sys/unpcb.h>
	#include <sys/jail.h>
	#include <sys/syslog.h>
	#include <netinet/in.h>
	#include <netinet/in_pcb.h>
	#include <netinet/tcp.h>

	#include <net/vnet.h>

	#include <security/mac/mac_framework.h>

	#include <vm/uma.h>

	#ifdef COMPAT_FREEBSD32
	#include <sys/mount.h>
	#include <sys/sysent.h>
	#include <compat/freebsd32/freebsd32.h>
	#endif

	static int soreceive_rcvoob(struct socket so, struct uio uio,
	int flags);
	static void so_rdknl_lock(void *);
	static void so_rdknl_unlock(void *);
	static void so_rdknl_assert_locked(void *);
	static void so_rdknl_assert_unlocked(void *);
	static void so_wrknl_lock(void *);
	static void so_wrknl_unlock(void *);
	static void so_wrknl_assert_locked(void *);
	static void so_wrknl_assert_unlocked(void *);

	static void filt_sordetach(struct knote *kn);
	static int filt_soread(struct knote *kn, long hint);
	static void filt_sowdetach(struct knote *kn);
	static int filt_sowrite(struct knote *kn, long hint);
	static int filt_soempty(struct knote *kn, long hint);
	static int inline hhook_run_socket(struct socket so, void hctx, int32_t h_id);
	fo_kqfilter_t soo_kqfilter;

	static struct filterops soread_filtops = {
	.f_isfd = 1,
	.f_detach = filt_sordetach,
	.f_event = filt_soread,
	};
	static struct filterops sowrite_filtops = {
	.f_isfd = 1,
	.f_detach = filt_sowdetach,
	.f_event = filt_sowrite,
	};
	static struct filterops soempty_filtops = {
	.f_isfd = 1,
	.f_detach = filt_sowdetach,
	.f_event = filt_soempty,
	};

	so_gen_t so_gencnt; /* generation count for sockets */

	MALLOC_DEFINE(M_SONAME, "soname", "socket name");
	MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");

	#define VNET_SO_ASSERT(so) \
	VNET_ASSERT(curvnet != NULL, \
	("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));

	VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]);
	#define V_socket_hhh VNET(socket_hhh)

	/*
	* Limit on the number of connections in the listen queue waiting
	* for accept(2).
	* NB: The original sysctl somaxconn is still available but hidden
	* to prevent confusion about the actual purpose of this number.
	*/
	static u_int somaxconn = SOMAXCONN;

	static int
	sysctl_somaxconn(SYSCTL_HANDLER_ARGS)
	{
	int error;
	int val;

	val = somaxconn;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error \|\| !req->newptr )
	return (error);

	/*
	* The purpose of the UINT_MAX / 3 limit, is so that the formula
	* 3 * so_qlimit / 2
	* below, will not overflow.
	*/

	if (val < 1 \|\| val > UINT_MAX / 3)
	return (EINVAL);

	somaxconn = val;
	return (0);
	}
	SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue,
	CTLTYPE_UINT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, 0, sizeof(int),
	sysctl_somaxconn, "I",
	"Maximum listen socket pending connection accept queue size");
	SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn,
	CTLTYPE_UINT \| CTLFLAG_RW \| CTLFLAG_SKIP \| CTLFLAG_NEEDGIANT, 0,
	sizeof(int), sysctl_somaxconn, "I",
	"Maximum listen socket pending connection accept queue size (compat)");

	static int numopensockets;
	SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
	&numopensockets, 0, "Number of open sockets");

	/*
	* accept_mtx locks down per-socket fields relating to accept queues. See
	* socketvar.h for an annotation of the protected fields of struct socket.
	*/
	struct mtx accept_mtx;
	MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF);

	/*
	* so_global_mtx protects so_gencnt, numopensockets, and the per-socket
	* so_gencnt field.
	*/
	static struct mtx so_global_mtx;
	MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF);

	/*
	* General IPC sysctl name space, used by sockets and a variety of other IPC
	* types.
	*/
	SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"IPC");

	/*
	* Initialize the socket subsystem and set up the socket
	* memory allocator.
	*/
	static uma_zone_t socket_zone;
	int maxsockets;

	static void
	socket_zone_change(void *tag)
	{

	maxsockets = uma_zone_set_max(socket_zone, maxsockets);
	}

	static void
	socket_hhook_register(int subtype)
	{

	if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype,
	&V_socket_hhh[subtype],
	HHOOK_NOWAIT\|HHOOK_HEADISINVNET) != 0)
	printf("%s: WARNING: unable to register hook\n", __func__);
	}

	static void
	socket_hhook_deregister(int subtype)
	{

	if (hhook_head_deregister(V_socket_hhh[subtype]) != 0)
	printf("%s: WARNING: unable to deregister hook\n", __func__);
	}

	static void
	socket_init(void *tag)
	{

	socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL,
	NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
	maxsockets = uma_zone_set_max(socket_zone, maxsockets);
	uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached");
	EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL,
	EVENTHANDLER_PRI_FIRST);
	}
	SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL);

	static void
	socket_vnet_init(const void *unused __unused)
	{
	int i;

	/* We expect a contiguous range */
	for (i = 0; i <= HHOOK_SOCKET_LAST; i++)
	socket_hhook_register(i);
	}
	VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY,
	socket_vnet_init, NULL);

	static void
	socket_vnet_uninit(const void *unused __unused)
	{
	int i;

	for (i = 0; i <= HHOOK_SOCKET_LAST; i++)
	socket_hhook_deregister(i);
	}
	VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY,
	socket_vnet_uninit, NULL);

	/*
	* Initialise maxsockets. This SYSINIT must be run after
	* tunable_mbinit().
	*/
	static void
	init_maxsockets(void *ignored)
	{

	TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets);
	maxsockets = imax(maxsockets, maxfiles);
	}
	SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL);

	/*
	* Sysctl to get and set the maximum global sockets limit. Notify protocols
	* of the change so that they can update their dependent limits as required.
	*/
	static int
	sysctl_maxsockets(SYSCTL_HANDLER_ARGS)
	{
	int error, newmaxsockets;

	newmaxsockets = maxsockets;
	error = sysctl_handle_int(oidp, &newmaxsockets, 0, req);
	if (error == 0 && req->newptr) {
	if (newmaxsockets > maxsockets &&
	newmaxsockets <= maxfiles) {
	maxsockets = newmaxsockets;
	EVENTHANDLER_INVOKE(maxsockets_change);
	} else
	error = EINVAL;
	}
	return (error);
	}
	SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets,
	CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT, &maxsockets, 0,
	sysctl_maxsockets, "IU",
	"Maximum number of sockets available");

	/*
	* Socket operation routines. These routines are called by the routines in
	* sys_socket.c or from a system process, and implement the semantics of
	* socket operations by switching out to the protocol specific routines.
	*/

	/*
	* Get a socket structure from our zone, and initialize it. Note that it
	* would probably be better to allocate socket and PCB at the same time, but
	* I'm not convinced that all the protocols can be easily modified to do
	* this.
	*
	* soalloc() returns a socket with a ref count of 0.
	*/
	static struct socket *
	soalloc(struct vnet *vnet)
	{
	struct socket *so;

	so = uma_zalloc(socket_zone, M_NOWAIT \| M_ZERO);
	if (so == NULL)
	return (NULL);
	#ifdef MAC
	if (mac_socket_init(so, M_NOWAIT) != 0) {
	uma_zfree(socket_zone, so);
	return (NULL);
	}
	#endif
	if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) {
	uma_zfree(socket_zone, so);
	return (NULL);
	}

	/*
	* The socket locking protocol allows to lock 2 sockets at a time,
	* however, the first one must be a listening socket. WITNESS lacks
	* a feature to change class of an existing lock, so we use DUPOK.
	*/
	mtx_init(&so->so_lock, "socket", NULL, MTX_DEF \| MTX_DUPOK);
	SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd");
	SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv");
	so->so_rcv.sb_sel = &so->so_rdsel;
	so->so_snd.sb_sel = &so->so_wrsel;
	sx_init(&so->so_snd.sb_sx, "so_snd_sx");
	sx_init(&so->so_rcv.sb_sx, "so_rcv_sx");
	TAILQ_INIT(&so->so_snd.sb_aiojobq);
	TAILQ_INIT(&so->so_rcv.sb_aiojobq);
	TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so);
	TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so);
	#ifdef VIMAGE
	VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p",
	__func__, __LINE__, so));
	so->so_vnet = vnet;
	#endif
	/* We shouldn't need the so_global_mtx */
	if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) {
	/* Do we need more comprehensive error returns? */
	uma_zfree(socket_zone, so);
	return (NULL);
	}
	mtx_lock(&so_global_mtx);
	so->so_gencnt = ++so_gencnt;
	++numopensockets;
	#ifdef VIMAGE
	vnet->vnet_sockcnt++;
	#endif
	mtx_unlock(&so_global_mtx);

	return (so);
	}

	/*
	* Free the storage associated with a socket at the socket layer, tear down
	* locks, labels, etc. All protocol state is assumed already to have been
	* torn down (and possibly never set up) by the caller.
	*/
	static void
	sodealloc(struct socket *so)
	{

	KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count));
	KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL"));

	mtx_lock(&so_global_mtx);
	so->so_gencnt = ++so_gencnt;
	--numopensockets; /* Could be below, but faster here. */
	#ifdef VIMAGE
	VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p",
	__func__, __LINE__, so));
	so->so_vnet->vnet_sockcnt--;
	#endif
	mtx_unlock(&so_global_mtx);
	#ifdef MAC
	mac_socket_destroy(so);
	#endif
	hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE);

	crfree(so->so_cred);
	khelp_destroy_osd(&so->osd);
	if (SOLISTENING(so)) {
	if (so->sol_accept_filter != NULL)
	accept_filt_setopt(so, NULL);
	} else {
	if (so->so_rcv.sb_hiwat)
	(void)chgsbsize(so->so_cred->cr_uidinfo,
	&so->so_rcv.sb_hiwat, 0, RLIM_INFINITY);
	if (so->so_snd.sb_hiwat)
	(void)chgsbsize(so->so_cred->cr_uidinfo,
	&so->so_snd.sb_hiwat, 0, RLIM_INFINITY);
	sx_destroy(&so->so_snd.sb_sx);
	sx_destroy(&so->so_rcv.sb_sx);
	SOCKBUF_LOCK_DESTROY(&so->so_snd);
	SOCKBUF_LOCK_DESTROY(&so->so_rcv);
	}
	mtx_destroy(&so->so_lock);
	uma_zfree(socket_zone, so);
	}

	/*
	* socreate returns a socket with a ref count of 1. The socket should be
	* closed with soclose().
	*/
	int
	socreate(int dom, struct socket **aso, int type, int proto,
	struct ucred cred, struct thread td)
	{
	struct protosw *prp;
	struct socket *so;
	int error;

	if (proto)
	prp = pffindproto(dom, proto, type);
	else
	prp = pffindtype(dom, type);

	if (prp == NULL) {
	/* No support for domain. */
	if (pffinddomain(dom) == NULL)
	return (EAFNOSUPPORT);
	/* No support for socket type. */
	if (proto == 0 && type != 0)
	return (EPROTOTYPE);
	return (EPROTONOSUPPORT);
	}
	if (prp->pr_usrreqs->pru_attach == NULL \|\|
	prp->pr_usrreqs->pru_attach == pru_attach_notsupp)
	return (EPROTONOSUPPORT);

	if (prison_check_af(cred, prp->pr_domain->dom_family) != 0)
	return (EPROTONOSUPPORT);

	if (prp->pr_type != type)
	return (EPROTOTYPE);
	so = soalloc(CRED_TO_VNET(cred));
	if (so == NULL)
	return (ENOBUFS);

	so->so_type = type;
	so->so_cred = crhold(cred);
	if ((prp->pr_domain->dom_family == PF_INET) \|\|
	(prp->pr_domain->dom_family == PF_INET6) \|\|
	(prp->pr_domain->dom_family == PF_ROUTE))
	so->so_fibnum = td->td_proc->p_fibnum;
	else
	so->so_fibnum = 0;
	so->so_proto = prp;
	#ifdef MAC
	mac_socket_create(cred, so);
	#endif
	knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
	so_rdknl_assert_locked, so_rdknl_assert_unlocked);
	knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
	so_wrknl_assert_locked, so_wrknl_assert_unlocked);
	/*
	* Auto-sizing of socket buffers is managed by the protocols and
	* the appropriate flags must be set in the pru_attach function.
	*/
	CURVNET_SET(so->so_vnet);
	error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
	CURVNET_RESTORE();
	if (error) {
	sodealloc(so);
	return (error);
	}
	soref(so);
	*aso = so;
	return (0);
	}

	#ifdef REGRESSION
	static int regression_sonewconn_earlytest = 1;
	SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW,
	&regression_sonewconn_earlytest, 0, "Perform early sonewconn limit test");
	#endif

	static struct timeval overinterval = { 60, 0 };
	SYSCTL_TIMEVAL_SEC(_kern_ipc, OID_AUTO, sooverinterval, CTLFLAG_RW,
	&overinterval,
	"Delay in seconds between warnings for listen socket overflows");

	/*
	* When an attempt at a new connection is noted on a socket which accepts
	* connections, sonewconn is called. If the connection is possible (subject
	* to space constraints, etc.) then we allocate a new structure, properly
	* linked into the data structure of the original socket, and return this.
	* Connstatus may be 0, or SS_ISCONFIRMING, or SS_ISCONNECTED.
	*
	* Note: the ref count on the socket is 0 on return.
	*/
	struct socket *
	sonewconn(struct socket *head, int connstatus)
	{
	struct sbuf descrsb;
	struct socket *so;
	int len, overcount;
	u_int qlen;
	const char localprefix[] = "local:";
	char descrbuf[SUNPATHLEN + sizeof(localprefix)];
	#if defined(INET6)
	char addrbuf[INET6_ADDRSTRLEN];
	#elif defined(INET)
	char addrbuf[INET_ADDRSTRLEN];
	#endif
	bool dolog, over;

	SOLISTEN_LOCK(head);
	over = (head->sol_qlen > 3 * head->sol_qlimit / 2);
	#ifdef REGRESSION
	if (regression_sonewconn_earlytest && over) {
	#else
	if (over) {
	#endif
	head->sol_overcount++;
	dolog = !!ratecheck(&head->sol_lastover, &overinterval);

	/*
	* If we're going to log, copy the overflow count and queue
	* length from the listen socket before dropping the lock.
	* Also, reset the overflow count.
	*/
	if (dolog) {
	overcount = head->sol_overcount;
	head->sol_overcount = 0;
	qlen = head->sol_qlen;
	}
	SOLISTEN_UNLOCK(head);

	if (dolog) {
	/*
	* Try to print something descriptive about the
	* socket for the error message.
	*/
	sbuf_new(&descrsb, descrbuf, sizeof(descrbuf),
	SBUF_FIXEDLEN);
	switch (head->so_proto->pr_domain->dom_family) {
	#if defined(INET) \|\| defined(INET6)
	#ifdef INET
	case AF_INET:
	#endif
	#ifdef INET6
	case AF_INET6:
	if (head->so_proto->pr_domain->dom_family ==
	AF_INET6 \|\|
	(sotoinpcb(head)->inp_inc.inc_flags &
	INC_ISIPV6)) {
	ip6_sprintf(addrbuf,
	&sotoinpcb(head)->inp_inc.inc6_laddr);
	sbuf_printf(&descrsb, "[%s]", addrbuf);
	} else
	#endif
	{
	#ifdef INET
	inet_ntoa_r(
	sotoinpcb(head)->inp_inc.inc_laddr,
	addrbuf);
	sbuf_cat(&descrsb, addrbuf);
	#endif
	}
	sbuf_printf(&descrsb, ":%hu (proto %u)",
	ntohs(sotoinpcb(head)->inp_inc.inc_lport),
	head->so_proto->pr_protocol);
	break;
	#endif /* INET \|\| INET6 */
	case AF_UNIX:
	sbuf_cat(&descrsb, localprefix);
	if (sotounpcb(head)->unp_addr != NULL)
	len =
	sotounpcb(head)->unp_addr->sun_len -
	offsetof(struct sockaddr_un,
	sun_path);
	else
	len = 0;
	if (len > 0)
	sbuf_bcat(&descrsb,
	sotounpcb(head)->unp_addr->sun_path,
	len);
	else
	sbuf_cat(&descrsb, "(unknown)");
	break;
	}

	/*
	* If we can't print something more specific, at least
	* print the domain name.
	*/
	if (sbuf_finish(&descrsb) != 0 \|\|
	sbuf_len(&descrsb) <= 0) {
	sbuf_clear(&descrsb);
	sbuf_cat(&descrsb,
	head->so_proto->pr_domain->dom_name ?:
	"unknown");
	sbuf_finish(&descrsb);
	}
	KASSERT(sbuf_len(&descrsb) > 0,
	("%s: sbuf creation failed", __func__));
	log(LOG_DEBUG,
	"%s: pcb %p (%s): Listen queue overflow: "
	"%i already in queue awaiting acceptance "
	"(%d occurrences)\n",
	__func__, head->so_pcb, sbuf_data(&descrsb),
	qlen, overcount);
	sbuf_delete(&descrsb);

	overcount = 0;
	}

	return (NULL);
	}
	SOLISTEN_UNLOCK(head);
	VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL",
	__func__, head));
	so = soalloc(head->so_vnet);
	if (so == NULL) {
	log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: "
	"limit reached or out of memory\n",
	__func__, head->so_pcb);
	return (NULL);
	}
	so->so_listen = head;
	so->so_type = head->so_type;
	so->so_linger = head->so_linger;
	so->so_state = head->so_state \| SS_NOFDREF;
	so->so_fibnum = head->so_fibnum;
	so->so_proto = head->so_proto;
	so->so_cred = crhold(head->so_cred);
	#ifdef MAC
	mac_socket_newconn(head, so);
	#endif
	knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
	so_rdknl_assert_locked, so_rdknl_assert_unlocked);
	knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
	so_wrknl_assert_locked, so_wrknl_assert_unlocked);
	VNET_SO_ASSERT(head);
	if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) {
	sodealloc(so);
	log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n",
	__func__, head->so_pcb);
	return (NULL);
	}
	if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
	sodealloc(so);
	log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n",
	__func__, head->so_pcb);
	return (NULL);
	}
	so->so_rcv.sb_lowat = head->sol_sbrcv_lowat;
	so->so_snd.sb_lowat = head->sol_sbsnd_lowat;
	so->so_rcv.sb_timeo = head->sol_sbrcv_timeo;
	so->so_snd.sb_timeo = head->sol_sbsnd_timeo;
	so->so_rcv.sb_flags \|= head->sol_sbrcv_flags & SB_AUTOSIZE;
	so->so_snd.sb_flags \|= head->sol_sbsnd_flags & SB_AUTOSIZE;

	SOLISTEN_LOCK(head);
	if (head->sol_accept_filter != NULL)
	connstatus = 0;
	so->so_state \|= connstatus;
	so->so_options = head->so_options & ~SO_ACCEPTCONN;
	soref(head); /* A socket on (in)complete queue refs head. */
	if (connstatus) {
	TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
	so->so_qstate = SQ_COMP;
	head->sol_qlen++;
	solisten_wakeup(head); /* unlocks */
	} else {
	/*
	* Keep removing sockets from the head until there's room for
	* us to insert on the tail. In pre-locking revisions, this
	* was a simple if(), but as we could be racing with other
	* threads and soabort() requires dropping locks, we must
	* loop waiting for the condition to be true.
	*/
	while (head->sol_incqlen > head->sol_qlimit) {
	struct socket *sp;

	sp = TAILQ_FIRST(&head->sol_incomp);
	TAILQ_REMOVE(&head->sol_incomp, sp, so_list);
	head->sol_incqlen--;
	SOCK_LOCK(sp);
	sp->so_qstate = SQ_NONE;
	sp->so_listen = NULL;
	SOCK_UNLOCK(sp);
	sorele(head); /* does SOLISTEN_UNLOCK, head stays */
	soabort(sp);
	SOLISTEN_LOCK(head);
	}
	TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list);
	so->so_qstate = SQ_INCOMP;
	head->sol_incqlen++;
	SOLISTEN_UNLOCK(head);
	}
	return (so);
	}

	#if defined(SCTP) \|\| defined(SCTP_SUPPORT)
	/*
	* Socket part of sctp_peeloff(). Detach a new socket from an
	* association. The new socket is returned with a reference.
	*/
	struct socket *
	sopeeloff(struct socket *head)
	{
	struct socket *so;

	VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p",
	__func__, __LINE__, head));
	so = soalloc(head->so_vnet);
	if (so == NULL) {
	log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: "
	"limit reached or out of memory\n",
	__func__, head->so_pcb);
	return (NULL);
	}
	so->so_type = head->so_type;
	so->so_options = head->so_options;
	so->so_linger = head->so_linger;
	so->so_state = (head->so_state & SS_NBIO) \| SS_ISCONNECTED;
	so->so_fibnum = head->so_fibnum;
	so->so_proto = head->so_proto;
	so->so_cred = crhold(head->so_cred);
	#ifdef MAC
	mac_socket_newconn(head, so);
	#endif
	knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
	so_rdknl_assert_locked, so_rdknl_assert_unlocked);
	knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
	so_wrknl_assert_locked, so_wrknl_assert_unlocked);
	VNET_SO_ASSERT(head);
	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
	sodealloc(so);
	log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n",
	__func__, head->so_pcb);
	return (NULL);
	}
	if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
	sodealloc(so);
	log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n",
	__func__, head->so_pcb);
	return (NULL);
	}
	so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
	so->so_snd.sb_lowat = head->so_snd.sb_lowat;
	so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
	so->so_snd.sb_timeo = head->so_snd.sb_timeo;
	so->so_rcv.sb_flags \|= head->so_rcv.sb_flags & SB_AUTOSIZE;
	so->so_snd.sb_flags \|= head->so_snd.sb_flags & SB_AUTOSIZE;

	soref(so);

	return (so);
	}
	#endif /* SCTP */

	int
	sobind(struct socket so, struct sockaddr nam, struct thread *td)
	{
	int error;

	CURVNET_SET(so->so_vnet);
	error = (*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td);
	CURVNET_RESTORE();
	return (error);
	}

	int
	sobindat(int fd, struct socket so, struct sockaddr nam, struct thread *td)
	{
	int error;

	CURVNET_SET(so->so_vnet);
	error = (*so->so_proto->pr_usrreqs->pru_bindat)(fd, so, nam, td);
	CURVNET_RESTORE();
	return (error);
	}

	/*
	* solisten() transitions a socket from a non-listening state to a listening
	* state, but can also be used to update the listen queue depth on an
	* existing listen socket. The protocol will call back into the sockets
	* layer using solisten_proto_check() and solisten_proto() to check and set
	* socket-layer listen state. Call backs are used so that the protocol can
	* acquire both protocol and socket layer locks in whatever order is required
	* by the protocol.
	*
	* Protocol implementors are advised to hold the socket lock across the
	* socket-layer test and set to avoid races at the socket layer.
	*/
	int
	solisten(struct socket so, int backlog, struct thread td)
	{
	int error;

	CURVNET_SET(so->so_vnet);
	error = (*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td);
	CURVNET_RESTORE();
	return (error);
	}

	int
	solisten_proto_check(struct socket *so)
	{

	SOCK_LOCK_ASSERT(so);

	if (so->so_state & (SS_ISCONNECTED \| SS_ISCONNECTING \|
	SS_ISDISCONNECTING))
	return (EINVAL);
	return (0);
	}

	void
	solisten_proto(struct socket *so, int backlog)
	{
	int sbrcv_lowat, sbsnd_lowat;
	u_int sbrcv_hiwat, sbsnd_hiwat;
	short sbrcv_flags, sbsnd_flags;
	sbintime_t sbrcv_timeo, sbsnd_timeo;

	SOCK_LOCK_ASSERT(so);

	if (SOLISTENING(so))
	goto listening;

	/*
	* Change this socket to listening state.
	*/
	sbrcv_lowat = so->so_rcv.sb_lowat;
	sbsnd_lowat = so->so_snd.sb_lowat;
	sbrcv_hiwat = so->so_rcv.sb_hiwat;
	sbsnd_hiwat = so->so_snd.sb_hiwat;
	sbrcv_flags = so->so_rcv.sb_flags;
	sbsnd_flags = so->so_snd.sb_flags;
	sbrcv_timeo = so->so_rcv.sb_timeo;
	sbsnd_timeo = so->so_snd.sb_timeo;

	sbdestroy(&so->so_snd, so);
	sbdestroy(&so->so_rcv, so);
	sx_destroy(&so->so_snd.sb_sx);
	sx_destroy(&so->so_rcv.sb_sx);
	SOCKBUF_LOCK_DESTROY(&so->so_snd);
	SOCKBUF_LOCK_DESTROY(&so->so_rcv);

	#ifdef INVARIANTS
	bzero(&so->so_rcv,
	sizeof(struct socket) - offsetof(struct socket, so_rcv));
	#endif

	so->sol_sbrcv_lowat = sbrcv_lowat;
	so->sol_sbsnd_lowat = sbsnd_lowat;
	so->sol_sbrcv_hiwat = sbrcv_hiwat;
	so->sol_sbsnd_hiwat = sbsnd_hiwat;
	so->sol_sbrcv_flags = sbrcv_flags;
	so->sol_sbsnd_flags = sbsnd_flags;
	so->sol_sbrcv_timeo = sbrcv_timeo;
	so->sol_sbsnd_timeo = sbsnd_timeo;

	so->sol_qlen = so->sol_incqlen = 0;
	TAILQ_INIT(&so->sol_incomp);
	TAILQ_INIT(&so->sol_comp);

	so->sol_accept_filter = NULL;
	so->sol_accept_filter_arg = NULL;
	so->sol_accept_filter_str = NULL;

	so->sol_upcall = NULL;
	so->sol_upcallarg = NULL;

	so->so_options \|= SO_ACCEPTCONN;

	listening:
	if (backlog < 0 \|\| backlog > somaxconn)
	backlog = somaxconn;
	so->sol_qlimit = backlog;
	}

	/*
	* Wakeup listeners/subsystems once we have a complete connection.
	* Enters with lock, returns unlocked.
	*/
	void
	solisten_wakeup(struct socket *sol)
	{

	if (sol->sol_upcall != NULL)
	(void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT);
	else {
	selwakeuppri(&sol->so_rdsel, PSOCK);
	KNOTE_LOCKED(&sol->so_rdsel.si_note, 0);
	}
	SOLISTEN_UNLOCK(sol);
	wakeup_one(&sol->sol_comp);
	if ((sol->so_state & SS_ASYNC) && sol->so_sigio != NULL)
	pgsigio(&sol->so_sigio, SIGIO, 0);
	}

	/*
	* Return single connection off a listening socket queue. Main consumer of
	* the function is kern_accept4(). Some modules, that do their own accept
	* management also use the function.
	*
	* Listening socket must be locked on entry and is returned unlocked on
	* return.
	* The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT.
	*/
	int
	solisten_dequeue(struct socket head, struct socket *ret, int flags)
	{
	struct socket *so;
	int error;

	SOLISTEN_LOCK_ASSERT(head);

	while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) &&
	head->so_error == 0) {
	error = msleep(&head->sol_comp, &head->so_lock, PSOCK \| PCATCH,
	"accept", 0);
	if (error != 0) {
	SOLISTEN_UNLOCK(head);
	return (error);
	}
	}
	if (head->so_error) {
	error = head->so_error;
	head->so_error = 0;
	} else if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp))
	error = EWOULDBLOCK;
	else
	error = 0;
	if (error) {
	SOLISTEN_UNLOCK(head);
	return (error);
	}
	so = TAILQ_FIRST(&head->sol_comp);
	SOCK_LOCK(so);
	KASSERT(so->so_qstate == SQ_COMP,
	("%s: so %p not SQ_COMP", __func__, so));
	soref(so);
	head->sol_qlen--;
	so->so_qstate = SQ_NONE;
	so->so_listen = NULL;
	TAILQ_REMOVE(&head->sol_comp, so, so_list);
	if (flags & ACCEPT4_INHERIT)
	so->so_state \|= (head->so_state & SS_NBIO);
	else
	so->so_state \|= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
	SOCK_UNLOCK(so);
	sorele(head);

	*ret = so;
	return (0);
	}

	/*
	* Evaluate the reference count and named references on a socket; if no
	* references remain, free it. This should be called whenever a reference is
	* released, such as in sorele(), but also when named reference flags are
	* cleared in socket or protocol code.
	*
	* sofree() will free the socket if:
	*
	* - There are no outstanding file descriptor references or related consumers
	* (so_count == 0).
	*
	* - The socket has been closed by user space, if ever open (SS_NOFDREF).
	*
	* - The protocol does not have an outstanding strong reference on the socket
	* (SS_PROTOREF).
	*
	* - The socket is not in a completed connection queue, so a process has been
	* notified that it is present. If it is removed, the user process may
	* block in accept() despite select() saying the socket was ready.
	*/
	void
	sofree(struct socket *so)
	{
	struct protosw *pr = so->so_proto;

	SOCK_LOCK_ASSERT(so);

	if ((so->so_state & SS_NOFDREF) == 0 \|\| so->so_count != 0 \|\|
	(so->so_state & SS_PROTOREF) \|\| (so->so_qstate == SQ_COMP)) {
	SOCK_UNLOCK(so);
	return;
	}

	if (!SOLISTENING(so) && so->so_qstate == SQ_INCOMP) {
	struct socket *sol;

	sol = so->so_listen;
	KASSERT(sol, ("%s: so %p on incomp of NULL", __func__, so));

	/*
	* To solve race between close of a listening socket and
	* a socket on its incomplete queue, we need to lock both.
	* The order is first listening socket, then regular.
	* Since we don't have SS_NOFDREF neither SS_PROTOREF, this
	* function and the listening socket are the only pointers
	* to so. To preserve so and sol, we reference both and then
	* relock.
	* After relock the socket may not move to so_comp since it
	* doesn't have PCB already, but it may be removed from
	* so_incomp. If that happens, we share responsiblity on
	* freeing the socket, but soclose() has already removed
	* it from queue.
	*/
	soref(sol);
	soref(so);
	SOCK_UNLOCK(so);
	SOLISTEN_LOCK(sol);
	SOCK_LOCK(so);
	if (so->so_qstate == SQ_INCOMP) {
	KASSERT(so->so_listen == sol,
	("%s: so %p migrated out of sol %p",
	__func__, so, sol));
	TAILQ_REMOVE(&sol->sol_incomp, so, so_list);
	sol->sol_incqlen--;
	/* This is guarenteed not to be the last. */
	refcount_release(&sol->so_count);
	so->so_qstate = SQ_NONE;
	so->so_listen = NULL;
	} else
	KASSERT(so->so_listen == NULL,
	("%s: so %p not on (in)comp with so_listen",
	__func__, so));
	sorele(sol);
	KASSERT(so->so_count == 1,
	("%s: so %p count %u", __func__, so, so->so_count));
	so->so_count = 0;
	}
	if (SOLISTENING(so))
	so->so_error = ECONNABORTED;
	SOCK_UNLOCK(so);

	if (so->so_dtor != NULL)
	so->so_dtor(so);

	VNET_SO_ASSERT(so);
	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
	(*pr->pr_domain->dom_dispose)(so);
	if (pr->pr_usrreqs->pru_detach != NULL)
	(*pr->pr_usrreqs->pru_detach)(so);

	/*
	* From this point on, we assume that no other references to this
	* socket exist anywhere else in the stack. Therefore, no locks need
	* to be acquired or held.
	*
	* We used to do a lot of socket buffer and socket locking here, as
	* well as invoke sorflush() and perform wakeups. The direct call to
	* dom_dispose() and sbdestroy() are an inlining of what was
	* necessary from sorflush().
	*
	* Notice that the socket buffer and kqueue state are torn down
	* before calling pru_detach. This means that protocols shold not
	* assume they can perform socket wakeups, etc, in their detach code.
	*/
	if (!SOLISTENING(so)) {
	sbdestroy(&so->so_snd, so);
	sbdestroy(&so->so_rcv, so);
	}
	seldrain(&so->so_rdsel);
	seldrain(&so->so_wrsel);
	knlist_destroy(&so->so_rdsel.si_note);
	knlist_destroy(&so->so_wrsel.si_note);
	sodealloc(so);
	}

	/*
	* Close a socket on last file table reference removal. Initiate disconnect
	* if connected. Free socket when disconnect complete.
	*
	* This function will sorele() the socket. Note that soclose() may be called
	* prior to the ref count reaching zero. The actual socket structure will
	* not be freed until the ref count reaches zero.
	*/
	int
	soclose(struct socket *so)
	{
	struct accept_queue lqueue;
	bool listening;
	int error = 0;

	KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter"));

	CURVNET_SET(so->so_vnet);
	funsetown(&so->so_sigio);
	if (so->so_state & SS_ISCONNECTED) {
	if ((so->so_state & SS_ISDISCONNECTING) == 0) {
	error = sodisconnect(so);
	if (error) {
	if (error == ENOTCONN)
	error = 0;
	goto drop;
	}
	}
	if (so->so_options & SO_LINGER) {
	if ((so->so_state & SS_ISDISCONNECTING) &&
	(so->so_state & SS_NBIO))
	goto drop;
	while (so->so_state & SS_ISCONNECTED) {
	error = tsleep(&so->so_timeo,
	PSOCK \| PCATCH, "soclos",
	so->so_linger * hz);
	if (error)
	break;
	}
	}
	}

	drop:
	if (so->so_proto->pr_usrreqs->pru_close != NULL)
	(*so->so_proto->pr_usrreqs->pru_close)(so);

	SOCK_LOCK(so);
	if ((listening = (so->so_options & SO_ACCEPTCONN))) {
	struct socket *sp;

	TAILQ_INIT(&lqueue);
	TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list);
	TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list);

	so->sol_qlen = so->sol_incqlen = 0;

	TAILQ_FOREACH(sp, &lqueue, so_list) {
	SOCK_LOCK(sp);
	sp->so_qstate = SQ_NONE;
	sp->so_listen = NULL;
	SOCK_UNLOCK(sp);
	/* Guaranteed not to be the last. */
	refcount_release(&so->so_count);
	}
	}
	KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF"));
	so->so_state \|= SS_NOFDREF;
	sorele(so);
	if (listening) {
	struct socket sp, tsp;

	TAILQ_FOREACH_SAFE(sp, &lqueue, so_list, tsp) {
	SOCK_LOCK(sp);
	if (sp->so_count == 0) {
	SOCK_UNLOCK(sp);
	soabort(sp);
	} else
	/* sp is now in sofree() */
	SOCK_UNLOCK(sp);
	}
	}
	CURVNET_RESTORE();
	return (error);
	}

	/*
	* soabort() is used to abruptly tear down a connection, such as when a
	* resource limit is reached (listen queue depth exceeded), or if a listen
	* socket is closed while there are sockets waiting to be accepted.
	*
	* This interface is tricky, because it is called on an unreferenced socket,
	* and must be called only by a thread that has actually removed the socket
	* from the listen queue it was on, or races with other threads are risked.
	*
	* This interface will call into the protocol code, so must not be called
	* with any socket locks held. Protocols do call it while holding their own
	* recursible protocol mutexes, but this is something that should be subject
	* to review in the future.
	*/
	void
	soabort(struct socket *so)
	{

	/*
	* In as much as is possible, assert that no references to this
	* socket are held. This is not quite the same as asserting that the
	* current thread is responsible for arranging for no references, but
	* is as close as we can get for now.
	*/
	KASSERT(so->so_count == 0, ("soabort: so_count"));
	KASSERT((so->so_state & SS_PROTOREF) == 0, ("soabort: SS_PROTOREF"));
	KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF"));
	VNET_SO_ASSERT(so);

	if (so->so_proto->pr_usrreqs->pru_abort != NULL)
	(*so->so_proto->pr_usrreqs->pru_abort)(so);
	SOCK_LOCK(so);
	sofree(so);
	}

	int
	soaccept(struct socket so, struct sockaddr *nam)
	{
	int error;

	SOCK_LOCK(so);
	KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF"));
	so->so_state &= ~SS_NOFDREF;
	SOCK_UNLOCK(so);

	CURVNET_SET(so->so_vnet);
	error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam);
	CURVNET_RESTORE();
	return (error);
	}

	int
	soconnect(struct socket so, struct sockaddr nam, struct thread *td)
	{

	return (soconnectat(AT_FDCWD, so, nam, td));
	}

	int
	soconnectat(int fd, struct socket so, struct sockaddr nam, struct thread *td)
	{
	int error;

	if (so->so_options & SO_ACCEPTCONN)
	return (EOPNOTSUPP);

	CURVNET_SET(so->so_vnet);
	/*
	* If protocol is connection-based, can only connect once.
	* Otherwise, if connected, try to disconnect first. This allows
	* user to disconnect by connecting to, e.g., a null address.
	*/
	if (so->so_state & (SS_ISCONNECTED\|SS_ISCONNECTING) &&
	((so->so_proto->pr_flags & PR_CONNREQUIRED) \|\|
	(error = sodisconnect(so)))) {
	error = EISCONN;
	} else {
	/*
	* Prevent accumulated error from previous connection from
	* biting us.
	*/
	so->so_error = 0;
	if (fd == AT_FDCWD) {
	error = (*so->so_proto->pr_usrreqs->pru_connect)(so,
	nam, td);
	} else {
	error = (*so->so_proto->pr_usrreqs->pru_connectat)(fd,
	so, nam, td);
	}
	}
	CURVNET_RESTORE();

	return (error);
	}

	int
	soconnect2(struct socket so1, struct socket so2)
	{
	int error;

	CURVNET_SET(so1->so_vnet);
	error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2);
	CURVNET_RESTORE();
	return (error);
	}

	int
	sodisconnect(struct socket *so)
	{
	int error;

	if ((so->so_state & SS_ISCONNECTED) == 0)
	return (ENOTCONN);
	if (so->so_state & SS_ISDISCONNECTING)
	return (EALREADY);
	VNET_SO_ASSERT(so);
	error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so);
	return (error);
	}

	#define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT)

	int
	sosend_dgram(struct socket so, struct sockaddr addr, struct uio *uio,
	struct mbuf top, struct mbuf control, int flags, struct thread *td)
	{
	long space;
	ssize_t resid;
	int clen = 0, error, dontroute;

	KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM"));
	KASSERT(so->so_proto->pr_flags & PR_ATOMIC,
	("sosend_dgram: !PR_ATOMIC"));

	if (uio != NULL)
	resid = uio->uio_resid;
	else
	resid = top->m_pkthdr.len;
	/*
	* In theory resid should be unsigned. However, space must be
	* signed, as it might be less than 0 if we over-committed, and we
	* must use a signed comparison of space and resid. On the other
	* hand, a negative resid causes us to loop sending 0-length
	* segments to the protocol.
	*/
	if (resid < 0) {
	error = EINVAL;
	goto out;
	}

	dontroute =
	(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0;
	if (td != NULL)
	td->td_ru.ru_msgsnd++;
	if (control != NULL)
	clen = control->m_len;

	SOCKBUF_LOCK(&so->so_snd);
	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
	SOCKBUF_UNLOCK(&so->so_snd);
	error = EPIPE;
	goto out;
	}
	if (so->so_error) {
	error = so->so_error;
	so->so_error = 0;
	SOCKBUF_UNLOCK(&so->so_snd);
	goto out;
	}
	if ((so->so_state & SS_ISCONNECTED) == 0) {
	/*
	* `sendto' and `sendmsg' is allowed on a connection-based
	* socket if it supports implied connect. Return ENOTCONN if
	* not connected and no address is supplied.
	*/
	if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
	(so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
	if ((so->so_state & SS_ISCONFIRMING) == 0 &&
	!(resid == 0 && clen != 0)) {
	SOCKBUF_UNLOCK(&so->so_snd);
	error = ENOTCONN;
	goto out;
	}
	} else if (addr == NULL) {
	if (so->so_proto->pr_flags & PR_CONNREQUIRED)
	error = ENOTCONN;
	else
	error = EDESTADDRREQ;
	SOCKBUF_UNLOCK(&so->so_snd);
	goto out;
	}
	}

	/*
	* Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a
	* problem and need fixing.
	*/
	space = sbspace(&so->so_snd);
	if (flags & MSG_OOB)
	space += 1024;
	space -= clen;
	SOCKBUF_UNLOCK(&so->so_snd);
	if (resid > space) {
	error = EMSGSIZE;
	goto out;
	}
	if (uio == NULL) {
	resid = 0;
	if (flags & MSG_EOR)
	top->m_flags \|= M_EOR;
	} else {
	/*
	* Copy the data from userland into a mbuf chain.
	* If no data is to be copied in, a single empty mbuf
	* is returned.
	*/
	top = m_uiotombuf(uio, M_WAITOK, space, max_hdr,
	(M_PKTHDR \| ((flags & MSG_EOR) ? M_EOR : 0)));
	if (top == NULL) {
	error = EFAULT; /* only possible error */
	goto out;
	}
	space -= resid - uio->uio_resid;
	resid = uio->uio_resid;
	}
	KASSERT(resid == 0, ("sosend_dgram: resid != 0"));
	/*
	* XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock
	* than with.
	*/
	if (dontroute) {
	SOCK_LOCK(so);
	so->so_options \|= SO_DONTROUTE;
	SOCK_UNLOCK(so);
	}
	/*
	* XXX all the SBS_CANTSENDMORE checks previously done could be out
	* of date. We could have received a reset packet in an interrupt or
	* maybe we slept while doing page faults in uiomove() etc. We could
	* probably recheck again inside the locking protection here, but
	* there are probably other places that this also happens. We must
	* rethink this.
	*/
	VNET_SO_ASSERT(so);
	error = (*so->so_proto->pr_usrreqs->pru_send)(so,
	(flags & MSG_OOB) ? PRUS_OOB :
	/*
	* If the user set MSG_EOF, the protocol understands this flag and
	* nothing left to send then use PRU_SEND_EOF instead of PRU_SEND.
	*/
	((flags & MSG_EOF) &&
	(so->so_proto->pr_flags & PR_IMPLOPCL) &&
	(resid <= 0)) ?
	PRUS_EOF :
	/* If there is more to send set PRUS_MORETOCOME */
	(flags & MSG_MORETOCOME) \|\|
	(resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
	top, addr, control, td);
	if (dontroute) {
	SOCK_LOCK(so);
	so->so_options &= ~SO_DONTROUTE;
	SOCK_UNLOCK(so);
	}
	clen = 0;
	control = NULL;
	top = NULL;
	out:
	if (top != NULL)
	m_freem(top);
	if (control != NULL)
	m_freem(control);
	return (error);
	}

	/*
	* Send on a socket. If send must go all at once and message is larger than
	* send buffering, then hard error. Lock against other senders. If must go
	* all at once and not enough room now, then inform user that this would
	* block and do nothing. Otherwise, if nonblocking, send as much as
	* possible. The data to be sent is described by "uio" if nonzero, otherwise
	* by the mbuf chain "top" (which must be null if uio is not). Data provided
	* in mbuf chain must be small enough to send all at once.
	*
	* Returns nonzero on error, timeout or signal; callers must check for short
	* counts if EINTR/ERESTART are returned. Data and control buffers are freed
	* on return.
	*/
	int
	sosend_generic(struct socket so, struct sockaddr addr, struct uio *uio,
	struct mbuf top, struct mbuf control, int flags, struct thread *td)
	{
	long space;
	ssize_t resid;
	int clen = 0, error, dontroute;
	int atomic = sosendallatonce(so) \|\| top;
	int pru_flag;
	#ifdef KERN_TLS
	struct ktls_session *tls;
	int tls_enq_cnt, tls_pruflag;
	uint8_t tls_rtype;

	tls = NULL;
	tls_rtype = TLS_RLTYPE_APP;
	#endif
	if (uio != NULL)
	resid = uio->uio_resid;
	else if ((top->m_flags & M_PKTHDR) != 0)
	resid = top->m_pkthdr.len;
	else
	resid = m_length(top, NULL);
	/*
	* In theory resid should be unsigned. However, space must be
	* signed, as it might be less than 0 if we over-committed, and we
	* must use a signed comparison of space and resid. On the other
	* hand, a negative resid causes us to loop sending 0-length
	* segments to the protocol.
	*
	* Also check to make sure that MSG_EOR isn't used on SOCK_STREAM
	* type sockets since that's an error.
	*/
	if (resid < 0 \|\| (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
	error = EINVAL;
	goto out;
	}

	dontroute =
	(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
	(so->so_proto->pr_flags & PR_ATOMIC);
	if (td != NULL)
	td->td_ru.ru_msgsnd++;
	if (control != NULL)
	clen = control->m_len;

	error = sblock(&so->so_snd, SBLOCKWAIT(flags));
	if (error)
	goto out;

	#ifdef KERN_TLS
	tls_pruflag = 0;
	tls = ktls_hold(so->so_snd.sb_tls_info);
	if (tls != NULL) {
	if (tls->mode == TCP_TLS_MODE_SW)
	tls_pruflag = PRUS_NOTREADY;

	if (control != NULL) {
	struct cmsghdr cm = mtod(control, struct cmsghdr );

	if (clen >= sizeof(*cm) &&
	cm->cmsg_type == TLS_SET_RECORD_TYPE) {
	tls_rtype = ((uint8_t )CMSG_DATA(cm));
	clen = 0;
	m_freem(control);
	control = NULL;
	atomic = 1;
	}
	}
	}
	#endif

	restart:
	do {
	SOCKBUF_LOCK(&so->so_snd);
	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
	SOCKBUF_UNLOCK(&so->so_snd);
	error = EPIPE;
	goto release;
	}
	if (so->so_error) {
	error = so->so_error;
	so->so_error = 0;
	SOCKBUF_UNLOCK(&so->so_snd);
	goto release;
	}
	if ((so->so_state & SS_ISCONNECTED) == 0) {
	/*
	* `sendto' and `sendmsg' is allowed on a connection-
	* based socket if it supports implied connect.
	* Return ENOTCONN if not connected and no address is
	* supplied.
	*/
	if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
	(so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
	if ((so->so_state & SS_ISCONFIRMING) == 0 &&
	!(resid == 0 && clen != 0)) {
	SOCKBUF_UNLOCK(&so->so_snd);
	error = ENOTCONN;
	goto release;
	}
	} else if (addr == NULL) {
	SOCKBUF_UNLOCK(&so->so_snd);
	if (so->so_proto->pr_flags & PR_CONNREQUIRED)
	error = ENOTCONN;
	else
	error = EDESTADDRREQ;
	goto release;
	}
	}
	space = sbspace(&so->so_snd);
	if (flags & MSG_OOB)
	space += 1024;
	if ((atomic && resid > so->so_snd.sb_hiwat) \|\|
	clen > so->so_snd.sb_hiwat) {
	SOCKBUF_UNLOCK(&so->so_snd);
	error = EMSGSIZE;
	goto release;
	}
	if (space < resid + clen &&
	(atomic \|\| space < so->so_snd.sb_lowat \|\| space < clen)) {
	if ((so->so_state & SS_NBIO) \|\|
	(flags & (MSG_NBIO \| MSG_DONTWAIT)) != 0) {
	SOCKBUF_UNLOCK(&so->so_snd);
	error = EWOULDBLOCK;
	goto release;
	}
	error = sbwait(&so->so_snd);
	SOCKBUF_UNLOCK(&so->so_snd);
	if (error)
	goto release;
	goto restart;
	}
	SOCKBUF_UNLOCK(&so->so_snd);
	space -= clen;
	do {
	if (uio == NULL) {
	resid = 0;
	if (flags & MSG_EOR)
	top->m_flags \|= M_EOR;
	#ifdef KERN_TLS
	if (tls != NULL) {
	ktls_frame(top, tls, &tls_enq_cnt,
	tls_rtype);
	tls_rtype = TLS_RLTYPE_APP;
	}
	#endif
	} else {
	/*
	* Copy the data from userland into a mbuf
	* chain. If resid is 0, which can happen
	* only if we have control to send, then
	* a single empty mbuf is returned. This
	* is a workaround to prevent protocol send
	* methods to panic.
	*/
	#ifdef KERN_TLS
	if (tls != NULL) {
	top = m_uiotombuf(uio, M_WAITOK, space,
	tls->params.max_frame_len,
	M_EXTPG \|
	((flags & MSG_EOR) ? M_EOR : 0));
	if (top != NULL) {
	ktls_frame(top, tls,
	&tls_enq_cnt, tls_rtype);
	}
	tls_rtype = TLS_RLTYPE_APP;
	} else
	#endif
	top = m_uiotombuf(uio, M_WAITOK, space,
	(atomic ? max_hdr : 0),
	(atomic ? M_PKTHDR : 0) \|
	((flags & MSG_EOR) ? M_EOR : 0));
	if (top == NULL) {
	error = EFAULT; /* only possible error */
	goto release;
	}
	space -= resid - uio->uio_resid;
	resid = uio->uio_resid;
	}
	if (dontroute) {
	SOCK_LOCK(so);
	so->so_options \|= SO_DONTROUTE;
	SOCK_UNLOCK(so);
	}
	/*
	* XXX all the SBS_CANTSENDMORE checks previously
	* done could be out of date. We could have received
	* a reset packet in an interrupt or maybe we slept
	* while doing page faults in uiomove() etc. We
	* could probably recheck again inside the locking
	* protection here, but there are probably other
	* places that this also happens. We must rethink
	* this.
	*/
	VNET_SO_ASSERT(so);

	pru_flag = (flags & MSG_OOB) ? PRUS_OOB :
	/*
	* If the user set MSG_EOF, the protocol understands
	* this flag and nothing left to send then use
	* PRU_SEND_EOF instead of PRU_SEND.
	*/
	((flags & MSG_EOF) &&
	(so->so_proto->pr_flags & PR_IMPLOPCL) &&
	(resid <= 0)) ?
	PRUS_EOF :
	/* If there is more to send set PRUS_MORETOCOME. */
	(flags & MSG_MORETOCOME) \|\|
	(resid > 0 && space > 0) ? PRUS_MORETOCOME : 0;

	#ifdef KERN_TLS
	pru_flag \|= tls_pruflag;
	#endif

	error = (*so->so_proto->pr_usrreqs->pru_send)(so,
	pru_flag, top, addr, control, td);

	if (dontroute) {
	SOCK_LOCK(so);
	so->so_options &= ~SO_DONTROUTE;
	SOCK_UNLOCK(so);
	}

	#ifdef KERN_TLS
	if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
	/*
	* Note that error is intentionally
	* ignored.
	*
	* Like sendfile(), we rely on the
	* completion routine (pru_ready())
	* to free the mbufs in the event that
	* pru_send() encountered an error and
	* did not append them to the sockbuf.
	*/
	soref(so);
	ktls_enqueue(top, so, tls_enq_cnt);
	}
	#endif
	clen = 0;
	control = NULL;
	top = NULL;
	if (error)
	goto release;
	} while (resid && space > 0);
	} while (resid);

	release:
	sbunlock(&so->so_snd);
	out:
	#ifdef KERN_TLS
	if (tls != NULL)
	ktls_free(tls);
	#endif
	if (top != NULL)
	m_freem(top);
	if (control != NULL)
	m_freem(control);
	return (error);
	}

	int
	sosend(struct socket so, struct sockaddr addr, struct uio *uio,
	struct mbuf top, struct mbuf control, int flags, struct thread *td)
	{
	int error;

	CURVNET_SET(so->so_vnet);
	if (!SOLISTENING(so))
	error = so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio,
	top, control, flags, td);
	else {
	m_freem(top);
	m_freem(control);
	error = ENOTCONN;
	}
	CURVNET_RESTORE();
	return (error);
	}

	/*
	* The part of soreceive() that implements reading non-inline out-of-band
	* data from a socket. For more complete comments, see soreceive(), from
	* which this code originated.
	*
	* Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is
	* unable to return an mbuf chain to the caller.
	*/
	static int
	soreceive_rcvoob(struct socket so, struct uio uio, int flags)
	{
	struct protosw *pr = so->so_proto;
	struct mbuf *m;
	int error;

	KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
	VNET_SO_ASSERT(so);

	m = m_get(M_WAITOK, MT_DATA);
	error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
	if (error)
	goto bad;
	do {
	error = uiomove(mtod(m, void *),
	(int) min(uio->uio_resid, m->m_len), uio);
	m = m_free(m);
	} while (uio->uio_resid && error == 0 && m);
	bad:
	if (m != NULL)
	m_freem(m);
	return (error);
	}

	/*
	* Following replacement or removal of the first mbuf on the first mbuf chain
	* of a socket buffer, push necessary state changes back into the socket
	* buffer so that other consumers see the values consistently. 'nextrecord'
	* is the callers locally stored value of the original value of
	* sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes.
	* NOTE: 'nextrecord' may be NULL.
	*/
	static __inline void
	sockbuf_pushsync(struct sockbuf sb, struct mbuf nextrecord)
	{

	SOCKBUF_LOCK_ASSERT(sb);
	/*
	* First, update for the new value of nextrecord. If necessary, make
	* it the first record.
	*/
	if (sb->sb_mb != NULL)
	sb->sb_mb->m_nextpkt = nextrecord;
	else
	sb->sb_mb = nextrecord;

	/*
	* Now update any dependent socket buffer fields to reflect the new
	* state. This is an expanded inline of SB_EMPTY_FIXUP(), with the
	* addition of a second clause that takes care of the case where
	* sb_mb has been updated, but remains the last record.
	*/
	if (sb->sb_mb == NULL) {
	sb->sb_mbtail = NULL;
	sb->sb_lastrecord = NULL;
	} else if (sb->sb_mb->m_nextpkt == NULL)
	sb->sb_lastrecord = sb->sb_mb;
	}

	/*
	* Implement receive operations on a socket. We depend on the way that
	* records are added to the sockbuf by sbappend. In particular, each record
	* (mbufs linked through m_next) must begin with an address if the protocol
	* so specifies, followed by an optional mbuf or mbufs containing ancillary
	* data, and then zero or more mbufs of data. In order to allow parallelism
	* between network receive and copying to user space, as well as avoid
	* sleeping with a mutex held, we release the socket buffer mutex during the
	* user space copy. Although the sockbuf is locked, new data may still be
	* appended, and thus we must maintain consistency of the sockbuf during that
	* time.
	*
	* The caller may receive the data as a single mbuf chain by supplying an
	* mbuf **mp0 for use in returning the chain. The uio is then used only for
	* the count in uio_resid.
	*/
	int
	soreceive_generic(struct socket so, struct sockaddr psa, struct uio uio,
	struct mbuf mp0, struct mbuf controlp, int *flagsp)
	{
	struct mbuf m, *mp;
	int flags, error, offset;
	ssize_t len;
	struct protosw *pr = so->so_proto;
	struct mbuf *nextrecord;
	int moff, type = 0;
	ssize_t orig_resid = uio->uio_resid;

	mp = mp0;
	if (psa != NULL)
	*psa = NULL;
	if (controlp != NULL)
	*controlp = NULL;
	if (flagsp != NULL)
	flags = *flagsp &~ MSG_EOR;
	else
	flags = 0;
	if (flags & MSG_OOB)
	return (soreceive_rcvoob(so, uio, flags));
	if (mp != NULL)
	*mp = NULL;
	if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING)
	&& uio->uio_resid) {
	VNET_SO_ASSERT(so);
	(*pr->pr_usrreqs->pru_rcvd)(so, 0);
	}

	error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
	if (error)
	return (error);

	restart:
	SOCKBUF_LOCK(&so->so_rcv);
	m = so->so_rcv.sb_mb;
	/*
	* If we have less data than requested, block awaiting more (subject
	* to any timeout) if:
	* 1. the current count is less than the low water mark, or
	* 2. MSG_DONTWAIT is not set
	*/
	if (m == NULL \|\| (((flags & MSG_DONTWAIT) == 0 &&
	sbavail(&so->so_rcv) < uio->uio_resid) &&
	sbavail(&so->so_rcv) < so->so_rcv.sb_lowat &&
	m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
	KASSERT(m != NULL \|\| !sbavail(&so->so_rcv),
	("receive: m == %p sbavail == %u",
	m, sbavail(&so->so_rcv)));
	if (so->so_error) {
	if (m != NULL)
	goto dontblock;
	error = so->so_error;
	if ((flags & MSG_PEEK) == 0)
	so->so_error = 0;
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto release;
	}
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
	- if (m == NULL && so->so_rcv.sb_tlsdcc == 0 &&
	+ if (m != NULL)
	+ goto dontblock;
	+#ifdef KERN_TLS
	+ else if (so->so_rcv.sb_tlsdcc == 0 &&
	so->so_rcv.sb_tlscc == 0) {
	+#else
	+ else {
	+#endif
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto release;
	- } else
	- goto dontblock;
	+ }
	}
	for (; m != NULL; m = m->m_next)
	if (m->m_type == MT_OOBDATA \|\| (m->m_flags & M_EOR)) {
	m = so->so_rcv.sb_mb;
	goto dontblock;
	}
	if ((so->so_state & (SS_ISCONNECTING \| SS_ISCONNECTED \|
	SS_ISDISCONNECTING \| SS_ISDISCONNECTED)) == 0 &&
	(so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	error = ENOTCONN;
	goto release;
	}
	if (uio->uio_resid == 0) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto release;
	}
	if ((so->so_state & SS_NBIO) \|\|
	(flags & (MSG_DONTWAIT\|MSG_NBIO))) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	error = EWOULDBLOCK;
	goto release;
	}
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	error = sbwait(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_rcv);
	if (error)
	goto release;
	goto restart;
	}
	dontblock:
	/*
	* From this point onward, we maintain 'nextrecord' as a cache of the
	* pointer to the next record in the socket buffer. We must keep the
	* various socket buffer pointers and local stack versions of the
	* pointers in sync, pushing out modifications before dropping the
	* socket buffer mutex, and re-reading them when picking it up.
	*
	* Otherwise, we will race with the network stack appending new data
	* or records onto the socket buffer by using inconsistent/stale
	* versions of the field, possibly resulting in socket buffer
	* corruption.
	*
	* By holding the high-level sblock(), we prevent simultaneous
	* readers from pulling off the front of the socket buffer.
	*/
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (uio->uio_td)
	uio->uio_td->td_ru.ru_msgrcv++;
	KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb"));
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	nextrecord = m->m_nextpkt;
	if (pr->pr_flags & PR_ADDR) {
	KASSERT(m->m_type == MT_SONAME,
	("m->m_type == %d", m->m_type));
	orig_resid = 0;
	if (psa != NULL)
	psa = sodupsockaddr(mtod(m, struct sockaddr ),
	M_NOWAIT);
	if (flags & MSG_PEEK) {
	m = m->m_next;
	} else {
	sbfree(&so->so_rcv, m);
	so->so_rcv.sb_mb = m_free(m);
	m = so->so_rcv.sb_mb;
	sockbuf_pushsync(&so->so_rcv, nextrecord);
	}
	}

	/*
	* Process one or more MT_CONTROL mbufs present before any data mbufs
	* in the first mbuf chain on the socket buffer. If MSG_PEEK, we
	* just copy the data; if !MSG_PEEK, we call into the protocol to
	* perform externalization (or freeing if controlp == NULL).
	*/
	if (m != NULL && m->m_type == MT_CONTROL) {
	struct mbuf cm = NULL, cmn;
	struct mbuf **cme = &cm;

	do {
	if (flags & MSG_PEEK) {
	if (controlp != NULL) {
	*controlp = m_copym(m, 0, m->m_len,
	M_NOWAIT);
	controlp = &(*controlp)->m_next;
	}
	m = m->m_next;
	} else {
	sbfree(&so->so_rcv, m);
	so->so_rcv.sb_mb = m->m_next;
	m->m_next = NULL;
	*cme = m;
	cme = &(*cme)->m_next;
	m = so->so_rcv.sb_mb;
	}
	} while (m != NULL && m->m_type == MT_CONTROL);
	if ((flags & MSG_PEEK) == 0)
	sockbuf_pushsync(&so->so_rcv, nextrecord);
	while (cm != NULL) {
	cmn = cm->m_next;
	cm->m_next = NULL;
	if (pr->pr_domain->dom_externalize != NULL) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	VNET_SO_ASSERT(so);
	error = (*pr->pr_domain->dom_externalize)
	(cm, controlp, flags);
	SOCKBUF_LOCK(&so->so_rcv);
	} else if (controlp != NULL)
	*controlp = cm;
	else
	m_freem(cm);
	if (controlp != NULL) {
	orig_resid = 0;
	while (*controlp != NULL)
	controlp = &(*controlp)->m_next;
	}
	cm = cmn;
	}
	if (m != NULL)
	nextrecord = so->so_rcv.sb_mb->m_nextpkt;
	else
	nextrecord = so->so_rcv.sb_mb;
	orig_resid = 0;
	}
	if (m != NULL) {
	if ((flags & MSG_PEEK) == 0) {
	KASSERT(m->m_nextpkt == nextrecord,
	("soreceive: post-control, nextrecord !sync"));
	if (nextrecord == NULL) {
	KASSERT(so->so_rcv.sb_mb == m,
	("soreceive: post-control, sb_mb!=m"));
	KASSERT(so->so_rcv.sb_lastrecord == m,
	("soreceive: post-control, lastrecord!=m"));
	}
	}
	type = m->m_type;
	if (type == MT_OOBDATA)
	flags \|= MSG_OOB;
	} else {
	if ((flags & MSG_PEEK) == 0) {
	KASSERT(so->so_rcv.sb_mb == nextrecord,
	("soreceive: sb_mb != nextrecord"));
	if (so->so_rcv.sb_mb == NULL) {
	KASSERT(so->so_rcv.sb_lastrecord == NULL,
	("soreceive: sb_lastercord != NULL"));
	}
	}
	}
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);

	/*
	* Now continue to read any data mbufs off of the head of the socket
	* buffer until the read request is satisfied. Note that 'type' is
	* used to store the type of any mbuf reads that have happened so far
	* such that soreceive() can stop reading if the type changes, which
	* causes soreceive() to return only one of regular data and inline
	* out-of-band data in a single socket receive operation.
	*/
	moff = 0;
	offset = 0;
	while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0
	&& error == 0) {
	/*
	* If the type of mbuf has changed since the last mbuf
	* examined ('type'), end the receive operation.
	*/
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (m->m_type == MT_OOBDATA \|\| m->m_type == MT_CONTROL) {
	if (type != m->m_type)
	break;
	} else if (type == MT_OOBDATA)
	break;
	else
	KASSERT(m->m_type == MT_DATA,
	("m->m_type == %d", m->m_type));
	so->so_rcv.sb_state &= ~SBS_RCVATMARK;
	len = uio->uio_resid;
	if (so->so_oobmark && len > so->so_oobmark - offset)
	len = so->so_oobmark - offset;
	if (len > m->m_len - moff)
	len = m->m_len - moff;
	/*
	* If mp is set, just pass back the mbufs. Otherwise copy
	* them out via the uio, then free. Sockbuf must be
	* consistent here (points to current mbuf, it points to next
	* record) when we drop priority; we must note any additions
	* to the sockbuf when we block interrupts again.
	*/
	if (mp == NULL) {
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_rcv);
	if ((m->m_flags & M_EXTPG) != 0)
	error = m_unmappedtouio(m, moff, uio, (int)len);
	else
	error = uiomove(mtod(m, char *) + moff,
	(int)len, uio);
	SOCKBUF_LOCK(&so->so_rcv);
	if (error) {
	/*
	* The MT_SONAME mbuf has already been removed
	* from the record, so it is necessary to
	* remove the data mbufs, if any, to preserve
	* the invariant in the case of PR_ADDR that
	* requires MT_SONAME mbufs at the head of
	* each record.
	*/
	if (pr->pr_flags & PR_ATOMIC &&
	((flags & MSG_PEEK) == 0))
	(void)sbdroprecord_locked(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto release;
	}
	} else
	uio->uio_resid -= len;
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (len == m->m_len - moff) {
	if (m->m_flags & M_EOR)
	flags \|= MSG_EOR;
	if (flags & MSG_PEEK) {
	m = m->m_next;
	moff = 0;
	} else {
	nextrecord = m->m_nextpkt;
	sbfree(&so->so_rcv, m);
	if (mp != NULL) {
	m->m_nextpkt = NULL;
	*mp = m;
	mp = &m->m_next;
	so->so_rcv.sb_mb = m = m->m_next;
	*mp = NULL;
	} else {
	so->so_rcv.sb_mb = m_free(m);
	m = so->so_rcv.sb_mb;
	}
	sockbuf_pushsync(&so->so_rcv, nextrecord);
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	}
	} else {
	if (flags & MSG_PEEK)
	moff += len;
	else {
	if (mp != NULL) {
	if (flags & MSG_DONTWAIT) {
	*mp = m_copym(m, 0, len,
	M_NOWAIT);
	if (*mp == NULL) {
	/*
	* m_copym() couldn't
	* allocate an mbuf.
	* Adjust uio_resid back
	* (it was adjusted
	* down by len bytes,
	* which we didn't end
	* up "copying" over).
	*/
	uio->uio_resid += len;
	break;
	}
	} else {
	SOCKBUF_UNLOCK(&so->so_rcv);
	*mp = m_copym(m, 0, len,
	M_WAITOK);
	SOCKBUF_LOCK(&so->so_rcv);
	}
	}
	sbcut_locked(&so->so_rcv, len);
	}
	}
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (so->so_oobmark) {
	if ((flags & MSG_PEEK) == 0) {
	so->so_oobmark -= len;
	if (so->so_oobmark == 0) {
	so->so_rcv.sb_state \|= SBS_RCVATMARK;
	break;
	}
	} else {
	offset += len;
	if (offset == so->so_oobmark)
	break;
	}
	}
	if (flags & MSG_EOR)
	break;
	/*
	* If the MSG_WAITALL flag is set (for non-atomic socket), we
	* must not quit until "uio->uio_resid == 0" or an error
	* termination. If a signal/timeout occurs, return with a
	* short count but without error. Keep sockbuf locked
	* against other readers.
	*/
	while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
	!sosendallatonce(so) && nextrecord == NULL) {
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (so->so_error \|\|
	so->so_rcv.sb_state & SBS_CANTRCVMORE)
	break;
	/*
	* Notify the protocol that some data has been
	* drained before blocking.
	*/
	if (pr->pr_flags & PR_WANTRCVD) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	VNET_SO_ASSERT(so);
	(*pr->pr_usrreqs->pru_rcvd)(so, flags);
	SOCKBUF_LOCK(&so->so_rcv);
	}
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	/*
	* We could receive some data while was notifying
	* the protocol. Skip blocking in this case.
	*/
	if (so->so_rcv.sb_mb == NULL) {
	error = sbwait(&so->so_rcv);
	if (error) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto release;
	}
	}
	m = so->so_rcv.sb_mb;
	if (m != NULL)
	nextrecord = m->m_nextpkt;
	}
	}

	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (m != NULL && pr->pr_flags & PR_ATOMIC) {
	flags \|= MSG_TRUNC;
	if ((flags & MSG_PEEK) == 0)
	(void) sbdroprecord_locked(&so->so_rcv);
	}
	if ((flags & MSG_PEEK) == 0) {
	if (m == NULL) {
	/*
	* First part is an inline SB_EMPTY_FIXUP(). Second
	* part makes sure sb_lastrecord is up-to-date if
	* there is still data in the socket buffer.
	*/
	so->so_rcv.sb_mb = nextrecord;
	if (so->so_rcv.sb_mb == NULL) {
	so->so_rcv.sb_mbtail = NULL;
	so->so_rcv.sb_lastrecord = NULL;
	} else if (nextrecord->m_nextpkt == NULL)
	so->so_rcv.sb_lastrecord = nextrecord;
	}
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	/*
	* If soreceive() is being done from the socket callback,
	* then don't need to generate ACK to peer to update window,
	* since ACK will be generated on return to TCP.
	*/
	if (!(flags & MSG_SOCALLBCK) &&
	(pr->pr_flags & PR_WANTRCVD)) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	VNET_SO_ASSERT(so);
	(*pr->pr_usrreqs->pru_rcvd)(so, flags);
	SOCKBUF_LOCK(&so->so_rcv);
	}
	}
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	if (orig_resid == uio->uio_resid && orig_resid &&
	(flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto restart;
	}
	SOCKBUF_UNLOCK(&so->so_rcv);

	if (flagsp != NULL)
	*flagsp \|= flags;
	release:
	sbunlock(&so->so_rcv);
	return (error);
	}

	/*
	* Optimized version of soreceive() for stream (TCP) sockets.
	*/
	int
	soreceive_stream(struct socket so, struct sockaddr psa, struct uio uio,
	struct mbuf mp0, struct mbuf controlp, int *flagsp)
	{
	int len = 0, error = 0, flags, oresid;
	struct sockbuf *sb;
	struct mbuf m, n = NULL;

	/* We only do stream sockets. */
	if (so->so_type != SOCK_STREAM)
	return (EINVAL);
	if (psa != NULL)
	*psa = NULL;
	if (flagsp != NULL)
	flags = *flagsp &~ MSG_EOR;
	else
	flags = 0;
	if (controlp != NULL)
	*controlp = NULL;
	if (flags & MSG_OOB)
	return (soreceive_rcvoob(so, uio, flags));
	if (mp0 != NULL)
	*mp0 = NULL;

	sb = &so->so_rcv;

	#ifdef KERN_TLS
	/*
	* KTLS store TLS records as records with a control message to
	* describe the framing.
	*
	* We check once here before acquiring locks to optimize the
	* common case.
	*/
	if (sb->sb_tls_info != NULL)
	return (soreceive_generic(so, psa, uio, mp0, controlp,
	flagsp));
	#endif

	/* Prevent other readers from entering the socket. */
	error = sblock(sb, SBLOCKWAIT(flags));
	if (error)
	return (error);
	SOCKBUF_LOCK(sb);

	#ifdef KERN_TLS
	if (sb->sb_tls_info != NULL) {
	SOCKBUF_UNLOCK(sb);
	sbunlock(sb);
	return (soreceive_generic(so, psa, uio, mp0, controlp,
	flagsp));
	}
	#endif

	/* Easy one, no space to copyout anything. */
	if (uio->uio_resid == 0) {
	error = EINVAL;
	goto out;
	}
	oresid = uio->uio_resid;

	/* We will never ever get anything unless we are or were connected. */
	if (!(so->so_state & (SS_ISCONNECTED\|SS_ISDISCONNECTED))) {
	error = ENOTCONN;
	goto out;
	}

	restart:
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);

	/* Abort if socket has reported problems. */
	if (so->so_error) {
	if (sbavail(sb) > 0)
	goto deliver;
	if (oresid > uio->uio_resid)
	goto out;
	error = so->so_error;
	if (!(flags & MSG_PEEK))
	so->so_error = 0;
	goto out;
	}

	/* Door is closed. Deliver what is left, if any. */
	if (sb->sb_state & SBS_CANTRCVMORE) {
	if (sbavail(sb) > 0)
	goto deliver;
	else
	goto out;
	}

	/* Socket buffer is empty and we shall not block. */
	if (sbavail(sb) == 0 &&
	((so->so_state & SS_NBIO) \|\| (flags & (MSG_DONTWAIT\|MSG_NBIO)))) {
	error = EAGAIN;
	goto out;
	}

	/* Socket buffer got some data that we shall deliver now. */
	if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) &&
	((so->so_state & SS_NBIO) \|\|
	(flags & (MSG_DONTWAIT\|MSG_NBIO)) \|\|
	sbavail(sb) >= sb->sb_lowat \|\|
	sbavail(sb) >= uio->uio_resid \|\|
	sbavail(sb) >= sb->sb_hiwat) ) {
	goto deliver;
	}

	/* On MSG_WAITALL we must wait until all data or error arrives. */
	if ((flags & MSG_WAITALL) &&
	(sbavail(sb) >= uio->uio_resid \|\| sbavail(sb) >= sb->sb_hiwat))
	goto deliver;

	/*
	* Wait and block until (more) data comes in.
	* NB: Drops the sockbuf lock during wait.
	*/
	error = sbwait(sb);
	if (error)
	goto out;
	goto restart;

	deliver:
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__));
	KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__));

	/* Statistics. */
	if (uio->uio_td)
	uio->uio_td->td_ru.ru_msgrcv++;

	/* Fill uio until full or current end of socket buffer is reached. */
	len = min(uio->uio_resid, sbavail(sb));
	if (mp0 != NULL) {
	/* Dequeue as many mbufs as possible. */
	if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) {
	if (*mp0 == NULL)
	*mp0 = sb->sb_mb;
	else
	m_cat(*mp0, sb->sb_mb);
	for (m = sb->sb_mb;
	m != NULL && m->m_len <= len;
	m = m->m_next) {
	KASSERT(!(m->m_flags & M_NOTAVAIL),
	("%s: m %p not available", __func__, m));
	len -= m->m_len;
	uio->uio_resid -= m->m_len;
	sbfree(sb, m);
	n = m;
	}
	n->m_next = NULL;
	sb->sb_mb = m;
	sb->sb_lastrecord = sb->sb_mb;
	if (sb->sb_mb == NULL)
	SB_EMPTY_FIXUP(sb);
	}
	/* Copy the remainder. */
	if (len > 0) {
	KASSERT(sb->sb_mb != NULL,
	("%s: len > 0 && sb->sb_mb empty", __func__));

	m = m_copym(sb->sb_mb, 0, len, M_NOWAIT);
	if (m == NULL)
	len = 0; /* Don't flush data from sockbuf. */
	else
	uio->uio_resid -= len;
	if (*mp0 != NULL)
	m_cat(*mp0, m);
	else
	*mp0 = m;
	if (*mp0 == NULL) {
	error = ENOBUFS;
	goto out;
	}
	}
	} else {
	/* NB: Must unlock socket buffer as uiomove may sleep. */
	SOCKBUF_UNLOCK(sb);
	error = m_mbuftouio(uio, sb->sb_mb, len);
	SOCKBUF_LOCK(sb);
	if (error)
	goto out;
	}
	SBLASTRECORDCHK(sb);
	SBLASTMBUFCHK(sb);

	/*
	* Remove the delivered data from the socket buffer unless we
	* were only peeking.
	*/
	if (!(flags & MSG_PEEK)) {
	if (len > 0)
	sbdrop_locked(sb, len);

	/* Notify protocol that we drained some data. */
	if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
	(((flags & MSG_WAITALL) && uio->uio_resid > 0) \|\|
	!(flags & MSG_SOCALLBCK))) {
	SOCKBUF_UNLOCK(sb);
	VNET_SO_ASSERT(so);
	(*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags);
	SOCKBUF_LOCK(sb);
	}
	}

	/*
	* For MSG_WAITALL we may have to loop again and wait for
	* more data to come in.
	*/
	if ((flags & MSG_WAITALL) && uio->uio_resid > 0)
	goto restart;
	out:
	SOCKBUF_LOCK_ASSERT(sb);
	SBLASTRECORDCHK(sb);
	SBLASTMBUFCHK(sb);
	SOCKBUF_UNLOCK(sb);
	sbunlock(sb);
	return (error);
	}

	/*
	* Optimized version of soreceive() for simple datagram cases from userspace.
	* Unlike in the stream case, we're able to drop a datagram if copyout()
	* fails, and because we handle datagrams atomically, we don't need to use a
	* sleep lock to prevent I/O interlacing.
	*/
	int
	soreceive_dgram(struct socket so, struct sockaddr psa, struct uio uio,
	struct mbuf mp0, struct mbuf controlp, int *flagsp)
	{
	struct mbuf m, m2;
	int flags, error;
	ssize_t len;
	struct protosw *pr = so->so_proto;
	struct mbuf *nextrecord;

	if (psa != NULL)
	*psa = NULL;
	if (controlp != NULL)
	*controlp = NULL;
	if (flagsp != NULL)
	flags = *flagsp &~ MSG_EOR;
	else
	flags = 0;

	/*
	* For any complicated cases, fall back to the full
	* soreceive_generic().
	*/
	if (mp0 != NULL \|\| (flags & MSG_PEEK) \|\| (flags & MSG_OOB))
	return (soreceive_generic(so, psa, uio, mp0, controlp,
	flagsp));

	/*
	* Enforce restrictions on use.
	*/
	KASSERT((pr->pr_flags & PR_WANTRCVD) == 0,
	("soreceive_dgram: wantrcvd"));
	KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic"));
	KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0,
	("soreceive_dgram: SBS_RCVATMARK"));
	KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0,
	("soreceive_dgram: P_CONNREQUIRED"));

	/*
	* Loop blocking while waiting for a datagram.
	*/
	SOCKBUF_LOCK(&so->so_rcv);
	while ((m = so->so_rcv.sb_mb) == NULL) {
	KASSERT(sbavail(&so->so_rcv) == 0,
	("soreceive_dgram: sb_mb NULL but sbavail %u",
	sbavail(&so->so_rcv)));
	if (so->so_error) {
	error = so->so_error;
	so->so_error = 0;
	SOCKBUF_UNLOCK(&so->so_rcv);
	return (error);
	}
	if (so->so_rcv.sb_state & SBS_CANTRCVMORE \|\|
	uio->uio_resid == 0) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	return (0);
	}
	if ((so->so_state & SS_NBIO) \|\|
	(flags & (MSG_DONTWAIT\|MSG_NBIO))) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	return (EWOULDBLOCK);
	}
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	error = sbwait(&so->so_rcv);
	if (error) {
	SOCKBUF_UNLOCK(&so->so_rcv);
	return (error);
	}
	}
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);

	if (uio->uio_td)
	uio->uio_td->td_ru.ru_msgrcv++;
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	nextrecord = m->m_nextpkt;
	if (nextrecord == NULL) {
	KASSERT(so->so_rcv.sb_lastrecord == m,
	("soreceive_dgram: lastrecord != m"));
	}

	KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord,
	("soreceive_dgram: m_nextpkt != nextrecord"));

	/*
	* Pull 'm' and its chain off the front of the packet queue.
	*/
	so->so_rcv.sb_mb = NULL;
	sockbuf_pushsync(&so->so_rcv, nextrecord);

	/*
	* Walk 'm's chain and free that many bytes from the socket buffer.
	*/
	for (m2 = m; m2 != NULL; m2 = m2->m_next)
	sbfree(&so->so_rcv, m2);

	/*
	* Do a few last checks before we let go of the lock.
	*/
	SBLASTRECORDCHK(&so->so_rcv);
	SBLASTMBUFCHK(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_rcv);

	if (pr->pr_flags & PR_ADDR) {
	KASSERT(m->m_type == MT_SONAME,
	("m->m_type == %d", m->m_type));
	if (psa != NULL)
	psa = sodupsockaddr(mtod(m, struct sockaddr ),
	M_NOWAIT);
	m = m_free(m);
	}
	if (m == NULL) {
	/* XXXRW: Can this happen? */
	return (0);
	}

	/*
	* Packet to copyout() is now in 'm' and it is disconnected from the
	* queue.
	*
	* Process one or more MT_CONTROL mbufs present before any data mbufs
	* in the first mbuf chain on the socket buffer. We call into the
	* protocol to perform externalization (or freeing if controlp ==
	* NULL). In some cases there can be only MT_CONTROL mbufs without
	* MT_DATA mbufs.
	*/
	if (m->m_type == MT_CONTROL) {
	struct mbuf cm = NULL, cmn;
	struct mbuf **cme = &cm;

	do {
	m2 = m->m_next;
	m->m_next = NULL;
	*cme = m;
	cme = &(*cme)->m_next;
	m = m2;
	} while (m != NULL && m->m_type == MT_CONTROL);
	while (cm != NULL) {
	cmn = cm->m_next;
	cm->m_next = NULL;
	if (pr->pr_domain->dom_externalize != NULL) {
	error = (*pr->pr_domain->dom_externalize)
	(cm, controlp, flags);
	} else if (controlp != NULL)
	*controlp = cm;
	else
	m_freem(cm);
	if (controlp != NULL) {
	while (*controlp != NULL)
	controlp = &(*controlp)->m_next;
	}
	cm = cmn;
	}
	}
	KASSERT(m == NULL \|\| m->m_type == MT_DATA,
	("soreceive_dgram: !data"));
	while (m != NULL && uio->uio_resid > 0) {
	len = uio->uio_resid;
	if (len > m->m_len)
	len = m->m_len;
	error = uiomove(mtod(m, char *), (int)len, uio);
	if (error) {
	m_freem(m);
	return (error);
	}
	if (len == m->m_len)
	m = m_free(m);
	else {
	m->m_data += len;
	m->m_len -= len;
	}
	}
	if (m != NULL) {
	flags \|= MSG_TRUNC;
	m_freem(m);
	}
	if (flagsp != NULL)
	*flagsp \|= flags;
	return (0);
	}

	int
	soreceive(struct socket so, struct sockaddr psa, struct uio uio,
	struct mbuf mp0, struct mbuf controlp, int *flagsp)
	{
	int error;

	CURVNET_SET(so->so_vnet);
	if (!SOLISTENING(so))
	error = (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio,
	mp0, controlp, flagsp));
	else
	error = ENOTCONN;
	CURVNET_RESTORE();
	return (error);
	}

	int
	soshutdown(struct socket *so, int how)
	{
	struct protosw *pr = so->so_proto;
	int error, soerror_enotconn;

	if (!(how == SHUT_RD \|\| how == SHUT_WR \|\| how == SHUT_RDWR))
	return (EINVAL);

	soerror_enotconn = 0;
	if ((so->so_state &
	(SS_ISCONNECTED \| SS_ISCONNECTING \| SS_ISDISCONNECTING)) == 0) {
	/*
	* POSIX mandates us to return ENOTCONN when shutdown(2) is
	* invoked on a datagram sockets, however historically we would
	* actually tear socket down. This is known to be leveraged by
	* some applications to unblock process waiting in recvXXX(2)
	* by other process that it shares that socket with. Try to meet
	* both backward-compatibility and POSIX requirements by forcing
	* ENOTCONN but still asking protocol to perform pru_shutdown().
	*/
	if (so->so_type != SOCK_DGRAM && !SOLISTENING(so))
	return (ENOTCONN);
	soerror_enotconn = 1;
	}

	if (SOLISTENING(so)) {
	if (how != SHUT_WR) {
	SOLISTEN_LOCK(so);
	so->so_error = ECONNABORTED;
	solisten_wakeup(so); /* unlocks so */
	}
	goto done;
	}

	CURVNET_SET(so->so_vnet);
	if (pr->pr_usrreqs->pru_flush != NULL)
	(*pr->pr_usrreqs->pru_flush)(so, how);
	if (how != SHUT_WR)
	sorflush(so);
	if (how != SHUT_RD) {
	error = (*pr->pr_usrreqs->pru_shutdown)(so);
	wakeup(&so->so_timeo);
	CURVNET_RESTORE();
	return ((error == 0 && soerror_enotconn) ? ENOTCONN : error);
	}
	wakeup(&so->so_timeo);
	CURVNET_RESTORE();

	done:
	return (soerror_enotconn ? ENOTCONN : 0);
	}

	void
	sorflush(struct socket *so)
	{
	struct sockbuf *sb = &so->so_rcv;
	struct protosw *pr = so->so_proto;
	struct socket aso;

	VNET_SO_ASSERT(so);

	/*
	* In order to avoid calling dom_dispose with the socket buffer mutex
	* held, and in order to generally avoid holding the lock for a long
	* time, we make a copy of the socket buffer and clear the original
	* (except locks, state). The new socket buffer copy won't have
	* initialized locks so we can only call routines that won't use or
	* assert those locks.
	*
	* Dislodge threads currently blocked in receive and wait to acquire
	* a lock against other simultaneous readers before clearing the
	* socket buffer. Don't let our acquire be interrupted by a signal
	* despite any existing socket disposition on interruptable waiting.
	*/
	socantrcvmore(so);
	(void) sblock(sb, SBL_WAIT \| SBL_NOINTR);

	/*
	* Invalidate/clear most of the sockbuf structure, but leave selinfo
	* and mutex data unchanged.
	*/
	SOCKBUF_LOCK(sb);
	bzero(&aso, sizeof(aso));
	aso.so_pcb = so->so_pcb;
	bcopy(&sb->sb_startzero, &aso.so_rcv.sb_startzero,
	sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
	bzero(&sb->sb_startzero,
	sizeof(*sb) - offsetof(struct sockbuf, sb_startzero));
	SOCKBUF_UNLOCK(sb);
	sbunlock(sb);

	/*
	* Dispose of special rights and flush the copied socket. Don't call
	* any unsafe routines (that rely on locks being initialized) on aso.
	*/
	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
	(*pr->pr_domain->dom_dispose)(&aso);
	sbrelease_internal(&aso.so_rcv, so);
	}

	/*
	* Wrapper for Socket established helper hook.
	* Parameters: socket, context of the hook point, hook id.
	*/
	static int inline
	hhook_run_socket(struct socket so, void hctx, int32_t h_id)
	{
	struct socket_hhook_data hhook_data = {
	.so = so,
	.hctx = hctx,
	.m = NULL,
	.status = 0
	};

	CURVNET_SET(so->so_vnet);
	HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd);
	CURVNET_RESTORE();

	/* Ugly but needed, since hhooks return void for now */
	return (hhook_data.status);
	}

	/*
	* Perhaps this routine, and sooptcopyout(), below, ought to come in an
	* additional variant to handle the case where the option value needs to be
	* some kind of integer, but not a specific size. In addition to their use
	* here, these functions are also called by the protocol-level pr_ctloutput()
	* routines.
	*/
	int
	sooptcopyin(struct sockopt sopt, void buf, size_t len, size_t minlen)
	{
	size_t valsize;

	/*
	* If the user gives us more than we wanted, we ignore it, but if we
	* don't get the minimum length the caller wants, we return EINVAL.
	* On success, sopt->sopt_valsize is set to however much we actually
	* retrieved.
	*/
	if ((valsize = sopt->sopt_valsize) < minlen)
	return EINVAL;
	if (valsize > len)
	sopt->sopt_valsize = valsize = len;

	if (sopt->sopt_td != NULL)
	return (copyin(sopt->sopt_val, buf, valsize));

	bcopy(sopt->sopt_val, buf, valsize);
	return (0);
	}

	/*
	* Kernel version of setsockopt(2).
	*
	* XXX: optlen is size_t, not socklen_t
	*/
	int
	so_setsockopt(struct socket so, int level, int optname, void optval,
	size_t optlen)
	{
	struct sockopt sopt;

	sopt.sopt_level = level;
	sopt.sopt_name = optname;
	sopt.sopt_dir = SOPT_SET;
	sopt.sopt_val = optval;
	sopt.sopt_valsize = optlen;
	sopt.sopt_td = NULL;
	return (sosetopt(so, &sopt));
	}

	int
	sosetopt(struct socket so, struct sockopt sopt)
	{
	int error, optval;
	struct linger l;
	struct timeval tv;
	sbintime_t val;
	uint32_t val32;
	#ifdef MAC
	struct mac extmac;
	#endif

	CURVNET_SET(so->so_vnet);
	error = 0;
	if (sopt->sopt_level != SOL_SOCKET) {
	if (so->so_proto->pr_ctloutput != NULL)
	error = (*so->so_proto->pr_ctloutput)(so, sopt);
	else
	error = ENOPROTOOPT;
	} else {
	switch (sopt->sopt_name) {
	case SO_ACCEPTFILTER:
	error = accept_filt_setopt(so, sopt);
	if (error)
	goto bad;
	break;

	case SO_LINGER:
	error = sooptcopyin(sopt, &l, sizeof l, sizeof l);
	if (error)
	goto bad;
	if (l.l_linger < 0 \|\|
	l.l_linger > USHRT_MAX \|\|
	l.l_linger > (INT_MAX / hz)) {
	error = EDOM;
	goto bad;
	}
	SOCK_LOCK(so);
	so->so_linger = l.l_linger;
	if (l.l_onoff)
	so->so_options \|= SO_LINGER;
	else
	so->so_options &= ~SO_LINGER;
	SOCK_UNLOCK(so);
	break;

	case SO_DEBUG:
	case SO_KEEPALIVE:
	case SO_DONTROUTE:
	case SO_USELOOPBACK:
	case SO_BROADCAST:
	case SO_REUSEADDR:
	case SO_REUSEPORT:
	case SO_REUSEPORT_LB:
	case SO_OOBINLINE:
	case SO_TIMESTAMP:
	case SO_BINTIME:
	case SO_NOSIGPIPE:
	case SO_NO_DDP:
	case SO_NO_OFFLOAD:
	error = sooptcopyin(sopt, &optval, sizeof optval,
	sizeof optval);
	if (error)
	goto bad;
	SOCK_LOCK(so);
	if (optval)
	so->so_options \|= sopt->sopt_name;
	else
	so->so_options &= ~sopt->sopt_name;
	SOCK_UNLOCK(so);
	break;

	case SO_SETFIB:
	error = sooptcopyin(sopt, &optval, sizeof optval,
	sizeof optval);
	if (error)
	goto bad;

	if (optval < 0 \|\| optval >= rt_numfibs) {
	error = EINVAL;
	goto bad;
	}
	if (((so->so_proto->pr_domain->dom_family == PF_INET) \|\|
	(so->so_proto->pr_domain->dom_family == PF_INET6) \|\|
	(so->so_proto->pr_domain->dom_family == PF_ROUTE)))
	so->so_fibnum = optval;
	else
	so->so_fibnum = 0;
	break;

	case SO_USER_COOKIE:
	error = sooptcopyin(sopt, &val32, sizeof val32,
	sizeof val32);
	if (error)
	goto bad;
	so->so_user_cookie = val32;
	break;

	case SO_SNDBUF:
	case SO_RCVBUF:
	case SO_SNDLOWAT:
	case SO_RCVLOWAT:
	error = sooptcopyin(sopt, &optval, sizeof optval,
	sizeof optval);
	if (error)
	goto bad;

	/*
	* Values < 1 make no sense for any of these options,
	* so disallow them.
	*/
	if (optval < 1) {
	error = EINVAL;
	goto bad;
	}

	error = sbsetopt(so, sopt->sopt_name, optval);
	break;

	case SO_SNDTIMEO:
	case SO_RCVTIMEO:
	#ifdef COMPAT_FREEBSD32
	if (SV_CURPROC_FLAG(SV_ILP32)) {
	struct timeval32 tv32;

	error = sooptcopyin(sopt, &tv32, sizeof tv32,
	sizeof tv32);
	CP(tv32, tv, tv_sec);
	CP(tv32, tv, tv_usec);
	} else
	#endif
	error = sooptcopyin(sopt, &tv, sizeof tv,
	sizeof tv);
	if (error)
	goto bad;
	if (tv.tv_sec < 0 \|\| tv.tv_usec < 0 \|\|
	tv.tv_usec >= 1000000) {
	error = EDOM;
	goto bad;
	}
	if (tv.tv_sec > INT32_MAX)
	val = SBT_MAX;
	else
	val = tvtosbt(tv);
	switch (sopt->sopt_name) {
	case SO_SNDTIMEO:
	so->so_snd.sb_timeo = val;
	break;
	case SO_RCVTIMEO:
	so->so_rcv.sb_timeo = val;
	break;
	}
	break;

	case SO_LABEL:
	#ifdef MAC
	error = sooptcopyin(sopt, &extmac, sizeof extmac,
	sizeof extmac);
	if (error)
	goto bad;
	error = mac_setsockopt_label(sopt->sopt_td->td_ucred,
	so, &extmac);
	#else
	error = EOPNOTSUPP;
	#endif
	break;

	case SO_TS_CLOCK:
	error = sooptcopyin(sopt, &optval, sizeof optval,
	sizeof optval);
	if (error)
	goto bad;
	if (optval < 0 \|\| optval > SO_TS_CLOCK_MAX) {
	error = EINVAL;
	goto bad;
	}
	so->so_ts_clock = optval;
	break;

	case SO_MAX_PACING_RATE:
	error = sooptcopyin(sopt, &val32, sizeof(val32),
	sizeof(val32));
	if (error)
	goto bad;
	so->so_max_pacing_rate = val32;
	break;

	default:
	if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0)
	error = hhook_run_socket(so, sopt,
	HHOOK_SOCKET_OPT);
	else
	error = ENOPROTOOPT;
	break;
	}
	if (error == 0 && so->so_proto->pr_ctloutput != NULL)
	(void)(*so->so_proto->pr_ctloutput)(so, sopt);
	}
	bad:
	CURVNET_RESTORE();
	return (error);
	}

	/*
	* Helper routine for getsockopt.
	*/
	int
	sooptcopyout(struct sockopt sopt, const void buf, size_t len)
	{
	int error;
	size_t valsize;

	error = 0;

	/*
	* Documented get behavior is that we always return a value, possibly
	* truncated to fit in the user's buffer. Traditional behavior is
	* that we always tell the user precisely how much we copied, rather
	* than something useful like the total amount we had available for
	* her. Note that this interface is not idempotent; the entire
	* answer must be generated ahead of time.
	*/
	valsize = min(len, sopt->sopt_valsize);
	sopt->sopt_valsize = valsize;
	if (sopt->sopt_val != NULL) {
	if (sopt->sopt_td != NULL)
	error = copyout(buf, sopt->sopt_val, valsize);
	else
	bcopy(buf, sopt->sopt_val, valsize);
	}
	return (error);
	}

	int
	sogetopt(struct socket so, struct sockopt sopt)
	{
	int error, optval;
	struct linger l;
	struct timeval tv;
	#ifdef MAC
	struct mac extmac;
	#endif

	CURVNET_SET(so->so_vnet);
	error = 0;
	if (sopt->sopt_level != SOL_SOCKET) {
	if (so->so_proto->pr_ctloutput != NULL)
	error = (*so->so_proto->pr_ctloutput)(so, sopt);
	else
	error = ENOPROTOOPT;
	CURVNET_RESTORE();
	return (error);
	} else {
	switch (sopt->sopt_name) {
	case SO_ACCEPTFILTER:
	error = accept_filt_getopt(so, sopt);
	break;

	case SO_LINGER:
	SOCK_LOCK(so);
	l.l_onoff = so->so_options & SO_LINGER;
	l.l_linger = so->so_linger;
	SOCK_UNLOCK(so);
	error = sooptcopyout(sopt, &l, sizeof l);
	break;

	case SO_USELOOPBACK:
	case SO_DONTROUTE:
	case SO_DEBUG:
	case SO_KEEPALIVE:
	case SO_REUSEADDR:
	case SO_REUSEPORT:
	case SO_REUSEPORT_LB:
	case SO_BROADCAST:
	case SO_OOBINLINE:
	case SO_ACCEPTCONN:
	case SO_TIMESTAMP:
	case SO_BINTIME:
	case SO_NOSIGPIPE:
	case SO_NO_DDP:
	case SO_NO_OFFLOAD:
	optval = so->so_options & sopt->sopt_name;
	integer:
	error = sooptcopyout(sopt, &optval, sizeof optval);
	break;

	case SO_DOMAIN:
	optval = so->so_proto->pr_domain->dom_family;
	goto integer;

	case SO_TYPE:
	optval = so->so_type;
	goto integer;

	case SO_PROTOCOL:
	optval = so->so_proto->pr_protocol;
	goto integer;

	case SO_ERROR:
	SOCK_LOCK(so);
	optval = so->so_error;
	so->so_error = 0;
	SOCK_UNLOCK(so);
	goto integer;

	case SO_SNDBUF:
	optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat :
	so->so_snd.sb_hiwat;
	goto integer;

	case SO_RCVBUF:
	optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat :
	so->so_rcv.sb_hiwat;
	goto integer;

	case SO_SNDLOWAT:
	optval = SOLISTENING(so) ? so->sol_sbsnd_lowat :
	so->so_snd.sb_lowat;
	goto integer;

	case SO_RCVLOWAT:
	optval = SOLISTENING(so) ? so->sol_sbrcv_lowat :
	so->so_rcv.sb_lowat;
	goto integer;

	case SO_SNDTIMEO:
	case SO_RCVTIMEO:
	tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ?
	so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
	#ifdef COMPAT_FREEBSD32
	if (SV_CURPROC_FLAG(SV_ILP32)) {
	struct timeval32 tv32;

	CP(tv, tv32, tv_sec);
	CP(tv, tv32, tv_usec);
	error = sooptcopyout(sopt, &tv32, sizeof tv32);
	} else
	#endif
	error = sooptcopyout(sopt, &tv, sizeof tv);
	break;

	case SO_LABEL:
	#ifdef MAC
	error = sooptcopyin(sopt, &extmac, sizeof(extmac),
	sizeof(extmac));
	if (error)
	goto bad;
	error = mac_getsockopt_label(sopt->sopt_td->td_ucred,
	so, &extmac);
	if (error)
	goto bad;
	error = sooptcopyout(sopt, &extmac, sizeof extmac);
	#else
	error = EOPNOTSUPP;
	#endif
	break;

	case SO_PEERLABEL:
	#ifdef MAC
	error = sooptcopyin(sopt, &extmac, sizeof(extmac),
	sizeof(extmac));
	if (error)
	goto bad;
	error = mac_getsockopt_peerlabel(
	sopt->sopt_td->td_ucred, so, &extmac);
	if (error)
	goto bad;
	error = sooptcopyout(sopt, &extmac, sizeof extmac);
	#else
	error = EOPNOTSUPP;
	#endif
	break;

	case SO_LISTENQLIMIT:
	optval = SOLISTENING(so) ? so->sol_qlimit : 0;
	goto integer;

	case SO_LISTENQLEN:
	optval = SOLISTENING(so) ? so->sol_qlen : 0;
	goto integer;

	case SO_LISTENINCQLEN:
	optval = SOLISTENING(so) ? so->sol_incqlen : 0;
	goto integer;

	case SO_TS_CLOCK:
	optval = so->so_ts_clock;
	goto integer;

	case SO_MAX_PACING_RATE:
	optval = so->so_max_pacing_rate;
	goto integer;

	default:
	if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0)
	error = hhook_run_socket(so, sopt,
	HHOOK_SOCKET_OPT);
	else
	error = ENOPROTOOPT;
	break;
	}
	}
	#ifdef MAC
	bad:
	#endif
	CURVNET_RESTORE();
	return (error);
	}

	int
	soopt_getm(struct sockopt sopt, struct mbuf *mp)
	{
	struct mbuf m, m_prev;
	int sopt_size = sopt->sopt_valsize;

	MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
	if (m == NULL)
	return ENOBUFS;
	if (sopt_size > MLEN) {
	MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT);
	if ((m->m_flags & M_EXT) == 0) {
	m_free(m);
	return ENOBUFS;
	}
	m->m_len = min(MCLBYTES, sopt_size);
	} else {
	m->m_len = min(MLEN, sopt_size);
	}
	sopt_size -= m->m_len;
	*mp = m;
	m_prev = m;

	while (sopt_size) {
	MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
	if (m == NULL) {
	m_freem(*mp);
	return ENOBUFS;
	}
	if (sopt_size > MLEN) {
	MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK :
	M_NOWAIT);
	if ((m->m_flags & M_EXT) == 0) {
	m_freem(m);
	m_freem(*mp);
	return ENOBUFS;
	}
	m->m_len = min(MCLBYTES, sopt_size);
	} else {
	m->m_len = min(MLEN, sopt_size);
	}
	sopt_size -= m->m_len;
	m_prev->m_next = m;
	m_prev = m;
	}
	return (0);
	}

	int
	soopt_mcopyin(struct sockopt sopt, struct mbuf m)
	{
	struct mbuf *m0 = m;

	if (sopt->sopt_val == NULL)
	return (0);
	while (m != NULL && sopt->sopt_valsize >= m->m_len) {
	if (sopt->sopt_td != NULL) {
	int error;

	error = copyin(sopt->sopt_val, mtod(m, char *),
	m->m_len);
	if (error != 0) {
	m_freem(m0);
	return(error);
	}
	} else
	bcopy(sopt->sopt_val, mtod(m, char *), m->m_len);
	sopt->sopt_valsize -= m->m_len;
	sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
	m = m->m_next;
	}
	if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */
	panic("ip6_sooptmcopyin");
	return (0);
	}

	int
	soopt_mcopyout(struct sockopt sopt, struct mbuf m)
	{
	struct mbuf *m0 = m;
	size_t valsize = 0;

	if (sopt->sopt_val == NULL)
	return (0);
	while (m != NULL && sopt->sopt_valsize >= m->m_len) {
	if (sopt->sopt_td != NULL) {
	int error;

	error = copyout(mtod(m, char *), sopt->sopt_val,
	m->m_len);
	if (error != 0) {
	m_freem(m0);
	return(error);
	}
	} else
	bcopy(mtod(m, char *), sopt->sopt_val, m->m_len);
	sopt->sopt_valsize -= m->m_len;
	sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
	valsize += m->m_len;
	m = m->m_next;
	}
	if (m != NULL) {
	/* enough soopt buffer should be given from user-land */
	m_freem(m0);
	return(EINVAL);
	}
	sopt->sopt_valsize = valsize;
	return (0);
	}

	/*
	* sohasoutofband(): protocol notifies socket layer of the arrival of new
	* out-of-band data, which will then notify socket consumers.
	*/
	void
	sohasoutofband(struct socket *so)
	{

	if (so->so_sigio != NULL)
	pgsigio(&so->so_sigio, SIGURG, 0);
	selwakeuppri(&so->so_rdsel, PSOCK);
	}

	int
	sopoll(struct socket so, int events, struct ucred active_cred,
	struct thread *td)
	{

	/*
	* We do not need to set or assert curvnet as long as everyone uses
	* sopoll_generic().
	*/
	return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred,
	td));
	}

	int
	sopoll_generic(struct socket so, int events, struct ucred active_cred,
	struct thread *td)
	{
	int revents;

	SOCK_LOCK(so);
	if (SOLISTENING(so)) {
	if (!(events & (POLLIN \| POLLRDNORM)))
	revents = 0;
	else if (!TAILQ_EMPTY(&so->sol_comp))
	revents = events & (POLLIN \| POLLRDNORM);
	else if ((events & POLLINIGNEOF) == 0 && so->so_error)
	revents = (events & (POLLIN \| POLLRDNORM)) \| POLLHUP;
	else {
	selrecord(td, &so->so_rdsel);
	revents = 0;
	}
	} else {
	revents = 0;
	SOCKBUF_LOCK(&so->so_snd);
	SOCKBUF_LOCK(&so->so_rcv);
	if (events & (POLLIN \| POLLRDNORM))
	if (soreadabledata(so))
	revents \|= events & (POLLIN \| POLLRDNORM);
	if (events & (POLLOUT \| POLLWRNORM))
	if (sowriteable(so))
	revents \|= events & (POLLOUT \| POLLWRNORM);
	if (events & (POLLPRI \| POLLRDBAND))
	if (so->so_oobmark \|\|
	(so->so_rcv.sb_state & SBS_RCVATMARK))
	revents \|= events & (POLLPRI \| POLLRDBAND);
	if ((events & POLLINIGNEOF) == 0) {
	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
	revents \|= events & (POLLIN \| POLLRDNORM);
	if (so->so_snd.sb_state & SBS_CANTSENDMORE)
	revents \|= POLLHUP;
	}
	}
	if (revents == 0) {
	if (events &
	(POLLIN \| POLLPRI \| POLLRDNORM \| POLLRDBAND)) {
	selrecord(td, &so->so_rdsel);
	so->so_rcv.sb_flags \|= SB_SEL;
	}
	if (events & (POLLOUT \| POLLWRNORM)) {
	selrecord(td, &so->so_wrsel);
	so->so_snd.sb_flags \|= SB_SEL;
	}
	}
	SOCKBUF_UNLOCK(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_snd);
	}
	SOCK_UNLOCK(so);
	return (revents);
	}

	int
	soo_kqfilter(struct file fp, struct knote kn)
	{
	struct socket *so = kn->kn_fp->f_data;
	struct sockbuf *sb;
	struct knlist *knl;

	switch (kn->kn_filter) {
	case EVFILT_READ:
	kn->kn_fop = &soread_filtops;
	knl = &so->so_rdsel.si_note;
	sb = &so->so_rcv;
	break;
	case EVFILT_WRITE:
	kn->kn_fop = &sowrite_filtops;
	knl = &so->so_wrsel.si_note;
	sb = &so->so_snd;
	break;
	case EVFILT_EMPTY:
	kn->kn_fop = &soempty_filtops;
	knl = &so->so_wrsel.si_note;
	sb = &so->so_snd;
	break;
	default:
	return (EINVAL);
	}

	SOCK_LOCK(so);
	if (SOLISTENING(so)) {
	knlist_add(knl, kn, 1);
	} else {
	SOCKBUF_LOCK(sb);
	knlist_add(knl, kn, 1);
	sb->sb_flags \|= SB_KNOTE;
	SOCKBUF_UNLOCK(sb);
	}
	SOCK_UNLOCK(so);
	return (0);
	}

	/*
	* Some routines that return EOPNOTSUPP for entry points that are not
	* supported by a protocol. Fill in as needed.
	*/
	int
	pru_accept_notsupp(struct socket so, struct sockaddr *nam)
	{

	return EOPNOTSUPP;
	}

	int
	pru_aio_queue_notsupp(struct socket so, struct kaiocb job)
	{

	return EOPNOTSUPP;
	}

	int
	pru_attach_notsupp(struct socket so, int proto, struct thread td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_bind_notsupp(struct socket so, struct sockaddr nam, struct thread *td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_bindat_notsupp(int fd, struct socket so, struct sockaddr nam,
	struct thread *td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_connect_notsupp(struct socket so, struct sockaddr nam, struct thread *td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_connectat_notsupp(int fd, struct socket so, struct sockaddr nam,
	struct thread *td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_connect2_notsupp(struct socket so1, struct socket so2)
	{

	return EOPNOTSUPP;
	}

	int
	pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data,
	struct ifnet ifp, struct thread td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_disconnect_notsupp(struct socket *so)
	{

	return EOPNOTSUPP;
	}

	int
	pru_listen_notsupp(struct socket so, int backlog, struct thread td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_peeraddr_notsupp(struct socket so, struct sockaddr *nam)
	{

	return EOPNOTSUPP;
	}

	int
	pru_rcvd_notsupp(struct socket *so, int flags)
	{

	return EOPNOTSUPP;
	}

	int
	pru_rcvoob_notsupp(struct socket so, struct mbuf m, int flags)
	{

	return EOPNOTSUPP;
	}

	int
	pru_send_notsupp(struct socket so, int flags, struct mbuf m,
	struct sockaddr addr, struct mbuf control, struct thread *td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_ready_notsupp(struct socket so, struct mbuf m, int count)
	{

	return (EOPNOTSUPP);
	}

	/*
	* This isn't really a ``null'' operation, but it's the default one and
	* doesn't do anything destructive.
	*/
	int
	pru_sense_null(struct socket so, struct stat sb)
	{

	sb->st_blksize = so->so_snd.sb_hiwat;
	return 0;
	}

	int
	pru_shutdown_notsupp(struct socket *so)
	{

	return EOPNOTSUPP;
	}

	int
	pru_sockaddr_notsupp(struct socket so, struct sockaddr *nam)
	{

	return EOPNOTSUPP;
	}

	int
	pru_sosend_notsupp(struct socket so, struct sockaddr addr, struct uio *uio,
	struct mbuf top, struct mbuf control, int flags, struct thread *td)
	{

	return EOPNOTSUPP;
	}

	int
	pru_soreceive_notsupp(struct socket so, struct sockaddr *paddr,
	struct uio uio, struct mbuf mp0, struct mbuf controlp, int flagsp)
	{

	return EOPNOTSUPP;
	}

	int
	pru_sopoll_notsupp(struct socket so, int events, struct ucred cred,
	struct thread *td)
	{

	return EOPNOTSUPP;
	}

	static void
	filt_sordetach(struct knote *kn)
	{
	struct socket *so = kn->kn_fp->f_data;

	so_rdknl_lock(so);
	knlist_remove(&so->so_rdsel.si_note, kn, 1);
	if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note))
	so->so_rcv.sb_flags &= ~SB_KNOTE;
	so_rdknl_unlock(so);
	}

	/ARGSUSED/
	static int
	filt_soread(struct knote *kn, long hint)
	{
	struct socket *so;

	so = kn->kn_fp->f_data;

	if (SOLISTENING(so)) {
	SOCK_LOCK_ASSERT(so);
	kn->kn_data = so->sol_qlen;
	if (so->so_error) {
	kn->kn_flags \|= EV_EOF;
	kn->kn_fflags = so->so_error;
	return (1);
	}
	return (!TAILQ_EMPTY(&so->sol_comp));
	}

	SOCKBUF_LOCK_ASSERT(&so->so_rcv);

	kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl;
	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
	kn->kn_flags \|= EV_EOF;
	kn->kn_fflags = so->so_error;
	return (1);
	} else if (so->so_error) /* temporary udp error */
	return (1);

	if (kn->kn_sfflags & NOTE_LOWAT) {
	if (kn->kn_data >= kn->kn_sdata)
	return (1);
	} else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat)
	return (1);

	/* This hook returning non-zero indicates an event, not error */
	return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD));
	}

	static void
	filt_sowdetach(struct knote *kn)
	{
	struct socket *so = kn->kn_fp->f_data;

	so_wrknl_lock(so);
	knlist_remove(&so->so_wrsel.si_note, kn, 1);
	if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note))
	so->so_snd.sb_flags &= ~SB_KNOTE;
	so_wrknl_unlock(so);
	}

	/ARGSUSED/
	static int
	filt_sowrite(struct knote *kn, long hint)
	{
	struct socket *so;

	so = kn->kn_fp->f_data;

	if (SOLISTENING(so))
	return (0);

	SOCKBUF_LOCK_ASSERT(&so->so_snd);
	kn->kn_data = sbspace(&so->so_snd);

	hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE);

	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
	kn->kn_flags \|= EV_EOF;
	kn->kn_fflags = so->so_error;
	return (1);
	} else if (so->so_error) /* temporary udp error */
	return (1);
	else if (((so->so_state & SS_ISCONNECTED) == 0) &&
	(so->so_proto->pr_flags & PR_CONNREQUIRED))
	return (0);
	else if (kn->kn_sfflags & NOTE_LOWAT)
	return (kn->kn_data >= kn->kn_sdata);
	else
	return (kn->kn_data >= so->so_snd.sb_lowat);
	}

	static int
	filt_soempty(struct knote *kn, long hint)
	{
	struct socket *so;

	so = kn->kn_fp->f_data;

	if (SOLISTENING(so))
	return (1);

	SOCKBUF_LOCK_ASSERT(&so->so_snd);
	kn->kn_data = sbused(&so->so_snd);

	if (kn->kn_data == 0)
	return (1);
	else
	return (0);
	}

	int
	socheckuid(struct socket *so, uid_t uid)
	{

	if (so == NULL)
	return (EPERM);
	if (so->so_cred->cr_uid != uid)
	return (EPERM);
	return (0);
	}

	/*
	* These functions are used by protocols to notify the socket layer (and its
	* consumers) of state changes in the sockets driven by protocol-side events.
	*/

	/*
	* Procedures to manipulate state flags of socket and do appropriate wakeups.
	*
	* Normal sequence from the active (originating) side is that
	* soisconnecting() is called during processing of connect() call, resulting
	* in an eventual call to soisconnected() if/when the connection is
	* established. When the connection is torn down soisdisconnecting() is
	* called during processing of disconnect() call, and soisdisconnected() is
	* called when the connection to the peer is totally severed. The semantics
	* of these routines are such that connectionless protocols can call
	* soisconnected() and soisdisconnected() only, bypassing the in-progress
	* calls when setting up a ``connection'' takes no time.
	*
	* From the passive side, a socket is created with two queues of sockets:
	* so_incomp for connections in progress and so_comp for connections already
	* made and awaiting user acceptance. As a protocol is preparing incoming
	* connections, it creates a socket structure queued on so_incomp by calling
	* sonewconn(). When the connection is established, soisconnected() is
	* called, and transfers the socket structure to so_comp, making it available
	* to accept().
	*
	* If a socket is closed with sockets on either so_incomp or so_comp, these
	* sockets are dropped.
	*
	* If higher-level protocols are implemented in the kernel, the wakeups done
	* here will sometimes cause software-interrupt process scheduling.
	*/
	void
	soisconnecting(struct socket *so)
	{

	SOCK_LOCK(so);
	so->so_state &= ~(SS_ISCONNECTED\|SS_ISDISCONNECTING);
	so->so_state \|= SS_ISCONNECTING;
	SOCK_UNLOCK(so);
	}

	void
	soisconnected(struct socket *so)
	{

	SOCK_LOCK(so);
	so->so_state &= ~(SS_ISCONNECTING\|SS_ISDISCONNECTING\|SS_ISCONFIRMING);
	so->so_state \|= SS_ISCONNECTED;

	if (so->so_qstate == SQ_INCOMP) {
	struct socket *head = so->so_listen;
	int ret;

	KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so));
	/*
	* Promoting a socket from incomplete queue to complete, we
	* need to go through reverse order of locking. We first do
	* trylock, and if that doesn't succeed, we go the hard way
	* leaving a reference and rechecking consistency after proper
	* locking.
	*/
	if (__predict_false(SOLISTEN_TRYLOCK(head) == 0)) {
	soref(head);
	SOCK_UNLOCK(so);
	SOLISTEN_LOCK(head);
	SOCK_LOCK(so);
	if (__predict_false(head != so->so_listen)) {
	/*
	* The socket went off the listen queue,
	* should be lost race to close(2) of sol.
	* The socket is about to soabort().
	*/
	SOCK_UNLOCK(so);
	sorele(head);
	return;
	}
	/* Not the last one, as so holds a ref. */
	refcount_release(&head->so_count);
	}
	again:
	if ((so->so_options & SO_ACCEPTFILTER) == 0) {
	TAILQ_REMOVE(&head->sol_incomp, so, so_list);
	head->sol_incqlen--;
	TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
	head->sol_qlen++;
	so->so_qstate = SQ_COMP;
	SOCK_UNLOCK(so);
	solisten_wakeup(head); /* unlocks */
	} else {
	SOCKBUF_LOCK(&so->so_rcv);
	soupcall_set(so, SO_RCV,
	head->sol_accept_filter->accf_callback,
	head->sol_accept_filter_arg);
	so->so_options &= ~SO_ACCEPTFILTER;
	ret = head->sol_accept_filter->accf_callback(so,
	head->sol_accept_filter_arg, M_NOWAIT);
	if (ret == SU_ISCONNECTED) {
	soupcall_clear(so, SO_RCV);
	SOCKBUF_UNLOCK(&so->so_rcv);
	goto again;
	}
	SOCKBUF_UNLOCK(&so->so_rcv);
	SOCK_UNLOCK(so);
	SOLISTEN_UNLOCK(head);
	}
	return;
	}
	SOCK_UNLOCK(so);
	wakeup(&so->so_timeo);
	sorwakeup(so);
	sowwakeup(so);
	}

	void
	soisdisconnecting(struct socket *so)
	{

	SOCK_LOCK(so);
	so->so_state &= ~SS_ISCONNECTING;
	so->so_state \|= SS_ISDISCONNECTING;

	if (!SOLISTENING(so)) {
	SOCKBUF_LOCK(&so->so_rcv);
	socantrcvmore_locked(so);
	SOCKBUF_LOCK(&so->so_snd);
	socantsendmore_locked(so);
	}
	SOCK_UNLOCK(so);
	wakeup(&so->so_timeo);
	}

	void
	soisdisconnected(struct socket *so)
	{

	SOCK_LOCK(so);

	/*
	* There is at least one reader of so_state that does not
	* acquire socket lock, namely soreceive_generic(). Ensure
	* that it never sees all flags that track connection status
	* cleared, by ordering the update with a barrier semantic of
	* our release thread fence.
	*/
	so->so_state \|= SS_ISDISCONNECTED;
	atomic_thread_fence_rel();
	so->so_state &= ~(SS_ISCONNECTING\|SS_ISCONNECTED\|SS_ISDISCONNECTING);

	if (!SOLISTENING(so)) {
	SOCK_UNLOCK(so);
	SOCKBUF_LOCK(&so->so_rcv);
	socantrcvmore_locked(so);
	SOCKBUF_LOCK(&so->so_snd);
	sbdrop_locked(&so->so_snd, sbused(&so->so_snd));
	socantsendmore_locked(so);
	} else
	SOCK_UNLOCK(so);
	wakeup(&so->so_timeo);
	}

	/*
	* Make a copy of a sockaddr in a malloced buffer of type M_SONAME.
	*/
	struct sockaddr *
	sodupsockaddr(const struct sockaddr *sa, int mflags)
	{
	struct sockaddr *sa2;

	sa2 = malloc(sa->sa_len, M_SONAME, mflags);
	if (sa2)
	bcopy(sa, sa2, sa->sa_len);
	return sa2;
	}

	/*
	* Register per-socket destructor.
	*/
	void
	sodtor_set(struct socket so, so_dtor_t func)
	{

	SOCK_LOCK_ASSERT(so);
	so->so_dtor = func;
	}

	/*
	* Register per-socket buffer upcalls.
	*/
	void
	soupcall_set(struct socket so, int which, so_upcall_t func, void arg)
	{
	struct sockbuf *sb;

	KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));

	switch (which) {
	case SO_RCV:
	sb = &so->so_rcv;
	break;
	case SO_SND:
	sb = &so->so_snd;
	break;
	default:
	panic("soupcall_set: bad which");
	}
	SOCKBUF_LOCK_ASSERT(sb);
	sb->sb_upcall = func;
	sb->sb_upcallarg = arg;
	sb->sb_flags \|= SB_UPCALL;
	}

	void
	soupcall_clear(struct socket *so, int which)
	{
	struct sockbuf *sb;

	KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));

	switch (which) {
	case SO_RCV:
	sb = &so->so_rcv;
	break;
	case SO_SND:
	sb = &so->so_snd;
	break;
	default:
	panic("soupcall_clear: bad which");
	}
	SOCKBUF_LOCK_ASSERT(sb);
	KASSERT(sb->sb_upcall != NULL,
	("%s: so %p no upcall to clear", __func__, so));
	sb->sb_upcall = NULL;
	sb->sb_upcallarg = NULL;
	sb->sb_flags &= ~SB_UPCALL;
	}

	void
	solisten_upcall_set(struct socket so, so_upcall_t func, void arg)
	{

	SOLISTEN_LOCK_ASSERT(so);
	so->sol_upcall = func;
	so->sol_upcallarg = arg;
	}

	static void
	so_rdknl_lock(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_LOCK(so);
	else
	SOCKBUF_LOCK(&so->so_rcv);
	}

	static void
	so_rdknl_unlock(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_UNLOCK(so);
	else
	SOCKBUF_UNLOCK(&so->so_rcv);
	}

	static void
	so_rdknl_assert_locked(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_LOCK_ASSERT(so);
	else
	SOCKBUF_LOCK_ASSERT(&so->so_rcv);
	}

	static void
	so_rdknl_assert_unlocked(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_UNLOCK_ASSERT(so);
	else
	SOCKBUF_UNLOCK_ASSERT(&so->so_rcv);
	}

	static void
	so_wrknl_lock(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_LOCK(so);
	else
	SOCKBUF_LOCK(&so->so_snd);
	}

	static void
	so_wrknl_unlock(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_UNLOCK(so);
	else
	SOCKBUF_UNLOCK(&so->so_snd);
	}

	static void
	so_wrknl_assert_locked(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_LOCK_ASSERT(so);
	else
	SOCKBUF_LOCK_ASSERT(&so->so_snd);
	}

	static void
	so_wrknl_assert_unlocked(void *arg)
	{
	struct socket *so = arg;

	if (SOLISTENING(so))
	SOCK_UNLOCK_ASSERT(so);
	else
	SOCKBUF_UNLOCK_ASSERT(&so->so_snd);
	}

	/*
	* Create an external-format (``xsocket'') structure using the information in
	* the kernel-format socket structure pointed to by so. This is done to
	* reduce the spew of irrelevant information over this interface, to isolate
	* user code from changes in the kernel structure, and potentially to provide
	* information-hiding if we decide that some of this information should be
	* hidden from users.
	*/
	void
	sotoxsocket(struct socket so, struct xsocket xso)
	{

	bzero(xso, sizeof(*xso));
	xso->xso_len = sizeof *xso;
	xso->xso_so = (uintptr_t)so;
	xso->so_type = so->so_type;
	xso->so_options = so->so_options;
	xso->so_linger = so->so_linger;
	xso->so_state = so->so_state;
	xso->so_pcb = (uintptr_t)so->so_pcb;
	xso->xso_protocol = so->so_proto->pr_protocol;
	xso->xso_family = so->so_proto->pr_domain->dom_family;
	xso->so_timeo = so->so_timeo;
	xso->so_error = so->so_error;
	xso->so_uid = so->so_cred->cr_uid;
	xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0;
	if (SOLISTENING(so)) {
	xso->so_qlen = so->sol_qlen;
	xso->so_incqlen = so->sol_incqlen;
	xso->so_qlimit = so->sol_qlimit;
	xso->so_oobmark = 0;
	} else {
	xso->so_state \|= so->so_qstate;
	xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0;
	xso->so_oobmark = so->so_oobmark;
	sbtoxsockbuf(&so->so_snd, &xso->so_snd);
	sbtoxsockbuf(&so->so_rcv, &xso->so_rcv);
	}
	}

	struct sockbuf *
	so_sockbuf_rcv(struct socket *so)
	{

	return (&so->so_rcv);
	}

	struct sockbuf *
	so_sockbuf_snd(struct socket *so)
	{

	return (&so->so_snd);
	}

	int
	so_state_get(const struct socket *so)
	{

	return (so->so_state);
	}

	void
	so_state_set(struct socket *so, int val)
	{

	so->so_state = val;
	}

	int
	so_options_get(const struct socket *so)
	{

	return (so->so_options);
	}

	void
	so_options_set(struct socket *so, int val)
	{

	so->so_options = val;
	}

	int
	so_error_get(const struct socket *so)
	{

	return (so->so_error);
	}

	void
	so_error_set(struct socket *so, int val)
	{

	so->so_error = val;
	}

	int
	so_linger_get(const struct socket *so)
	{

	return (so->so_linger);
	}

	void
	so_linger_set(struct socket *so, int val)
	{

	KASSERT(val >= 0 && val <= USHRT_MAX && val <= (INT_MAX / hz),
	("%s: val %d out of range", __func__, val));

	so->so_linger = val;
	}

	struct protosw *
	so_protosw_get(const struct socket *so)
	{

	return (so->so_proto);
	}

	void
	so_protosw_set(struct socket so, struct protosw val)
	{

	so->so_proto = val;
	}

	void
	so_sorwakeup(struct socket *so)
	{

	sorwakeup(so);
	}

	void
	so_sowwakeup(struct socket *so)
	{

	sowwakeup(so);
	}

	void
	so_sorwakeup_locked(struct socket *so)
	{

	sorwakeup_locked(so);
	}

	void
	so_sowwakeup_locked(struct socket *so)
	{

	sowwakeup_locked(so);
	}

	void
	so_lock(struct socket *so)
	{

	SOCK_LOCK(so);
	}

	void
	so_unlock(struct socket *so)
	{

	SOCK_UNLOCK(so);
	}
	diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
	index bd9447d0fb55..efd592f82fe2 100644
	--- a/sys/kern/uipc_usrreq.c
	+++ b/sys/kern/uipc_usrreq.c
	@@ -1,2972 +1,2973 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1982, 1986, 1989, 1991, 1993
	* The Regents of the University of California. All Rights Reserved.
	* Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved.
	* Copyright (c) 2018 Matthew Macy
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* From: @(#)uipc_usrreq.c 8.3 (Berkeley) 1/4/94
	*/

	/*
	* UNIX Domain (Local) Sockets
	*
	* This is an implementation of UNIX (local) domain sockets. Each socket has
	* an associated struct unpcb (UNIX protocol control block). Stream sockets
	* may be connected to 0 or 1 other socket. Datagram sockets may be
	* connected to 0, 1, or many other sockets. Sockets may be created and
	* connected in pairs (socketpair(2)), or bound/connected to using the file
	* system name space. For most purposes, only the receive socket buffer is
	* used, as sending on one socket delivers directly to the receive socket
	* buffer of a second socket.
	*
	* The implementation is substantially complicated by the fact that
	* "ancillary data", such as file descriptors or credentials, may be passed
	* across UNIX domain sockets. The potential for passing UNIX domain sockets
	* over other UNIX domain sockets requires the implementation of a simple
	* garbage collector to find and tear down cycles of disconnected sockets.
	*
	* TODO:
	* RDM
	* rethink name space problems
	* need a proper out-of-band
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_ddb.h"

	#include <sys/param.h>
	#include <sys/capsicum.h>
	#include <sys/domain.h>
	#include <sys/fcntl.h>
	#include <sys/malloc.h> /* XXX must be before <sys/file.h> */
	#include <sys/eventhandler.h>
	#include <sys/file.h>
	#include <sys/filedesc.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/mbuf.h>
	#include <sys/mount.h>
	#include <sys/mutex.h>
	#include <sys/namei.h>
	#include <sys/proc.h>
	#include <sys/protosw.h>
	#include <sys/queue.h>
	#include <sys/resourcevar.h>
	#include <sys/rwlock.h>
	#include <sys/socket.h>
	#include <sys/socketvar.h>
	#include <sys/signalvar.h>
	#include <sys/stat.h>
	#include <sys/sx.h>
	#include <sys/sysctl.h>
	#include <sys/systm.h>
	#include <sys/taskqueue.h>
	#include <sys/un.h>
	#include <sys/unpcb.h>
	#include <sys/vnode.h>

	#include <net/vnet.h>

	#ifdef DDB
	#include <ddb/ddb.h>
	#endif

	#include <security/mac/mac_framework.h>

	#include <vm/uma.h>

	MALLOC_DECLARE(M_FILECAPS);

	/*
	* Locking key:
	* (l) Locked using list lock
	* (g) Locked using linkage lock
	*/

	static uma_zone_t unp_zone;
	static unp_gen_t unp_gencnt; /* (l) */
	static u_int unp_count; /* (l) Count of local sockets. */
	static ino_t unp_ino; /* Prototype for fake inode numbers. */
	static int unp_rights; /* (g) File descriptors in flight. */
	static struct unp_head unp_shead; /* (l) List of stream sockets. */
	static struct unp_head unp_dhead; /* (l) List of datagram sockets. */
	static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */

	struct unp_defer {
	SLIST_ENTRY(unp_defer) ud_link;
	struct file *ud_fp;
	};
	static SLIST_HEAD(, unp_defer) unp_defers;
	static int unp_defers_count;

	static const struct sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL };

	/*
	* Garbage collection of cyclic file descriptor/socket references occurs
	* asynchronously in a taskqueue context in order to avoid recursion and
	* reentrance in the UNIX domain socket, file descriptor, and socket layer
	* code. See unp_gc() for a full description.
	*/
	static struct timeout_task unp_gc_task;

	/*
	* The close of unix domain sockets attached as SCM_RIGHTS is
	* postponed to the taskqueue, to avoid arbitrary recursion depth.
	* The attached sockets might have another sockets attached.
	*/
	static struct task unp_defer_task;

	/*
	* Both send and receive buffers are allocated PIPSIZ bytes of buffering for
	* stream sockets, although the total for sender and receiver is actually
	* only PIPSIZ.
	*
	* Datagram sockets really use the sendspace as the maximum datagram size,
	* and don't really want to reserve the sendspace. Their recvspace should be
	* large enough for at least one max-size datagram plus address.
	*/
	#ifndef PIPSIZ
	#define PIPSIZ 8192
	#endif
	static u_long unpst_sendspace = PIPSIZ;
	static u_long unpst_recvspace = PIPSIZ;
	static u_long unpdg_sendspace = 21024; / really max datagram size */
	static u_long unpdg_recvspace = 4*1024;
	static u_long unpsp_sendspace = PIPSIZ; /* really max datagram size */
	static u_long unpsp_recvspace = PIPSIZ;

	static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Local domain");
	static SYSCTL_NODE(_net_local, SOCK_STREAM, stream,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"SOCK_STREAM");
	static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"SOCK_DGRAM");
	static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"SOCK_SEQPACKET");

	SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW,
	&unpst_sendspace, 0, "Default stream send space.");
	SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW,
	&unpst_recvspace, 0, "Default stream receive space.");
	SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW,
	&unpdg_sendspace, 0, "Default datagram send space.");
	SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW,
	&unpdg_recvspace, 0, "Default datagram receive space.");
	SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW,
	&unpsp_sendspace, 0, "Default seqpacket send space.");
	SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW,
	&unpsp_recvspace, 0, "Default seqpacket receive space.");
	SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0,
	"File descriptors in flight.");
	SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD,
	&unp_defers_count, 0,
	"File descriptors deferred to taskqueue for close.");

	/*
	* Locking and synchronization:
	*
	* Three types of locks exist in the local domain socket implementation: a
	* a global linkage rwlock, the mtxpool lock, and per-unpcb mutexes.
	* The linkage lock protects the socket count, global generation number,
	* and stream/datagram global lists.
	*
	* The mtxpool lock protects the vnode from being modified while referenced.
	* Lock ordering requires that it be acquired before any unpcb locks.
	*
	* The unpcb lock (unp_mtx) protects all fields in the unpcb. Of particular
	* note is that this includes the unp_conn field. So long as the unpcb lock
	* is held the reference to the unpcb pointed to by unp_conn is valid. If we
	* require that the unpcb pointed to by unp_conn remain live in cases where
	* we need to drop the unp_mtx as when we need to acquire the lock for a
	* second unpcb the caller must first acquire an additional reference on the
	* second unpcb and then revalidate any state (typically check that unp_conn
	* is non-NULL) upon requiring the initial unpcb lock. The lock ordering
	* between unpcbs is the conventional ascending address order. Two helper
	* routines exist for this:
	*
	* - unp_pcb_lock2(unp, unp2) - which just acquires the two locks in the
	* safe ordering.
	*
	* - unp_pcb_owned_lock2(unp, unp2, freed) - the lock for unp is held
	* when called. If unp is unlocked and unp2 is subsequently freed
	* freed will be set to 1.
	*
	* The helper routines for references are:
	*
	* - unp_pcb_hold(unp): Can be called any time we currently hold a valid
	* reference to unp.
	*
	* - unp_pcb_rele(unp): The caller must hold the unp lock. If we are
	* releasing the last reference, detach must have been called thus
	* unp->unp_socket be NULL.
	*
	* UNIX domain sockets each have an unpcb hung off of their so_pcb pointer,
	* allocated in pru_attach() and freed in pru_detach(). The validity of that
	* pointer is an invariant, so no lock is required to dereference the so_pcb
	* pointer if a valid socket reference is held by the caller. In practice,
	* this is always true during operations performed on a socket. Each unpcb
	* has a back-pointer to its socket, unp_socket, which will be stable under
	* the same circumstances.
	*
	* This pointer may only be safely dereferenced as long as a valid reference
	* to the unpcb is held. Typically, this reference will be from the socket,
	* or from another unpcb when the referring unpcb's lock is held (in order
	* that the reference not be invalidated during use). For example, to follow
	* unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee
	* that detach is not run clearing unp_socket.
	*
	* Blocking with UNIX domain sockets is a tricky issue: unlike most network
	* protocols, bind() is a non-atomic operation, and connect() requires
	* potential sleeping in the protocol, due to potentially waiting on local or
	* distributed file systems. We try to separate "lookup" operations, which
	* may sleep, and the IPC operations themselves, which typically can occur
	* with relative atomicity as locks can be held over the entire operation.
	*
	* Another tricky issue is simultaneous multi-threaded or multi-process
	* access to a single UNIX domain socket. These are handled by the flags
	* UNP_CONNECTING and UNP_BINDING, which prevent concurrent connecting or
	* binding, both of which involve dropping UNIX domain socket locks in order
	* to perform namei() and other file system operations.
	*/
	static struct rwlock unp_link_rwlock;
	static struct mtx unp_defers_lock;

	#define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \
	"unp_link_rwlock")

	#define UNP_LINK_LOCK_ASSERT() rw_assert(&unp_link_rwlock, \
	RA_LOCKED)
	#define UNP_LINK_UNLOCK_ASSERT() rw_assert(&unp_link_rwlock, \
	RA_UNLOCKED)

	#define UNP_LINK_RLOCK() rw_rlock(&unp_link_rwlock)
	#define UNP_LINK_RUNLOCK() rw_runlock(&unp_link_rwlock)
	#define UNP_LINK_WLOCK() rw_wlock(&unp_link_rwlock)
	#define UNP_LINK_WUNLOCK() rw_wunlock(&unp_link_rwlock)
	#define UNP_LINK_WLOCK_ASSERT() rw_assert(&unp_link_rwlock, \
	RA_WLOCKED)
	#define UNP_LINK_WOWNED() rw_wowned(&unp_link_rwlock)

	#define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \
	"unp_defer", NULL, MTX_DEF)
	#define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock)
	#define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock)

	#define UNP_REF_LIST_LOCK() UNP_DEFERRED_LOCK();
	#define UNP_REF_LIST_UNLOCK() UNP_DEFERRED_UNLOCK();

	#define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \
	"unp", "unp", \
	MTX_DUPOK\|MTX_DEF)
	#define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx)
	#define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx)
	#define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx)
	#define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx)
	#define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx)
	#define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED)
	#define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED)

	static int uipc_connect2(struct socket , struct socket );
	static int uipc_ctloutput(struct socket , struct sockopt );
	static int unp_connect(struct socket , struct sockaddr ,
	struct thread *);
	static int unp_connectat(int, struct socket , struct sockaddr ,
	struct thread *);
	static int unp_connect2(struct socket so, struct socket so2, int);
	static void unp_disconnect(struct unpcb unp, struct unpcb unp2);
	static void unp_dispose(struct socket *so);
	static void unp_dispose_mbuf(struct mbuf *);
	static void unp_shutdown(struct unpcb *);
	static void unp_drop(struct unpcb *);
	static void unp_gc(__unused void *, int);
	static void unp_scan(struct mbuf , void ()(struct filedescent **, int));
	static void unp_discard(struct file *);
	static void unp_freerights(struct filedescent **, int);
	static void unp_init(void);
	static int unp_internalize(struct mbuf *, struct thread );
	static void unp_internalize_fp(struct file *);
	static int unp_externalize(struct mbuf , struct mbuf *, int);
	static int unp_externalize_fp(struct file *);
	static struct mbuf unp_addsockcred(struct thread , struct mbuf *);
	static void unp_process_defers(void * __unused, int);

	static void
	unp_pcb_hold(struct unpcb *unp)
	{
	MPASS(unp->unp_refcount);
	refcount_acquire(&unp->unp_refcount);
	}

	static int
	unp_pcb_rele(struct unpcb *unp)
	{
	int freed;

	UNP_PCB_LOCK_ASSERT(unp);
	MPASS(unp->unp_refcount);
	if ((freed = refcount_release(&unp->unp_refcount))) {
	/* we got here with having detached? */
	MPASS(unp->unp_socket == NULL);
	UNP_PCB_UNLOCK(unp);
	UNP_PCB_LOCK_DESTROY(unp);
	uma_zfree(unp_zone, unp);
	}
	return (freed);
	}

	static void
	unp_pcb_lock2(struct unpcb unp, struct unpcb unp2)
	{
	MPASS(unp != unp2);
	UNP_PCB_UNLOCK_ASSERT(unp);
	UNP_PCB_UNLOCK_ASSERT(unp2);
	if ((uintptr_t)unp2 > (uintptr_t)unp) {
	UNP_PCB_LOCK(unp);
	UNP_PCB_LOCK(unp2);
	} else {
	UNP_PCB_LOCK(unp2);
	UNP_PCB_LOCK(unp);
	}
	}

	static __noinline void
	unp_pcb_owned_lock2_slowpath(struct unpcb unp, struct unpcb *unp2p,
	int *freed)
	{
	struct unpcb *unp2;

	unp2 = *unp2p;
	unp_pcb_hold(unp2);
	UNP_PCB_UNLOCK(unp);
	UNP_PCB_LOCK(unp2);
	UNP_PCB_LOCK(unp);
	*freed = unp_pcb_rele(unp2);
	if (*freed)
	*unp2p = NULL;
	}

	#define unp_pcb_owned_lock2(unp, unp2, freed) do { \
	freed = 0; \
	UNP_PCB_LOCK_ASSERT(unp); \
	UNP_PCB_UNLOCK_ASSERT(unp2); \
	MPASS((unp) != (unp2)); \
	if (__predict_true(UNP_PCB_TRYLOCK(unp2))) \
	break; \
	else if ((uintptr_t)(unp2) > (uintptr_t)(unp)) \
	UNP_PCB_LOCK(unp2); \
	else \
	unp_pcb_owned_lock2_slowpath((unp), &(unp2), &freed); \
	} while (0)

	/*
	* Definitions of protocols supported in the LOCAL domain.
	*/
	static struct domain localdomain;
	static struct pr_usrreqs uipc_usrreqs_dgram, uipc_usrreqs_stream;
	static struct pr_usrreqs uipc_usrreqs_seqpacket;
	static struct protosw localsw[] = {
	{
	.pr_type = SOCK_STREAM,
	.pr_domain = &localdomain,
	.pr_flags = PR_CONNREQUIRED\|PR_WANTRCVD\|PR_RIGHTS,
	.pr_ctloutput = &uipc_ctloutput,
	.pr_usrreqs = &uipc_usrreqs_stream
	},
	{
	.pr_type = SOCK_DGRAM,
	.pr_domain = &localdomain,
	.pr_flags = PR_ATOMIC\|PR_ADDR\|PR_RIGHTS,
	.pr_ctloutput = &uipc_ctloutput,
	.pr_usrreqs = &uipc_usrreqs_dgram
	},
	{
	.pr_type = SOCK_SEQPACKET,
	.pr_domain = &localdomain,

	/*
	* XXXRW: For now, PR_ADDR because soreceive will bump into them
	* due to our use of sbappendaddr. A new sbappend variants is needed
	* that supports both atomic record writes and control data.
	*/
	.pr_flags = PR_ADDR\|PR_ATOMIC\|PR_CONNREQUIRED\|PR_WANTRCVD\|
	PR_RIGHTS,
	.pr_ctloutput = &uipc_ctloutput,
	.pr_usrreqs = &uipc_usrreqs_seqpacket,
	},
	};

	static struct domain localdomain = {
	.dom_family = AF_LOCAL,
	.dom_name = "local",
	.dom_init = unp_init,
	.dom_externalize = unp_externalize,
	.dom_dispose = unp_dispose,
	.dom_protosw = localsw,
	.dom_protoswNPROTOSW = &localsw[nitems(localsw)]
	};
	DOMAIN_SET(local);

	static void
	uipc_abort(struct socket *so)
	{
	struct unpcb unp, unp2;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_abort: unp == NULL"));
	UNP_PCB_UNLOCK_ASSERT(unp);

	UNP_PCB_LOCK(unp);
	unp2 = unp->unp_conn;
	if (unp2 != NULL) {
	unp_pcb_hold(unp2);
	UNP_PCB_UNLOCK(unp);
	unp_drop(unp2);
	} else
	UNP_PCB_UNLOCK(unp);
	}

	static int
	uipc_accept(struct socket so, struct sockaddr *nam)
	{
	struct unpcb unp, unp2;
	const struct sockaddr *sa;

	/*
	* Pass back name of connected socket, if it was bound and we are
	* still connected (our peer may have closed already!).
	*/
	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_accept: unp == NULL"));

	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
	UNP_LINK_RLOCK();
	unp2 = unp->unp_conn;
	if (unp2 != NULL && unp2->unp_addr != NULL) {
	UNP_PCB_LOCK(unp2);
	sa = (struct sockaddr *) unp2->unp_addr;
	bcopy(sa, *nam, sa->sa_len);
	UNP_PCB_UNLOCK(unp2);
	} else {
	sa = &sun_noname;
	bcopy(sa, *nam, sa->sa_len);
	}
	UNP_LINK_RUNLOCK();
	return (0);
	}

	static int
	uipc_attach(struct socket so, int proto, struct thread td)
	{
	u_long sendspace, recvspace;
	struct unpcb *unp;
	int error;
	bool locked;

	KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL"));
	if (so->so_snd.sb_hiwat == 0 \|\| so->so_rcv.sb_hiwat == 0) {
	switch (so->so_type) {
	case SOCK_STREAM:
	sendspace = unpst_sendspace;
	recvspace = unpst_recvspace;
	break;

	case SOCK_DGRAM:
	sendspace = unpdg_sendspace;
	recvspace = unpdg_recvspace;
	break;

	case SOCK_SEQPACKET:
	sendspace = unpsp_sendspace;
	recvspace = unpsp_recvspace;
	break;

	default:
	panic("uipc_attach");
	}
	error = soreserve(so, sendspace, recvspace);
	if (error)
	return (error);
	}
	unp = uma_zalloc(unp_zone, M_NOWAIT \| M_ZERO);
	if (unp == NULL)
	return (ENOBUFS);
	LIST_INIT(&unp->unp_refs);
	UNP_PCB_LOCK_INIT(unp);
	unp->unp_socket = so;
	so->so_pcb = unp;
	unp->unp_refcount = 1;

	if ((locked = UNP_LINK_WOWNED()) == false)
	UNP_LINK_WLOCK();

	unp->unp_gencnt = ++unp_gencnt;
	unp->unp_ino = ++unp_ino;
	unp_count++;
	switch (so->so_type) {
	case SOCK_STREAM:
	LIST_INSERT_HEAD(&unp_shead, unp, unp_link);
	break;

	case SOCK_DGRAM:
	LIST_INSERT_HEAD(&unp_dhead, unp, unp_link);
	break;

	case SOCK_SEQPACKET:
	LIST_INSERT_HEAD(&unp_sphead, unp, unp_link);
	break;

	default:
	panic("uipc_attach");
	}

	if (locked == false)
	UNP_LINK_WUNLOCK();

	return (0);
	}

	static int
	uipc_bindat(int fd, struct socket so, struct sockaddr nam, struct thread *td)
	{
	struct sockaddr_un soun = (struct sockaddr_un )nam;
	struct vattr vattr;
	int error, namelen;
	struct nameidata nd;
	struct unpcb *unp;
	struct vnode *vp;
	struct mount *mp;
	cap_rights_t rights;
	char *buf;

	if (nam->sa_family != AF_UNIX)
	return (EAFNOSUPPORT);

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_bind: unp == NULL"));

	if (soun->sun_len > sizeof(struct sockaddr_un))
	return (EINVAL);
	namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path);
	if (namelen <= 0)
	return (EINVAL);

	/*
	* We don't allow simultaneous bind() calls on a single UNIX domain
	* socket, so flag in-progress operations, and return an error if an
	* operation is already in progress.
	*
	* Historically, we have not allowed a socket to be rebound, so this
	* also returns an error. Not allowing re-binding simplifies the
	* implementation and avoids a great many possible failure modes.
	*/
	UNP_PCB_LOCK(unp);
	if (unp->unp_vnode != NULL) {
	UNP_PCB_UNLOCK(unp);
	return (EINVAL);
	}
	if (unp->unp_flags & UNP_BINDING) {
	UNP_PCB_UNLOCK(unp);
	return (EALREADY);
	}
	unp->unp_flags \|= UNP_BINDING;
	UNP_PCB_UNLOCK(unp);

	buf = malloc(namelen + 1, M_TEMP, M_WAITOK);
	bcopy(soun->sun_path, buf, namelen);
	buf[namelen] = 0;

	restart:
	NDINIT_ATRIGHTS(&nd, CREATE, NOFOLLOW \| LOCKPARENT \| SAVENAME \| NOCACHE,
	UIO_SYSSPACE, buf, fd, cap_rights_init(&rights, CAP_BINDAT), td);
	/* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */
	error = namei(&nd);
	if (error)
	goto error;
	vp = nd.ni_vp;
	if (vp != NULL \|\| vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (nd.ni_dvp == vp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	if (vp != NULL) {
	vrele(vp);
	error = EADDRINUSE;
	goto error;
	}
	error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH);
	if (error)
	goto error;
	goto restart;
	}
	VATTR_NULL(&vattr);
	vattr.va_type = VSOCK;
	vattr.va_mode = (ACCESSPERMS & ~td->td_proc->p_fd->fd_cmask);
	#ifdef MAC
	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
	&vattr);
	#endif
	if (error == 0)
	error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if (error) {
	vn_finished_write(mp);
	goto error;
	}
	vp = nd.ni_vp;
	ASSERT_VOP_ELOCKED(vp, "uipc_bind");
	soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK);

	UNP_PCB_LOCK(unp);
	VOP_UNP_BIND(vp, unp);
	unp->unp_vnode = vp;
	unp->unp_addr = soun;
	unp->unp_flags &= ~UNP_BINDING;
	UNP_PCB_UNLOCK(unp);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	free(buf, M_TEMP);
	return (0);

	error:
	UNP_PCB_LOCK(unp);
	unp->unp_flags &= ~UNP_BINDING;
	UNP_PCB_UNLOCK(unp);
	free(buf, M_TEMP);
	return (error);
	}

	static int
	uipc_bind(struct socket so, struct sockaddr nam, struct thread *td)
	{

	return (uipc_bindat(AT_FDCWD, so, nam, td));
	}

	static int
	uipc_connect(struct socket so, struct sockaddr nam, struct thread *td)
	{
	int error;

	KASSERT(td == curthread, ("uipc_connect: td != curthread"));
	error = unp_connect(so, nam, td);
	return (error);
	}

	static int
	uipc_connectat(int fd, struct socket so, struct sockaddr nam,
	struct thread *td)
	{
	int error;

	KASSERT(td == curthread, ("uipc_connectat: td != curthread"));
	error = unp_connectat(fd, so, nam, td);
	return (error);
	}

	static void
	uipc_close(struct socket *so)
	{
	struct unpcb unp, unp2;
	struct vnode *vp = NULL;
	struct mtx *vplock;
	int freed;
	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_close: unp == NULL"));

	vplock = NULL;
	if ((vp = unp->unp_vnode) != NULL) {
	vplock = mtx_pool_find(mtxpool_sleep, vp);
	mtx_lock(vplock);
	}
	UNP_PCB_LOCK(unp);
	if (vp && unp->unp_vnode == NULL) {
	mtx_unlock(vplock);
	vp = NULL;
	}
	if (vp != NULL) {
	VOP_UNP_DETACH(vp);
	unp->unp_vnode = NULL;
	}
	unp2 = unp->unp_conn;
	unp_pcb_hold(unp);
	if (__predict_false(unp == unp2)) {
	unp_disconnect(unp, unp2);
	} else if (unp2 != NULL) {
	unp_pcb_hold(unp2);
	unp_pcb_owned_lock2(unp, unp2, freed);
	unp_disconnect(unp, unp2);
	if (unp_pcb_rele(unp2) == 0)
	UNP_PCB_UNLOCK(unp2);
	}
	if (unp_pcb_rele(unp) == 0)
	UNP_PCB_UNLOCK(unp);
	if (vp) {
	mtx_unlock(vplock);
	vrele(vp);
	}
	}

	static int
	uipc_connect2(struct socket so1, struct socket so2)
	{
	struct unpcb unp, unp2;
	int error;

	unp = so1->so_pcb;
	KASSERT(unp != NULL, ("uipc_connect2: unp == NULL"));
	unp2 = so2->so_pcb;
	KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL"));
	if (unp != unp2)
	unp_pcb_lock2(unp, unp2);
	else
	UNP_PCB_LOCK(unp);
	error = unp_connect2(so1, so2, PRU_CONNECT2);
	if (unp != unp2)
	UNP_PCB_UNLOCK(unp2);
	UNP_PCB_UNLOCK(unp);
	return (error);
	}

	static void
	uipc_detach(struct socket *so)
	{
	struct unpcb unp, unp2;
	struct mtx *vplock;
	struct vnode *vp;
	int freeunp, local_unp_rights;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_detach: unp == NULL"));

	vp = NULL;
	vplock = NULL;
	local_unp_rights = 0;

	SOCK_LOCK(so);
	if (!SOLISTENING(so)) {
	/*
	* Once the socket is removed from the global lists,
	* uipc_ready() will not be able to locate its socket buffer, so
	* clear the buffer now. At this point internalized rights have
	* already been disposed of.
	*/
	sbrelease(&so->so_rcv, so);
	}
	SOCK_UNLOCK(so);

	UNP_LINK_WLOCK();
	LIST_REMOVE(unp, unp_link);
	if (unp->unp_gcflag & UNPGC_DEAD)
	LIST_REMOVE(unp, unp_dead);
	unp->unp_gencnt = ++unp_gencnt;
	--unp_count;
	UNP_LINK_WUNLOCK();

	UNP_PCB_UNLOCK_ASSERT(unp);
	restart:
	if ((vp = unp->unp_vnode) != NULL) {
	vplock = mtx_pool_find(mtxpool_sleep, vp);
	mtx_lock(vplock);
	}
	UNP_PCB_LOCK(unp);
	if (unp->unp_vnode != vp && unp->unp_vnode != NULL) {
	if (vplock)
	mtx_unlock(vplock);
	UNP_PCB_UNLOCK(unp);
	goto restart;
	}
	if ((vp = unp->unp_vnode) != NULL) {
	VOP_UNP_DETACH(vp);
	unp->unp_vnode = NULL;
	}
	if (__predict_false(unp == unp->unp_conn)) {
	unp_disconnect(unp, unp);
	unp2 = NULL;
	} else {
	if ((unp2 = unp->unp_conn) != NULL) {
	unp_pcb_owned_lock2(unp, unp2, freeunp);
	if (freeunp)
	unp2 = NULL;
	}
	unp_pcb_hold(unp);
	if (unp2 != NULL) {
	unp_pcb_hold(unp2);
	unp_disconnect(unp, unp2);
	if (unp_pcb_rele(unp2) == 0)
	UNP_PCB_UNLOCK(unp2);
	}
	}
	UNP_PCB_UNLOCK(unp);
	UNP_REF_LIST_LOCK();
	while (!LIST_EMPTY(&unp->unp_refs)) {
	struct unpcb *ref = LIST_FIRST(&unp->unp_refs);

	unp_pcb_hold(ref);
	UNP_REF_LIST_UNLOCK();

	MPASS(ref != unp);
	UNP_PCB_UNLOCK_ASSERT(ref);
	unp_drop(ref);
	UNP_REF_LIST_LOCK();
	}

	UNP_REF_LIST_UNLOCK();
	UNP_PCB_LOCK(unp);
	freeunp = unp_pcb_rele(unp);
	MPASS(freeunp == 0);
	local_unp_rights = unp_rights;
	unp->unp_socket->so_pcb = NULL;
	unp->unp_socket = NULL;
	free(unp->unp_addr, M_SONAME);
	unp->unp_addr = NULL;
	if (!unp_pcb_rele(unp))
	UNP_PCB_UNLOCK(unp);
	if (vp) {
	mtx_unlock(vplock);
	vrele(vp);
	}
	if (local_unp_rights)
	taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1);
	}

	static int
	uipc_disconnect(struct socket *so)
	{
	struct unpcb unp, unp2;
	int freed;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL"));

	UNP_PCB_LOCK(unp);
	if ((unp2 = unp->unp_conn) == NULL) {
	UNP_PCB_UNLOCK(unp);
	return (0);
	}
	if (__predict_true(unp != unp2)) {
	unp_pcb_owned_lock2(unp, unp2, freed);
	if (__predict_false(freed)) {
	UNP_PCB_UNLOCK(unp);
	return (0);
	}
	unp_pcb_hold(unp2);
	}
	unp_pcb_hold(unp);
	unp_disconnect(unp, unp2);
	if (unp_pcb_rele(unp) == 0)
	UNP_PCB_UNLOCK(unp);
	if ((unp != unp2) && unp_pcb_rele(unp2) == 0)
	UNP_PCB_UNLOCK(unp2);
	return (0);
	}

	static int
	uipc_listen(struct socket so, int backlog, struct thread td)
	{
	struct unpcb *unp;
	int error;

	if (so->so_type != SOCK_STREAM && so->so_type != SOCK_SEQPACKET)
	return (EOPNOTSUPP);

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_listen: unp == NULL"));

	UNP_PCB_LOCK(unp);
	if (unp->unp_vnode == NULL) {
	/* Already connected or not bound to an address. */
	error = unp->unp_conn != NULL ? EINVAL : EDESTADDRREQ;
	UNP_PCB_UNLOCK(unp);
	return (error);
	}

	SOCK_LOCK(so);
	error = solisten_proto_check(so);
	if (error == 0) {
	cru2xt(td, &unp->unp_peercred);
	solisten_proto(so, backlog);
	}
	SOCK_UNLOCK(so);
	UNP_PCB_UNLOCK(unp);
	return (error);
	}

	static int
	uipc_peeraddr(struct socket so, struct sockaddr *nam)
	{
	struct unpcb unp, unp2;
	const struct sockaddr *sa;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_peeraddr: unp == NULL"));

	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
	UNP_LINK_RLOCK();
	/*
	* XXX: It seems that this test always fails even when connection is
	* established. So, this else clause is added as workaround to
	* return PF_LOCAL sockaddr.
	*/
	unp2 = unp->unp_conn;
	if (unp2 != NULL) {
	UNP_PCB_LOCK(unp2);
	if (unp2->unp_addr != NULL)
	sa = (struct sockaddr *) unp2->unp_addr;
	else
	sa = &sun_noname;
	bcopy(sa, *nam, sa->sa_len);
	UNP_PCB_UNLOCK(unp2);
	} else {
	sa = &sun_noname;
	bcopy(sa, *nam, sa->sa_len);
	}
	UNP_LINK_RUNLOCK();
	return (0);
	}

	static int
	uipc_rcvd(struct socket *so, int flags)
	{
	struct unpcb unp, unp2;
	struct socket *so2;
	u_int mbcnt, sbcc;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("%s: unp == NULL", __func__));
	KASSERT(so->so_type == SOCK_STREAM \|\| so->so_type == SOCK_SEQPACKET,
	("%s: socktype %d", __func__, so->so_type));

	/*
	* Adjust backpressure on sender and wakeup any waiting to write.
	*
	* The unp lock is acquired to maintain the validity of the unp_conn
	* pointer; no lock on unp2 is required as unp2->unp_socket will be
	* static as long as we don't permit unp2 to disconnect from unp,
	* which is prevented by the lock on unp. We cache values from
	* so_rcv to avoid holding the so_rcv lock over the entire
	* transaction on the remote so_snd.
	*/
	SOCKBUF_LOCK(&so->so_rcv);
	mbcnt = so->so_rcv.sb_mbcnt;
	sbcc = sbavail(&so->so_rcv);
	SOCKBUF_UNLOCK(&so->so_rcv);
	/*
	* There is a benign race condition at this point. If we're planning to
	* clear SB_STOP, but uipc_send is called on the connected socket at
	* this instant, it might add data to the sockbuf and set SB_STOP. Then
	* we would erroneously clear SB_STOP below, even though the sockbuf is
	* full. The race is benign because the only ill effect is to allow the
	* sockbuf to exceed its size limit, and the size limits are not
	* strictly guaranteed anyway.
	*/
	UNP_PCB_LOCK(unp);
	unp2 = unp->unp_conn;
	if (unp2 == NULL) {
	UNP_PCB_UNLOCK(unp);
	return (0);
	}
	so2 = unp2->unp_socket;
	SOCKBUF_LOCK(&so2->so_snd);
	if (sbcc < so2->so_snd.sb_hiwat && mbcnt < so2->so_snd.sb_mbmax)
	so2->so_snd.sb_flags &= ~SB_STOP;
	sowwakeup_locked(so2);
	UNP_PCB_UNLOCK(unp);
	return (0);
	}

	static int
	connect_internal(struct socket so, struct sockaddr nam, struct thread *td)
	{
	int error;
	struct unpcb *unp;

	unp = so->so_pcb;
	if (unp->unp_conn != NULL)
	return (EISCONN);
	error = unp_connect(so, nam, td);
	if (error)
	return (error);
	UNP_PCB_LOCK(unp);
	if (unp->unp_conn == NULL) {
	UNP_PCB_UNLOCK(unp);
	if (error == 0)
	error = ENOTCONN;
	}
	return (error);
	}

	static int
	uipc_send(struct socket so, int flags, struct mbuf m, struct sockaddr *nam,
	struct mbuf control, struct thread td)
	{
	struct unpcb unp, unp2;
	struct socket *so2;
	u_int mbcnt, sbcc;
	int freed, error;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("%s: unp == NULL", __func__));
	KASSERT(so->so_type == SOCK_STREAM \|\| so->so_type == SOCK_DGRAM \|\|
	so->so_type == SOCK_SEQPACKET,
	("%s: socktype %d", __func__, so->so_type));

	freed = error = 0;
	if (flags & PRUS_OOB) {
	error = EOPNOTSUPP;
	goto release;
	}
	if (control != NULL && (error = unp_internalize(&control, td)))
	goto release;

	unp2 = NULL;
	switch (so->so_type) {
	case SOCK_DGRAM:
	{
	const struct sockaddr *from;

	if (nam != NULL) {
	/*
	* We return with UNP_PCB_LOCK_HELD so we know that
	* the reference is live if the pointer is valid.
	*/
	if ((error = connect_internal(so, nam, td)))
	break;
	MPASS(unp->unp_conn != NULL);
	unp2 = unp->unp_conn;
	} else {
	UNP_PCB_LOCK(unp);

	/*
	* Because connect() and send() are non-atomic in a sendto()
	* with a target address, it's possible that the socket will
	* have disconnected before the send() can run. In that case
	* return the slightly counter-intuitive but otherwise
	* correct error that the socket is not connected.
	*/
	if ((unp2 = unp->unp_conn) == NULL) {
	UNP_PCB_UNLOCK(unp);
	error = ENOTCONN;
	break;
	}
	}
	if (__predict_false(unp == unp2)) {
	if (unp->unp_socket == NULL) {
	error = ENOTCONN;
	break;
	}
	goto connect_self;
	}
	unp_pcb_owned_lock2(unp, unp2, freed);
	if (__predict_false(freed)) {
	UNP_PCB_UNLOCK(unp);
	error = ENOTCONN;
	break;
	}
	/*
	* The socket referencing unp2 may have been closed
	* or unp may have been disconnected if the unp lock
	* was dropped to acquire unp2.
	*/
	if (__predict_false(unp->unp_conn == NULL) \|\|
	unp2->unp_socket == NULL) {
	UNP_PCB_UNLOCK(unp);
	if (unp_pcb_rele(unp2) == 0)
	UNP_PCB_UNLOCK(unp2);
	error = ENOTCONN;
	break;
	}
	connect_self:
	if (unp2->unp_flags & UNP_WANTCRED)
	control = unp_addsockcred(td, control);
	if (unp->unp_addr != NULL)
	from = (struct sockaddr *)unp->unp_addr;
	else
	from = &sun_noname;
	so2 = unp2->unp_socket;
	SOCKBUF_LOCK(&so2->so_rcv);
	if (sbappendaddr_locked(&so2->so_rcv, from, m,
	control)) {
	sorwakeup_locked(so2);
	m = NULL;
	control = NULL;
	} else {
	SOCKBUF_UNLOCK(&so2->so_rcv);
	error = ENOBUFS;
	}
	if (nam != NULL)
	unp_disconnect(unp, unp2);
	if (__predict_true(unp != unp2))
	UNP_PCB_UNLOCK(unp2);
	UNP_PCB_UNLOCK(unp);
	break;
	}

	case SOCK_SEQPACKET:
	case SOCK_STREAM:
	if ((so->so_state & SS_ISCONNECTED) == 0) {
	if (nam != NULL) {
	error = connect_internal(so, nam, td);
	if (error != 0)
	break;
	} else {
	error = ENOTCONN;
	break;
	}
	} else {
	UNP_PCB_LOCK(unp);
	}

	if ((unp2 = unp->unp_conn) == NULL) {
	UNP_PCB_UNLOCK(unp);
	error = ENOTCONN;
	break;
	} else if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
	UNP_PCB_UNLOCK(unp);
	error = EPIPE;
	break;
	} else if ((unp2 = unp->unp_conn) == NULL) {
	UNP_PCB_UNLOCK(unp);
	error = ENOTCONN;
	break;
	}
	unp_pcb_owned_lock2(unp, unp2, freed);
	UNP_PCB_UNLOCK(unp);
	if (__predict_false(freed)) {
	error = ENOTCONN;
	break;
	}
	if ((so2 = unp2->unp_socket) == NULL) {
	UNP_PCB_UNLOCK(unp2);
	error = ENOTCONN;
	break;
	}
	SOCKBUF_LOCK(&so2->so_rcv);
	if (unp2->unp_flags & UNP_WANTCRED) {
	/*
	* Credentials are passed only once on SOCK_STREAM
	* and SOCK_SEQPACKET.
	*/
	unp2->unp_flags &= ~UNP_WANTCRED;
	control = unp_addsockcred(td, control);
	}

	/*
	* Send to paired receive port and wake up readers. Don't
	* check for space available in the receive buffer if we're
	* attaching ancillary data; Unix domain sockets only check
	* for space in the sending sockbuf, and that check is
	* performed one level up the stack. At that level we cannot
	* precisely account for the amount of buffer space used
	* (e.g., because control messages are not yet internalized).
	*/
	switch (so->so_type) {
	case SOCK_STREAM:
	if (control != NULL) {
	sbappendcontrol_locked(&so2->so_rcv, m,
	control, flags);
	control = NULL;
	} else
	sbappend_locked(&so2->so_rcv, m, flags);
	break;

	case SOCK_SEQPACKET:
	if (sbappendaddr_nospacecheck_locked(&so2->so_rcv,
	&sun_noname, m, control))
	control = NULL;
	break;
	}

	mbcnt = so2->so_rcv.sb_mbcnt;
	sbcc = sbavail(&so2->so_rcv);
	if (sbcc)
	sorwakeup_locked(so2);
	else
	SOCKBUF_UNLOCK(&so2->so_rcv);

	/*
	* The PCB lock on unp2 protects the SB_STOP flag. Without it,
	* it would be possible for uipc_rcvd to be called at this
	* point, drain the receiving sockbuf, clear SB_STOP, and then
	* we would set SB_STOP below. That could lead to an empty
	* sockbuf having SB_STOP set
	*/
	SOCKBUF_LOCK(&so->so_snd);
	if (sbcc >= so->so_snd.sb_hiwat \|\| mbcnt >= so->so_snd.sb_mbmax)
	so->so_snd.sb_flags \|= SB_STOP;
	SOCKBUF_UNLOCK(&so->so_snd);
	UNP_PCB_UNLOCK(unp2);
	m = NULL;
	break;
	}

	/*
	* PRUS_EOF is equivalent to pru_send followed by pru_shutdown.
	*/
	if (flags & PRUS_EOF) {
	UNP_PCB_LOCK(unp);
	socantsendmore(so);
	unp_shutdown(unp);
	UNP_PCB_UNLOCK(unp);
	}
	if (control != NULL && error != 0)
	unp_dispose_mbuf(control);

	release:
	if (control != NULL)
	m_freem(control);
	/*
	* In case of PRUS_NOTREADY, uipc_ready() is responsible
	* for freeing memory.
	*/
	if (m != NULL && (flags & PRUS_NOTREADY) == 0)
	m_freem(m);
	return (error);
	}

	static bool
	uipc_ready_scan(struct socket so, struct mbuf m, int count, int *errorp)
	{
	struct mbuf mb, n;
	struct sockbuf *sb;

	SOCK_LOCK(so);
	if (SOLISTENING(so)) {
	SOCK_UNLOCK(so);
	return (false);
	}
	mb = NULL;
	sb = &so->so_rcv;
	SOCKBUF_LOCK(sb);
	if (sb->sb_fnrdy != NULL) {
	for (mb = sb->sb_mb, n = mb->m_nextpkt; mb != NULL;) {
	if (mb == m) {
	*errorp = sbready(sb, m, count);
	break;
	}
	mb = mb->m_next;
	if (mb == NULL) {
	mb = n;
	- n = mb->m_nextpkt;
	+ if (mb != NULL)
	+ n = mb->m_nextpkt;
	}
	}
	}
	SOCKBUF_UNLOCK(sb);
	SOCK_UNLOCK(so);
	return (mb != NULL);
	}

	static int
	uipc_ready(struct socket so, struct mbuf m, int count)
	{
	struct unpcb unp, unp2;
	struct socket *so2;
	int error, i;

	unp = sotounpcb(so);

	KASSERT(so->so_type == SOCK_STREAM,
	("%s: unexpected socket type for %p", __func__, so));

	UNP_PCB_LOCK(unp);
	if ((unp2 = unp->unp_conn) == NULL) {
	UNP_PCB_UNLOCK(unp);
	goto search;
	}
	if (unp != unp2) {
	if (UNP_PCB_TRYLOCK(unp2) == 0) {
	unp_pcb_hold(unp2);
	UNP_PCB_UNLOCK(unp);
	UNP_PCB_LOCK(unp2);
	if (unp_pcb_rele(unp2))
	goto search;
	} else
	UNP_PCB_UNLOCK(unp);
	}
	so2 = unp2->unp_socket;
	SOCKBUF_LOCK(&so2->so_rcv);
	if ((error = sbready(&so2->so_rcv, m, count)) == 0)
	sorwakeup_locked(so2);
	else
	SOCKBUF_UNLOCK(&so2->so_rcv);
	UNP_PCB_UNLOCK(unp2);
	return (error);

	search:
	/*
	* The receiving socket has been disconnected, but may still be valid.
	* In this case, the now-ready mbufs are still present in its socket
	* buffer, so perform an exhaustive search before giving up and freeing
	* the mbufs.
	*/
	UNP_LINK_RLOCK();
	LIST_FOREACH(unp, &unp_shead, unp_link) {
	if (uipc_ready_scan(unp->unp_socket, m, count, &error))
	break;
	}
	UNP_LINK_RUNLOCK();

	if (unp == NULL) {
	for (i = 0; i < count; i++)
	m = m_free(m);
	error = ECONNRESET;
	}
	return (error);
	}

	static int
	uipc_sense(struct socket so, struct stat sb)
	{
	struct unpcb *unp;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_sense: unp == NULL"));

	sb->st_blksize = so->so_snd.sb_hiwat;
	sb->st_dev = NODEV;
	sb->st_ino = unp->unp_ino;
	return (0);
	}

	static int
	uipc_shutdown(struct socket *so)
	{
	struct unpcb *unp;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_shutdown: unp == NULL"));

	UNP_PCB_LOCK(unp);
	socantsendmore(so);
	unp_shutdown(unp);
	UNP_PCB_UNLOCK(unp);
	return (0);
	}

	static int
	uipc_sockaddr(struct socket so, struct sockaddr *nam)
	{
	struct unpcb *unp;
	const struct sockaddr *sa;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_sockaddr: unp == NULL"));

	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
	UNP_PCB_LOCK(unp);
	if (unp->unp_addr != NULL)
	sa = (struct sockaddr *) unp->unp_addr;
	else
	sa = &sun_noname;
	bcopy(sa, *nam, sa->sa_len);
	UNP_PCB_UNLOCK(unp);
	return (0);
	}

	static struct pr_usrreqs uipc_usrreqs_dgram = {
	.pru_abort = uipc_abort,
	.pru_accept = uipc_accept,
	.pru_attach = uipc_attach,
	.pru_bind = uipc_bind,
	.pru_bindat = uipc_bindat,
	.pru_connect = uipc_connect,
	.pru_connectat = uipc_connectat,
	.pru_connect2 = uipc_connect2,
	.pru_detach = uipc_detach,
	.pru_disconnect = uipc_disconnect,
	.pru_listen = uipc_listen,
	.pru_peeraddr = uipc_peeraddr,
	.pru_rcvd = uipc_rcvd,
	.pru_send = uipc_send,
	.pru_sense = uipc_sense,
	.pru_shutdown = uipc_shutdown,
	.pru_sockaddr = uipc_sockaddr,
	.pru_soreceive = soreceive_dgram,
	.pru_close = uipc_close,
	};

	static struct pr_usrreqs uipc_usrreqs_seqpacket = {
	.pru_abort = uipc_abort,
	.pru_accept = uipc_accept,
	.pru_attach = uipc_attach,
	.pru_bind = uipc_bind,
	.pru_bindat = uipc_bindat,
	.pru_connect = uipc_connect,
	.pru_connectat = uipc_connectat,
	.pru_connect2 = uipc_connect2,
	.pru_detach = uipc_detach,
	.pru_disconnect = uipc_disconnect,
	.pru_listen = uipc_listen,
	.pru_peeraddr = uipc_peeraddr,
	.pru_rcvd = uipc_rcvd,
	.pru_send = uipc_send,
	.pru_sense = uipc_sense,
	.pru_shutdown = uipc_shutdown,
	.pru_sockaddr = uipc_sockaddr,
	.pru_soreceive = soreceive_generic, /* XXX: or...? */
	.pru_close = uipc_close,
	};

	static struct pr_usrreqs uipc_usrreqs_stream = {
	.pru_abort = uipc_abort,
	.pru_accept = uipc_accept,
	.pru_attach = uipc_attach,
	.pru_bind = uipc_bind,
	.pru_bindat = uipc_bindat,
	.pru_connect = uipc_connect,
	.pru_connectat = uipc_connectat,
	.pru_connect2 = uipc_connect2,
	.pru_detach = uipc_detach,
	.pru_disconnect = uipc_disconnect,
	.pru_listen = uipc_listen,
	.pru_peeraddr = uipc_peeraddr,
	.pru_rcvd = uipc_rcvd,
	.pru_send = uipc_send,
	.pru_ready = uipc_ready,
	.pru_sense = uipc_sense,
	.pru_shutdown = uipc_shutdown,
	.pru_sockaddr = uipc_sockaddr,
	.pru_soreceive = soreceive_generic,
	.pru_close = uipc_close,
	};

	static int
	uipc_ctloutput(struct socket so, struct sockopt sopt)
	{
	struct unpcb *unp;
	struct xucred xu;
	int error, optval;

	if (sopt->sopt_level != 0)
	return (EINVAL);

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("uipc_ctloutput: unp == NULL"));
	error = 0;
	switch (sopt->sopt_dir) {
	case SOPT_GET:
	switch (sopt->sopt_name) {
	case LOCAL_PEERCRED:
	UNP_PCB_LOCK(unp);
	if (unp->unp_flags & UNP_HAVEPC)
	xu = unp->unp_peercred;
	else {
	if (so->so_type == SOCK_STREAM)
	error = ENOTCONN;
	else
	error = EINVAL;
	}
	UNP_PCB_UNLOCK(unp);
	if (error == 0)
	error = sooptcopyout(sopt, &xu, sizeof(xu));
	break;

	case LOCAL_CREDS:
	/* Unlocked read. */
	optval = unp->unp_flags & UNP_WANTCRED ? 1 : 0;
	error = sooptcopyout(sopt, &optval, sizeof(optval));
	break;

	case LOCAL_CONNWAIT:
	/* Unlocked read. */
	optval = unp->unp_flags & UNP_CONNWAIT ? 1 : 0;
	error = sooptcopyout(sopt, &optval, sizeof(optval));
	break;

	default:
	error = EOPNOTSUPP;
	break;
	}
	break;

	case SOPT_SET:
	switch (sopt->sopt_name) {
	case LOCAL_CREDS:
	case LOCAL_CONNWAIT:
	error = sooptcopyin(sopt, &optval, sizeof(optval),
	sizeof(optval));
	if (error)
	break;

	#define OPTSET(bit) do { \
	UNP_PCB_LOCK(unp); \
	if (optval) \
	unp->unp_flags \|= bit; \
	else \
	unp->unp_flags &= ~bit; \
	UNP_PCB_UNLOCK(unp); \
	} while (0)

	switch (sopt->sopt_name) {
	case LOCAL_CREDS:
	OPTSET(UNP_WANTCRED);
	break;

	case LOCAL_CONNWAIT:
	OPTSET(UNP_CONNWAIT);
	break;

	default:
	break;
	}
	break;
	#undef OPTSET
	default:
	error = ENOPROTOOPT;
	break;
	}
	break;

	default:
	error = EOPNOTSUPP;
	break;
	}
	return (error);
	}

	static int
	unp_connect(struct socket so, struct sockaddr nam, struct thread *td)
	{

	return (unp_connectat(AT_FDCWD, so, nam, td));
	}

	static int
	unp_connectat(int fd, struct socket so, struct sockaddr nam,
	struct thread *td)
	{
	struct sockaddr_un soun = (struct sockaddr_un )nam;
	struct vnode *vp;
	struct socket *so2;
	struct unpcb unp, unp2, *unp3;
	struct nameidata nd;
	char buf[SOCK_MAXADDRLEN];
	struct sockaddr *sa;
	cap_rights_t rights;
	int error, len, freed;
	struct mtx *vplock;

	if (nam->sa_family != AF_UNIX)
	return (EAFNOSUPPORT);
	if (nam->sa_len > sizeof(struct sockaddr_un))
	return (EINVAL);
	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
	if (len <= 0)
	return (EINVAL);
	bcopy(soun->sun_path, buf, len);
	buf[len] = 0;

	unp = sotounpcb(so);
	UNP_PCB_LOCK(unp);
	if (unp->unp_flags & UNP_CONNECTING) {
	UNP_PCB_UNLOCK(unp);
	return (EALREADY);
	}
	unp->unp_flags \|= UNP_CONNECTING;
	UNP_PCB_UNLOCK(unp);

	sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW \| LOCKSHARED \| LOCKLEAF,
	UIO_SYSSPACE, buf, fd, cap_rights_init(&rights, CAP_CONNECTAT), td);
	error = namei(&nd);
	if (error)
	vp = NULL;
	else
	vp = nd.ni_vp;
	ASSERT_VOP_LOCKED(vp, "unp_connect");
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (error)
	goto bad;

	if (vp->v_type != VSOCK) {
	error = ENOTSOCK;
	goto bad;
	}
	#ifdef MAC
	error = mac_vnode_check_open(td->td_ucred, vp, VWRITE \| VREAD);
	if (error)
	goto bad;
	#endif
	error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td);
	if (error)
	goto bad;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("unp_connect: unp == NULL"));

	vplock = mtx_pool_find(mtxpool_sleep, vp);
	mtx_lock(vplock);
	VOP_UNP_CONNECT(vp, &unp2);
	if (unp2 == NULL) {
	error = ECONNREFUSED;
	goto bad2;
	}
	so2 = unp2->unp_socket;
	if (so->so_type != so2->so_type) {
	error = EPROTOTYPE;
	goto bad2;
	}
	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
	if (so2->so_options & SO_ACCEPTCONN) {
	CURVNET_SET(so2->so_vnet);
	so2 = sonewconn(so2, 0);
	CURVNET_RESTORE();
	} else
	so2 = NULL;
	if (so2 == NULL) {
	error = ECONNREFUSED;
	goto bad2;
	}
	unp3 = sotounpcb(so2);
	unp_pcb_lock2(unp2, unp3);
	if (unp2->unp_addr != NULL) {
	bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len);
	unp3->unp_addr = (struct sockaddr_un *) sa;
	sa = NULL;
	}

	unp_copy_peercred(td, unp3, unp, unp2);

	UNP_PCB_UNLOCK(unp2);
	unp2 = unp3;
	unp_pcb_owned_lock2(unp2, unp, freed);
	if (__predict_false(freed)) {
	UNP_PCB_UNLOCK(unp2);
	error = ECONNREFUSED;
	goto bad2;
	}
	#ifdef MAC
	mac_socketpeer_set_from_socket(so, so2);
	mac_socketpeer_set_from_socket(so2, so);
	#endif
	} else {
	if (unp == unp2)
	UNP_PCB_LOCK(unp);
	else
	unp_pcb_lock2(unp, unp2);
	}
	KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 &&
	sotounpcb(so2) == unp2,
	("%s: unp2 %p so2 %p", __func__, unp2, so2));
	error = unp_connect2(so, so2, PRU_CONNECT);
	if (unp != unp2)
	UNP_PCB_UNLOCK(unp2);
	UNP_PCB_UNLOCK(unp);
	bad2:
	mtx_unlock(vplock);
	bad:
	if (vp != NULL) {
	vput(vp);
	}
	free(sa, M_SONAME);
	UNP_PCB_LOCK(unp);
	unp->unp_flags &= ~UNP_CONNECTING;
	UNP_PCB_UNLOCK(unp);
	return (error);
	}

	/*
	* Set socket peer credentials at connection time.
	*
	* The client's PCB credentials are copied from its process structure. The
	* server's PCB credentials are copied from the socket on which it called
	* listen(2). uipc_listen cached that process's credentials at the time.
	*/
	void
	unp_copy_peercred(struct thread td, struct unpcb client_unp,
	struct unpcb server_unp, struct unpcb listen_unp)
	{
	cru2xt(td, &client_unp->unp_peercred);
	client_unp->unp_flags \|= UNP_HAVEPC;

	memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred,
	sizeof(server_unp->unp_peercred));
	server_unp->unp_flags \|= UNP_HAVEPC;
	if (listen_unp->unp_flags & UNP_WANTCRED)
	client_unp->unp_flags \|= UNP_WANTCRED;
	}

	static int
	unp_connect2(struct socket so, struct socket so2, int req)
	{
	struct unpcb *unp;
	struct unpcb *unp2;

	unp = sotounpcb(so);
	KASSERT(unp != NULL, ("unp_connect2: unp == NULL"));
	unp2 = sotounpcb(so2);
	KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL"));

	UNP_PCB_LOCK_ASSERT(unp);
	UNP_PCB_LOCK_ASSERT(unp2);

	if (so2->so_type != so->so_type)
	return (EPROTOTYPE);
	unp->unp_conn = unp2;
	unp_pcb_hold(unp2);
	unp_pcb_hold(unp);
	switch (so->so_type) {
	case SOCK_DGRAM:
	UNP_REF_LIST_LOCK();
	LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink);
	UNP_REF_LIST_UNLOCK();
	soisconnected(so);
	break;

	case SOCK_STREAM:
	case SOCK_SEQPACKET:
	unp2->unp_conn = unp;
	if (req == PRU_CONNECT &&
	((unp->unp_flags \| unp2->unp_flags) & UNP_CONNWAIT))
	soisconnecting(so);
	else
	soisconnected(so);
	soisconnected(so2);
	break;

	default:
	panic("unp_connect2");
	}
	return (0);
	}

	static void
	unp_disconnect(struct unpcb unp, struct unpcb unp2)
	{
	struct socket so, so2;
	int freed __unused;

	KASSERT(unp2 != NULL, ("unp_disconnect: unp2 == NULL"));

	UNP_PCB_LOCK_ASSERT(unp);
	UNP_PCB_LOCK_ASSERT(unp2);

	if (unp->unp_conn == NULL && unp2->unp_conn == NULL)
	return;

	MPASS(unp->unp_conn == unp2);
	unp->unp_conn = NULL;
	so = unp->unp_socket;
	so2 = unp2->unp_socket;
	switch (unp->unp_socket->so_type) {
	case SOCK_DGRAM:
	UNP_REF_LIST_LOCK();
	LIST_REMOVE(unp, unp_reflink);
	UNP_REF_LIST_UNLOCK();
	if (so) {
	SOCK_LOCK(so);
	so->so_state &= ~SS_ISCONNECTED;
	SOCK_UNLOCK(so);
	}
	break;

	case SOCK_STREAM:
	case SOCK_SEQPACKET:
	if (so)
	soisdisconnected(so);
	MPASS(unp2->unp_conn == unp);
	unp2->unp_conn = NULL;
	if (so2)
	soisdisconnected(so2);
	break;
	}
	freed = unp_pcb_rele(unp);
	MPASS(freed == 0);
	freed = unp_pcb_rele(unp2);
	MPASS(freed == 0);
	}

	/*
	* unp_pcblist() walks the global list of struct unpcb's to generate a
	* pointer list, bumping the refcount on each unpcb. It then copies them out
	* sequentially, validating the generation number on each to see if it has
	* been detached. All of this is necessary because copyout() may sleep on
	* disk I/O.
	*/
	static int
	unp_pcblist(SYSCTL_HANDLER_ARGS)
	{
	struct unpcb unp, *unp_list;
	unp_gen_t gencnt;
	struct xunpgen *xug;
	struct unp_head *head;
	struct xunpcb *xu;
	u_int i;
	int error, freeunp, n;

	switch ((intptr_t)arg1) {
	case SOCK_STREAM:
	head = &unp_shead;
	break;

	case SOCK_DGRAM:
	head = &unp_dhead;
	break;

	case SOCK_SEQPACKET:
	head = &unp_sphead;
	break;

	default:
	panic("unp_pcblist: arg1 %d", (int)(intptr_t)arg1);
	}

	/*
	* The process of preparing the PCB list is too time-consuming and
	* resource-intensive to repeat twice on every request.
	*/
	if (req->oldptr == NULL) {
	n = unp_count;
	req->oldidx = 2 * (sizeof *xug)
	+ (n + n/8) * sizeof(struct xunpcb);
	return (0);
	}

	if (req->newptr != NULL)
	return (EPERM);

	/*
	* OK, now we're committed to doing something.
	*/
	xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK \| M_ZERO);
	UNP_LINK_RLOCK();
	gencnt = unp_gencnt;
	n = unp_count;
	UNP_LINK_RUNLOCK();

	xug->xug_len = sizeof *xug;
	xug->xug_count = n;
	xug->xug_gen = gencnt;
	xug->xug_sogen = so_gencnt;
	error = SYSCTL_OUT(req, xug, sizeof *xug);
	if (error) {
	free(xug, M_TEMP);
	return (error);
	}

	unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK);

	UNP_LINK_RLOCK();
	for (unp = LIST_FIRST(head), i = 0; unp && i < n;
	unp = LIST_NEXT(unp, unp_link)) {
	UNP_PCB_LOCK(unp);
	if (unp->unp_gencnt <= gencnt) {
	if (cr_cansee(req->td->td_ucred,
	unp->unp_socket->so_cred)) {
	UNP_PCB_UNLOCK(unp);
	continue;
	}
	unp_list[i++] = unp;
	unp_pcb_hold(unp);
	}
	UNP_PCB_UNLOCK(unp);
	}
	UNP_LINK_RUNLOCK();
	n = i; /* In case we lost some during malloc. */

	error = 0;
	xu = malloc(sizeof(*xu), M_TEMP, M_WAITOK \| M_ZERO);
	for (i = 0; i < n; i++) {
	unp = unp_list[i];
	UNP_PCB_LOCK(unp);
	freeunp = unp_pcb_rele(unp);

	if (freeunp == 0 && unp->unp_gencnt <= gencnt) {
	xu->xu_len = sizeof *xu;
	xu->xu_unpp = (uintptr_t)unp;
	/*
	* XXX - need more locking here to protect against
	* connect/disconnect races for SMP.
	*/
	if (unp->unp_addr != NULL)
	bcopy(unp->unp_addr, &xu->xu_addr,
	unp->unp_addr->sun_len);
	else
	bzero(&xu->xu_addr, sizeof(xu->xu_addr));
	if (unp->unp_conn != NULL &&
	unp->unp_conn->unp_addr != NULL)
	bcopy(unp->unp_conn->unp_addr,
	&xu->xu_caddr,
	unp->unp_conn->unp_addr->sun_len);
	else
	bzero(&xu->xu_caddr, sizeof(xu->xu_caddr));
	xu->unp_vnode = (uintptr_t)unp->unp_vnode;
	xu->unp_conn = (uintptr_t)unp->unp_conn;
	xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs);
	xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink);
	xu->unp_gencnt = unp->unp_gencnt;
	sotoxsocket(unp->unp_socket, &xu->xu_socket);
	UNP_PCB_UNLOCK(unp);
	error = SYSCTL_OUT(req, xu, sizeof *xu);
	} else if (freeunp == 0)
	UNP_PCB_UNLOCK(unp);
	}
	free(xu, M_TEMP);
	if (!error) {
	/*
	* Give the user an updated idea of our state. If the
	* generation differs from what we told her before, she knows
	* that something happened while we were processing this
	* request, and it might be necessary to retry.
	*/
	xug->xug_gen = unp_gencnt;
	xug->xug_sogen = so_gencnt;
	xug->xug_count = unp_count;
	error = SYSCTL_OUT(req, xug, sizeof *xug);
	}
	free(unp_list, M_TEMP);
	free(xug, M_TEMP);
	return (error);
	}

	SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist,
	CTLTYPE_OPAQUE \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	(void *)(intptr_t)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb",
	"List of active local datagram sockets");
	SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist,
	CTLTYPE_OPAQUE \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	(void *)(intptr_t)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb",
	"List of active local stream sockets");
	SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist,
	CTLTYPE_OPAQUE \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	(void *)(intptr_t)SOCK_SEQPACKET, 0, unp_pcblist, "S,xunpcb",
	"List of active local seqpacket sockets");

	static void
	unp_shutdown(struct unpcb *unp)
	{
	struct unpcb *unp2;
	struct socket *so;

	UNP_PCB_LOCK_ASSERT(unp);

	unp2 = unp->unp_conn;
	if ((unp->unp_socket->so_type == SOCK_STREAM \|\|
	(unp->unp_socket->so_type == SOCK_SEQPACKET)) && unp2 != NULL) {
	so = unp2->unp_socket;
	if (so != NULL)
	socantrcvmore(so);
	}
	}

	static void
	unp_drop(struct unpcb *unp)
	{
	struct socket *so = unp->unp_socket;
	struct unpcb *unp2;
	int freed;

	/*
	* Regardless of whether the socket's peer dropped the connection
	* with this socket by aborting or disconnecting, POSIX requires
	* that ECONNRESET is returned.
	*/
	/* acquire a reference so that unp isn't freed from underneath us */

	UNP_PCB_LOCK(unp);
	if (so)
	so->so_error = ECONNRESET;
	unp2 = unp->unp_conn;
	if (unp2 == unp) {
	unp_disconnect(unp, unp2);
	} else if (unp2 != NULL) {
	unp_pcb_hold(unp2);
	unp_pcb_owned_lock2(unp, unp2, freed);
	unp_disconnect(unp, unp2);
	if (unp_pcb_rele(unp2) == 0)
	UNP_PCB_UNLOCK(unp2);
	}
	if (unp_pcb_rele(unp) == 0)
	UNP_PCB_UNLOCK(unp);
	}

	static void
	unp_freerights(struct filedescent **fdep, int fdcount)
	{
	struct file *fp;
	int i;

	KASSERT(fdcount > 0, ("%s: fdcount %d", __func__, fdcount));

	for (i = 0; i < fdcount; i++) {
	fp = fdep[i]->fde_file;
	filecaps_free(&fdep[i]->fde_caps);
	unp_discard(fp);
	}
	free(fdep[0], M_FILECAPS);
	}

	static int
	unp_externalize(struct mbuf control, struct mbuf *controlp, int flags)
	{
	struct thread td = curthread; / XXX */
	struct cmsghdr cm = mtod(control, struct cmsghdr );
	int i;
	int *fdp;
	struct filedesc *fdesc = td->td_proc->p_fd;
	struct filedescent **fdep;
	void *data;
	socklen_t clen = control->m_len, datalen;
	int error, newfds;
	u_int newlen;

	UNP_LINK_UNLOCK_ASSERT();

	error = 0;
	if (controlp != NULL) /* controlp == NULL => free control messages */
	*controlp = NULL;
	while (cm != NULL) {
	if (sizeof(*cm) > clen \|\| cm->cmsg_len > clen) {
	error = EINVAL;
	break;
	}
	data = CMSG_DATA(cm);
	datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data;
	if (cm->cmsg_level == SOL_SOCKET
	&& cm->cmsg_type == SCM_RIGHTS) {
	newfds = datalen / sizeof(*fdep);
	if (newfds == 0)
	goto next;
	fdep = data;

	/* If we're not outputting the descriptors free them. */
	if (error \|\| controlp == NULL) {
	unp_freerights(fdep, newfds);
	goto next;
	}
	FILEDESC_XLOCK(fdesc);

	/*
	* Now change each pointer to an fd in the global
	* table to an integer that is the index to the local
	* fd table entry that we set up to point to the
	* global one we are transferring.
	*/
	newlen = newfds * sizeof(int);
	*controlp = sbcreatecontrol(NULL, newlen,
	SCM_RIGHTS, SOL_SOCKET);
	if (*controlp == NULL) {
	FILEDESC_XUNLOCK(fdesc);
	error = E2BIG;
	unp_freerights(fdep, newfds);
	goto next;
	}

	fdp = (int *)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	if (fdallocn(td, 0, fdp, newfds) != 0) {
	FILEDESC_XUNLOCK(fdesc);
	error = EMSGSIZE;
	unp_freerights(fdep, newfds);
	m_freem(*controlp);
	*controlp = NULL;
	goto next;
	}
	for (i = 0; i < newfds; i++, fdp++) {
	_finstall(fdesc, fdep[i]->fde_file, *fdp,
	(flags & MSG_CMSG_CLOEXEC) != 0 ? UF_EXCLOSE : 0,
	&fdep[i]->fde_caps);
	unp_externalize_fp(fdep[i]->fde_file);
	}

	/*
	* The new type indicates that the mbuf data refers to
	* kernel resources that may need to be released before
	* the mbuf is freed.
	*/
	m_chtype(*controlp, MT_EXTCONTROL);
	FILEDESC_XUNLOCK(fdesc);
	free(fdep[0], M_FILECAPS);
	} else {
	/* We can just copy anything else across. */
	if (error \|\| controlp == NULL)
	goto next;
	*controlp = sbcreatecontrol(NULL, datalen,
	cm->cmsg_type, cm->cmsg_level);
	if (*controlp == NULL) {
	error = ENOBUFS;
	goto next;
	}
	bcopy(data,
	CMSG_DATA(mtod(controlp, struct cmsghdr )),
	datalen);
	}
	controlp = &(*controlp)->m_next;

	next:
	if (CMSG_SPACE(datalen) < clen) {
	clen -= CMSG_SPACE(datalen);
	cm = (struct cmsghdr *)
	((caddr_t)cm + CMSG_SPACE(datalen));
	} else {
	clen = 0;
	cm = NULL;
	}
	}

	m_freem(control);
	return (error);
	}

	static void
	unp_zone_change(void *tag)
	{

	uma_zone_set_max(unp_zone, maxsockets);
	}

	static void
	unp_init(void)
	{

	#ifdef VIMAGE
	if (!IS_DEFAULT_VNET(curvnet))
	return;
	#endif
	unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, NULL,
	NULL, NULL, UMA_ALIGN_CACHE, 0);
	if (unp_zone == NULL)
	panic("unp_init");
	uma_zone_set_max(unp_zone, maxsockets);
	uma_zone_set_warning(unp_zone, "kern.ipc.maxsockets limit reached");
	EVENTHANDLER_REGISTER(maxsockets_change, unp_zone_change,
	NULL, EVENTHANDLER_PRI_ANY);
	LIST_INIT(&unp_dhead);
	LIST_INIT(&unp_shead);
	LIST_INIT(&unp_sphead);
	SLIST_INIT(&unp_defers);
	TIMEOUT_TASK_INIT(taskqueue_thread, &unp_gc_task, 0, unp_gc, NULL);
	TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL);
	UNP_LINK_LOCK_INIT();
	UNP_DEFERRED_LOCK_INIT();
	}

	static void
	unp_internalize_cleanup_rights(struct mbuf *control)
	{
	struct cmsghdr *cp;
	struct mbuf *m;
	void *data;
	socklen_t datalen;

	for (m = control; m != NULL; m = m->m_next) {
	cp = mtod(m, struct cmsghdr *);
	if (cp->cmsg_level != SOL_SOCKET \|\|
	cp->cmsg_type != SCM_RIGHTS)
	continue;
	data = CMSG_DATA(cp);
	datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data;
	unp_freerights(data, datalen / sizeof(struct filedesc *));
	}
	}

	static int
	unp_internalize(struct mbuf *controlp, struct thread td)
	{
	struct mbuf control, *initial_controlp;
	struct proc *p;
	struct filedesc *fdesc;
	struct bintime *bt;
	struct cmsghdr *cm;
	struct cmsgcred *cmcred;
	struct filedescent fde, fdep, fdev;
	struct file *fp;
	struct timeval *tv;
	struct timespec *ts;
	void *data;
	socklen_t clen, datalen;
	int i, j, error, *fdp, oldfds;
	u_int newlen;

	UNP_LINK_UNLOCK_ASSERT();

	p = td->td_proc;
	fdesc = p->p_fd;
	error = 0;
	control = *controlp;
	clen = control->m_len;
	*controlp = NULL;
	initial_controlp = controlp;
	for (cm = mtod(control, struct cmsghdr *); cm != NULL;) {
	if (sizeof(*cm) > clen \|\| cm->cmsg_level != SOL_SOCKET
	\|\| cm->cmsg_len > clen \|\| cm->cmsg_len < sizeof(*cm)) {
	error = EINVAL;
	goto out;
	}
	data = CMSG_DATA(cm);
	datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data;

	switch (cm->cmsg_type) {
	/*
	* Fill in credential information.
	*/
	case SCM_CREDS:
	controlp = sbcreatecontrol(NULL, sizeof(cmcred),
	SCM_CREDS, SOL_SOCKET);
	if (*controlp == NULL) {
	error = ENOBUFS;
	goto out;
	}
	cmcred = (struct cmsgcred *)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	cmcred->cmcred_pid = p->p_pid;
	cmcred->cmcred_uid = td->td_ucred->cr_ruid;
	cmcred->cmcred_gid = td->td_ucred->cr_rgid;
	cmcred->cmcred_euid = td->td_ucred->cr_uid;
	cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups,
	CMGROUP_MAX);
	for (i = 0; i < cmcred->cmcred_ngroups; i++)
	cmcred->cmcred_groups[i] =
	td->td_ucred->cr_groups[i];
	break;

	case SCM_RIGHTS:
	oldfds = datalen / sizeof (int);
	if (oldfds == 0)
	break;
	/*
	* Check that all the FDs passed in refer to legal
	* files. If not, reject the entire operation.
	*/
	fdp = data;
	FILEDESC_SLOCK(fdesc);
	for (i = 0; i < oldfds; i++, fdp++) {
	fp = fget_locked(fdesc, *fdp);
	if (fp == NULL) {
	FILEDESC_SUNLOCK(fdesc);
	error = EBADF;
	goto out;
	}
	if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) {
	FILEDESC_SUNLOCK(fdesc);
	error = EOPNOTSUPP;
	goto out;
	}

	}

	/*
	* Now replace the integer FDs with pointers to the
	* file structure and capability rights.
	*/
	newlen = oldfds * sizeof(fdep[0]);
	*controlp = sbcreatecontrol(NULL, newlen,
	SCM_RIGHTS, SOL_SOCKET);
	if (*controlp == NULL) {
	FILEDESC_SUNLOCK(fdesc);
	error = E2BIG;
	goto out;
	}
	fdp = data;
	for (i = 0; i < oldfds; i++, fdp++) {
	if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) {
	fdp = data;
	for (j = 0; j < i; j++, fdp++) {
	fdrop(fdesc->fd_ofiles[*fdp].
	fde_file, td);
	}
	FILEDESC_SUNLOCK(fdesc);
	error = EBADF;
	goto out;
	}
	}
	fdp = data;
	fdep = (struct filedescent **)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	fdev = malloc(sizeof(fdev) oldfds, M_FILECAPS,
	M_WAITOK);
	for (i = 0; i < oldfds; i++, fdev++, fdp++) {
	fde = &fdesc->fd_ofiles[*fdp];
	fdep[i] = fdev;
	fdep[i]->fde_file = fde->fde_file;
	filecaps_copy(&fde->fde_caps,
	&fdep[i]->fde_caps, true);
	unp_internalize_fp(fdep[i]->fde_file);
	}
	FILEDESC_SUNLOCK(fdesc);
	break;

	case SCM_TIMESTAMP:
	controlp = sbcreatecontrol(NULL, sizeof(tv),
	SCM_TIMESTAMP, SOL_SOCKET);
	if (*controlp == NULL) {
	error = ENOBUFS;
	goto out;
	}
	tv = (struct timeval *)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	microtime(tv);
	break;

	case SCM_BINTIME:
	controlp = sbcreatecontrol(NULL, sizeof(bt),
	SCM_BINTIME, SOL_SOCKET);
	if (*controlp == NULL) {
	error = ENOBUFS;
	goto out;
	}
	bt = (struct bintime *)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	bintime(bt);
	break;

	case SCM_REALTIME:
	controlp = sbcreatecontrol(NULL, sizeof(ts),
	SCM_REALTIME, SOL_SOCKET);
	if (*controlp == NULL) {
	error = ENOBUFS;
	goto out;
	}
	ts = (struct timespec *)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	nanotime(ts);
	break;

	case SCM_MONOTONIC:
	controlp = sbcreatecontrol(NULL, sizeof(ts),
	SCM_MONOTONIC, SOL_SOCKET);
	if (*controlp == NULL) {
	error = ENOBUFS;
	goto out;
	}
	ts = (struct timespec *)
	CMSG_DATA(mtod(controlp, struct cmsghdr ));
	nanouptime(ts);
	break;

	default:
	error = EINVAL;
	goto out;
	}

	if (*controlp != NULL)
	controlp = &(*controlp)->m_next;
	if (CMSG_SPACE(datalen) < clen) {
	clen -= CMSG_SPACE(datalen);
	cm = (struct cmsghdr *)
	((caddr_t)cm + CMSG_SPACE(datalen));
	} else {
	clen = 0;
	cm = NULL;
	}
	}

	out:
	if (error != 0 && initial_controlp != NULL)
	unp_internalize_cleanup_rights(*initial_controlp);
	m_freem(control);
	return (error);
	}

	static struct mbuf *
	unp_addsockcred(struct thread td, struct mbuf control)
	{
	struct mbuf m, n, *n_prev;
	struct sockcred *sc;
	const struct cmsghdr *cm;
	int ngroups;
	int i;

	ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX);
	m = sbcreatecontrol(NULL, SOCKCREDSIZE(ngroups), SCM_CREDS, SOL_SOCKET);
	if (m == NULL)
	return (control);

	sc = (struct sockcred ) CMSG_DATA(mtod(m, struct cmsghdr ));
	sc->sc_uid = td->td_ucred->cr_ruid;
	sc->sc_euid = td->td_ucred->cr_uid;
	sc->sc_gid = td->td_ucred->cr_rgid;
	sc->sc_egid = td->td_ucred->cr_gid;
	sc->sc_ngroups = ngroups;
	for (i = 0; i < sc->sc_ngroups; i++)
	sc->sc_groups[i] = td->td_ucred->cr_groups[i];

	/*
	* Unlink SCM_CREDS control messages (struct cmsgcred), since just
	* created SCM_CREDS control message (struct sockcred) has another
	* format.
	*/
	if (control != NULL)
	for (n = control, n_prev = NULL; n != NULL;) {
	cm = mtod(n, struct cmsghdr *);
	if (cm->cmsg_level == SOL_SOCKET &&
	cm->cmsg_type == SCM_CREDS) {
	if (n_prev == NULL)
	control = n->m_next;
	else
	n_prev->m_next = n->m_next;
	n = m_free(n);
	} else {
	n_prev = n;
	n = n->m_next;
	}
	}

	/* Prepend it to the head. */
	m->m_next = control;
	return (m);
	}

	static struct unpcb *
	fptounp(struct file *fp)
	{
	struct socket *so;

	if (fp->f_type != DTYPE_SOCKET)
	return (NULL);
	if ((so = fp->f_data) == NULL)
	return (NULL);
	if (so->so_proto->pr_domain != &localdomain)
	return (NULL);
	return sotounpcb(so);
	}

	static void
	unp_discard(struct file *fp)
	{
	struct unp_defer *dr;

	if (unp_externalize_fp(fp)) {
	dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK);
	dr->ud_fp = fp;
	UNP_DEFERRED_LOCK();
	SLIST_INSERT_HEAD(&unp_defers, dr, ud_link);
	UNP_DEFERRED_UNLOCK();
	atomic_add_int(&unp_defers_count, 1);
	taskqueue_enqueue(taskqueue_thread, &unp_defer_task);
	} else
	(void) closef(fp, (struct thread *)NULL);
	}

	static void
	unp_process_defers(void *arg __unused, int pending)
	{
	struct unp_defer *dr;
	SLIST_HEAD(, unp_defer) drl;
	int count;

	SLIST_INIT(&drl);
	for (;;) {
	UNP_DEFERRED_LOCK();
	if (SLIST_FIRST(&unp_defers) == NULL) {
	UNP_DEFERRED_UNLOCK();
	break;
	}
	SLIST_SWAP(&unp_defers, &drl, unp_defer);
	UNP_DEFERRED_UNLOCK();
	count = 0;
	while ((dr = SLIST_FIRST(&drl)) != NULL) {
	SLIST_REMOVE_HEAD(&drl, ud_link);
	closef(dr->ud_fp, NULL);
	free(dr, M_TEMP);
	count++;
	}
	atomic_add_int(&unp_defers_count, -count);
	}
	}

	static void
	unp_internalize_fp(struct file *fp)
	{
	struct unpcb *unp;

	UNP_LINK_WLOCK();
	if ((unp = fptounp(fp)) != NULL) {
	unp->unp_file = fp;
	unp->unp_msgcount++;
	}
	unp_rights++;
	UNP_LINK_WUNLOCK();
	}

	static int
	unp_externalize_fp(struct file *fp)
	{
	struct unpcb *unp;
	int ret;

	UNP_LINK_WLOCK();
	if ((unp = fptounp(fp)) != NULL) {
	unp->unp_msgcount--;
	ret = 1;
	} else
	ret = 0;
	unp_rights--;
	UNP_LINK_WUNLOCK();
	return (ret);
	}

	/*
	* unp_defer indicates whether additional work has been defered for a future
	* pass through unp_gc(). It is thread local and does not require explicit
	* synchronization.
	*/
	static int unp_marked;

	static void
	unp_remove_dead_ref(struct filedescent **fdep, int fdcount)
	{
	struct unpcb *unp;
	struct file *fp;
	int i;

	/*
	* This function can only be called from the gc task.
	*/
	KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0,
	("%s: not on gc callout", __func__));
	UNP_LINK_LOCK_ASSERT();

	for (i = 0; i < fdcount; i++) {
	fp = fdep[i]->fde_file;
	if ((unp = fptounp(fp)) == NULL)
	continue;
	if ((unp->unp_gcflag & UNPGC_DEAD) == 0)
	continue;
	unp->unp_gcrefs--;
	}
	}

	static void
	unp_restore_undead_ref(struct filedescent **fdep, int fdcount)
	{
	struct unpcb *unp;
	struct file *fp;
	int i;

	/*
	* This function can only be called from the gc task.
	*/
	KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0,
	("%s: not on gc callout", __func__));
	UNP_LINK_LOCK_ASSERT();

	for (i = 0; i < fdcount; i++) {
	fp = fdep[i]->fde_file;
	if ((unp = fptounp(fp)) == NULL)
	continue;
	if ((unp->unp_gcflag & UNPGC_DEAD) == 0)
	continue;
	unp->unp_gcrefs++;
	unp_marked++;
	}
	}

	static void
	unp_gc_scan(struct unpcb unp, void (op)(struct filedescent **, int))
	{
	struct socket so, soa;

	so = unp->unp_socket;
	SOCK_LOCK(so);
	if (SOLISTENING(so)) {
	/*
	* Mark all sockets in our accept queue.
	*/
	TAILQ_FOREACH(soa, &so->sol_comp, so_list) {
	if (sotounpcb(soa)->unp_gcflag & UNPGC_IGNORE_RIGHTS)
	continue;
	SOCKBUF_LOCK(&soa->so_rcv);
	unp_scan(soa->so_rcv.sb_mb, op);
	SOCKBUF_UNLOCK(&soa->so_rcv);
	}
	} else {
	/*
	* Mark all sockets we reference with RIGHTS.
	*/
	if ((unp->unp_gcflag & UNPGC_IGNORE_RIGHTS) == 0) {
	SOCKBUF_LOCK(&so->so_rcv);
	unp_scan(so->so_rcv.sb_mb, op);
	SOCKBUF_UNLOCK(&so->so_rcv);
	}
	}
	SOCK_UNLOCK(so);
	}

	static int unp_recycled;
	SYSCTL_INT(_net_local, OID_AUTO, recycled, CTLFLAG_RD, &unp_recycled, 0,
	"Number of unreachable sockets claimed by the garbage collector.");

	static int unp_taskcount;
	SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0,
	"Number of times the garbage collector has run.");

	SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0,
	"Number of active local sockets.");

	static void
	unp_gc(__unused void *arg, int pending)
	{
	struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead,
	NULL };
	struct unp_head **head;
	struct unp_head unp_deadhead; /* List of potentially-dead sockets. */
	struct file f, *unref;
	struct unpcb unp, unptmp;
	int i, total, unp_unreachable;

	LIST_INIT(&unp_deadhead);
	unp_taskcount++;
	UNP_LINK_RLOCK();
	/*
	* First determine which sockets may be in cycles.
	*/
	unp_unreachable = 0;

	for (head = heads; *head != NULL; head++)
	LIST_FOREACH(unp, *head, unp_link) {

	KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0,
	("%s: unp %p has unexpected gc flags 0x%x",
	__func__, unp, (unsigned int)unp->unp_gcflag));

	f = unp->unp_file;

	/*
	* Check for an unreachable socket potentially in a
	* cycle. It must be in a queue as indicated by
	* msgcount, and this must equal the file reference
	* count. Note that when msgcount is 0 the file is
	* NULL.
	*/
	if (f != NULL && unp->unp_msgcount != 0 &&
	f->f_count == unp->unp_msgcount) {
	LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead);
	unp->unp_gcflag \|= UNPGC_DEAD;
	unp->unp_gcrefs = unp->unp_msgcount;
	unp_unreachable++;
	}
	}

	/*
	* Scan all sockets previously marked as potentially being in a cycle
	* and remove the references each socket holds on any UNPGC_DEAD
	* sockets in its queue. After this step, all remaining references on
	* sockets marked UNPGC_DEAD should not be part of any cycle.
	*/
	LIST_FOREACH(unp, &unp_deadhead, unp_dead)
	unp_gc_scan(unp, unp_remove_dead_ref);

	/*
	* If a socket still has a non-negative refcount, it cannot be in a
	* cycle. In this case increment refcount of all children iteratively.
	* Stop the scan once we do a complete loop without discovering
	* a new reachable socket.
	*/
	do {
	unp_marked = 0;
	LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp)
	if (unp->unp_gcrefs > 0) {
	unp->unp_gcflag &= ~UNPGC_DEAD;
	LIST_REMOVE(unp, unp_dead);
	KASSERT(unp_unreachable > 0,
	("%s: unp_unreachable underflow.",
	__func__));
	unp_unreachable--;
	unp_gc_scan(unp, unp_restore_undead_ref);
	}
	} while (unp_marked);

	UNP_LINK_RUNLOCK();

	if (unp_unreachable == 0)
	return;

	/*
	* Allocate space for a local array of dead unpcbs.
	* TODO: can this path be simplified by instead using the local
	* dead list at unp_deadhead, after taking out references
	* on the file object and/or unpcb and dropping the link lock?
	*/
	unref = malloc(unp_unreachable * sizeof(struct file *),
	M_TEMP, M_WAITOK);

	/*
	* Iterate looking for sockets which have been specifically marked
	* as unreachable and store them locally.
	*/
	UNP_LINK_RLOCK();
	total = 0;
	LIST_FOREACH(unp, &unp_deadhead, unp_dead) {
	KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0,
	("%s: unp %p not marked UNPGC_DEAD", __func__, unp));
	unp->unp_gcflag &= ~UNPGC_DEAD;
	f = unp->unp_file;
	if (unp->unp_msgcount == 0 \|\| f == NULL \|\|
	f->f_count != unp->unp_msgcount \|\|
	!fhold(f))
	continue;
	unref[total++] = f;
	KASSERT(total <= unp_unreachable,
	("%s: incorrect unreachable count.", __func__));
	}
	UNP_LINK_RUNLOCK();

	/*
	* Now flush all sockets, free'ing rights. This will free the
	* struct files associated with these sockets but leave each socket
	* with one remaining ref.
	*/
	for (i = 0; i < total; i++) {
	struct socket *so;

	so = unref[i]->f_data;
	CURVNET_SET(so->so_vnet);
	sorflush(so);
	CURVNET_RESTORE();
	}

	/*
	* And finally release the sockets so they can be reclaimed.
	*/
	for (i = 0; i < total; i++)
	fdrop(unref[i], NULL);
	unp_recycled += total;
	free(unref, M_TEMP);
	}

	static void
	unp_dispose_mbuf(struct mbuf *m)
	{

	if (m)
	unp_scan(m, unp_freerights);
	}

	/*
	* Synchronize against unp_gc, which can trip over data as we are freeing it.
	*/
	static void
	unp_dispose(struct socket *so)
	{
	struct unpcb *unp;

	unp = sotounpcb(so);
	UNP_LINK_WLOCK();
	unp->unp_gcflag \|= UNPGC_IGNORE_RIGHTS;
	UNP_LINK_WUNLOCK();
	if (!SOLISTENING(so))
	unp_dispose_mbuf(so->so_rcv.sb_mb);
	}

	static void
	unp_scan(struct mbuf m0, void (op)(struct filedescent **, int))
	{
	struct mbuf *m;
	struct cmsghdr *cm;
	void *data;
	socklen_t clen, datalen;

	while (m0 != NULL) {
	for (m = m0; m; m = m->m_next) {
	if (m->m_type != MT_CONTROL)
	continue;

	cm = mtod(m, struct cmsghdr *);
	clen = m->m_len;

	while (cm != NULL) {
	if (sizeof(*cm) > clen \|\| cm->cmsg_len > clen)
	break;

	data = CMSG_DATA(cm);
	datalen = (caddr_t)cm + cm->cmsg_len
	- (caddr_t)data;

	if (cm->cmsg_level == SOL_SOCKET &&
	cm->cmsg_type == SCM_RIGHTS) {
	(*op)(data, datalen /
	sizeof(struct filedescent *));
	}

	if (CMSG_SPACE(datalen) < clen) {
	clen -= CMSG_SPACE(datalen);
	cm = (struct cmsghdr *)
	((caddr_t)cm + CMSG_SPACE(datalen));
	} else {
	clen = 0;
	cm = NULL;
	}
	}
	}
	m0 = m0->m_nextpkt;
	}
	}

	/*
	* A helper function called by VFS before socket-type vnode reclamation.
	* For an active vnode it clears unp_vnode pointer and decrements unp_vnode
	* use count.
	*/
	void
	vfs_unp_reclaim(struct vnode *vp)
	{
	struct unpcb *unp;
	int active;
	struct mtx *vplock;

	ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim");
	KASSERT(vp->v_type == VSOCK,
	("vfs_unp_reclaim: vp->v_type != VSOCK"));

	active = 0;
	vplock = mtx_pool_find(mtxpool_sleep, vp);
	mtx_lock(vplock);
	VOP_UNP_CONNECT(vp, &unp);
	if (unp == NULL)
	goto done;
	UNP_PCB_LOCK(unp);
	if (unp->unp_vnode == vp) {
	VOP_UNP_DETACH(vp);
	unp->unp_vnode = NULL;
	active = 1;
	}
	UNP_PCB_UNLOCK(unp);
	done:
	mtx_unlock(vplock);
	if (active)
	vunref(vp);
	}

	#ifdef DDB
	static void
	db_print_indent(int indent)
	{
	int i;

	for (i = 0; i < indent; i++)
	db_printf(" ");
	}

	static void
	db_print_unpflags(int unp_flags)
	{
	int comma;

	comma = 0;
	if (unp_flags & UNP_HAVEPC) {
	db_printf("%sUNP_HAVEPC", comma ? ", " : "");
	comma = 1;
	}
	if (unp_flags & UNP_WANTCRED) {
	db_printf("%sUNP_WANTCRED", comma ? ", " : "");
	comma = 1;
	}
	if (unp_flags & UNP_CONNWAIT) {
	db_printf("%sUNP_CONNWAIT", comma ? ", " : "");
	comma = 1;
	}
	if (unp_flags & UNP_CONNECTING) {
	db_printf("%sUNP_CONNECTING", comma ? ", " : "");
	comma = 1;
	}
	if (unp_flags & UNP_BINDING) {
	db_printf("%sUNP_BINDING", comma ? ", " : "");
	comma = 1;
	}
	}

	static void
	db_print_xucred(int indent, struct xucred *xu)
	{
	int comma, i;

	db_print_indent(indent);
	db_printf("cr_version: %u cr_uid: %u cr_pid: %d cr_ngroups: %d\n",
	xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups);
	db_print_indent(indent);
	db_printf("cr_groups: ");
	comma = 0;
	for (i = 0; i < xu->cr_ngroups; i++) {
	db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]);
	comma = 1;
	}
	db_printf("\n");
	}

	static void
	db_print_unprefs(int indent, struct unp_head *uh)
	{
	struct unpcb *unp;
	int counter;

	counter = 0;
	LIST_FOREACH(unp, uh, unp_reflink) {
	if (counter % 4 == 0)
	db_print_indent(indent);
	db_printf("%p ", unp);
	if (counter % 4 == 3)
	db_printf("\n");
	counter++;
	}
	if (counter != 0 && counter % 4 != 0)
	db_printf("\n");
	}

	DB_SHOW_COMMAND(unpcb, db_show_unpcb)
	{
	struct unpcb *unp;

	if (!have_addr) {
	db_printf("usage: show unpcb <addr>\n");
	return;
	}
	unp = (struct unpcb *)addr;

	db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket,
	unp->unp_vnode);

	db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino,
	unp->unp_conn);

	db_printf("unp_refs:\n");
	db_print_unprefs(2, &unp->unp_refs);

	/* XXXRW: Would be nice to print the full address, if any. */
	db_printf("unp_addr: %p\n", unp->unp_addr);

	db_printf("unp_gencnt: %llu\n",
	(unsigned long long)unp->unp_gencnt);

	db_printf("unp_flags: %x (", unp->unp_flags);
	db_print_unpflags(unp->unp_flags);
	db_printf(")\n");

	db_printf("unp_peercred:\n");
	db_print_xucred(2, &unp->unp_peercred);

	db_printf("unp_refcount: %u\n", unp->unp_refcount);
	}
	#endif
	diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
	index 0deb5ff785c4..524554cdd48f 100644
	--- a/sys/kern/vfs_bio.c
	+++ b/sys/kern/vfs_bio.c
	@@ -1,5478 +1,5473 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2004 Poul-Henning Kamp
	* Copyright (c) 1994,1997 John S. Dyson
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* Portions of this software were developed by Konstantin Belousov
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/*
	* this file contains a new buffer I/O scheme implementing a coherent
	* VM object and buffer cache scheme. Pains have been taken to make
	* sure that the performance degradation associated with schemes such
	* as this is not realized.
	*
	* Author: John S. Dyson
	* Significant help during the development and debugging phases
	* had been provided by David Greenman, also of the FreeBSD core team.
	*
	* see man buf(9) for more info.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bio.h>
	#include <sys/bitset.h>
	#include <sys/conf.h>
	#include <sys/counter.h>
	#include <sys/buf.h>
	#include <sys/devicestat.h>
	#include <sys/eventhandler.h>
	#include <sys/fail.h>
	#include <sys/ktr.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/mount.h>
	#include <sys/mutex.h>
	#include <sys/kernel.h>
	#include <sys/kthread.h>
	#include <sys/proc.h>
	#include <sys/racct.h>
	#include <sys/refcount.h>
	#include <sys/resourcevar.h>
	#include <sys/rwlock.h>
	#include <sys/smp.h>
	#include <sys/sysctl.h>
	#include <sys/syscallsubr.h>
	#include <sys/vmem.h>
	#include <sys/vmmeter.h>
	#include <sys/vnode.h>
	#include <sys/watchdog.h>
	#include <geom/geom.h>
	#include <vm/vm.h>
	#include <vm/vm_param.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pageout.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_map.h>
	#include <vm/swap_pager.h>

	static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");

	struct bio_ops bioops; /* I/O operation notification */

	struct buf_ops buf_ops_bio = {
	.bop_name = "buf_ops_bio",
	.bop_write = bufwrite,
	.bop_strategy = bufstrategy,
	.bop_sync = bufsync,
	.bop_bdflush = bufbdflush,
	};

	struct bufqueue {
	struct mtx_padalign bq_lock;
	TAILQ_HEAD(, buf) bq_queue;
	uint8_t bq_index;
	uint16_t bq_subqueue;
	int bq_len;
	} __aligned(CACHE_LINE_SIZE);

	#define BQ_LOCKPTR(bq) (&(bq)->bq_lock)
	#define BQ_LOCK(bq) mtx_lock(BQ_LOCKPTR((bq)))
	#define BQ_UNLOCK(bq) mtx_unlock(BQ_LOCKPTR((bq)))
	#define BQ_ASSERT_LOCKED(bq) mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)

	struct bufdomain {
	struct bufqueue bd_subq[MAXCPU + 1]; /* Per-cpu sub queues + global */
	struct bufqueue bd_dirtyq;
	struct bufqueue *bd_cleanq;
	struct mtx_padalign bd_run_lock;
	/* Constants */
	long bd_maxbufspace;
	long bd_hibufspace;
	long bd_lobufspace;
	long bd_bufspacethresh;
	int bd_hifreebuffers;
	int bd_lofreebuffers;
	int bd_hidirtybuffers;
	int bd_lodirtybuffers;
	int bd_dirtybufthresh;
	int bd_lim;
	/* atomics */
	int bd_wanted;
	int __aligned(CACHE_LINE_SIZE) bd_numdirtybuffers;
	int __aligned(CACHE_LINE_SIZE) bd_running;
	long __aligned(CACHE_LINE_SIZE) bd_bufspace;
	int __aligned(CACHE_LINE_SIZE) bd_freebuffers;
	} __aligned(CACHE_LINE_SIZE);

	#define BD_LOCKPTR(bd) (&(bd)->bd_cleanq->bq_lock)
	#define BD_LOCK(bd) mtx_lock(BD_LOCKPTR((bd)))
	#define BD_UNLOCK(bd) mtx_unlock(BD_LOCKPTR((bd)))
	#define BD_ASSERT_LOCKED(bd) mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)
	#define BD_RUN_LOCKPTR(bd) (&(bd)->bd_run_lock)
	#define BD_RUN_LOCK(bd) mtx_lock(BD_RUN_LOCKPTR((bd)))
	#define BD_RUN_UNLOCK(bd) mtx_unlock(BD_RUN_LOCKPTR((bd)))
	#define BD_DOMAIN(bd) (bd - bdomain)

	static struct buf buf; / buffer header pool */
	extern struct buf swbuf; / Swap buffer header pool. */
	caddr_t __read_mostly unmapped_buf;

	/* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
	struct proc *bufdaemonproc;

	static int inmem(struct vnode *vp, daddr_t blkno);
	static void vm_hold_free_pages(struct buf *bp, int newbsize);
	static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
	vm_offset_t to);
	static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
	static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
	vm_page_t m);
	static void vfs_clean_pages_dirty_buf(struct buf *bp);
	static void vfs_setdirty_range(struct buf *bp);
	static void vfs_vmio_invalidate(struct buf *bp);
	static void vfs_vmio_truncate(struct buf *bp, int npages);
	static void vfs_vmio_extend(struct buf *bp, int npages, int size);
	static int vfs_bio_clcheck(struct vnode *vp, int size,
	daddr_t lblkno, daddr_t blkno);
	static void breada(struct vnode , daddr_t , int , int, struct ucred , int,
	void ()(struct buf ));
	static int buf_flush(struct vnode vp, struct bufdomain , int);
	static int flushbufqueues(struct vnode , struct bufdomain , int, int);
	static void buf_daemon(void);
	static __inline void bd_wakeup(void);
	static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
	static void bufkva_reclaim(vmem_t *, int);
	static void bufkva_free(struct buf *);
	static int buf_import(void , void *, int, int, int);
	static void buf_release(void , void *, int);
	static void maxbcachebuf_adjust(void);
	static inline struct bufdomain bufdomain(struct buf );
	static void bq_remove(struct bufqueue bq, struct buf bp);
	static void bq_insert(struct bufqueue bq, struct buf bp, bool unlock);
	static int buf_recycle(struct bufdomain *, bool kva);
	static void bq_init(struct bufqueue *bq, int qindex, int cpu,
	const char *lockname);
	static void bd_init(struct bufdomain *bd);
	static int bd_flushall(struct bufdomain *bd);
	static int sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS);
	static int sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS);

	static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
	int vmiodirenable = TRUE;
	SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
	"Use the VM system for directory writes");
	long runningbufspace;
	SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
	"Amount of presently outstanding async buffer io");
	SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG\|CTLFLAG_MPSAFE\|CTLFLAG_RD,
	NULL, 0, sysctl_bufspace, "L", "Physical memory used for buffers");
	static counter_u64_t bufkvaspace;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace,
	"Kernel virtual memory used for buffers");
	static long maxbufspace;
	SYSCTL_PROC(_vfs, OID_AUTO, maxbufspace,
	CTLTYPE_LONG\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &maxbufspace,
	__offsetof(struct bufdomain, bd_maxbufspace), sysctl_bufdomain_long, "L",
	"Maximum allowed value of bufspace (including metadata)");
	static long bufmallocspace;
	SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
	"Amount of malloced memory for buffers");
	static long maxbufmallocspace;
	SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
	0, "Maximum amount of malloced memory for buffers");
	static long lobufspace;
	SYSCTL_PROC(_vfs, OID_AUTO, lobufspace,
	CTLTYPE_LONG\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &lobufspace,
	__offsetof(struct bufdomain, bd_lobufspace), sysctl_bufdomain_long, "L",
	"Minimum amount of buffers we want to have");
	long hibufspace;
	SYSCTL_PROC(_vfs, OID_AUTO, hibufspace,
	CTLTYPE_LONG\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &hibufspace,
	__offsetof(struct bufdomain, bd_hibufspace), sysctl_bufdomain_long, "L",
	"Maximum allowed value of bufspace (excluding metadata)");
	long bufspacethresh;
	SYSCTL_PROC(_vfs, OID_AUTO, bufspacethresh,
	CTLTYPE_LONG\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &bufspacethresh,
	__offsetof(struct bufdomain, bd_bufspacethresh), sysctl_bufdomain_long, "L",
	"Bufspace consumed before waking the daemon to free some");
	static counter_u64_t buffreekvacnt;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt,
	"Number of times we have freed the KVA space from some buffer");
	static counter_u64_t bufdefragcnt;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt,
	"Number of times we have had to repeat buffer allocation to defragment");
	static long lorunningspace;
	SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG \| CTLFLAG_MPSAFE \|
	CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
	"Minimum preferred space used for in-progress I/O");
	static long hirunningspace;
	SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG \| CTLFLAG_MPSAFE \|
	CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
	"Maximum amount of space to use for in-progress I/O");
	int dirtybufferflushes;
	SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
	0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
	int bdwriteskip;
	SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
	0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
	int altbufferflushes;
	SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW \| CTLFLAG_STATS,
	&altbufferflushes, 0, "Number of fsync flushes to limit dirty buffers");
	static int recursiveflushes;
	SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW \| CTLFLAG_STATS,
	&recursiveflushes, 0, "Number of flushes skipped due to being recursive");
	static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS);
	SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers,
	CTLTYPE_INT\|CTLFLAG_MPSAFE\|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I",
	"Number of buffers that are dirty (has unwritten changes) at the moment");
	static int lodirtybuffers;
	SYSCTL_PROC(_vfs, OID_AUTO, lodirtybuffers,
	CTLTYPE_INT\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &lodirtybuffers,
	__offsetof(struct bufdomain, bd_lodirtybuffers), sysctl_bufdomain_int, "I",
	"How many buffers we want to have free before bufdaemon can sleep");
	static int hidirtybuffers;
	SYSCTL_PROC(_vfs, OID_AUTO, hidirtybuffers,
	CTLTYPE_INT\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &hidirtybuffers,
	__offsetof(struct bufdomain, bd_hidirtybuffers), sysctl_bufdomain_int, "I",
	"When the number of dirty buffers is considered severe");
	int dirtybufthresh;
	SYSCTL_PROC(_vfs, OID_AUTO, dirtybufthresh,
	CTLTYPE_INT\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &dirtybufthresh,
	__offsetof(struct bufdomain, bd_dirtybufthresh), sysctl_bufdomain_int, "I",
	"Number of bdwrite to bawrite conversions to clear dirty buffers");
	static int numfreebuffers;
	SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
	"Number of free buffers");
	static int lofreebuffers;
	SYSCTL_PROC(_vfs, OID_AUTO, lofreebuffers,
	CTLTYPE_INT\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &lofreebuffers,
	__offsetof(struct bufdomain, bd_lofreebuffers), sysctl_bufdomain_int, "I",
	"Target number of free buffers");
	static int hifreebuffers;
	SYSCTL_PROC(_vfs, OID_AUTO, hifreebuffers,
	CTLTYPE_INT\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &hifreebuffers,
	__offsetof(struct bufdomain, bd_hifreebuffers), sysctl_bufdomain_int, "I",
	"Threshold for clean buffer recycling");
	static counter_u64_t getnewbufcalls;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD,
	&getnewbufcalls, "Number of calls to getnewbuf");
	static counter_u64_t getnewbufrestarts;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RD,
	&getnewbufrestarts,
	"Number of times getnewbuf has had to restart a buffer acquisition");
	static counter_u64_t mappingrestarts;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RD,
	&mappingrestarts,
	"Number of times getblk has had to restart a buffer mapping for "
	"unmapped buffer");
	static counter_u64_t numbufallocfails;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW,
	&numbufallocfails, "Number of times buffer allocations failed");
	static int flushbufqtarget = 100;
	SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
	"Amount of work to do in flushbufqueues when helping bufdaemon");
	static counter_u64_t notbufdflushes;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, &notbufdflushes,
	"Number of dirty buffer flushes done by the bufdaemon helpers");
	static long barrierwrites;
	SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW \| CTLFLAG_STATS,
	&barrierwrites, 0, "Number of barrier writes");
	SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed, CTLFLAG_RD,
	&unmapped_buf_allowed, 0,
	"Permit the use of the unmapped i/o");
	int maxbcachebuf = MAXBCACHEBUF;
	SYSCTL_INT(_vfs, OID_AUTO, maxbcachebuf, CTLFLAG_RDTUN, &maxbcachebuf, 0,
	"Maximum size of a buffer cache block");

	/*
	* This lock synchronizes access to bd_request.
	*/
	static struct mtx_padalign __exclusive_cache_line bdlock;

	/*
	* This lock protects the runningbufreq and synchronizes runningbufwakeup and
	* waitrunningbufspace().
	*/
	static struct mtx_padalign __exclusive_cache_line rbreqlock;

	/*
	* Lock that protects bdirtywait.
	*/
	static struct mtx_padalign __exclusive_cache_line bdirtylock;

	/*
	* Wakeup point for bufdaemon, as well as indicator of whether it is already
	* active. Set to 1 when the bufdaemon is already "on" the queue, 0 when it
	* is idling.
	*/
	static int bd_request;

	/*
	* Request for the buf daemon to write more buffers than is indicated by
	* lodirtybuf. This may be necessary to push out excess dependencies or
	* defragment the address space where a simple count of the number of dirty
	* buffers is insufficient to characterize the demand for flushing them.
	*/
	static int bd_speedupreq;

	/*
	* Synchronization (sleep/wakeup) variable for active buffer space requests.
	* Set when wait starts, cleared prior to wakeup().
	* Used in runningbufwakeup() and waitrunningbufspace().
	*/
	static int runningbufreq;

	/*
	* Synchronization for bwillwrite() waiters.
	*/
	static int bdirtywait;

	/*
	* Definitions for the buffer free lists.
	*/
	#define QUEUE_NONE 0 /* on no queue */
	#define QUEUE_EMPTY 1 /* empty buffer headers */
	#define QUEUE_DIRTY 2 /* B_DELWRI buffers */
	#define QUEUE_CLEAN 3 /* non-B_DELWRI buffers */
	#define QUEUE_SENTINEL 4 /* not an queue index, but mark for sentinel */

	/* Maximum number of buffer domains. */
	#define BUF_DOMAINS 8

	struct bufdomainset bdlodirty; /* Domains > lodirty */
	struct bufdomainset bdhidirty; /* Domains > hidirty */

	/* Configured number of clean queues. */
	static int __read_mostly buf_domains;

	BITSET_DEFINE(bufdomainset, BUF_DOMAINS);
	struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS];
	struct bufqueue __exclusive_cache_line bqempty;

	/*
	* per-cpu empty buffer cache.
	*/
	uma_zone_t buf_zone;

	/*
	* Single global constant for BUF_WMESG, to avoid getting multiple references.
	* buf_wmesg is referred from macros.
	*/
	const char *buf_wmesg = BUF_WMESG;

	static int
	sysctl_runningspace(SYSCTL_HANDLER_ARGS)
	{
	long value;
	int error;

	value = (long )arg1;
	error = sysctl_handle_long(oidp, &value, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	mtx_lock(&rbreqlock);
	if (arg1 == &hirunningspace) {
	if (value < lorunningspace)
	error = EINVAL;
	else
	hirunningspace = value;
	} else {
	KASSERT(arg1 == &lorunningspace,
	("%s: unknown arg1", __func__));
	if (value > hirunningspace)
	error = EINVAL;
	else
	lorunningspace = value;
	}
	mtx_unlock(&rbreqlock);
	return (error);
	}

	static int
	sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)
	{
	int error;
	int value;
	int i;

	value = (int )arg1;
	error = sysctl_handle_int(oidp, &value, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	(int )arg1 = value;
	for (i = 0; i < buf_domains; i++)
	(int )(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
	value / buf_domains;

	return (error);
	}

	static int
	sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)
	{
	long value;
	int error;
	int i;

	value = (long )arg1;
	error = sysctl_handle_long(oidp, &value, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	(long )arg1 = value;
	for (i = 0; i < buf_domains; i++)
	(long )(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
	value / buf_domains;

	return (error);
	}

	#if defined(COMPAT_FREEBSD4) \|\| defined(COMPAT_FREEBSD5) \|\| \
	defined(COMPAT_FREEBSD6) \|\| defined(COMPAT_FREEBSD7)
	static int
	sysctl_bufspace(SYSCTL_HANDLER_ARGS)
	{
	long lvalue;
	int ivalue;
	int i;

	lvalue = 0;
	for (i = 0; i < buf_domains; i++)
	lvalue += bdomain[i].bd_bufspace;
	if (sizeof(int) == sizeof(long) \|\| req->oldlen >= sizeof(long))
	return (sysctl_handle_long(oidp, &lvalue, 0, req));
	if (lvalue > INT_MAX)
	/* On overflow, still write out a long to trigger ENOMEM. */
	return (sysctl_handle_long(oidp, &lvalue, 0, req));
	ivalue = lvalue;
	return (sysctl_handle_int(oidp, &ivalue, 0, req));
	}
	#else
	static int
	sysctl_bufspace(SYSCTL_HANDLER_ARGS)
	{
	long lvalue;
	int i;

	lvalue = 0;
	for (i = 0; i < buf_domains; i++)
	lvalue += bdomain[i].bd_bufspace;
	return (sysctl_handle_long(oidp, &lvalue, 0, req));
	}
	#endif

	static int
	sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)
	{
	int value;
	int i;

	value = 0;
	for (i = 0; i < buf_domains; i++)
	value += bdomain[i].bd_numdirtybuffers;
	return (sysctl_handle_int(oidp, &value, 0, req));
	}

	/*
	* bdirtywakeup:
	*
	* Wakeup any bwillwrite() waiters.
	*/
	static void
	bdirtywakeup(void)
	{
	mtx_lock(&bdirtylock);
	if (bdirtywait) {
	bdirtywait = 0;
	wakeup(&bdirtywait);
	}
	mtx_unlock(&bdirtylock);
	}

	/*
	* bd_clear:
	*
	* Clear a domain from the appropriate bitsets when dirtybuffers
	* is decremented.
	*/
	static void
	bd_clear(struct bufdomain *bd)
	{

	mtx_lock(&bdirtylock);
	if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)
	BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
	if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)
	BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
	mtx_unlock(&bdirtylock);
	}

	/*
	* bd_set:
	*
	* Set a domain in the appropriate bitsets when dirtybuffers
	* is incremented.
	*/
	static void
	bd_set(struct bufdomain *bd)
	{

	mtx_lock(&bdirtylock);
	if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)
	BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
	if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)
	BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
	mtx_unlock(&bdirtylock);
	}

	/*
	* bdirtysub:
	*
	* Decrement the numdirtybuffers count by one and wakeup any
	* threads blocked in bwillwrite().
	*/
	static void
	bdirtysub(struct buf *bp)
	{
	struct bufdomain *bd;
	int num;

	bd = bufdomain(bp);
	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);
	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
	bdirtywakeup();
	if (num == bd->bd_lodirtybuffers \|\| num == bd->bd_hidirtybuffers)
	bd_clear(bd);
	}

	/*
	* bdirtyadd:
	*
	* Increment the numdirtybuffers count by one and wakeup the buf
	* daemon if needed.
	*/
	static void
	bdirtyadd(struct buf *bp)
	{
	struct bufdomain *bd;
	int num;

	/*
	* Only do the wakeup once as we cross the boundary. The
	* buf daemon will keep running until the condition clears.
	*/
	bd = bufdomain(bp);
	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);
	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
	bd_wakeup();
	if (num == bd->bd_lodirtybuffers \|\| num == bd->bd_hidirtybuffers)
	bd_set(bd);
	}

	/*
	* bufspace_daemon_wakeup:
	*
	* Wakeup the daemons responsible for freeing clean bufs.
	*/
	static void
	bufspace_daemon_wakeup(struct bufdomain *bd)
	{

	/*
	* avoid the lock if the daemon is running.
	*/
	if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {
	BD_RUN_LOCK(bd);
	atomic_store_int(&bd->bd_running, 1);
	wakeup(&bd->bd_running);
	BD_RUN_UNLOCK(bd);
	}
	}

	/*
	* bufspace_daemon_wait:
	*
	* Sleep until the domain falls below a limit or one second passes.
	*/
	static void
	bufspace_daemon_wait(struct bufdomain *bd)
	{
	/*
	* Re-check our limits and sleep. bd_running must be
	* cleared prior to checking the limits to avoid missed
	* wakeups. The waker will adjust one of bufspace or
	* freebuffers prior to checking bd_running.
	*/
	BD_RUN_LOCK(bd);
	atomic_store_int(&bd->bd_running, 0);
	if (bd->bd_bufspace < bd->bd_bufspacethresh &&
	bd->bd_freebuffers > bd->bd_lofreebuffers) {
	msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd), PRIBIO\|PDROP,
	"-", hz);
	} else {
	/* Avoid spurious wakeups while running. */
	atomic_store_int(&bd->bd_running, 1);
	BD_RUN_UNLOCK(bd);
	}
	}

	/*
	* bufspace_adjust:
	*
	* Adjust the reported bufspace for a KVA managed buffer, possibly
	* waking any waiters.
	*/
	static void
	bufspace_adjust(struct buf *bp, int bufsize)
	{
	struct bufdomain *bd;
	long space;
	int diff;

	KASSERT((bp->b_flags & B_MALLOC) == 0,
	("bufspace_adjust: malloc buf %p", bp));
	bd = bufdomain(bp);
	diff = bufsize - bp->b_bufsize;
	if (diff < 0) {
	atomic_subtract_long(&bd->bd_bufspace, -diff);
	} else if (diff > 0) {
	space = atomic_fetchadd_long(&bd->bd_bufspace, diff);
	/* Wake up the daemon on the transition. */
	if (space < bd->bd_bufspacethresh &&
	space + diff >= bd->bd_bufspacethresh)
	bufspace_daemon_wakeup(bd);
	}
	bp->b_bufsize = bufsize;
	}

	/*
	* bufspace_reserve:
	*
	* Reserve bufspace before calling allocbuf(). metadata has a
	* different space limit than data.
	*/
	static int
	bufspace_reserve(struct bufdomain *bd, int size, bool metadata)
	{
	long limit, new;
	long space;

	if (metadata)
	limit = bd->bd_maxbufspace;
	else
	limit = bd->bd_hibufspace;
	space = atomic_fetchadd_long(&bd->bd_bufspace, size);
	new = space + size;
	if (new > limit) {
	atomic_subtract_long(&bd->bd_bufspace, size);
	return (ENOSPC);
	}

	/* Wake up the daemon on the transition. */
	if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)
	bufspace_daemon_wakeup(bd);

	return (0);
	}

	/*
	* bufspace_release:
	*
	* Release reserved bufspace after bufspace_adjust() has consumed it.
	*/
	static void
	bufspace_release(struct bufdomain *bd, int size)
	{

	atomic_subtract_long(&bd->bd_bufspace, size);
	}

	/*
	* bufspace_wait:
	*
	* Wait for bufspace, acting as the buf daemon if a locked vnode is
	* supplied. bd_wanted must be set prior to polling for space. The
	* operation must be re-tried on return.
	*/
	static void
	bufspace_wait(struct bufdomain bd, struct vnode vp, int gbflags,
	int slpflag, int slptimeo)
	{
	struct thread *td;
	int error, fl, norunbuf;

	if ((gbflags & GB_NOWAIT_BD) != 0)
	return;

	td = curthread;
	BD_LOCK(bd);
	while (bd->bd_wanted) {
	if (vp != NULL && vp->v_type != VCHR &&
	(td->td_pflags & TDP_BUFNEED) == 0) {
	BD_UNLOCK(bd);
	/*
	* getblk() is called with a vnode locked, and
	* some majority of the dirty buffers may as
	* well belong to the vnode. Flushing the
	* buffers there would make a progress that
	* cannot be achieved by the buf_daemon, that
	* cannot lock the vnode.
	*/
	norunbuf = ~(TDP_BUFNEED \| TDP_NORUNNINGBUF) \|
	(td->td_pflags & TDP_NORUNNINGBUF);

	/*
	* Play bufdaemon. The getnewbuf() function
	* may be called while the thread owns lock
	* for another dirty buffer for the same
	* vnode, which makes it impossible to use
	* VOP_FSYNC() there, due to the buffer lock
	* recursion.
	*/
	td->td_pflags \|= TDP_BUFNEED \| TDP_NORUNNINGBUF;
	fl = buf_flush(vp, bd, flushbufqtarget);
	td->td_pflags &= norunbuf;
	BD_LOCK(bd);
	if (fl != 0)
	continue;
	if (bd->bd_wanted == 0)
	break;
	}
	error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
	(PRIBIO + 4) \| slpflag, "newbuf", slptimeo);
	if (error != 0)
	break;
	}
	BD_UNLOCK(bd);
	}

	/*
	* bufspace_daemon:
	*
	* buffer space management daemon. Tries to maintain some marginal
	* amount of free buffer space so that requesting processes neither
	* block nor work to reclaim buffers.
	*/
	static void
	bufspace_daemon(void *arg)
	{
	struct bufdomain *bd;

	EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
	SHUTDOWN_PRI_LAST + 100);

	bd = arg;
	for (;;) {
	kthread_suspend_check();

	/*
	* Free buffers from the clean queue until we meet our
	* targets.
	*
	* Theory of operation: The buffer cache is most efficient
	* when some free buffer headers and space are always
	* available to getnewbuf(). This daemon attempts to prevent
	* the excessive blocking and synchronization associated
	* with shortfall. It goes through three phases according
	* demand:
	*
	* 1) The daemon wakes up voluntarily once per-second
	* during idle periods when the counters are below
	* the wakeup thresholds (bufspacethresh, lofreebuffers).
	*
	* 2) The daemon wakes up as we cross the thresholds
	* ahead of any potential blocking. This may bounce
	* slightly according to the rate of consumption and
	* release.
	*
	* 3) The daemon and consumers are starved for working
	* clean buffers. This is the 'bufspace' sleep below
	* which will inefficiently trade bufs with bqrelse
	* until we return to condition 2.
	*/
	while (bd->bd_bufspace > bd->bd_lobufspace \|\|
	bd->bd_freebuffers < bd->bd_hifreebuffers) {
	if (buf_recycle(bd, false) != 0) {
	if (bd_flushall(bd))
	continue;
	/*
	* Speedup dirty if we've run out of clean
	* buffers. This is possible in particular
	* because softdep may held many bufs locked
	* pending writes to other bufs which are
	* marked for delayed write, exhausting
	* clean space until they are written.
	*/
	bd_speedup();
	BD_LOCK(bd);
	if (bd->bd_wanted) {
	msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
	PRIBIO\|PDROP, "bufspace", hz/10);
	} else
	BD_UNLOCK(bd);
	}
	maybe_yield();
	}
	bufspace_daemon_wait(bd);
	}
	}

	/*
	* bufmallocadjust:
	*
	* Adjust the reported bufspace for a malloc managed buffer, possibly
	* waking any waiters.
	*/
	static void
	bufmallocadjust(struct buf *bp, int bufsize)
	{
	int diff;

	KASSERT((bp->b_flags & B_MALLOC) != 0,
	("bufmallocadjust: non-malloc buf %p", bp));
	diff = bufsize - bp->b_bufsize;
	if (diff < 0)
	atomic_subtract_long(&bufmallocspace, -diff);
	else
	atomic_add_long(&bufmallocspace, diff);
	bp->b_bufsize = bufsize;
	}

	/*
	* runningwakeup:
	*
	* Wake up processes that are waiting on asynchronous writes to fall
	* below lorunningspace.
	*/
	static void
	runningwakeup(void)
	{

	mtx_lock(&rbreqlock);
	if (runningbufreq) {
	runningbufreq = 0;
	wakeup(&runningbufreq);
	}
	mtx_unlock(&rbreqlock);
	}

	/*
	* runningbufwakeup:
	*
	* Decrement the outstanding write count according.
	*/
	void
	runningbufwakeup(struct buf *bp)
	{
	long space, bspace;

	bspace = bp->b_runningbufspace;
	if (bspace == 0)
	return;
	space = atomic_fetchadd_long(&runningbufspace, -bspace);
	KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
	space, bspace));
	bp->b_runningbufspace = 0;
	/*
	* Only acquire the lock and wakeup on the transition from exceeding
	* the threshold to falling below it.
	*/
	if (space < lorunningspace)
	return;
	if (space - bspace > lorunningspace)
	return;
	runningwakeup();
	}

	/*
	* waitrunningbufspace()
	*
	* runningbufspace is a measure of the amount of I/O currently
	* running. This routine is used in async-write situations to
	* prevent creating huge backups of pending writes to a device.
	* Only asynchronous writes are governed by this function.
	*
	* This does NOT turn an async write into a sync write. It waits
	* for earlier writes to complete and generally returns before the
	* caller's write has reached the device.
	*/
	void
	waitrunningbufspace(void)
	{

	mtx_lock(&rbreqlock);
	while (runningbufspace > hirunningspace) {
	runningbufreq = 1;
	msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
	}
	mtx_unlock(&rbreqlock);
	}

	/*
	* vfs_buf_test_cache:
	*
	* Called when a buffer is extended. This function clears the B_CACHE
	* bit if the newly extended portion of the buffer does not contain
	* valid data.
	*/
	static __inline void
	vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
	vm_offset_t size, vm_page_t m)
	{

	/*
	* This function and its results are protected by higher level
	* synchronization requiring vnode and buf locks to page in and
	* validate pages.
	*/
	if (bp->b_flags & B_CACHE) {
	int base = (foff + off) & PAGE_MASK;
	if (vm_page_is_valid(m, base, size) == 0)
	bp->b_flags &= ~B_CACHE;
	}
	}

	/* Wake up the buffer daemon if necessary */
	static void
	bd_wakeup(void)
	{

	mtx_lock(&bdlock);
	if (bd_request == 0) {
	bd_request = 1;
	wakeup(&bd_request);
	}
	mtx_unlock(&bdlock);
	}

	/*
	* Adjust the maxbcachbuf tunable.
	*/
	static void
	maxbcachebuf_adjust(void)
	{
	int i;

	/*
	* maxbcachebuf must be a power of 2 >= MAXBSIZE.
	*/
	i = 2;
	while (i * 2 <= maxbcachebuf)
	i *= 2;
	maxbcachebuf = i;
	if (maxbcachebuf < MAXBSIZE)
	maxbcachebuf = MAXBSIZE;
	if (maxbcachebuf > MAXPHYS)
	maxbcachebuf = MAXPHYS;
	if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF)
	printf("maxbcachebuf=%d\n", maxbcachebuf);
	}

	/*
	* bd_speedup - speedup the buffer cache flushing code
	*/
	void
	bd_speedup(void)
	{
	int needwake;

	mtx_lock(&bdlock);
	needwake = 0;
	if (bd_speedupreq == 0 \|\| bd_request == 0)
	needwake = 1;
	bd_speedupreq = 1;
	bd_request = 1;
	if (needwake)
	wakeup(&bd_request);
	mtx_unlock(&bdlock);
	}

	#ifdef __i386__
	#define TRANSIENT_DENOM 5
	#else
	#define TRANSIENT_DENOM 10
	#endif

	/*
	* Calculating buffer cache scaling values and reserve space for buffer
	* headers. This is called during low level kernel initialization and
	* may be called more then once. We CANNOT write to the memory area
	* being reserved at this time.
	*/
	caddr_t
	kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est)
	{
	int tuned_nbuf;
	long maxbuf, maxbuf_sz, buf_sz, biotmap_sz;

	/*
	* physmem_est is in pages. Convert it to kilobytes (assumes
	* PAGE_SIZE is >= 1K)
	*/
	physmem_est = physmem_est * (PAGE_SIZE / 1024);

	maxbcachebuf_adjust();
	/*
	* The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
	* For the first 64MB of ram nominally allocate sufficient buffers to
	* cover 1/4 of our ram. Beyond the first 64MB allocate additional
	* buffers to cover 1/10 of our ram over 64MB. When auto-sizing
	* the buffer cache we limit the eventual kva reservation to
	* maxbcache bytes.
	*
	* factor represents the 1/4 x ram conversion.
	*/
	if (nbuf == 0) {
	int factor = 4 * BKVASIZE / 1024;

	nbuf = 50;
	if (physmem_est > 4096)
	nbuf += min((physmem_est - 4096) / factor,
	65536 / factor);
	if (physmem_est > 65536)
	nbuf += min((physmem_est - 65536) * 2 / (factor * 5),
	32 * 1024 * 1024 / (factor * 5));

	if (maxbcache && nbuf > maxbcache / BKVASIZE)
	nbuf = maxbcache / BKVASIZE;
	tuned_nbuf = 1;
	} else
	tuned_nbuf = 0;

	/* XXX Avoid unsigned long overflows later on with maxbufspace. */
	maxbuf = (LONG_MAX / 3) / BKVASIZE;
	if (nbuf > maxbuf) {
	if (!tuned_nbuf)
	printf("Warning: nbufs lowered from %d to %ld\n", nbuf,
	maxbuf);
	nbuf = maxbuf;
	}

	/*
	* Ideal allocation size for the transient bio submap is 10%
	* of the maximal space buffer map. This roughly corresponds
	* to the amount of the buffer mapped for typical UFS load.
	*
	* Clip the buffer map to reserve space for the transient
	* BIOs, if its extent is bigger than 90% (80% on i386) of the
	* maximum buffer map extent on the platform.
	*
	* The fall-back to the maxbuf in case of maxbcache unset,
	* allows to not trim the buffer KVA for the architectures
	* with ample KVA space.
	*/
	if (bio_transient_maxcnt == 0 && unmapped_buf_allowed) {
	maxbuf_sz = maxbcache != 0 ? maxbcache : maxbuf * BKVASIZE;
	buf_sz = (long)nbuf * BKVASIZE;
	if (buf_sz < maxbuf_sz / TRANSIENT_DENOM *
	(TRANSIENT_DENOM - 1)) {
	/*
	* There is more KVA than memory. Do not
	* adjust buffer map size, and assign the rest
	* of maxbuf to transient map.
	*/
	biotmap_sz = maxbuf_sz - buf_sz;
	} else {
	/*
	* Buffer map spans all KVA we could afford on
	* this platform. Give 10% (20% on i386) of
	* the buffer map to the transient bio map.
	*/
	biotmap_sz = buf_sz / TRANSIENT_DENOM;
	buf_sz -= biotmap_sz;
	}
	if (biotmap_sz / INT_MAX > MAXPHYS)
	bio_transient_maxcnt = INT_MAX;
	else
	bio_transient_maxcnt = biotmap_sz / MAXPHYS;
	/*
	* Artificially limit to 1024 simultaneous in-flight I/Os
	* using the transient mapping.
	*/
	if (bio_transient_maxcnt > 1024)
	bio_transient_maxcnt = 1024;
	if (tuned_nbuf)
	nbuf = buf_sz / BKVASIZE;
	}

	if (nswbuf == 0) {
	nswbuf = min(nbuf / 4, 256);
	if (nswbuf < NSWBUF_MIN)
	nswbuf = NSWBUF_MIN;
	}

	/*
	* Reserve space for the buffer cache buffers
	*/
	buf = (void *)v;
	v = (caddr_t)(buf + nbuf);

	return(v);
	}

	/* Initialize the buffer subsystem. Called before use of any buffers. */
	void
	bufinit(void)
	{
	struct buf *bp;
	int i;

	KASSERT(maxbcachebuf >= MAXBSIZE,
	("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf,
	MAXBSIZE));
	bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock");
	mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
	mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
	mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF);

	unmapped_buf = (caddr_t)kva_alloc(MAXPHYS);

	/* finally, initialize each buffer header and stick on empty q */
	for (i = 0; i < nbuf; i++) {
	bp = &buf[i];
	bzero(bp, sizeof *bp);
	bp->b_flags = B_INVAL;
	bp->b_rcred = NOCRED;
	bp->b_wcred = NOCRED;
	bp->b_qindex = QUEUE_NONE;
	bp->b_domain = -1;
	bp->b_subqueue = mp_maxid + 1;
	bp->b_xflags = 0;
	bp->b_data = bp->b_kvabase = unmapped_buf;
	LIST_INIT(&bp->b_dep);
	BUF_LOCKINIT(bp);
	bq_insert(&bqempty, bp, false);
	}

	/*
	* maxbufspace is the absolute maximum amount of buffer space we are
	* allowed to reserve in KVM and in real terms. The absolute maximum
	* is nominally used by metadata. hibufspace is the nominal maximum
	* used by most other requests. The differential is required to
	* ensure that metadata deadlocks don't occur.
	*
	* maxbufspace is based on BKVASIZE. Allocating buffers larger then
	* this may result in KVM fragmentation which is not handled optimally
	* by the system. XXX This is less true with vmem. We could use
	* PAGE_SIZE.
	*/
	maxbufspace = (long)nbuf * BKVASIZE;
	hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - maxbcachebuf * 10);
	lobufspace = (hibufspace / 20) * 19; /* 95% */
	bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;

	/*
	* Note: The 16 MiB upper limit for hirunningspace was chosen
	* arbitrarily and may need further tuning. It corresponds to
	* 128 outstanding write IO requests (if IO size is 128 KiB),
	* which fits with many RAID controllers' tagged queuing limits.
	* The lower 1 MiB limit is the historical upper limit for
	* hirunningspace.
	*/
	hirunningspace = lmax(lmin(roundup(hibufspace / 64, maxbcachebuf),
	16 * 1024 * 1024), 1024 * 1024);
	lorunningspace = roundup((hirunningspace * 2) / 3, maxbcachebuf);

	/*
	* Limit the amount of malloc memory since it is wired permanently into
	* the kernel space. Even though this is accounted for in the buffer
	* allocation, we don't want the malloced region to grow uncontrolled.
	* The malloc scheme improves memory utilization significantly on
	* average (small) directories.
	*/
	maxbufmallocspace = hibufspace / 20;

	/*
	* Reduce the chance of a deadlock occurring by limiting the number
	* of delayed-write dirty buffers we allow to stack up.
	*/
	hidirtybuffers = nbuf / 4 + 20;
	dirtybufthresh = hidirtybuffers * 9 / 10;
	/*
	* To support extreme low-memory systems, make sure hidirtybuffers
	* cannot eat up all available buffer space. This occurs when our
	* minimum cannot be met. We try to size hidirtybuffers to 3/4 our
	* buffer space assuming BKVASIZE'd buffers.
	*/
	while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
	hidirtybuffers >>= 1;
	}
	lodirtybuffers = hidirtybuffers / 2;

	/*
	* lofreebuffers should be sufficient to avoid stalling waiting on
	* buf headers under heavy utilization. The bufs in per-cpu caches
	* are counted as free but will be unavailable to threads executing
	* on other cpus.
	*
	* hifreebuffers is the free target for the bufspace daemon. This
	* should be set appropriately to limit work per-iteration.
	*/
	lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
	hifreebuffers = (3 * lofreebuffers) / 2;
	numfreebuffers = nbuf;

	/* Setup the kva and free list allocators. */
	vmem_set_reclaim(buffer_arena, bufkva_reclaim);
	buf_zone = uma_zcache_create("buf free cache", sizeof(struct buf),
	NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);

	/*
	* Size the clean queue according to the amount of buffer space.
	* One queue per-256mb up to the max. More queues gives better
	* concurrency but less accurate LRU.
	*/
	buf_domains = MIN(howmany(maxbufspace, 25610241024), BUF_DOMAINS);
	for (i = 0 ; i < buf_domains; i++) {
	struct bufdomain *bd;

	bd = &bdomain[i];
	bd_init(bd);
	bd->bd_freebuffers = nbuf / buf_domains;
	bd->bd_hifreebuffers = hifreebuffers / buf_domains;
	bd->bd_lofreebuffers = lofreebuffers / buf_domains;
	bd->bd_bufspace = 0;
	bd->bd_maxbufspace = maxbufspace / buf_domains;
	bd->bd_hibufspace = hibufspace / buf_domains;
	bd->bd_lobufspace = lobufspace / buf_domains;
	bd->bd_bufspacethresh = bufspacethresh / buf_domains;
	bd->bd_numdirtybuffers = 0;
	bd->bd_hidirtybuffers = hidirtybuffers / buf_domains;
	bd->bd_lodirtybuffers = lodirtybuffers / buf_domains;
	bd->bd_dirtybufthresh = dirtybufthresh / buf_domains;
	/* Don't allow more than 2% of bufs in the per-cpu caches. */
	bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus;
	}
	getnewbufcalls = counter_u64_alloc(M_WAITOK);
	getnewbufrestarts = counter_u64_alloc(M_WAITOK);
	mappingrestarts = counter_u64_alloc(M_WAITOK);
	numbufallocfails = counter_u64_alloc(M_WAITOK);
	notbufdflushes = counter_u64_alloc(M_WAITOK);
	buffreekvacnt = counter_u64_alloc(M_WAITOK);
	bufdefragcnt = counter_u64_alloc(M_WAITOK);
	bufkvaspace = counter_u64_alloc(M_WAITOK);
	}

	#ifdef INVARIANTS
	static inline void
	vfs_buf_check_mapped(struct buf *bp)
	{

	KASSERT(bp->b_kvabase != unmapped_buf,
	("mapped buf: b_kvabase was not updated %p", bp));
	KASSERT(bp->b_data != unmapped_buf,
	("mapped buf: b_data was not updated %p", bp));
	KASSERT(bp->b_data < unmapped_buf \|\| bp->b_data >= unmapped_buf +
	MAXPHYS, ("b_data + b_offset unmapped %p", bp));
	}

	static inline void
	vfs_buf_check_unmapped(struct buf *bp)
	{

	KASSERT(bp->b_data == unmapped_buf,
	("unmapped buf: corrupted b_data %p", bp));
	}

	#define BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
	#define BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
	#else
	#define BUF_CHECK_MAPPED(bp) do {} while (0)
	#define BUF_CHECK_UNMAPPED(bp) do {} while (0)
	#endif

	static int
	isbufbusy(struct buf *bp)
	{
	if (((bp->b_flags & B_INVAL) == 0 && BUF_ISLOCKED(bp)) \|\|
	((bp->b_flags & (B_DELWRI \| B_INVAL)) == B_DELWRI))
	return (1);
	return (0);
	}

	/*
	* Shutdown the system cleanly to prepare for reboot, halt, or power off.
	*/
	void
	bufshutdown(int show_busybufs)
	{
	static int first_buf_printf = 1;
	struct buf *bp;
	int iter, nbusy, pbusy;
	#ifndef PREEMPTION
	int subiter;
	#endif

	/*
	* Sync filesystems for shutdown
	*/
	wdog_kern_pat(WD_LASTVAL);
	kern_sync(curthread);

	/*
	* With soft updates, some buffers that are
	* written will be remarked as dirty until other
	* buffers are written.
	*/
	for (iter = pbusy = 0; iter < 20; iter++) {
	nbusy = 0;
	for (bp = &buf[nbuf]; --bp >= buf; )
	if (isbufbusy(bp))
	nbusy++;
	if (nbusy == 0) {
	if (first_buf_printf)
	printf("All buffers synced.");
	break;
	}
	if (first_buf_printf) {
	printf("Syncing disks, buffers remaining... ");
	first_buf_printf = 0;
	}
	printf("%d ", nbusy);
	if (nbusy < pbusy)
	iter = 0;
	pbusy = nbusy;

	wdog_kern_pat(WD_LASTVAL);
	kern_sync(curthread);

	#ifdef PREEMPTION
	/*
	* Spin for a while to allow interrupt threads to run.
	*/
	DELAY(50000 * iter);
	#else
	/*
	* Context switch several times to allow interrupt
	* threads to run.
	*/
	for (subiter = 0; subiter < 50 * iter; subiter++) {
	thread_lock(curthread);
	mi_switch(SW_VOL);
	DELAY(1000);
	}
	#endif
	}
	printf("\n");
	/*
	* Count only busy local buffers to prevent forcing
	* a fsck if we're just a client of a wedged NFS server
	*/
	nbusy = 0;
	for (bp = &buf[nbuf]; --bp >= buf; ) {
	if (isbufbusy(bp)) {
	#if 0
	/* XXX: This is bogus. We should probably have a BO_REMOTE flag instead */
	if (bp->b_dev == NULL) {
	TAILQ_REMOVE(&mountlist,
	bp->b_vp->v_mount, mnt_list);
	continue;
	}
	#endif
	nbusy++;
	if (show_busybufs > 0) {
	printf(
	"%d: buf:%p, vnode:%p, flags:%0x, blkno:%jd, lblkno:%jd, buflock:",
	nbusy, bp, bp->b_vp, bp->b_flags,
	(intmax_t)bp->b_blkno,
	(intmax_t)bp->b_lblkno);
	BUF_LOCKPRINTINFO(bp);
	if (show_busybufs > 1)
	vn_printf(bp->b_vp,
	"vnode content: ");
	}
	}
	}
	if (nbusy) {
	/*
	* Failed to sync all blocks. Indicate this and don't
	* unmount filesystems (thus forcing an fsck on reboot).
	*/
	printf("Giving up on %d buffers\n", nbusy);
	DELAY(5000000); /* 5 seconds */
	} else {
	if (!first_buf_printf)
	printf("Final sync complete\n");
	/*
	* Unmount filesystems
	*/
	if (!KERNEL_PANICKED())
	vfs_unmountall();
	}
	swapoff_all();
	DELAY(100000); /* wait for console output to finish */
	}

	static void
	bpmap_qenter(struct buf *bp)
	{

	BUF_CHECK_MAPPED(bp);

	/*
	* bp->b_data is relative to bp->b_offset, but
	* bp->b_offset may be offset into the first page.
	*/
	bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
	pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
	bp->b_data = (caddr_t)((vm_offset_t)bp->b_data \|
	(vm_offset_t)(bp->b_offset & PAGE_MASK));
	}

	static inline struct bufdomain *
	bufdomain(struct buf *bp)
	{

	return (&bdomain[bp->b_domain]);
	}

	static struct bufqueue *
	bufqueue(struct buf *bp)
	{

	switch (bp->b_qindex) {
	case QUEUE_NONE:
	/* FALLTHROUGH */
	case QUEUE_SENTINEL:
	return (NULL);
	case QUEUE_EMPTY:
	return (&bqempty);
	case QUEUE_DIRTY:
	return (&bufdomain(bp)->bd_dirtyq);
	case QUEUE_CLEAN:
	return (&bufdomain(bp)->bd_subq[bp->b_subqueue]);
	default:
	break;
	}
	panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex);
	}

	/*
	* Return the locked bufqueue that bp is a member of.
	*/
	static struct bufqueue *
	bufqueue_acquire(struct buf *bp)
	{
	struct bufqueue bq, nbq;

	/*
	* bp can be pushed from a per-cpu queue to the
	* cleanq while we're waiting on the lock. Retry
	* if the queues don't match.
	*/
	bq = bufqueue(bp);
	BQ_LOCK(bq);
	for (;;) {
	nbq = bufqueue(bp);
	if (bq == nbq)
	break;
	BQ_UNLOCK(bq);
	BQ_LOCK(nbq);
	bq = nbq;
	}
	return (bq);
	}

	/*
	* binsfree:
	*
	* Insert the buffer into the appropriate free list. Requires a
	* locked buffer on entry and buffer is unlocked before return.
	*/
	static void
	binsfree(struct buf *bp, int qindex)
	{
	struct bufdomain *bd;
	struct bufqueue *bq;

	KASSERT(qindex == QUEUE_CLEAN \|\| qindex == QUEUE_DIRTY,
	("binsfree: Invalid qindex %d", qindex));
	BUF_ASSERT_XLOCKED(bp);

	/*
	* Handle delayed bremfree() processing.
	*/
	if (bp->b_flags & B_REMFREE) {
	if (bp->b_qindex == qindex) {
	bp->b_flags \|= B_REUSE;
	bp->b_flags &= ~B_REMFREE;
	BUF_UNLOCK(bp);
	return;
	}
	bq = bufqueue_acquire(bp);
	bq_remove(bq, bp);
	BQ_UNLOCK(bq);
	}
	bd = bufdomain(bp);
	if (qindex == QUEUE_CLEAN) {
	if (bd->bd_lim != 0)
	bq = &bd->bd_subq[PCPU_GET(cpuid)];
	else
	bq = bd->bd_cleanq;
	} else
	bq = &bd->bd_dirtyq;
	bq_insert(bq, bp, true);
	}

	/*
	* buf_free:
	*
	* Free a buffer to the buf zone once it no longer has valid contents.
	*/
	static void
	buf_free(struct buf *bp)
	{

	if (bp->b_flags & B_REMFREE)
	bremfreef(bp);
	if (bp->b_vflags & BV_BKGRDINPROG)
	panic("losing buffer 1");
	if (bp->b_rcred != NOCRED) {
	crfree(bp->b_rcred);
	bp->b_rcred = NOCRED;
	}
	if (bp->b_wcred != NOCRED) {
	crfree(bp->b_wcred);
	bp->b_wcred = NOCRED;
	}
	if (!LIST_EMPTY(&bp->b_dep))
	buf_deallocate(bp);
	bufkva_free(bp);
	atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1);
	BUF_UNLOCK(bp);
	uma_zfree(buf_zone, bp);
	}

	/*
	* buf_import:
	*
	* Import bufs into the uma cache from the buf list. The system still
	* expects a static array of bufs and much of the synchronization
	* around bufs assumes type stable storage. As a result, UMA is used
	* only as a per-cpu cache of bufs still maintained on a global list.
	*/
	static int
	buf_import(void arg, void *store, int cnt, int domain, int flags)
	{
	struct buf *bp;
	int i;

	BQ_LOCK(&bqempty);
	for (i = 0; i < cnt; i++) {
	bp = TAILQ_FIRST(&bqempty.bq_queue);
	if (bp == NULL)
	break;
	bq_remove(&bqempty, bp);
	store[i] = bp;
	}
	BQ_UNLOCK(&bqempty);

	return (i);
	}

	/*
	* buf_release:
	*
	* Release bufs from the uma cache back to the buffer queues.
	*/
	static void
	buf_release(void arg, void *store, int cnt)
	{
	struct bufqueue *bq;
	struct buf *bp;
	int i;

	bq = &bqempty;
	BQ_LOCK(bq);
	for (i = 0; i < cnt; i++) {
	bp = store[i];
	/* Inline bq_insert() to batch locking. */
	TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
	bp->b_flags &= ~(B_AGE \| B_REUSE);
	bq->bq_len++;
	bp->b_qindex = bq->bq_index;
	}
	BQ_UNLOCK(bq);
	}

	/*
	* buf_alloc:
	*
	* Allocate an empty buffer header.
	*/
	static struct buf *
	buf_alloc(struct bufdomain *bd)
	{
	struct buf *bp;
	int freebufs;

	/*
	* We can only run out of bufs in the buf zone if the average buf
	* is less than BKVASIZE. In this case the actual wait/block will
	* come from buf_reycle() failing to flush one of these small bufs.
	*/
	bp = NULL;
	freebufs = atomic_fetchadd_int(&bd->bd_freebuffers, -1);
	if (freebufs > 0)
	bp = uma_zalloc(buf_zone, M_NOWAIT);
	if (bp == NULL) {
	atomic_add_int(&bd->bd_freebuffers, 1);
	bufspace_daemon_wakeup(bd);
	counter_u64_add(numbufallocfails, 1);
	return (NULL);
	}
	/*
	* Wake-up the bufspace daemon on transition below threshold.
	*/
	if (freebufs == bd->bd_lofreebuffers)
	bufspace_daemon_wakeup(bd);

	if (BUF_LOCK(bp, LK_EXCLUSIVE \| LK_NOWAIT, NULL) != 0)
	panic("getnewbuf_empty: Locked buf %p on free queue.", bp);

	KASSERT(bp->b_vp == NULL,
	("bp: %p still has vnode %p.", bp, bp->b_vp));
	KASSERT((bp->b_flags & (B_DELWRI \| B_NOREUSE)) == 0,
	("invalid buffer %p flags %#x", bp, bp->b_flags));
	KASSERT((bp->b_xflags & (BX_VNCLEAN\|BX_VNDIRTY)) == 0,
	("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
	KASSERT(bp->b_npages == 0,
	("bp: %p still has %d vm pages\n", bp, bp->b_npages));
	KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
	KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));

	bp->b_domain = BD_DOMAIN(bd);
	bp->b_flags = 0;
	bp->b_ioflags = 0;
	bp->b_xflags = 0;
	bp->b_vflags = 0;
	bp->b_vp = NULL;
	bp->b_blkno = bp->b_lblkno = 0;
	bp->b_offset = NOOFFSET;
	bp->b_iodone = 0;
	bp->b_error = 0;
	bp->b_resid = 0;
	bp->b_bcount = 0;
	bp->b_npages = 0;
	bp->b_dirtyoff = bp->b_dirtyend = 0;
	bp->b_bufobj = NULL;
	bp->b_data = bp->b_kvabase = unmapped_buf;
	bp->b_fsprivate1 = NULL;
	bp->b_fsprivate2 = NULL;
	bp->b_fsprivate3 = NULL;
	LIST_INIT(&bp->b_dep);

	return (bp);
	}

	/*
	* buf_recycle:
	*
	* Free a buffer from the given bufqueue. kva controls whether the
	* freed buf must own some kva resources. This is used for
	* defragmenting.
	*/
	static int
	buf_recycle(struct bufdomain *bd, bool kva)
	{
	struct bufqueue *bq;
	struct buf bp, nbp;

	if (kva)
	counter_u64_add(bufdefragcnt, 1);
	nbp = NULL;
	bq = bd->bd_cleanq;
	BQ_LOCK(bq);
	KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),
	("buf_recycle: Locks don't match"));
	nbp = TAILQ_FIRST(&bq->bq_queue);

	/*
	* Run scan, possibly freeing data and/or kva mappings on the fly
	* depending.
	*/
	while ((bp = nbp) != NULL) {
	/*
	* Calculate next bp (we can only use it if we do not
	* release the bqlock).
	*/
	nbp = TAILQ_NEXT(bp, b_freelist);

	/*
	* If we are defragging then we need a buffer with
	* some kva to reclaim.
	*/
	if (kva && bp->b_kvasize == 0)
	continue;

	if (BUF_LOCK(bp, LK_EXCLUSIVE \| LK_NOWAIT, NULL) != 0)
	continue;

	/*
	* Implement a second chance algorithm for frequently
	* accessed buffers.
	*/
	if ((bp->b_flags & B_REUSE) != 0) {
	TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
	TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
	bp->b_flags &= ~B_REUSE;
	BUF_UNLOCK(bp);
	continue;
	}

	/*
	* Skip buffers with background writes in progress.
	*/
	if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
	BUF_UNLOCK(bp);
	continue;
	}

	KASSERT(bp->b_qindex == QUEUE_CLEAN,
	("buf_recycle: inconsistent queue %d bp %p",
	bp->b_qindex, bp));
	KASSERT(bp->b_domain == BD_DOMAIN(bd),
	("getnewbuf: queue domain %d doesn't match request %d",
	bp->b_domain, (int)BD_DOMAIN(bd)));
	/*
	* NOTE: nbp is now entirely invalid. We can only restart
	* the scan from this point on.
	*/
	bq_remove(bq, bp);
	BQ_UNLOCK(bq);

	/*
	* Requeue the background write buffer with error and
	* restart the scan.
	*/
	if ((bp->b_vflags & BV_BKGRDERR) != 0) {
	bqrelse(bp);
	BQ_LOCK(bq);
	nbp = TAILQ_FIRST(&bq->bq_queue);
	continue;
	}
	bp->b_flags \|= B_INVAL;
	brelse(bp);
	return (0);
	}
	bd->bd_wanted = 1;
	BQ_UNLOCK(bq);

	return (ENOBUFS);
	}

	/*
	* bremfree:
	*
	* Mark the buffer for removal from the appropriate free list.
	*
	*/
	void
	bremfree(struct buf *bp)
	{

	CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	KASSERT((bp->b_flags & B_REMFREE) == 0,
	("bremfree: buffer %p already marked for delayed removal.", bp));
	KASSERT(bp->b_qindex != QUEUE_NONE,
	("bremfree: buffer %p not on a queue.", bp));
	BUF_ASSERT_XLOCKED(bp);

	bp->b_flags \|= B_REMFREE;
	}

	/*
	* bremfreef:
	*
	* Force an immediate removal from a free list. Used only in nfs when
	* it abuses the b_freelist pointer.
	*/
	void
	bremfreef(struct buf *bp)
	{
	struct bufqueue *bq;

	bq = bufqueue_acquire(bp);
	bq_remove(bq, bp);
	BQ_UNLOCK(bq);
	}

	static void
	bq_init(struct bufqueue bq, int qindex, int subqueue, const char lockname)
	{

	mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF);
	TAILQ_INIT(&bq->bq_queue);
	bq->bq_len = 0;
	bq->bq_index = qindex;
	bq->bq_subqueue = subqueue;
	}

	static void
	bd_init(struct bufdomain *bd)
	{
	int i;

	bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1];
	bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock");
	bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock");
	for (i = 0; i <= mp_maxid; i++)
	bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i,
	"bufq clean subqueue lock");
	mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF);
	}

	/*
	* bq_remove:
	*
	* Removes a buffer from the free list, must be called with the
	* correct qlock held.
	*/
	static void
	bq_remove(struct bufqueue bq, struct buf bp)
	{

	CTR3(KTR_BUF, "bq_remove(%p) vp %p flags %X",
	bp, bp->b_vp, bp->b_flags);
	KASSERT(bp->b_qindex != QUEUE_NONE,
	("bq_remove: buffer %p not on a queue.", bp));
	KASSERT(bufqueue(bp) == bq,
	("bq_remove: Remove buffer %p from wrong queue.", bp));

	BQ_ASSERT_LOCKED(bq);
	if (bp->b_qindex != QUEUE_EMPTY) {
	BUF_ASSERT_XLOCKED(bp);
	}
	KASSERT(bq->bq_len >= 1,
	("queue %d underflow", bp->b_qindex));
	TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
	bq->bq_len--;
	bp->b_qindex = QUEUE_NONE;
	bp->b_flags &= ~(B_REMFREE \| B_REUSE);
	}

	static void
	bd_flush(struct bufdomain bd, struct bufqueue bq)
	{
	struct buf *bp;

	BQ_ASSERT_LOCKED(bq);
	if (bq != bd->bd_cleanq) {
	BD_LOCK(bd);
	while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) {
	TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
	TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp,
	b_freelist);
	bp->b_subqueue = bd->bd_cleanq->bq_subqueue;
	}
	bd->bd_cleanq->bq_len += bq->bq_len;
	bq->bq_len = 0;
	}
	if (bd->bd_wanted) {
	bd->bd_wanted = 0;
	wakeup(&bd->bd_wanted);
	}
	if (bq != bd->bd_cleanq)
	BD_UNLOCK(bd);
	}

	static int
	bd_flushall(struct bufdomain *bd)
	{
	struct bufqueue *bq;
	int flushed;
	int i;

	if (bd->bd_lim == 0)
	return (0);
	flushed = 0;
	for (i = 0; i <= mp_maxid; i++) {
	bq = &bd->bd_subq[i];
	if (bq->bq_len == 0)
	continue;
	BQ_LOCK(bq);
	bd_flush(bd, bq);
	BQ_UNLOCK(bq);
	flushed++;
	}

	return (flushed);
	}

	static void
	bq_insert(struct bufqueue bq, struct buf bp, bool unlock)
	{
	struct bufdomain *bd;

	if (bp->b_qindex != QUEUE_NONE)
	panic("bq_insert: free buffer %p onto another queue?", bp);

	bd = bufdomain(bp);
	if (bp->b_flags & B_AGE) {
	/* Place this buf directly on the real queue. */
	if (bq->bq_index == QUEUE_CLEAN)
	bq = bd->bd_cleanq;
	BQ_LOCK(bq);
	TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist);
	} else {
	BQ_LOCK(bq);
	TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
	}
	bp->b_flags &= ~(B_AGE \| B_REUSE);
	bq->bq_len++;
	bp->b_qindex = bq->bq_index;
	bp->b_subqueue = bq->bq_subqueue;

	/*
	* Unlock before we notify so that we don't wakeup a waiter that
	* fails a trylock on the buf and sleeps again.
	*/
	if (unlock)
	BUF_UNLOCK(bp);

	if (bp->b_qindex == QUEUE_CLEAN) {
	/*
	* Flush the per-cpu queue and notify any waiters.
	*/
	if (bd->bd_wanted \|\| (bq != bd->bd_cleanq &&
	bq->bq_len >= bd->bd_lim))
	bd_flush(bd, bq);
	}
	BQ_UNLOCK(bq);
	}

	/*
	* bufkva_free:
	*
	* Free the kva allocation for a buffer.
	*
	*/
	static void
	bufkva_free(struct buf *bp)
	{

	#ifdef INVARIANTS
	if (bp->b_kvasize == 0) {
	KASSERT(bp->b_kvabase == unmapped_buf &&
	bp->b_data == unmapped_buf,
	("Leaked KVA space on %p", bp));
	} else if (buf_mapped(bp))
	BUF_CHECK_MAPPED(bp);
	else
	BUF_CHECK_UNMAPPED(bp);
	#endif
	if (bp->b_kvasize == 0)
	return;

	vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
	counter_u64_add(bufkvaspace, -bp->b_kvasize);
	counter_u64_add(buffreekvacnt, 1);
	bp->b_data = bp->b_kvabase = unmapped_buf;
	bp->b_kvasize = 0;
	}

	/*
	* bufkva_alloc:
	*
	* Allocate the buffer KVA and set b_kvasize and b_kvabase.
	*/
	static int
	bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
	{
	vm_offset_t addr;
	int error;

	KASSERT((gbflags & GB_UNMAPPED) == 0 \|\| (gbflags & GB_KVAALLOC) != 0,
	("Invalid gbflags 0x%x in %s", gbflags, __func__));

	bufkva_free(bp);

	addr = 0;
	error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT \| M_NOWAIT, &addr);
	if (error != 0) {
	/*
	* Buffer map is too fragmented. Request the caller
	* to defragment the map.
	*/
	return (error);
	}
	bp->b_kvabase = (caddr_t)addr;
	bp->b_kvasize = maxsize;
	counter_u64_add(bufkvaspace, bp->b_kvasize);
	if ((gbflags & GB_UNMAPPED) != 0) {
	bp->b_data = unmapped_buf;
	BUF_CHECK_UNMAPPED(bp);
	} else {
	bp->b_data = bp->b_kvabase;
	BUF_CHECK_MAPPED(bp);
	}
	return (0);
	}

	/*
	* bufkva_reclaim:
	*
	* Reclaim buffer kva by freeing buffers holding kva. This is a vmem
	* callback that fires to avoid returning failure.
	*/
	static void
	bufkva_reclaim(vmem_t *vmem, int flags)
	{
	bool done;
	int q;
	int i;

	done = false;
	for (i = 0; i < 5; i++) {
	for (q = 0; q < buf_domains; q++)
	if (buf_recycle(&bdomain[q], true) != 0)
	done = true;
	if (done)
	break;
	}
	return;
	}

	/*
	* Attempt to initiate asynchronous I/O on read-ahead blocks. We must
	* clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
	* the buffer is valid and we do not have to do anything.
	*/
	static void
	breada(struct vnode * vp, daddr_t * rablkno, int * rabsize, int cnt,
	struct ucred * cred, int flags, void (ckhashfunc)(struct buf ))
	{
	struct buf *rabp;
	struct thread *td;
	int i;

	td = curthread;

	for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
	if (inmem(vp, *rablkno))
	continue;
	rabp = getblk(vp, rablkno, rabsize, 0, 0, 0);
	if ((rabp->b_flags & B_CACHE) != 0) {
	brelse(rabp);
	continue;
	}
	#ifdef RACCT
	if (racct_enable) {
	PROC_LOCK(curproc);
	racct_add_buf(curproc, rabp, 0);
	PROC_UNLOCK(curproc);
	}
	#endif /* RACCT */
	td->td_ru.ru_inblock++;
	rabp->b_flags \|= B_ASYNC;
	rabp->b_flags &= ~B_INVAL;
	if ((flags & GB_CKHASH) != 0) {
	rabp->b_flags \|= B_CKHASH;
	rabp->b_ckhashcalc = ckhashfunc;
	}
	rabp->b_ioflags &= ~BIO_ERROR;
	rabp->b_iocmd = BIO_READ;
	if (rabp->b_rcred == NOCRED && cred != NOCRED)
	rabp->b_rcred = crhold(cred);
	vfs_busy_pages(rabp, 0);
	BUF_KERNPROC(rabp);
	rabp->b_iooffset = dbtob(rabp->b_blkno);
	bstrategy(rabp);
	}
	}

	/*
	* Entry point for bread() and breadn() via #defines in sys/buf.h.
	*
	* Get a buffer with the specified data. Look in the cache first. We
	* must clear BIO_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
	* is set, the buffer is valid and we do not have to do anything, see
	* getblk(). Also starts asynchronous I/O on read-ahead blocks.
	*
	* Always return a NULL buffer pointer (in bpp) when returning an error.
	*
	* The blkno parameter is the logical block being requested. Normally
	* the mapping of logical block number to disk block address is done
	* by calling VOP_BMAP(). However, if the mapping is already known, the
	* disk block address can be passed using the dblkno parameter. If the
	* disk block address is not known, then the same value should be passed
	* for blkno and dblkno.
	*/
	int
	breadn_flags(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size,
	daddr_t rablkno, int rabsize, int cnt, struct ucred *cred, int flags,
	void (ckhashfunc)(struct buf ), struct buf **bpp)
	{
	struct buf *bp;
	struct thread *td;
	int error, readwait, rv;

	CTR3(KTR_BUF, "breadn(%p, %jd, %d)", vp, blkno, size);
	td = curthread;
	/*
	* Can only return NULL if GB_LOCK_NOWAIT or GB_SPARSE flags
	* are specified.
	*/
	error = getblkx(vp, blkno, dblkno, size, 0, 0, flags, &bp);
	if (error != 0) {
	*bpp = NULL;
	return (error);
	}
	KASSERT(blkno == bp->b_lblkno,
	("getblkx returned buffer for blkno %jd instead of blkno %jd",
	(intmax_t)bp->b_lblkno, (intmax_t)blkno));
	flags &= ~GB_NOSPARSE;
	*bpp = bp;

	/*
	* If not found in cache, do some I/O
	*/
	readwait = 0;
	if ((bp->b_flags & B_CACHE) == 0) {
	#ifdef RACCT
	if (racct_enable) {
	PROC_LOCK(td->td_proc);
	racct_add_buf(td->td_proc, bp, 0);
	PROC_UNLOCK(td->td_proc);
	}
	#endif /* RACCT */
	td->td_ru.ru_inblock++;
	bp->b_iocmd = BIO_READ;
	bp->b_flags &= ~B_INVAL;
	if ((flags & GB_CKHASH) != 0) {
	bp->b_flags \|= B_CKHASH;
	bp->b_ckhashcalc = ckhashfunc;
	}
	if ((flags & GB_CVTENXIO) != 0)
	bp->b_xflags \|= BX_CVTENXIO;
	bp->b_ioflags &= ~BIO_ERROR;
	if (bp->b_rcred == NOCRED && cred != NOCRED)
	bp->b_rcred = crhold(cred);
	vfs_busy_pages(bp, 0);
	bp->b_iooffset = dbtob(bp->b_blkno);
	bstrategy(bp);
	++readwait;
	}

	/*
	* Attempt to initiate asynchronous I/O on read-ahead blocks.
	*/
	breada(vp, rablkno, rabsize, cnt, cred, flags, ckhashfunc);

	rv = 0;
	if (readwait) {
	rv = bufwait(bp);
	if (rv != 0) {
	brelse(bp);
	*bpp = NULL;
	}
	}
	return (rv);
	}

	/*
	* Write, release buffer on completion. (Done by iodone
	* if async). Do not bother writing anything if the buffer
	* is invalid.
	*
	* Note that we set B_CACHE here, indicating that buffer is
	* fully valid and thus cacheable. This is true even of NFS
	* now so we set it generally. This could be set either here
	* or in biodone() since the I/O is synchronous. We put it
	* here.
	*/
	int
	bufwrite(struct buf *bp)
	{
	int oldflags;
	struct vnode *vp;
	long space;
	int vp_md;

	CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
	bp->b_flags \|= B_INVAL \| B_RELBUF;
	bp->b_flags &= ~B_CACHE;
	brelse(bp);
	return (ENXIO);
	}
	if (bp->b_flags & B_INVAL) {
	brelse(bp);
	return (0);
	}

	if (bp->b_flags & B_BARRIER)
	atomic_add_long(&barrierwrites, 1);

	oldflags = bp->b_flags;

	KASSERT(!(bp->b_vflags & BV_BKGRDINPROG),
	("FFS background buffer should not get here %p", bp));

	vp = bp->b_vp;
	if (vp)
	vp_md = vp->v_vflag & VV_MD;
	else
	vp_md = 0;

	/*
	* Mark the buffer clean. Increment the bufobj write count
	* before bundirty() call, to prevent other thread from seeing
	* empty dirty list and zero counter for writes in progress,
	* falsely indicating that the bufobj is clean.
	*/
	bufobj_wref(bp->b_bufobj);
	bundirty(bp);

	bp->b_flags &= ~B_DONE;
	bp->b_ioflags &= ~BIO_ERROR;
	bp->b_flags \|= B_CACHE;
	bp->b_iocmd = BIO_WRITE;

	vfs_busy_pages(bp, 1);

	/*
	* Normal bwrites pipeline writes
	*/
	bp->b_runningbufspace = bp->b_bufsize;
	space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);

	#ifdef RACCT
	if (racct_enable) {
	PROC_LOCK(curproc);
	racct_add_buf(curproc, bp, 1);
	PROC_UNLOCK(curproc);
	}
	#endif /* RACCT */
	curthread->td_ru.ru_oublock++;
	if (oldflags & B_ASYNC)
	BUF_KERNPROC(bp);
	bp->b_iooffset = dbtob(bp->b_blkno);
	buf_track(bp, __func__);
	bstrategy(bp);

	if ((oldflags & B_ASYNC) == 0) {
	int rtval = bufwait(bp);
	brelse(bp);
	return (rtval);
	} else if (space > hirunningspace) {
	/*
	* don't allow the async write to saturate the I/O
	* system. We will not deadlock here because
	* we are blocking waiting for I/O that is already in-progress
	* to complete. We do not block here if it is the update
	* or syncer daemon trying to clean up as that can lead
	* to deadlock.
	*/
	if ((curthread->td_pflags & TDP_NORUNNINGBUF) == 0 && !vp_md)
	waitrunningbufspace();
	}

	return (0);
	}

	void
	bufbdflush(struct bufobj bo, struct buf bp)
	{
	struct buf *nbp;

	if (bo->bo_dirty.bv_cnt > dirtybufthresh + 10) {
	(void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
	altbufferflushes++;
	} else if (bo->bo_dirty.bv_cnt > dirtybufthresh) {
	BO_LOCK(bo);
	/*
	* Try to find a buffer to flush.
	*/
	TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
	if ((nbp->b_vflags & BV_BKGRDINPROG) \|\|
	BUF_LOCK(nbp,
	LK_EXCLUSIVE \| LK_NOWAIT, NULL))
	continue;
	if (bp == nbp)
	panic("bdwrite: found ourselves");
	BO_UNLOCK(bo);
	/* Don't countdeps with the bo lock held. */
	if (buf_countdeps(nbp, 0)) {
	BO_LOCK(bo);
	BUF_UNLOCK(nbp);
	continue;
	}
	if (nbp->b_flags & B_CLUSTEROK) {
	vfs_bio_awrite(nbp);
	} else {
	bremfree(nbp);
	bawrite(nbp);
	}
	dirtybufferflushes++;
	break;
	}
	if (nbp == NULL)
	BO_UNLOCK(bo);
	}
	}

	/*
	* Delayed write. (Buffer is marked dirty). Do not bother writing
	* anything if the buffer is marked invalid.
	*
	* Note that since the buffer must be completely valid, we can safely
	* set B_CACHE. In fact, we have to set B_CACHE here rather then in
	* biodone() in order to prevent getblk from writing the buffer
	* out synchronously.
	*/
	void
	bdwrite(struct buf *bp)
	{
	struct thread *td = curthread;
	struct vnode *vp;
	struct bufobj *bo;

	CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
	KASSERT((bp->b_flags & B_BARRIER) == 0,
	("Barrier request in delayed write %p", bp));

	if (bp->b_flags & B_INVAL) {
	brelse(bp);
	return;
	}

	/*
	* If we have too many dirty buffers, don't create any more.
	* If we are wildly over our limit, then force a complete
	* cleanup. Otherwise, just keep the situation from getting
	* out of control. Note that we have to avoid a recursive
	* disaster and not try to clean up after our own cleanup!
	*/
	vp = bp->b_vp;
	bo = bp->b_bufobj;
	if ((td->td_pflags & (TDP_COWINPROGRESS\|TDP_INBDFLUSH)) == 0) {
	td->td_pflags \|= TDP_INBDFLUSH;
	BO_BDFLUSH(bo, bp);
	td->td_pflags &= ~TDP_INBDFLUSH;
	} else
	recursiveflushes++;

	bdirty(bp);
	/*
	* Set B_CACHE, indicating that the buffer is fully valid. This is
	* true even of NFS now.
	*/
	bp->b_flags \|= B_CACHE;

	/*
	* This bmap keeps the system from needing to do the bmap later,
	* perhaps when the system is attempting to do a sync. Since it
	* is likely that the indirect block -- or whatever other datastructure
	* that the filesystem needs is still in memory now, it is a good
	* thing to do this. Note also, that if the pageout daemon is
	* requesting a sync -- there might not be enough memory to do
	* the bmap then... So, this is important to do.
	*/
	if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
	VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
	}

	buf_track(bp, __func__);

	/*
	* Set the dirty buffer range based upon the VM system dirty
	* pages.
	*
	* Mark the buffer pages as clean. We need to do this here to
	* satisfy the vnode_pager and the pageout daemon, so that it
	* thinks that the pages have been "cleaned". Note that since
	* the pages are in a delayed write buffer -- the VFS layer
	* "will" see that the pages get written out on the next sync,
	* or perhaps the cluster will be completed.
	*/
	vfs_clean_pages_dirty_buf(bp);
	bqrelse(bp);

	/*
	* note: we cannot initiate I/O from a bdwrite even if we wanted to,
	* due to the softdep code.
	*/
	}

	/*
	* bdirty:
	*
	* Turn buffer into delayed write request. We must clear BIO_READ and
	* B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
	* itself to properly update it in the dirty/clean lists. We mark it
	* B_DONE to ensure that any asynchronization of the buffer properly
	* clears B_DONE ( else a panic will occur later ).
	*
	* bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
	* might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
	* should only be called if the buffer is known-good.
	*
	* Since the buffer is not on a queue, we do not update the numfreebuffers
	* count.
	*
	* The buffer must be on QUEUE_NONE.
	*/
	void
	bdirty(struct buf *bp)
	{

	CTR3(KTR_BUF, "bdirty(%p) vp %p flags %X",
	bp, bp->b_vp, bp->b_flags);
	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
	KASSERT(bp->b_flags & B_REMFREE \|\| bp->b_qindex == QUEUE_NONE,
	("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
	bp->b_flags &= ~(B_RELBUF);
	bp->b_iocmd = BIO_WRITE;

	if ((bp->b_flags & B_DELWRI) == 0) {
	bp->b_flags \|= /* XXX B_DONE \| */ B_DELWRI;
	reassignbuf(bp);
	bdirtyadd(bp);
	}
	}

	/*
	* bundirty:
	*
	* Clear B_DELWRI for buffer.
	*
	* Since the buffer is not on a queue, we do not update the numfreebuffers
	* count.
	*
	* The buffer must be on QUEUE_NONE.
	*/

	void
	bundirty(struct buf *bp)
	{

	CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
	KASSERT(bp->b_flags & B_REMFREE \|\| bp->b_qindex == QUEUE_NONE,
	("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));

	if (bp->b_flags & B_DELWRI) {
	bp->b_flags &= ~B_DELWRI;
	reassignbuf(bp);
	bdirtysub(bp);
	}
	/*
	* Since it is now being written, we can clear its deferred write flag.
	*/
	bp->b_flags &= ~B_DEFERRED;
	}

	/*
	* bawrite:
	*
	* Asynchronous write. Start output on a buffer, but do not wait for
	* it to complete. The buffer is released when the output completes.
	*
	* bwrite() ( or the VOP routine anyway ) is responsible for handling
	* B_INVAL buffers. Not us.
	*/
	void
	bawrite(struct buf *bp)
	{

	bp->b_flags \|= B_ASYNC;
	(void) bwrite(bp);
	}

	/*
	* babarrierwrite:
	*
	* Asynchronous barrier write. Start output on a buffer, but do not
	* wait for it to complete. Place a write barrier after this write so
	* that this buffer and all buffers written before it are committed to
	* the disk before any buffers written after this write are committed
	* to the disk. The buffer is released when the output completes.
	*/
	void
	babarrierwrite(struct buf *bp)
	{

	bp->b_flags \|= B_ASYNC \| B_BARRIER;
	(void) bwrite(bp);
	}

	/*
	* bbarrierwrite:
	*
	* Synchronous barrier write. Start output on a buffer and wait for
	* it to complete. Place a write barrier after this write so that
	* this buffer and all buffers written before it are committed to
	* the disk before any buffers written after this write are committed
	* to the disk. The buffer is released when the output completes.
	*/
	int
	bbarrierwrite(struct buf *bp)
	{

	bp->b_flags \|= B_BARRIER;
	return (bwrite(bp));
	}

	/*
	* bwillwrite:
	*
	* Called prior to the locking of any vnodes when we are expecting to
	* write. We do not want to starve the buffer cache with too many
	* dirty buffers so we block here. By blocking prior to the locking
	* of any vnodes we attempt to avoid the situation where a locked vnode
	* prevents the various system daemons from flushing related buffers.
	*/
	void
	bwillwrite(void)
	{

	if (buf_dirty_count_severe()) {
	mtx_lock(&bdirtylock);
	while (buf_dirty_count_severe()) {
	bdirtywait = 1;
	msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4),
	"flswai", 0);
	}
	mtx_unlock(&bdirtylock);
	}
	}

	/*
	* Return true if we have too many dirty buffers.
	*/
	int
	buf_dirty_count_severe(void)
	{

	return (!BIT_EMPTY(BUF_DOMAINS, &bdhidirty));
	}

	/*
	* brelse:
	*
	* Release a busy buffer and, if requested, free its resources. The
	* buffer will be stashed in the appropriate bufqueue[] allowing it
	* to be accessed later as a cache entity or reused for other purposes.
	*/
	void
	brelse(struct buf *bp)
	{
	struct mount *v_mnt;
	int qindex;

	/*
	* Many functions erroneously call brelse with a NULL bp under rare
	* error conditions. Simply return when called with a NULL bp.
	*/
	if (bp == NULL)
	return;
	CTR3(KTR_BUF, "brelse(%p) vp %p flags %X",
	bp, bp->b_vp, bp->b_flags);
	KASSERT(!(bp->b_flags & (B_CLUSTER\|B_PAGING)),
	("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
	KASSERT((bp->b_flags & B_VMIO) != 0 \|\| (bp->b_flags & B_NOREUSE) == 0,
	("brelse: non-VMIO buffer marked NOREUSE"));

	if (BUF_LOCKRECURSED(bp)) {
	/*
	* Do not process, in particular, do not handle the
	* B_INVAL/B_RELBUF and do not release to free list.
	*/
	BUF_UNLOCK(bp);
	return;
	}

	if (bp->b_flags & B_MANAGED) {
	bqrelse(bp);
	return;
	}

	if ((bp->b_vflags & (BV_BKGRDINPROG \| BV_BKGRDERR)) == BV_BKGRDERR) {
	BO_LOCK(bp->b_bufobj);
	bp->b_vflags &= ~BV_BKGRDERR;
	BO_UNLOCK(bp->b_bufobj);
	bdirty(bp);
	}

	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
	(bp->b_flags & B_INVALONERR)) {
	/*
	* Forced invalidation of dirty buffer contents, to be used
	* after a failed write in the rare case that the loss of the
	* contents is acceptable. The buffer is invalidated and
	* freed.
	*/
	bp->b_flags \|= B_INVAL \| B_RELBUF \| B_NOCACHE;
	bp->b_flags &= ~(B_ASYNC \| B_CACHE);
	}

	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
	(bp->b_error != ENXIO \|\| !LIST_EMPTY(&bp->b_dep)) &&
	!(bp->b_flags & B_INVAL)) {
	/*
	* Failed write, redirty. All errors except ENXIO (which
	* means the device is gone) are treated as being
	* transient.
	*
	* XXX Treating EIO as transient is not correct; the
	* contract with the local storage device drivers is that
	* they will only return EIO once the I/O is no longer
	* retriable. Network I/O also respects this through the
	* guarantees of TCP and/or the internal retries of NFS.
	* ENOMEM might be transient, but we also have no way of
	* knowing when its ok to retry/reschedule. In general,
	* this entire case should be made obsolete through better
	* error handling/recovery and resource scheduling.
	*
	* Do this also for buffers that failed with ENXIO, but have
	* non-empty dependencies - the soft updates code might need
	* to access the buffer to untangle them.
	*
	* Must clear BIO_ERROR to prevent pages from being scrapped.
	*/
	bp->b_ioflags &= ~BIO_ERROR;
	bdirty(bp);
	} else if ((bp->b_flags & (B_NOCACHE \| B_INVAL)) \|\|
	(bp->b_ioflags & BIO_ERROR) \|\| (bp->b_bufsize <= 0)) {
	/*
	* Either a failed read I/O, or we were asked to free or not
	* cache the buffer, or we failed to write to a device that's
	* no longer present.
	*/
	bp->b_flags \|= B_INVAL;
	if (!LIST_EMPTY(&bp->b_dep))
	buf_deallocate(bp);
	if (bp->b_flags & B_DELWRI)
	bdirtysub(bp);
	bp->b_flags &= ~(B_DELWRI \| B_CACHE);
	if ((bp->b_flags & B_VMIO) == 0) {
	allocbuf(bp, 0);
	if (bp->b_vp)
	brelvp(bp);
	}
	}

	/*
	* We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_truncate()
	* is called with B_DELWRI set, the underlying pages may wind up
	* getting freed causing a previous write (bdwrite()) to get 'lost'
	* because pages associated with a B_DELWRI bp are marked clean.
	*
	* We still allow the B_INVAL case to call vfs_vmio_truncate(), even
	* if B_DELWRI is set.
	*/
	if (bp->b_flags & B_DELWRI)
	bp->b_flags &= ~B_RELBUF;

	/*
	* VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
	* constituted, not even NFS buffers now. Two flags effect this. If
	* B_INVAL, the struct buf is invalidated but the VM object is kept
	* around ( i.e. so it is trivial to reconstitute the buffer later ).
	*
	* If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
	* invalidated. BIO_ERROR cannot be set for a failed write unless the
	* buffer is also B_INVAL because it hits the re-dirtying code above.
	*
	* Normally we can do this whether a buffer is B_DELWRI or not. If
	* the buffer is an NFS buffer, it is tracking piecemeal writes or
	* the commit state and we cannot afford to lose the buffer. If the
	* buffer has a background write in progress, we need to keep it
	* around to prevent it from being reconstituted and starting a second
	* background write.
	*/

	v_mnt = bp->b_vp != NULL ? bp->b_vp->v_mount : NULL;

	if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE \|\|
	(bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
	(v_mnt == NULL \|\| (v_mnt->mnt_vfc->vfc_flags & VFCF_NETWORK) == 0 \|\|
	vn_isdisk(bp->b_vp, NULL) \|\| (bp->b_flags & B_DELWRI) == 0)) {
	vfs_vmio_invalidate(bp);
	allocbuf(bp, 0);
	}

	if ((bp->b_flags & (B_INVAL \| B_RELBUF)) != 0 \|\|
	(bp->b_flags & (B_DELWRI \| B_NOREUSE)) == B_NOREUSE) {
	allocbuf(bp, 0);
	bp->b_flags &= ~B_NOREUSE;
	if (bp->b_vp != NULL)
	brelvp(bp);
	}

	/*
	* If the buffer has junk contents signal it and eventually
	* clean up B_DELWRI and diassociate the vnode so that gbincore()
	* doesn't find it.
	*/
	if (bp->b_bufsize == 0 \|\| (bp->b_ioflags & BIO_ERROR) != 0 \|\|
	(bp->b_flags & (B_INVAL \| B_NOCACHE \| B_RELBUF)) != 0)
	bp->b_flags \|= B_INVAL;
	if (bp->b_flags & B_INVAL) {
	if (bp->b_flags & B_DELWRI)
	bundirty(bp);
	if (bp->b_vp)
	brelvp(bp);
	}

	buf_track(bp, __func__);

	/* buffers with no memory */
	if (bp->b_bufsize == 0) {
	buf_free(bp);
	return;
	}
	/* buffers with junk contents */
	if (bp->b_flags & (B_INVAL \| B_NOCACHE \| B_RELBUF) \|\|
	(bp->b_ioflags & BIO_ERROR)) {
	bp->b_xflags &= ~(BX_BKGRDWRITE \| BX_ALTDATA);
	if (bp->b_vflags & BV_BKGRDINPROG)
	panic("losing buffer 2");
	qindex = QUEUE_CLEAN;
	bp->b_flags \|= B_AGE;
	/* remaining buffers */
	} else if (bp->b_flags & B_DELWRI)
	qindex = QUEUE_DIRTY;
	else
	qindex = QUEUE_CLEAN;

	if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
	panic("brelse: not dirty");

	bp->b_flags &= ~(B_ASYNC \| B_NOCACHE \| B_RELBUF \| B_DIRECT);
	bp->b_xflags &= ~(BX_CVTENXIO);
	/* binsfree unlocks bp. */
	binsfree(bp, qindex);
	}

	/*
	* Release a buffer back to the appropriate queue but do not try to free
	* it. The buffer is expected to be used again soon.
	*
	* bqrelse() is used by bdwrite() to requeue a delayed write, and used by
	* biodone() to requeue an async I/O on completion. It is also used when
	* known good buffers need to be requeued but we think we may need the data
	* again soon.
	*
	* XXX we should be able to leave the B_RELBUF hint set on completion.
	*/
	void
	bqrelse(struct buf *bp)
	{
	int qindex;

	CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	KASSERT(!(bp->b_flags & (B_CLUSTER\|B_PAGING)),
	("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));

	qindex = QUEUE_NONE;
	if (BUF_LOCKRECURSED(bp)) {
	/* do not release to free list */
	BUF_UNLOCK(bp);
	return;
	}
	bp->b_flags &= ~(B_ASYNC \| B_NOCACHE \| B_AGE \| B_RELBUF);
	bp->b_xflags &= ~(BX_CVTENXIO);

	if (bp->b_flags & B_MANAGED) {
	if (bp->b_flags & B_REMFREE)
	bremfreef(bp);
	goto out;
	}

	/* buffers with stale but valid contents */
	if ((bp->b_flags & B_DELWRI) != 0 \|\| (bp->b_vflags & (BV_BKGRDINPROG \|
	BV_BKGRDERR)) == BV_BKGRDERR) {
	BO_LOCK(bp->b_bufobj);
	bp->b_vflags &= ~BV_BKGRDERR;
	BO_UNLOCK(bp->b_bufobj);
	qindex = QUEUE_DIRTY;
	} else {
	if ((bp->b_flags & B_DELWRI) == 0 &&
	(bp->b_xflags & BX_VNDIRTY))
	panic("bqrelse: not dirty");
	if ((bp->b_flags & B_NOREUSE) != 0) {
	brelse(bp);
	return;
	}
	qindex = QUEUE_CLEAN;
	}
	buf_track(bp, __func__);
	/* binsfree unlocks bp. */
	binsfree(bp, qindex);
	return;

	out:
	buf_track(bp, __func__);
	/* unlock */
	BUF_UNLOCK(bp);
	}

	/*
	* Complete I/O to a VMIO backed page. Validate the pages as appropriate,
	* restore bogus pages.
	*/
	static void
	vfs_vmio_iodone(struct buf *bp)
	{
	vm_ooffset_t foff;
	vm_page_t m;
	vm_object_t obj;
	struct vnode *vp __unused;
	int i, iosize, resid;
	bool bogus;

	obj = bp->b_bufobj->bo_object;
	KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages,
	("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
	blockcount_read(&obj->paging_in_progress), bp->b_npages));

	vp = bp->b_vp;
	VNPASS(vp->v_holdcnt > 0, vp);
	VNPASS(vp->v_object != NULL, vp);

	foff = bp->b_offset;
	KASSERT(bp->b_offset != NOOFFSET,
	("vfs_vmio_iodone: bp %p has no buffer offset", bp));

	bogus = false;
	iosize = bp->b_bcount - bp->b_resid;
	for (i = 0; i < bp->b_npages; i++) {
	resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
	if (resid > iosize)
	resid = iosize;

	/*
	* cleanup bogus pages, restoring the originals
	*/
	m = bp->b_pages[i];
	if (m == bogus_page) {
	bogus = true;
	m = vm_page_relookup(obj, OFF_TO_IDX(foff));
	if (m == NULL)
	panic("biodone: page disappeared!");
	bp->b_pages[i] = m;
	} else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
	/*
	* In the write case, the valid and clean bits are
	* already changed correctly ( see bdwrite() ), so we
	* only need to do this here in the read case.
	*/
	KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
	resid)) == 0, ("vfs_vmio_iodone: page %p "
	"has unexpected dirty bits", m));
	vfs_page_set_valid(bp, foff, m);
	}
	KASSERT(OFF_TO_IDX(foff) == m->pindex,
	("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
	(intmax_t)foff, (uintmax_t)m->pindex));

	vm_page_sunbusy(m);
	foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
	iosize -= resid;
	}
	vm_object_pip_wakeupn(obj, bp->b_npages);
	if (bogus && buf_mapped(bp)) {
	BUF_CHECK_MAPPED(bp);
	pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
	bp->b_pages, bp->b_npages);
	}
	}

	/*
	* Perform page invalidation when a buffer is released. The fully invalid
	* pages will be reclaimed later in vfs_vmio_truncate().
	*/
	static void
	vfs_vmio_invalidate(struct buf *bp)
	{
	vm_object_t obj;
	vm_page_t m;
	int flags, i, resid, poffset, presid;

	if (buf_mapped(bp)) {
	BUF_CHECK_MAPPED(bp);
	pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
	} else
	BUF_CHECK_UNMAPPED(bp);
	/*
	* Get the base offset and length of the buffer. Note that
	* in the VMIO case if the buffer block size is not
	* page-aligned then b_data pointer may not be page-aligned.
	* But our b_pages[] array IS page aligned.
	*
	* block sizes less then DEV_BSIZE (usually 512) are not
	* supported due to the page granularity bits (m->valid,
	* m->dirty, etc...).
	*
	* See man buf(9) for more information
	*/
	flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
	obj = bp->b_bufobj->bo_object;
	resid = bp->b_bufsize;
	poffset = bp->b_offset & PAGE_MASK;
	VM_OBJECT_WLOCK(obj);
	for (i = 0; i < bp->b_npages; i++) {
	m = bp->b_pages[i];
	if (m == bogus_page)
	panic("vfs_vmio_invalidate: Unexpected bogus page.");
	bp->b_pages[i] = NULL;

	presid = resid > (PAGE_SIZE - poffset) ?
	(PAGE_SIZE - poffset) : resid;
	KASSERT(presid >= 0, ("brelse: extra page"));
	vm_page_busy_acquire(m, VM_ALLOC_SBUSY);
	if (pmap_page_wired_mappings(m) == 0)
	vm_page_set_invalid(m, poffset, presid);
	vm_page_sunbusy(m);
	vm_page_release_locked(m, flags);
	resid -= presid;
	poffset = 0;
	}
	VM_OBJECT_WUNLOCK(obj);
	bp->b_npages = 0;
	}

	/*
	* Page-granular truncation of an existing VMIO buffer.
	*/
	static void
	vfs_vmio_truncate(struct buf *bp, int desiredpages)
	{
	vm_object_t obj;
	vm_page_t m;
	int flags, i;

	if (bp->b_npages == desiredpages)
	return;

	if (buf_mapped(bp)) {
	BUF_CHECK_MAPPED(bp);
	pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
	(desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
	} else
	BUF_CHECK_UNMAPPED(bp);

	/*
	* The object lock is needed only if we will attempt to free pages.
	*/
	flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
	if ((bp->b_flags & B_DIRECT) != 0) {
	flags \|= VPR_TRYFREE;
	obj = bp->b_bufobj->bo_object;
	VM_OBJECT_WLOCK(obj);
	} else {
	obj = NULL;
	}
	for (i = desiredpages; i < bp->b_npages; i++) {
	m = bp->b_pages[i];
	KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
	bp->b_pages[i] = NULL;
	if (obj != NULL)
	vm_page_release_locked(m, flags);
	else
	vm_page_release(m, flags);
	}
	if (obj != NULL)
	VM_OBJECT_WUNLOCK(obj);
	bp->b_npages = desiredpages;
	}

	/*
	* Byte granular extension of VMIO buffers.
	*/
	static void
	vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
	{
	/*
	* We are growing the buffer, possibly in a
	* byte-granular fashion.
	*/
	vm_object_t obj;
	vm_offset_t toff;
	vm_offset_t tinc;
	vm_page_t m;

	/*
	* Step 1, bring in the VM pages from the object, allocating
	* them if necessary. We must clear B_CACHE if these pages
	* are not valid for the range covered by the buffer.
	*/
	obj = bp->b_bufobj->bo_object;
	if (bp->b_npages < desiredpages) {
	/*
	* We must allocate system pages since blocking
	* here could interfere with paging I/O, no
	* matter which process we are.
	*
	* Only exclusive busy can be tested here.
	* Blocking on shared busy might lead to
	* deadlocks once allocbuf() is called after
	* pages are vfs_busy_pages().
	*/
	(void)vm_page_grab_pages_unlocked(obj,
	OFF_TO_IDX(bp->b_offset) + bp->b_npages,
	VM_ALLOC_SYSTEM \| VM_ALLOC_IGN_SBUSY \|
	VM_ALLOC_NOBUSY \| VM_ALLOC_WIRED,
	&bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
	bp->b_npages = desiredpages;
	}

	/*
	* Step 2. We've loaded the pages into the buffer,
	* we have to figure out if we can still have B_CACHE
	* set. Note that B_CACHE is set according to the
	* byte-granular range ( bcount and size ), not the
	* aligned range ( newbsize ).
	*
	* The VM test is against m->valid, which is DEV_BSIZE
	* aligned. Needless to say, the validity of the data
	* needs to also be DEV_BSIZE aligned. Note that this
	* fails with NFS if the server or some other client
	* extends the file's EOF. If our buffer is resized,
	* B_CACHE may remain set! XXX
	*/
	toff = bp->b_bcount;
	tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
	while ((bp->b_flags & B_CACHE) && toff < size) {
	vm_pindex_t pi;

	if (tinc > (size - toff))
	tinc = size - toff;
	pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
	m = bp->b_pages[pi];
	vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
	toff += tinc;
	tinc = PAGE_SIZE;
	}

	/*
	* Step 3, fixup the KVA pmap.
	*/
	if (buf_mapped(bp))
	bpmap_qenter(bp);
	else
	BUF_CHECK_UNMAPPED(bp);
	}

	/*
	* Check to see if a block at a particular lbn is available for a clustered
	* write.
	*/
	static int
	vfs_bio_clcheck(struct vnode *vp, int size, daddr_t lblkno, daddr_t blkno)
	{
	struct buf *bpa;
	int match;

	match = 0;

	/* If the buf isn't in core skip it */
	if ((bpa = gbincore(&vp->v_bufobj, lblkno)) == NULL)
	return (0);

	/* If the buf is busy we don't want to wait for it */
	if (BUF_LOCK(bpa, LK_EXCLUSIVE \| LK_NOWAIT, NULL) != 0)
	return (0);

	/* Only cluster with valid clusterable delayed write buffers */
	if ((bpa->b_flags & (B_DELWRI \| B_CLUSTEROK \| B_INVAL)) !=
	(B_DELWRI \| B_CLUSTEROK))
	goto done;

	if (bpa->b_bufsize != size)
	goto done;

	/*
	* Check to see if it is in the expected place on disk and that the
	* block has been mapped.
	*/
	if ((bpa->b_blkno != bpa->b_lblkno) && (bpa->b_blkno == blkno))
	match = 1;
	done:
	BUF_UNLOCK(bpa);
	return (match);
	}

	/*
	* vfs_bio_awrite:
	*
	* Implement clustered async writes for clearing out B_DELWRI buffers.
	* This is much better then the old way of writing only one buffer at
	* a time. Note that we may not be presented with the buffers in the
	* correct order, so we search for the cluster in both directions.
	*/
	int
	vfs_bio_awrite(struct buf *bp)
	{
	struct bufobj *bo;
	int i;
	int j;
	daddr_t lblkno = bp->b_lblkno;
	struct vnode *vp = bp->b_vp;
	int ncl;
	int nwritten;
	int size;
	int maxcl;
	int gbflags;

	bo = &vp->v_bufobj;
	gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
	/*
	* right now we support clustered writing only to regular files. If
	* we find a clusterable block we could be in the middle of a cluster
	* rather then at the beginning.
	*/
	if ((vp->v_type == VREG) &&
	(vp->v_mount != 0) && /* Only on nodes that have the size info */
	(bp->b_flags & (B_CLUSTEROK \| B_INVAL)) == B_CLUSTEROK) {

	size = vp->v_mount->mnt_stat.f_iosize;
	maxcl = MAXPHYS / size;

	BO_RLOCK(bo);
	for (i = 1; i < maxcl; i++)
	if (vfs_bio_clcheck(vp, size, lblkno + i,
	bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
	break;

	for (j = 1; i + j <= maxcl && j <= lblkno; j++)
	if (vfs_bio_clcheck(vp, size, lblkno - j,
	bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
	break;
	BO_RUNLOCK(bo);
	--j;
	ncl = i + j;
	/*
	* this is a possible cluster write
	*/
	if (ncl != 1) {
	BUF_UNLOCK(bp);
	nwritten = cluster_wbuild(vp, size, lblkno - j, ncl,
	gbflags);
	return (nwritten);
	}
	}
	bremfree(bp);
	bp->b_flags \|= B_ASYNC;
	/*
	* default (old) behavior, writing out only one block
	*
	* XXX returns b_bufsize instead of b_bcount for nwritten?
	*/
	nwritten = bp->b_bufsize;
	(void) bwrite(bp);

	return (nwritten);
	}

	/*
	* getnewbuf_kva:
	*
	* Allocate KVA for an empty buf header according to gbflags.
	*/
	static int
	getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
	{

	if ((gbflags & (GB_UNMAPPED \| GB_KVAALLOC)) != GB_UNMAPPED) {
	/*
	* In order to keep fragmentation sane we only allocate kva
	* in BKVASIZE chunks. XXX with vmem we can do page size.
	*/
	maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;

	if (maxsize != bp->b_kvasize &&
	bufkva_alloc(bp, maxsize, gbflags))
	return (ENOSPC);
	}
	return (0);
	}

	/*
	* getnewbuf:
	*
	* Find and initialize a new buffer header, freeing up existing buffers
	* in the bufqueues as necessary. The new buffer is returned locked.
	*
	* We block if:
	* We have insufficient buffer headers
	* We have insufficient buffer space
	* buffer_arena is too fragmented ( space reservation fails )
	* If we have to flush dirty buffers ( but we try to avoid this )
	*
	* The caller is responsible for releasing the reserved bufspace after
	* allocbuf() is called.
	*/
	static struct buf *
	getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int maxsize, int gbflags)
	{
	struct bufdomain *bd;
	struct buf *bp;
	bool metadata, reserved;

	bp = NULL;
	KASSERT((gbflags & (GB_UNMAPPED \| GB_KVAALLOC)) != GB_KVAALLOC,
	("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
	if (!unmapped_buf_allowed)
	gbflags &= ~(GB_UNMAPPED \| GB_KVAALLOC);

	if (vp == NULL \|\| (vp->v_vflag & (VV_MD \| VV_SYSTEM)) != 0 \|\|
	vp->v_type == VCHR)
	metadata = true;
	else
	metadata = false;
	if (vp == NULL)
	bd = &bdomain[0];
	else
	bd = &bdomain[vp->v_bufobj.bo_domain];

	counter_u64_add(getnewbufcalls, 1);
	reserved = false;
	do {
	if (reserved == false &&
	bufspace_reserve(bd, maxsize, metadata) != 0) {
	counter_u64_add(getnewbufrestarts, 1);
	continue;
	}
	reserved = true;
	if ((bp = buf_alloc(bd)) == NULL) {
	counter_u64_add(getnewbufrestarts, 1);
	continue;
	}
	if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
	return (bp);
	break;
	} while (buf_recycle(bd, false) == 0);

	if (reserved)
	bufspace_release(bd, maxsize);
	if (bp != NULL) {
	bp->b_flags \|= B_INVAL;
	brelse(bp);
	}
	bufspace_wait(bd, vp, gbflags, slpflag, slptimeo);

	return (NULL);
	}

	/*
	* buf_daemon:
	*
	* buffer flushing daemon. Buffers are normally flushed by the
	* update daemon but if it cannot keep up this process starts to
	* take the load in an attempt to prevent getnewbuf() from blocking.
	*/
	static struct kproc_desc buf_kp = {
	"bufdaemon",
	buf_daemon,
	&bufdaemonproc
	};
	SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);

	static int
	buf_flush(struct vnode vp, struct bufdomain bd, int target)
	{
	int flushed;

	flushed = flushbufqueues(vp, bd, target, 0);
	if (flushed == 0) {
	/*
	* Could not find any buffers without rollback
	* dependencies, so just write the first one
	* in the hopes of eventually making progress.
	*/
	if (vp != NULL && target > 2)
	target /= 2;
	flushbufqueues(vp, bd, target, 1);
	}
	return (flushed);
	}

	static void
	buf_daemon()
	{
	struct bufdomain *bd;
	int speedupreq;
	int lodirty;
	int i;

	/*
	* This process needs to be suspended prior to shutdown sync.
	*/
	EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
	SHUTDOWN_PRI_LAST + 100);

	/*
	* Start the buf clean daemons as children threads.
	*/
	for (i = 0 ; i < buf_domains; i++) {
	int error;

	error = kthread_add((void ()(void ))bufspace_daemon,
	&bdomain[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i);
	if (error)
	panic("error %d spawning bufspace daemon", error);
	}

	/*
	* This process is allowed to take the buffer cache to the limit
	*/
	curthread->td_pflags \|= TDP_NORUNNINGBUF \| TDP_BUFNEED;
	mtx_lock(&bdlock);
	for (;;) {
	bd_request = 0;
	mtx_unlock(&bdlock);

	kthread_suspend_check();

	/*
	* Save speedupreq for this pass and reset to capture new
	* requests.
	*/
	speedupreq = bd_speedupreq;
	bd_speedupreq = 0;

	/*
	* Flush each domain sequentially according to its level and
	* the speedup request.
	*/
	for (i = 0; i < buf_domains; i++) {
	bd = &bdomain[i];
	if (speedupreq)
	lodirty = bd->bd_numdirtybuffers / 2;
	else
	lodirty = bd->bd_lodirtybuffers;
	while (bd->bd_numdirtybuffers > lodirty) {
	if (buf_flush(NULL, bd,
	bd->bd_numdirtybuffers - lodirty) == 0)
	break;
	kern_yield(PRI_USER);
	}
	}

	/*
	* Only clear bd_request if we have reached our low water
	* mark. The buf_daemon normally waits 1 second and
	* then incrementally flushes any dirty buffers that have
	* built up, within reason.
	*
	* If we were unable to hit our low water mark and couldn't
	* find any flushable buffers, we sleep for a short period
	* to avoid endless loops on unlockable buffers.
	*/
	mtx_lock(&bdlock);
	if (!BIT_EMPTY(BUF_DOMAINS, &bdlodirty)) {
	/*
	* We reached our low water mark, reset the
	* request and sleep until we are needed again.
	* The sleep is just so the suspend code works.
	*/
	bd_request = 0;
	/*
	* Do an extra wakeup in case dirty threshold
	* changed via sysctl and the explicit transition
	* out of shortfall was missed.
	*/
	bdirtywakeup();
	if (runningbufspace <= lorunningspace)
	runningwakeup();
	msleep(&bd_request, &bdlock, PVM, "psleep", hz);
	} else {
	/*
	* We couldn't find any flushable dirty buffers but
	* still have too many dirty buffers, we
	* have to sleep and try again. (rare)
	*/
	msleep(&bd_request, &bdlock, PVM, "qsleep", hz / 10);
	}
	}
	}

	/*
	* flushbufqueues:
	*
	* Try to flush a buffer in the dirty queue. We must be careful to
	* free up B_INVAL buffers instead of write them, which NFS is
	* particularly sensitive to.
	*/
	static int flushwithdeps = 0;
	SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW \| CTLFLAG_STATS,
	&flushwithdeps, 0,
	"Number of buffers flushed with dependecies that require rollbacks");

	static int
	flushbufqueues(struct vnode lvp, struct bufdomain bd, int target,
	int flushdeps)
	{
	struct bufqueue *bq;
	struct buf *sentinel;
	struct vnode *vp;
	struct mount *mp;
	struct buf *bp;
	int hasdeps;
	int flushed;
	int error;
	bool unlock;

	flushed = 0;
	bq = &bd->bd_dirtyq;
	bp = NULL;
	sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK \| M_ZERO);
	sentinel->b_qindex = QUEUE_SENTINEL;
	BQ_LOCK(bq);
	TAILQ_INSERT_HEAD(&bq->bq_queue, sentinel, b_freelist);
	BQ_UNLOCK(bq);
	while (flushed != target) {
	maybe_yield();
	BQ_LOCK(bq);
	bp = TAILQ_NEXT(sentinel, b_freelist);
	if (bp != NULL) {
	TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
	TAILQ_INSERT_AFTER(&bq->bq_queue, bp, sentinel,
	b_freelist);
	} else {
	BQ_UNLOCK(bq);
	break;
	}
	/*
	* Skip sentinels inserted by other invocations of the
	* flushbufqueues(), taking care to not reorder them.
	*
	* Only flush the buffers that belong to the
	* vnode locked by the curthread.
	*/
	if (bp->b_qindex == QUEUE_SENTINEL \|\| (lvp != NULL &&
	bp->b_vp != lvp)) {
	BQ_UNLOCK(bq);
	continue;
	}
	error = BUF_LOCK(bp, LK_EXCLUSIVE \| LK_NOWAIT, NULL);
	BQ_UNLOCK(bq);
	if (error != 0)
	continue;

	/*
	* BKGRDINPROG can only be set with the buf and bufobj
	* locks both held. We tolerate a race to clear it here.
	*/
	if ((bp->b_vflags & BV_BKGRDINPROG) != 0 \|\|
	(bp->b_flags & B_DELWRI) == 0) {
	BUF_UNLOCK(bp);
	continue;
	}
	if (bp->b_flags & B_INVAL) {
	bremfreef(bp);
	brelse(bp);
	flushed++;
	continue;
	}

	if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
	if (flushdeps == 0) {
	BUF_UNLOCK(bp);
	continue;
	}
	hasdeps = 1;
	} else
	hasdeps = 0;
	/*
	* We must hold the lock on a vnode before writing
	* one of its buffers. Otherwise we may confuse, or
	* in the case of a snapshot vnode, deadlock the
	* system.
	*
	* The lock order here is the reverse of the normal
	* of vnode followed by buf lock. This is ok because
	* the NOWAIT will prevent deadlock.
	*/
	vp = bp->b_vp;
	if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
	BUF_UNLOCK(bp);
	continue;
	}
	if (lvp == NULL) {
	unlock = true;
	error = vn_lock(vp, LK_EXCLUSIVE \| LK_NOWAIT);
	} else {
	ASSERT_VOP_LOCKED(vp, "getbuf");
	unlock = false;
	error = VOP_ISLOCKED(vp) == LK_EXCLUSIVE ? 0 :
	vn_lock(vp, LK_TRYUPGRADE);
	}
	if (error == 0) {
	CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
	bp, bp->b_vp, bp->b_flags);
	if (curproc == bufdaemonproc) {
	vfs_bio_awrite(bp);
	} else {
	bremfree(bp);
	bwrite(bp);
	counter_u64_add(notbufdflushes, 1);
	}
	vn_finished_write(mp);
	if (unlock)
	VOP_UNLOCK(vp);
	flushwithdeps += hasdeps;
	flushed++;

	/*
	* Sleeping on runningbufspace while holding
	* vnode lock leads to deadlock.
	*/
	if (curproc == bufdaemonproc &&
	runningbufspace > hirunningspace)
	waitrunningbufspace();
	continue;
	}
	vn_finished_write(mp);
	BUF_UNLOCK(bp);
	}
	BQ_LOCK(bq);
	TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
	BQ_UNLOCK(bq);
	free(sentinel, M_TEMP);
	return (flushed);
	}

	/*
	* Check to see if a block is currently memory resident.
	*/
	struct buf *
	incore(struct bufobj *bo, daddr_t blkno)
	{
	return (gbincore_unlocked(bo, blkno));
	}

	/*
	* Returns true if no I/O is needed to access the
	* associated VM object. This is like incore except
	* it also hunts around in the VM system for the data.
	*/

	static int
	inmem(struct vnode * vp, daddr_t blkno)
	{
	vm_object_t obj;
	vm_offset_t toff, tinc, size;
	vm_page_t m;
	vm_ooffset_t off;

	ASSERT_VOP_LOCKED(vp, "inmem");

	if (incore(&vp->v_bufobj, blkno))
	return 1;
	if (vp->v_mount == NULL)
	return 0;
	obj = vp->v_object;
	if (obj == NULL)
	return (0);

	size = PAGE_SIZE;
	if (size > vp->v_mount->mnt_stat.f_iosize)
	size = vp->v_mount->mnt_stat.f_iosize;
	off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;

	VM_OBJECT_RLOCK(obj);
	for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
	m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
	if (!m)
	goto notinmem;
	tinc = size;
	if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
	tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
	if (vm_page_is_valid(m,
	(vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
	goto notinmem;
	}
	VM_OBJECT_RUNLOCK(obj);
	return 1;

	notinmem:
	VM_OBJECT_RUNLOCK(obj);
	return (0);
	}

	/*
	* Set the dirty range for a buffer based on the status of the dirty
	* bits in the pages comprising the buffer. The range is limited
	* to the size of the buffer.
	*
	* Tell the VM system that the pages associated with this buffer
	* are clean. This is used for delayed writes where the data is
	* going to go to disk eventually without additional VM intevention.
	*
	* Note that while we only really need to clean through to b_bcount, we
	* just go ahead and clean through to b_bufsize.
	*/
	static void
	vfs_clean_pages_dirty_buf(struct buf *bp)
	{
	vm_ooffset_t foff, noff, eoff;
	vm_page_t m;
	int i;

	if ((bp->b_flags & B_VMIO) == 0 \|\| bp->b_bufsize == 0)
	return;

	foff = bp->b_offset;
	KASSERT(bp->b_offset != NOOFFSET,
	("vfs_clean_pages_dirty_buf: no buffer offset"));

	vfs_busy_pages_acquire(bp);
	vfs_setdirty_range(bp);
	for (i = 0; i < bp->b_npages; i++) {
	noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
	eoff = noff;
	if (eoff > bp->b_offset + bp->b_bufsize)
	eoff = bp->b_offset + bp->b_bufsize;
	m = bp->b_pages[i];
	vfs_page_set_validclean(bp, foff, m);
	/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
	foff = noff;
	}
	vfs_busy_pages_release(bp);
	}

	static void
	vfs_setdirty_range(struct buf *bp)
	{
	vm_offset_t boffset;
	vm_offset_t eoffset;
	int i;

	/*
	* test the pages to see if they have been modified directly
	* by users through the VM system.
	*/
	for (i = 0; i < bp->b_npages; i++)
	vm_page_test_dirty(bp->b_pages[i]);

	/*
	* Calculate the encompassing dirty range, boffset and eoffset,
	* (eoffset - boffset) bytes.
	*/

	for (i = 0; i < bp->b_npages; i++) {
	if (bp->b_pages[i]->dirty)
	break;
	}
	boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);

	for (i = bp->b_npages - 1; i >= 0; --i) {
	if (bp->b_pages[i]->dirty) {
	break;
	}
	}
	eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);

	/*
	* Fit it to the buffer.
	*/

	if (eoffset > bp->b_bcount)
	eoffset = bp->b_bcount;

	/*
	* If we have a good dirty range, merge with the existing
	* dirty range.
	*/

	if (boffset < eoffset) {
	if (bp->b_dirtyoff > boffset)
	bp->b_dirtyoff = boffset;
	if (bp->b_dirtyend < eoffset)
	bp->b_dirtyend = eoffset;
	}
	}

	/*
	* Allocate the KVA mapping for an existing buffer.
	* If an unmapped buffer is provided but a mapped buffer is requested, take
	* also care to properly setup mappings between pages and KVA.
	*/
	static void
	bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
	{
	int bsize, maxsize, need_mapping, need_kva;
	off_t offset;

	need_mapping = bp->b_data == unmapped_buf &&
	(gbflags & GB_UNMAPPED) == 0;
	need_kva = bp->b_kvabase == unmapped_buf &&
	bp->b_data == unmapped_buf &&
	(gbflags & GB_KVAALLOC) != 0;
	if (!need_mapping && !need_kva)
	return;

	BUF_CHECK_UNMAPPED(bp);

	if (need_mapping && bp->b_kvabase != unmapped_buf) {
	/*
	* Buffer is not mapped, but the KVA was already
	* reserved at the time of the instantiation. Use the
	* allocated space.
	*/
	goto has_addr;
	}

	/*
	* Calculate the amount of the address space we would reserve
	* if the buffer was mapped.
	*/
	bsize = vn_isdisk(bp->b_vp, NULL) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
	KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
	offset = blkno * bsize;
	maxsize = size + (offset & PAGE_MASK);
	maxsize = imax(maxsize, bsize);

	while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
	if ((gbflags & GB_NOWAIT_BD) != 0) {
	/*
	* XXXKIB: defragmentation cannot
	* succeed, not sure what else to do.
	*/
	panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
	}
	counter_u64_add(mappingrestarts, 1);
	bufspace_wait(bufdomain(bp), bp->b_vp, gbflags, 0, 0);
	}
	has_addr:
	if (need_mapping) {
	/* b_offset is handled by bpmap_qenter. */
	bp->b_data = bp->b_kvabase;
	BUF_CHECK_MAPPED(bp);
	bpmap_qenter(bp);
	}
	}

	struct buf *
	getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
	int flags)
	{
	struct buf *bp;
	int error;

	error = getblkx(vp, blkno, blkno, size, slpflag, slptimeo, flags, &bp);
	if (error != 0)
	return (NULL);
	return (bp);
	}

	/*
	* getblkx:
	*
	* Get a block given a specified block and offset into a file/device.
	* The buffers B_DONE bit will be cleared on return, making it almost
	* ready for an I/O initiation. B_INVAL may or may not be set on
	* return. The caller should clear B_INVAL prior to initiating a
	* READ.
	*
	* For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
	* an existing buffer.
	*
	* For a VMIO buffer, B_CACHE is modified according to the backing VM.
	* If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
	* and then cleared based on the backing VM. If the previous buffer is
	* non-0-sized but invalid, B_CACHE will be cleared.
	*
	* If getblk() must create a new buffer, the new buffer is returned with
	* both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
	* case it is returned with B_INVAL clear and B_CACHE set based on the
	* backing VM.
	*
	* getblk() also forces a bwrite() for any B_DELWRI buffer whose
	* B_CACHE bit is clear.
	*
	* What this means, basically, is that the caller should use B_CACHE to
	* determine whether the buffer is fully valid or not and should clear
	* B_INVAL prior to issuing a read. If the caller intends to validate
	* the buffer by loading its data area with something, the caller needs
	* to clear B_INVAL. If the caller does this without issuing an I/O,
	* the caller should set B_CACHE ( as an optimization ), else the caller
	* should issue the I/O and biodone() will set B_CACHE if the I/O was
	* a write attempt or if it was a successful read. If the caller
	* intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
	* prior to issuing the READ. biodone() will not clear B_INVAL.
	*
	* The blkno parameter is the logical block being requested. Normally
	* the mapping of logical block number to disk block address is done
	* by calling VOP_BMAP(). However, if the mapping is already known, the
	* disk block address can be passed using the dblkno parameter. If the
	* disk block address is not known, then the same value should be passed
	* for blkno and dblkno.
	*/
	int
	getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
	int slptimeo, int flags, struct buf **bpp)
	{
	struct buf *bp;
	struct bufobj *bo;
	daddr_t d_blkno;
	- int bsize, error, maxsize, vmio, lockflags;
	+ int bsize, error, maxsize, vmio;
	off_t offset;

	CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
	KASSERT((flags & (GB_UNMAPPED \| GB_KVAALLOC)) != GB_KVAALLOC,
	("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
	ASSERT_VOP_LOCKED(vp, "getblk");
	if (size > maxbcachebuf)
	panic("getblk: size(%d) > maxbcachebuf(%d)\n", size,
	maxbcachebuf);
	if (!unmapped_buf_allowed)
	flags &= ~(GB_UNMAPPED \| GB_KVAALLOC);

	bo = &vp->v_bufobj;
	d_blkno = dblkno;

	/* Attempt lockless lookup first. */
	bp = gbincore_unlocked(bo, blkno);
	if (bp == NULL)
	goto newbuf_unlocked;

	- lockflags = LK_EXCLUSIVE \| LK_SLEEPFAIL \|
	- ((flags & GB_LOCK_NOWAIT) ? LK_NOWAIT : 0);
	-
	- error = BUF_TIMELOCK(bp, lockflags, NULL, "getblku", slpflag,
	- slptimeo);
	- if (error == EINTR \|\| error == ERESTART)
	- return (error);
	- else if (error != 0)
	+ error = BUF_TIMELOCK(bp, LK_EXCLUSIVE \| LK_NOWAIT, NULL, "getblku", 0,
	+ 0);
	+ if (error != 0)
	goto loop;

	/* Verify buf identify has not changed since lookup. */
	if (bp->b_bufobj == bo && bp->b_lblkno == blkno)
	goto foundbuf_fastpath;

	/* It changed, fallback to locked lookup. */
	BUF_UNLOCK_RAW(bp);

	loop:
	BO_RLOCK(bo);
	bp = gbincore(bo, blkno);
	if (bp != NULL) {
	+ int lockflags;
	+
	/*
	* Buffer is in-core. If the buffer is not busy nor managed,
	* it must be on a queue.
	*/
	- lockflags = LK_EXCLUSIVE \| LK_SLEEPFAIL \| LK_INTERLOCK;
	-
	- if ((flags & GB_LOCK_NOWAIT) != 0)
	- lockflags \|= LK_NOWAIT;
	+ lockflags = LK_EXCLUSIVE \| LK_INTERLOCK \|
	+ ((flags & GB_LOCK_NOWAIT) ? LK_NOWAIT : LK_SLEEPFAIL);

	error = BUF_TIMELOCK(bp, lockflags,
	BO_LOCKPTR(bo), "getblk", slpflag, slptimeo);

	/*
	* If we slept and got the lock we have to restart in case
	* the buffer changed identities.
	*/
	if (error == ENOLCK)
	goto loop;
	/* We timed out or were interrupted. */
	else if (error != 0)
	return (error);

	foundbuf_fastpath:
	/* If recursed, assume caller knows the rules. */
	if (BUF_LOCKRECURSED(bp))
	goto end;

	/*
	* The buffer is locked. B_CACHE is cleared if the buffer is
	* invalid. Otherwise, for a non-VMIO buffer, B_CACHE is set
	* and for a VMIO buffer B_CACHE is adjusted according to the
	* backing VM cache.
	*/
	if (bp->b_flags & B_INVAL)
	bp->b_flags &= ~B_CACHE;
	else if ((bp->b_flags & (B_VMIO \| B_INVAL)) == 0)
	bp->b_flags \|= B_CACHE;
	if (bp->b_flags & B_MANAGED)
	MPASS(bp->b_qindex == QUEUE_NONE);
	else
	bremfree(bp);

	/*
	* check for size inconsistencies for non-VMIO case.
	*/
	if (bp->b_bcount != size) {
	if ((bp->b_flags & B_VMIO) == 0 \|\|
	(size > bp->b_kvasize)) {
	if (bp->b_flags & B_DELWRI) {
	bp->b_flags \|= B_NOCACHE;
	bwrite(bp);
	} else {
	if (LIST_EMPTY(&bp->b_dep)) {
	bp->b_flags \|= B_RELBUF;
	brelse(bp);
	} else {
	bp->b_flags \|= B_NOCACHE;
	bwrite(bp);
	}
	}
	goto loop;
	}
	}

	/*
	* Handle the case of unmapped buffer which should
	* become mapped, or the buffer for which KVA
	* reservation is requested.
	*/
	bp_unmapped_get_kva(bp, blkno, size, flags);

	/*
	* If the size is inconsistent in the VMIO case, we can resize
	* the buffer. This might lead to B_CACHE getting set or
	* cleared. If the size has not changed, B_CACHE remains
	* unchanged from its previous state.
	*/
	allocbuf(bp, size);

	KASSERT(bp->b_offset != NOOFFSET,
	("getblk: no buffer offset"));

	/*
	* A buffer with B_DELWRI set and B_CACHE clear must
	* be committed before we can return the buffer in
	* order to prevent the caller from issuing a read
	* ( due to B_CACHE not being set ) and overwriting
	* it.
	*
	* Most callers, including NFS and FFS, need this to
	* operate properly either because they assume they
	* can issue a read if B_CACHE is not set, or because
	* ( for example ) an uncached B_DELWRI might loop due
	* to softupdates re-dirtying the buffer. In the latter
	* case, B_CACHE is set after the first write completes,
	* preventing further loops.
	* NOTE! b*write() sets B_CACHE. If we cleared B_CACHE
	* above while extending the buffer, we cannot allow the
	* buffer to remain with B_CACHE set after the write
	* completes or it will represent a corrupt state. To
	* deal with this we set B_NOCACHE to scrap the buffer
	* after the write.
	*
	* We might be able to do something fancy, like setting
	* B_CACHE in bwrite() except if B_DELWRI is already set,
	* so the below call doesn't set B_CACHE, but that gets real
	* confusing. This is much easier.
	*/

	if ((bp->b_flags & (B_CACHE\|B_DELWRI)) == B_DELWRI) {
	bp->b_flags \|= B_NOCACHE;
	bwrite(bp);
	goto loop;
	}
	bp->b_flags &= ~B_DONE;
	} else {
	/*
	* Buffer is not in-core, create new buffer. The buffer
	* returned by getnewbuf() is locked. Note that the returned
	* buffer is also considered valid (not marked B_INVAL).
	*/
	BO_RUNLOCK(bo);
	newbuf_unlocked:
	/*
	* If the user does not want us to create the buffer, bail out
	* here.
	*/
	if (flags & GB_NOCREAT)
	return (EEXIST);

	bsize = vn_isdisk(vp, NULL) ? DEV_BSIZE : bo->bo_bsize;
	KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
	offset = blkno * bsize;
	vmio = vp->v_object != NULL;
	if (vmio) {
	maxsize = size + (offset & PAGE_MASK);
	} else {
	maxsize = size;
	/* Do not allow non-VMIO notmapped buffers. */
	flags &= ~(GB_UNMAPPED \| GB_KVAALLOC);
	}
	maxsize = imax(maxsize, bsize);
	if ((flags & GB_NOSPARSE) != 0 && vmio &&
	!vn_isdisk(vp, NULL)) {
	error = VOP_BMAP(vp, blkno, NULL, &d_blkno, 0, 0);
	KASSERT(error != EOPNOTSUPP,
	("GB_NOSPARSE from fs not supporting bmap, vp %p",
	vp));
	if (error != 0)
	return (error);
	if (d_blkno == -1)
	return (EJUSTRETURN);
	}

	bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
	if (bp == NULL) {
	if (slpflag \|\| slptimeo)
	return (ETIMEDOUT);
	/*
	* XXX This is here until the sleep path is diagnosed
	* enough to work under very low memory conditions.
	*
	* There's an issue on low memory, 4BSD+non-preempt
	* systems (eg MIPS routers with 32MB RAM) where buffer
	* exhaustion occurs without sleeping for buffer
	* reclaimation. This just sticks in a loop and
	* constantly attempts to allocate a buffer, which
	* hits exhaustion and tries to wakeup bufdaemon.
	* This never happens because we never yield.
	*
	* The real solution is to identify and fix these cases
	* so we aren't effectively busy-waiting in a loop
	* until the reclaimation path has cycles to run.
	*/
	kern_yield(PRI_USER);
	goto loop;
	}

	/*
	* This code is used to make sure that a buffer is not
	* created while the getnewbuf routine is blocked.
	* This can be a problem whether the vnode is locked or not.
	* If the buffer is created out from under us, we have to
	* throw away the one we just created.
	*
	* Note: this must occur before we associate the buffer
	* with the vp especially considering limitations in
	* the splay tree implementation when dealing with duplicate
	* lblkno's.
	*/
	BO_LOCK(bo);
	if (gbincore(bo, blkno)) {
	BO_UNLOCK(bo);
	bp->b_flags \|= B_INVAL;
	bufspace_release(bufdomain(bp), maxsize);
	brelse(bp);
	goto loop;
	}

	/*
	* Insert the buffer into the hash, so that it can
	* be found by incore.
	*/
	bp->b_lblkno = blkno;
	bp->b_blkno = d_blkno;
	bp->b_offset = offset;
	bgetvp(vp, bp);
	BO_UNLOCK(bo);

	/*
	* set B_VMIO bit. allocbuf() the buffer bigger. Since the
	* buffer size starts out as 0, B_CACHE will be set by
	* allocbuf() for the VMIO case prior to it testing the
	* backing store for validity.
	*/

	if (vmio) {
	bp->b_flags \|= B_VMIO;
	KASSERT(vp->v_object == bp->b_bufobj->bo_object,
	("ARGH! different b_bufobj->bo_object %p %p %p\n",
	bp, vp->v_object, bp->b_bufobj->bo_object));
	} else {
	bp->b_flags &= ~B_VMIO;
	KASSERT(bp->b_bufobj->bo_object == NULL,
	("ARGH! has b_bufobj->bo_object %p %p\n",
	bp, bp->b_bufobj->bo_object));
	BUF_CHECK_MAPPED(bp);
	}

	allocbuf(bp, size);
	bufspace_release(bufdomain(bp), maxsize);
	bp->b_flags &= ~B_DONE;
	}
	CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
	end:
	buf_track(bp, __func__);
	KASSERT(bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
	*bpp = bp;
	return (0);
	}

	/*
	* Get an empty, disassociated buffer of given size. The buffer is initially
	* set to B_INVAL.
	*/
	struct buf *
	geteblk(int size, int flags)
	{
	struct buf *bp;
	int maxsize;

	maxsize = (size + BKVAMASK) & ~BKVAMASK;
	while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
	if ((flags & GB_NOWAIT_BD) &&
	(curthread->td_pflags & TDP_BUFNEED) != 0)
	return (NULL);
	}
	allocbuf(bp, size);
	bufspace_release(bufdomain(bp), maxsize);
	bp->b_flags \|= B_INVAL; /* b_dep cleared by getnewbuf() */
	return (bp);
	}

	/*
	* Truncate the backing store for a non-vmio buffer.
	*/
	static void
	vfs_nonvmio_truncate(struct buf *bp, int newbsize)
	{

	if (bp->b_flags & B_MALLOC) {
	/*
	* malloced buffers are not shrunk
	*/
	if (newbsize == 0) {
	bufmallocadjust(bp, 0);
	free(bp->b_data, M_BIOBUF);
	bp->b_data = bp->b_kvabase;
	bp->b_flags &= ~B_MALLOC;
	}
	return;
	}
	vm_hold_free_pages(bp, newbsize);
	bufspace_adjust(bp, newbsize);
	}

	/*
	* Extend the backing for a non-VMIO buffer.
	*/
	static void
	vfs_nonvmio_extend(struct buf *bp, int newbsize)
	{
	caddr_t origbuf;
	int origbufsize;

	/*
	* We only use malloced memory on the first allocation.
	* and revert to page-allocated memory when the buffer
	* grows.
	*
	* There is a potential smp race here that could lead
	* to bufmallocspace slightly passing the max. It
	* is probably extremely rare and not worth worrying
	* over.
	*/
	if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
	bufmallocspace < maxbufmallocspace) {
	bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
	bp->b_flags \|= B_MALLOC;
	bufmallocadjust(bp, newbsize);
	return;
	}

	/*
	* If the buffer is growing on its other-than-first
	* allocation then we revert to the page-allocation
	* scheme.
	*/
	origbuf = NULL;
	origbufsize = 0;
	if (bp->b_flags & B_MALLOC) {
	origbuf = bp->b_data;
	origbufsize = bp->b_bufsize;
	bp->b_data = bp->b_kvabase;
	bufmallocadjust(bp, 0);
	bp->b_flags &= ~B_MALLOC;
	newbsize = round_page(newbsize);
	}
	vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
	(vm_offset_t) bp->b_data + newbsize);
	if (origbuf != NULL) {
	bcopy(origbuf, bp->b_data, origbufsize);
	free(origbuf, M_BIOBUF);
	}
	bufspace_adjust(bp, newbsize);
	}

	/*
	* This code constitutes the buffer memory from either anonymous system
	* memory (in the case of non-VMIO operations) or from an associated
	* VM object (in the case of VMIO operations). This code is able to
	* resize a buffer up or down.
	*
	* Note that this code is tricky, and has many complications to resolve
	* deadlock or inconsistent data situations. Tread lightly!!!
	* There are B_CACHE and B_DELWRI interactions that must be dealt with by
	* the caller. Calling this code willy nilly can result in the loss of data.
	*
	* allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
	* B_CACHE for the non-VMIO case.
	*/
	int
	allocbuf(struct buf *bp, int size)
	{
	int newbsize;

	if (bp->b_bcount == size)
	return (1);

	if (bp->b_kvasize != 0 && bp->b_kvasize < size)
	panic("allocbuf: buffer too small");

	newbsize = roundup2(size, DEV_BSIZE);
	if ((bp->b_flags & B_VMIO) == 0) {
	if ((bp->b_flags & B_MALLOC) == 0)
	newbsize = round_page(newbsize);
	/*
	* Just get anonymous memory from the kernel. Don't
	* mess with B_CACHE.
	*/
	if (newbsize < bp->b_bufsize)
	vfs_nonvmio_truncate(bp, newbsize);
	else if (newbsize > bp->b_bufsize)
	vfs_nonvmio_extend(bp, newbsize);
	} else {
	int desiredpages;

	desiredpages = (size == 0) ? 0 :
	num_pages((bp->b_offset & PAGE_MASK) + newbsize);

	if (bp->b_flags & B_MALLOC)
	panic("allocbuf: VMIO buffer can't be malloced");
	/*
	* Set B_CACHE initially if buffer is 0 length or will become
	* 0-length.
	*/
	if (size == 0 \|\| bp->b_bufsize == 0)
	bp->b_flags \|= B_CACHE;

	if (newbsize < bp->b_bufsize)
	vfs_vmio_truncate(bp, desiredpages);
	/* XXX This looks as if it should be newbsize > b_bufsize */
	else if (size > bp->b_bcount)
	vfs_vmio_extend(bp, desiredpages, size);
	bufspace_adjust(bp, newbsize);
	}
	bp->b_bcount = size; /* requested buffer size. */
	return (1);
	}

	extern int inflight_transient_maps;

	static struct bio_queue nondump_bios;

	void
	biodone(struct bio *bp)
	{
	struct mtx *mtxp;
	void (done)(struct bio );
	vm_offset_t start, end;

	biotrack(bp, __func__);

	/*
	* Avoid completing I/O when dumping after a panic since that may
	* result in a deadlock in the filesystem or pager code. Note that
	* this doesn't affect dumps that were started manually since we aim
	* to keep the system usable after it has been resumed.
	*/
	if (__predict_false(dumping && SCHEDULER_STOPPED())) {
	TAILQ_INSERT_HEAD(&nondump_bios, bp, bio_queue);
	return;
	}
	if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
	bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
	bp->bio_flags \|= BIO_UNMAPPED;
	start = trunc_page((vm_offset_t)bp->bio_data);
	end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
	bp->bio_data = unmapped_buf;
	pmap_qremove(start, atop(end - start));
	vmem_free(transient_arena, start, end - start);
	atomic_add_int(&inflight_transient_maps, -1);
	}
	done = bp->bio_done;
	if (done == NULL) {
	mtxp = mtx_pool_find(mtxpool_sleep, bp);
	mtx_lock(mtxp);
	bp->bio_flags \|= BIO_DONE;
	wakeup(bp);
	mtx_unlock(mtxp);
	} else
	done(bp);
	}

	/*
	* Wait for a BIO to finish.
	*/
	int
	biowait(struct bio bp, const char wchan)
	{
	struct mtx *mtxp;

	mtxp = mtx_pool_find(mtxpool_sleep, bp);
	mtx_lock(mtxp);
	while ((bp->bio_flags & BIO_DONE) == 0)
	msleep(bp, mtxp, PRIBIO, wchan, 0);
	mtx_unlock(mtxp);
	if (bp->bio_error != 0)
	return (bp->bio_error);
	if (!(bp->bio_flags & BIO_ERROR))
	return (0);
	return (EIO);
	}

	void
	biofinish(struct bio bp, struct devstat stat, int error)
	{

	if (error) {
	bp->bio_error = error;
	bp->bio_flags \|= BIO_ERROR;
	}
	if (stat != NULL)
	devstat_end_transaction_bio(stat, bp);
	biodone(bp);
	}

	#if defined(BUF_TRACKING) \|\| defined(FULL_BUF_TRACKING)
	void
	biotrack_buf(struct bio bp, const char location)
	{

	buf_track(bp->bio_track_bp, location);
	}
	#endif

	/*
	* bufwait:
	*
	* Wait for buffer I/O completion, returning error status. The buffer
	* is left locked and B_DONE on return. B_EINTR is converted into an EINTR
	* error and cleared.
	*/
	int
	bufwait(struct buf *bp)
	{
	if (bp->b_iocmd == BIO_READ)
	bwait(bp, PRIBIO, "biord");
	else
	bwait(bp, PRIBIO, "biowr");
	if (bp->b_flags & B_EINTR) {
	bp->b_flags &= ~B_EINTR;
	return (EINTR);
	}
	if (bp->b_ioflags & BIO_ERROR) {
	return (bp->b_error ? bp->b_error : EIO);
	} else {
	return (0);
	}
	}

	/*
	* bufdone:
	*
	* Finish I/O on a buffer, optionally calling a completion function.
	* This is usually called from an interrupt so process blocking is
	* not allowed.
	*
	* biodone is also responsible for setting B_CACHE in a B_VMIO bp.
	* In a non-VMIO bp, B_CACHE will be set on the next getblk()
	* assuming B_INVAL is clear.
	*
	* For the VMIO case, we set B_CACHE if the op was a read and no
	* read error occurred, or if the op was a write. B_CACHE is never
	* set if the buffer is invalid or otherwise uncacheable.
	*
	* bufdone does not mess with B_INVAL, allowing the I/O routine or the
	* initiator to leave B_INVAL set to brelse the buffer out of existence
	* in the biodone routine.
	*/
	void
	bufdone(struct buf *bp)
	{
	struct bufobj *dropobj;
	void (biodone)(struct buf );

	buf_track(bp, __func__);
	CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	dropobj = NULL;

	KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));

	runningbufwakeup(bp);
	if (bp->b_iocmd == BIO_WRITE)
	dropobj = bp->b_bufobj;
	/* call optional completion function if requested */
	if (bp->b_iodone != NULL) {
	biodone = bp->b_iodone;
	bp->b_iodone = NULL;
	(*biodone) (bp);
	if (dropobj)
	bufobj_wdrop(dropobj);
	return;
	}
	if (bp->b_flags & B_VMIO) {
	/*
	* Set B_CACHE if the op was a normal read and no error
	* occurred. B_CACHE is set for writes in the b*write()
	* routines.
	*/
	if (bp->b_iocmd == BIO_READ &&
	!(bp->b_flags & (B_INVAL\|B_NOCACHE)) &&
	!(bp->b_ioflags & BIO_ERROR))
	bp->b_flags \|= B_CACHE;
	vfs_vmio_iodone(bp);
	}
	if (!LIST_EMPTY(&bp->b_dep))
	buf_complete(bp);
	if ((bp->b_flags & B_CKHASH) != 0) {
	KASSERT(bp->b_iocmd == BIO_READ,
	("bufdone: b_iocmd %d not BIO_READ", bp->b_iocmd));
	KASSERT(buf_mapped(bp), ("bufdone: bp %p not mapped", bp));
	(*bp->b_ckhashcalc)(bp);
	}
	/*
	* For asynchronous completions, release the buffer now. The brelse
	* will do a wakeup there if necessary - so no need to do a wakeup
	* here in the async case. The sync case always needs to do a wakeup.
	*/
	if (bp->b_flags & B_ASYNC) {
	if ((bp->b_flags & (B_NOCACHE \| B_INVAL \| B_RELBUF)) \|\|
	(bp->b_ioflags & BIO_ERROR))
	brelse(bp);
	else
	bqrelse(bp);
	} else
	bdone(bp);
	if (dropobj)
	bufobj_wdrop(dropobj);
	}

	/*
	* This routine is called in lieu of iodone in the case of
	* incomplete I/O. This keeps the busy status for pages
	* consistent.
	*/
	void
	vfs_unbusy_pages(struct buf *bp)
	{
	int i;
	vm_object_t obj;
	vm_page_t m;

	runningbufwakeup(bp);
	if (!(bp->b_flags & B_VMIO))
	return;

	obj = bp->b_bufobj->bo_object;
	for (i = 0; i < bp->b_npages; i++) {
	m = bp->b_pages[i];
	if (m == bogus_page) {
	m = vm_page_relookup(obj, OFF_TO_IDX(bp->b_offset) + i);
	if (!m)
	panic("vfs_unbusy_pages: page missing\n");
	bp->b_pages[i] = m;
	if (buf_mapped(bp)) {
	BUF_CHECK_MAPPED(bp);
	pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
	bp->b_pages, bp->b_npages);
	} else
	BUF_CHECK_UNMAPPED(bp);
	}
	vm_page_sunbusy(m);
	}
	vm_object_pip_wakeupn(obj, bp->b_npages);
	}

	/*
	* vfs_page_set_valid:
	*
	* Set the valid bits in a page based on the supplied offset. The
	* range is restricted to the buffer's size.
	*
	* This routine is typically called after a read completes.
	*/
	static void
	vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
	{
	vm_ooffset_t eoff;

	/*
	* Compute the end offset, eoff, such that [off, eoff) does not span a
	* page boundary and eoff is not greater than the end of the buffer.
	* The end of the buffer, in this case, is our file EOF, not the
	* allocation size of the buffer.
	*/
	eoff = (off + PAGE_SIZE) & ~(vm_ooffset_t)PAGE_MASK;
	if (eoff > bp->b_offset + bp->b_bcount)
	eoff = bp->b_offset + bp->b_bcount;

	/*
	* Set valid range. This is typically the entire buffer and thus the
	* entire page.
	*/
	if (eoff > off)
	vm_page_set_valid_range(m, off & PAGE_MASK, eoff - off);
	}

	/*
	* vfs_page_set_validclean:
	*
	* Set the valid bits and clear the dirty bits in a page based on the
	* supplied offset. The range is restricted to the buffer's size.
	*/
	static void
	vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
	{
	vm_ooffset_t soff, eoff;

	/*
	* Start and end offsets in buffer. eoff - soff may not cross a
	* page boundary or cross the end of the buffer. The end of the
	* buffer, in this case, is our file EOF, not the allocation size
	* of the buffer.
	*/
	soff = off;
	eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
	if (eoff > bp->b_offset + bp->b_bcount)
	eoff = bp->b_offset + bp->b_bcount;

	/*
	* Set valid range. This is typically the entire buffer and thus the
	* entire page.
	*/
	if (eoff > soff) {
	vm_page_set_validclean(
	m,
	(vm_offset_t) (soff & PAGE_MASK),
	(vm_offset_t) (eoff - soff)
	);
	}
	}

	/*
	* Acquire a shared busy on all pages in the buf.
	*/
	void
	vfs_busy_pages_acquire(struct buf *bp)
	{
	int i;

	for (i = 0; i < bp->b_npages; i++)
	vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY);
	}

	void
	vfs_busy_pages_release(struct buf *bp)
	{
	int i;

	for (i = 0; i < bp->b_npages; i++)
	vm_page_sunbusy(bp->b_pages[i]);
	}

	/*
	* This routine is called before a device strategy routine.
	* It is used to tell the VM system that paging I/O is in
	* progress, and treat the pages associated with the buffer
	* almost as being exclusive busy. Also the object paging_in_progress
	* flag is handled to make sure that the object doesn't become
	* inconsistent.
	*
	* Since I/O has not been initiated yet, certain buffer flags
	* such as BIO_ERROR or B_INVAL may be in an inconsistent state
	* and should be ignored.
	*/
	void
	vfs_busy_pages(struct buf *bp, int clear_modify)
	{
	vm_object_t obj;
	vm_ooffset_t foff;
	vm_page_t m;
	int i;
	bool bogus;

	if (!(bp->b_flags & B_VMIO))
	return;

	obj = bp->b_bufobj->bo_object;
	foff = bp->b_offset;
	KASSERT(bp->b_offset != NOOFFSET,
	("vfs_busy_pages: no buffer offset"));
	if ((bp->b_flags & B_CLUSTER) == 0) {
	vm_object_pip_add(obj, bp->b_npages);
	vfs_busy_pages_acquire(bp);
	}
	if (bp->b_bufsize != 0)
	vfs_setdirty_range(bp);
	bogus = false;
	for (i = 0; i < bp->b_npages; i++) {
	m = bp->b_pages[i];
	vm_page_assert_sbusied(m);

	/*
	* When readying a buffer for a read ( i.e
	* clear_modify == 0 ), it is important to do
	* bogus_page replacement for valid pages in
	* partially instantiated buffers. Partially
	* instantiated buffers can, in turn, occur when
	* reconstituting a buffer from its VM backing store
	* base. We only have to do this if B_CACHE is
	* clear ( which causes the I/O to occur in the
	* first place ). The replacement prevents the read
	* I/O from overwriting potentially dirty VM-backed
	* pages. XXX bogus page replacement is, uh, bogus.
	* It may not work properly with small-block devices.
	* We need to find a better way.
	*/
	if (clear_modify) {
	pmap_remove_write(m);
	vfs_page_set_validclean(bp, foff, m);
	} else if (vm_page_all_valid(m) &&
	(bp->b_flags & B_CACHE) == 0) {
	bp->b_pages[i] = bogus_page;
	bogus = true;
	}
	foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
	}
	if (bogus && buf_mapped(bp)) {
	BUF_CHECK_MAPPED(bp);
	pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
	bp->b_pages, bp->b_npages);
	}
	}

	/*
	* vfs_bio_set_valid:
	*
	* Set the range within the buffer to valid. The range is
	* relative to the beginning of the buffer, b_offset. Note that
	* b_offset itself may be offset from the beginning of the first
	* page.
	*/
	void
	vfs_bio_set_valid(struct buf *bp, int base, int size)
	{
	int i, n;
	vm_page_t m;

	if (!(bp->b_flags & B_VMIO))
	return;

	/*
	* Fixup base to be relative to beginning of first page.
	* Set initial n to be the maximum number of bytes in the
	* first page that can be validated.
	*/
	base += (bp->b_offset & PAGE_MASK);
	n = PAGE_SIZE - (base & PAGE_MASK);

	/*
	* Busy may not be strictly necessary here because the pages are
	* unlikely to be fully valid and the vnode lock will synchronize
	* their access via getpages. It is grabbed for consistency with
	* other page validation.
	*/
	vfs_busy_pages_acquire(bp);
	for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
	m = bp->b_pages[i];
	if (n > size)
	n = size;
	vm_page_set_valid_range(m, base & PAGE_MASK, n);
	base += n;
	size -= n;
	n = PAGE_SIZE;
	}
	vfs_busy_pages_release(bp);
	}

	/*
	* vfs_bio_clrbuf:
	*
	* If the specified buffer is a non-VMIO buffer, clear the entire
	* buffer. If the specified buffer is a VMIO buffer, clear and
	* validate only the previously invalid portions of the buffer.
	* This routine essentially fakes an I/O, so we need to clear
	* BIO_ERROR and B_INVAL.
	*
	* Note that while we only theoretically need to clear through b_bcount,
	* we go ahead and clear through b_bufsize.
	*/
	void
	vfs_bio_clrbuf(struct buf *bp)
	{
	int i, j, mask, sa, ea, slide;

	if ((bp->b_flags & (B_VMIO \| B_MALLOC)) != B_VMIO) {
	clrbuf(bp);
	return;
	}
	bp->b_flags &= ~B_INVAL;
	bp->b_ioflags &= ~BIO_ERROR;
	vfs_busy_pages_acquire(bp);
	sa = bp->b_offset & PAGE_MASK;
	slide = 0;
	for (i = 0; i < bp->b_npages; i++, sa = 0) {
	slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
	ea = slide & PAGE_MASK;
	if (ea == 0)
	ea = PAGE_SIZE;
	if (bp->b_pages[i] == bogus_page)
	continue;
	j = sa / DEV_BSIZE;
	mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
	if ((bp->b_pages[i]->valid & mask) == mask)
	continue;
	if ((bp->b_pages[i]->valid & mask) == 0)
	pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
	else {
	for (; sa < ea; sa += DEV_BSIZE, j++) {
	if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
	pmap_zero_page_area(bp->b_pages[i],
	sa, DEV_BSIZE);
	}
	}
	}
	vm_page_set_valid_range(bp->b_pages[i], j * DEV_BSIZE,
	roundup2(ea - sa, DEV_BSIZE));
	}
	vfs_busy_pages_release(bp);
	bp->b_resid = 0;
	}

	void
	vfs_bio_bzero_buf(struct buf *bp, int base, int size)
	{
	vm_page_t m;
	int i, n;

	if (buf_mapped(bp)) {
	BUF_CHECK_MAPPED(bp);
	bzero(bp->b_data + base, size);
	} else {
	BUF_CHECK_UNMAPPED(bp);
	n = PAGE_SIZE - (base & PAGE_MASK);
	for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
	m = bp->b_pages[i];
	if (n > size)
	n = size;
	pmap_zero_page_area(m, base & PAGE_MASK, n);
	base += n;
	size -= n;
	n = PAGE_SIZE;
	}
	}
	}

	/*
	* Update buffer flags based on I/O request parameters, optionally releasing the
	* buffer. If it's VMIO or direct I/O, the buffer pages are released to the VM,
	* where they may be placed on a page queue (VMIO) or freed immediately (direct
	* I/O). Otherwise the buffer is released to the cache.
	*/
	static void
	b_io_dismiss(struct buf *bp, int ioflag, bool release)
	{

	KASSERT((ioflag & IO_NOREUSE) == 0 \|\| (ioflag & IO_VMIO) != 0,
	("buf %p non-VMIO noreuse", bp));

	if ((ioflag & IO_DIRECT) != 0)
	bp->b_flags \|= B_DIRECT;
	if ((ioflag & IO_EXT) != 0)
	bp->b_xflags \|= BX_ALTDATA;
	if ((ioflag & (IO_VMIO \| IO_DIRECT)) != 0 && LIST_EMPTY(&bp->b_dep)) {
	bp->b_flags \|= B_RELBUF;
	if ((ioflag & IO_NOREUSE) != 0)
	bp->b_flags \|= B_NOREUSE;
	if (release)
	brelse(bp);
	} else if (release)
	bqrelse(bp);
	}

	void
	vfs_bio_brelse(struct buf *bp, int ioflag)
	{

	b_io_dismiss(bp, ioflag, true);
	}

	void
	vfs_bio_set_flags(struct buf *bp, int ioflag)
	{

	b_io_dismiss(bp, ioflag, false);
	}

	/*
	* vm_hold_load_pages and vm_hold_free_pages get pages into
	* a buffers address space. The pages are anonymous and are
	* not associated with a file object.
	*/
	static void
	vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
	{
	vm_offset_t pg;
	vm_page_t p;
	int index;

	BUF_CHECK_MAPPED(bp);

	to = round_page(to);
	from = round_page(from);
	index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;

	for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
	/*
	* note: must allocate system pages since blocking here
	* could interfere with paging I/O, no matter which
	* process we are.
	*/
	p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM \| VM_ALLOC_NOOBJ \|
	VM_ALLOC_WIRED \| VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT) \|
	VM_ALLOC_WAITOK);
	pmap_qenter(pg, &p, 1);
	bp->b_pages[index] = p;
	}
	bp->b_npages = index;
	}

	/* Return pages associated with this buf to the vm system */
	static void
	vm_hold_free_pages(struct buf *bp, int newbsize)
	{
	vm_offset_t from;
	vm_page_t p;
	int index, newnpages;

	BUF_CHECK_MAPPED(bp);

	from = round_page((vm_offset_t)bp->b_data + newbsize);
	newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
	if (bp->b_npages > newnpages)
	pmap_qremove(from, bp->b_npages - newnpages);
	for (index = newnpages; index < bp->b_npages; index++) {
	p = bp->b_pages[index];
	bp->b_pages[index] = NULL;
	vm_page_unwire_noq(p);
	vm_page_free(p);
	}
	bp->b_npages = newnpages;
	}

	/*
	* Map an IO request into kernel virtual address space.
	*
	* All requests are (re)mapped into kernel VA space.
	* Notice that we use b_bufsize for the size of the buffer
	* to be mapped. b_bcount might be modified by the driver.
	*
	* Note that even if the caller determines that the address space should
	* be valid, a race or a smaller-file mapped into a larger space may
	* actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
	* check the return value.
	*
	* This function only works with pager buffers.
	*/
	int
	vmapbuf(struct buf *bp, int mapbuf)
	{
	vm_prot_t prot;
	int pidx;

	if (bp->b_bufsize < 0)
	return (-1);
	prot = VM_PROT_READ;
	if (bp->b_iocmd == BIO_READ)
	prot \|= VM_PROT_WRITE; /* Less backwards than it looks */
	if ((pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
	(vm_offset_t)bp->b_data, bp->b_bufsize, prot, bp->b_pages,
	btoc(MAXPHYS))) < 0)
	return (-1);
	bp->b_npages = pidx;
	bp->b_offset = ((vm_offset_t)bp->b_data) & PAGE_MASK;
	if (mapbuf \|\| !unmapped_buf_allowed) {
	pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
	bp->b_data = bp->b_kvabase + bp->b_offset;
	} else
	bp->b_data = unmapped_buf;
	return(0);
	}

	/*
	* Free the io map PTEs associated with this IO operation.
	* We also invalidate the TLB entries and restore the original b_addr.
	*
	* This function only works with pager buffers.
	*/
	void
	vunmapbuf(struct buf *bp)
	{
	int npages;

	npages = bp->b_npages;
	if (buf_mapped(bp))
	pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
	vm_page_unhold_pages(bp->b_pages, npages);

	bp->b_data = unmapped_buf;
	}

	void
	bdone(struct buf *bp)
	{
	struct mtx *mtxp;

	mtxp = mtx_pool_find(mtxpool_sleep, bp);
	mtx_lock(mtxp);
	bp->b_flags \|= B_DONE;
	wakeup(bp);
	mtx_unlock(mtxp);
	}

	void
	bwait(struct buf bp, u_char pri, const char wchan)
	{
	struct mtx *mtxp;

	mtxp = mtx_pool_find(mtxpool_sleep, bp);
	mtx_lock(mtxp);
	while ((bp->b_flags & B_DONE) == 0)
	msleep(bp, mtxp, pri, wchan, 0);
	mtx_unlock(mtxp);
	}

	int
	bufsync(struct bufobj *bo, int waitfor)
	{

	return (VOP_FSYNC(bo2vnode(bo), waitfor, curthread));
	}

	void
	bufstrategy(struct bufobj bo, struct buf bp)
	{
	int i __unused;
	struct vnode *vp;

	vp = bp->b_vp;
	KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy"));
	KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
	("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
	i = VOP_STRATEGY(vp, bp);
	KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
	}

	/*
	* Initialize a struct bufobj before use. Memory is assumed zero filled.
	*/
	void
	bufobj_init(struct bufobj bo, void private)
	{
	static volatile int bufobj_cleanq;

	bo->bo_domain =
	atomic_fetchadd_int(&bufobj_cleanq, 1) % buf_domains;
	rw_init(BO_LOCKPTR(bo), "bufobj interlock");
	bo->bo_private = private;
	TAILQ_INIT(&bo->bo_clean.bv_hd);
	TAILQ_INIT(&bo->bo_dirty.bv_hd);
	}

	void
	bufobj_wrefl(struct bufobj *bo)
	{

	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
	ASSERT_BO_WLOCKED(bo);
	bo->bo_numoutput++;
	}

	void
	bufobj_wref(struct bufobj *bo)
	{

	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
	BO_LOCK(bo);
	bo->bo_numoutput++;
	BO_UNLOCK(bo);
	}

	void
	bufobj_wdrop(struct bufobj *bo)
	{

	KASSERT(bo != NULL, ("NULL bo in bufobj_wdrop"));
	BO_LOCK(bo);
	KASSERT(bo->bo_numoutput > 0, ("bufobj_wdrop non-positive count"));
	if ((--bo->bo_numoutput == 0) && (bo->bo_flag & BO_WWAIT)) {
	bo->bo_flag &= ~BO_WWAIT;
	wakeup(&bo->bo_numoutput);
	}
	BO_UNLOCK(bo);
	}

	int
	bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
	{
	int error;

	KASSERT(bo != NULL, ("NULL bo in bufobj_wwait"));
	ASSERT_BO_WLOCKED(bo);
	error = 0;
	while (bo->bo_numoutput) {
	bo->bo_flag \|= BO_WWAIT;
	error = msleep(&bo->bo_numoutput, BO_LOCKPTR(bo),
	slpflag \| (PRIBIO + 1), "bo_wwait", timeo);
	if (error)
	break;
	}
	return (error);
	}

	/*
	* Set bio_data or bio_ma for struct bio from the struct buf.
	*/
	void
	bdata2bio(struct buf bp, struct bio bip)
	{

	if (!buf_mapped(bp)) {
	KASSERT(unmapped_buf_allowed, ("unmapped"));
	bip->bio_ma = bp->b_pages;
	bip->bio_ma_n = bp->b_npages;
	bip->bio_data = unmapped_buf;
	bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
	bip->bio_flags \|= BIO_UNMAPPED;
	KASSERT(round_page(bip->bio_ma_offset + bip->bio_length) /
	PAGE_SIZE == bp->b_npages,
	("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
	(long long)bip->bio_length, bip->bio_ma_n));
	} else {
	bip->bio_data = bp->b_data;
	bip->bio_ma = NULL;
	}
	}

	/*
	* The MIPS pmap code currently doesn't handle aliased pages.
	* The VIPT caches may not handle page aliasing themselves, leading
	* to data corruption.
	*
	* As such, this code makes a system extremely unhappy if said
	* system doesn't support unaliasing the above situation in hardware.
	* Some "recent" systems (eg some mips24k/mips74k cores) don't enable
	* this feature at build time, so it has to be handled in software.
	*
	* Once the MIPS pmap/cache code grows to support this function on
	* earlier chips, it should be flipped back off.
	*/
	#ifdef __mips__
	static int buf_pager_relbuf = 1;
	#else
	static int buf_pager_relbuf = 0;
	#endif
	SYSCTL_INT(_vfs, OID_AUTO, buf_pager_relbuf, CTLFLAG_RWTUN,
	&buf_pager_relbuf, 0,
	"Make buffer pager release buffers after reading");

	/*
	* The buffer pager. It uses buffer reads to validate pages.
	*
	* In contrast to the generic local pager from vm/vnode_pager.c, this
	* pager correctly and easily handles volumes where the underlying
	* device block size is greater than the machine page size. The
	* buffer cache transparently extends the requested page run to be
	* aligned at the block boundary, and does the necessary bogus page
	* replacements in the addends to avoid obliterating already valid
	* pages.
	*
	* The only non-trivial issue is that the exclusive busy state for
	* pages, which is assumed by the vm_pager_getpages() interface, is
	* incompatible with the VMIO buffer cache's desire to share-busy the
	* pages. This function performs a trivial downgrade of the pages'
	* state before reading buffers, and a less trivial upgrade from the
	* shared-busy to excl-busy state after the read.
	*/
	int
	vfs_bio_getpages(struct vnode vp, vm_page_t ma, int count,
	int rbehind, int rahead, vbg_get_lblkno_t get_lblkno,
	vbg_get_blksize_t get_blksize)
	{
	vm_page_t m;
	vm_object_t object;
	struct buf *bp;
	struct mount *mp;
	daddr_t lbn, lbnp;
	vm_ooffset_t la, lb, poff, poffe;
	long bsize;
	int bo_bs, br_flags, error, i, pgsin, pgsin_a, pgsin_b;
	bool redo, lpart;

	object = vp->v_object;
	mp = vp->v_mount;
	error = 0;
	la = IDX_TO_OFF(ma[count - 1]->pindex);
	if (la >= object->un_pager.vnp.vnp_size)
	return (VM_PAGER_BAD);

	/*
	* Change the meaning of la from where the last requested page starts
	* to where it ends, because that's the end of the requested region
	* and the start of the potential read-ahead region.
	*/
	la += PAGE_SIZE;
	lpart = la > object->un_pager.vnp.vnp_size;
	bo_bs = get_blksize(vp, get_lblkno(vp, IDX_TO_OFF(ma[0]->pindex)));

	/*
	* Calculate read-ahead, behind and total pages.
	*/
	pgsin = count;
	lb = IDX_TO_OFF(ma[0]->pindex);
	pgsin_b = OFF_TO_IDX(lb - rounddown2(lb, bo_bs));
	pgsin += pgsin_b;
	if (rbehind != NULL)
	*rbehind = pgsin_b;
	pgsin_a = OFF_TO_IDX(roundup2(la, bo_bs) - la);
	if (la + IDX_TO_OFF(pgsin_a) >= object->un_pager.vnp.vnp_size)
	pgsin_a = OFF_TO_IDX(roundup2(object->un_pager.vnp.vnp_size,
	PAGE_SIZE) - la);
	pgsin += pgsin_a;
	if (rahead != NULL)
	*rahead = pgsin_a;
	VM_CNT_INC(v_vnodein);
	VM_CNT_ADD(v_vnodepgsin, pgsin);

	br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)
	!= 0) ? GB_UNMAPPED : 0;
	again:
	for (i = 0; i < count; i++) {
	if (ma[i] != bogus_page)
	vm_page_busy_downgrade(ma[i]);
	}

	lbnp = -1;
	for (i = 0; i < count; i++) {
	m = ma[i];
	if (m == bogus_page)
	continue;

	/*
	* Pages are shared busy and the object lock is not
	* owned, which together allow for the pages'
	* invalidation. The racy test for validity avoids
	* useless creation of the buffer for the most typical
	* case when invalidation is not used in redo or for
	* parallel read. The shared->excl upgrade loop at
	* the end of the function catches the race in a
	* reliable way (protected by the object lock).
	*/
	if (vm_page_all_valid(m))
	continue;

	poff = IDX_TO_OFF(m->pindex);
	poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size);
	for (; poff < poffe; poff += bsize) {
	lbn = get_lblkno(vp, poff);
	if (lbn == lbnp)
	goto next_page;
	lbnp = lbn;

	bsize = get_blksize(vp, lbn);
	error = bread_gb(vp, lbn, bsize, curthread->td_ucred,
	br_flags, &bp);
	if (error != 0)
	goto end_pages;
	if (bp->b_rcred == curthread->td_ucred) {
	crfree(bp->b_rcred);
	bp->b_rcred = NOCRED;
	}
	if (LIST_EMPTY(&bp->b_dep)) {
	/*
	* Invalidation clears m->valid, but
	* may leave B_CACHE flag if the
	* buffer existed at the invalidation
	* time. In this case, recycle the
	* buffer to do real read on next
	* bread() after redo.
	*
	* Otherwise B_RELBUF is not strictly
	* necessary, enable to reduce buf
	* cache pressure.
	*/
	if (buf_pager_relbuf \|\|
	!vm_page_all_valid(m))
	bp->b_flags \|= B_RELBUF;

	bp->b_flags &= ~B_NOCACHE;
	brelse(bp);
	} else {
	bqrelse(bp);
	}
	}
	KASSERT(1 /* racy, enable for debugging */ \|\|
	vm_page_all_valid(m) \|\| i == count - 1,
	("buf %d %p invalid", i, m));
	if (i == count - 1 && lpart) {
	if (!vm_page_none_valid(m) &&
	!vm_page_all_valid(m))
	vm_page_zero_invalid(m, TRUE);
	}
	next_page:;
	}
	end_pages:

	redo = false;
	for (i = 0; i < count; i++) {
	if (ma[i] == bogus_page)
	continue;
	if (vm_page_busy_tryupgrade(ma[i]) == 0) {
	vm_page_sunbusy(ma[i]);
	ma[i] = vm_page_grab_unlocked(object, ma[i]->pindex,
	VM_ALLOC_NORMAL);
	}

	/*
	* Since the pages were only sbusy while neither the
	* buffer nor the object lock was held by us, or
	* reallocated while vm_page_grab() slept for busy
	* relinguish, they could have been invalidated.
	* Recheck the valid bits and re-read as needed.
	*
	* Note that the last page is made fully valid in the
	* read loop, and partial validity for the page at
	* index count - 1 could mean that the page was
	* invalidated or removed, so we must restart for
	* safety as well.
	*/
	if (!vm_page_all_valid(ma[i]))
	redo = true;
	}
	if (redo && error == 0)
	goto again;
	return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
	}

	#include "opt_ddb.h"
	#ifdef DDB
	#include <ddb/ddb.h>

	/* DDB command to show buffer data */
	DB_SHOW_COMMAND(buffer, db_show_buffer)
	{
	/* get args */
	struct buf bp = (struct buf )addr;
	#ifdef FULL_BUF_TRACKING
	uint32_t i, j;
	#endif

	if (!have_addr) {
	db_printf("usage: show buffer <addr>\n");
	return;
	}

	db_printf("buf at %p\n", bp);
	db_printf("b_flags = 0x%b, b_xflags=0x%b\n",
	(u_int)bp->b_flags, PRINT_BUF_FLAGS,
	(u_int)bp->b_xflags, PRINT_BUF_XFLAGS);
	db_printf("b_vflags=0x%b b_ioflags0x%b\n",
	(u_int)bp->b_vflags, PRINT_BUF_VFLAGS,
	(u_int)bp->b_ioflags, PRINT_BIO_FLAGS);
	db_printf(
	"b_error = %d, b_bufsize = %ld, b_bcount = %ld, b_resid = %ld\n"
	"b_bufobj = (%p), b_data = %p\n, b_blkno = %jd, b_lblkno = %jd, "
	"b_vp = %p, b_dep = %p\n",
	bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
	bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
	(intmax_t)bp->b_lblkno, bp->b_vp, bp->b_dep.lh_first);
	db_printf("b_kvabase = %p, b_kvasize = %d\n",
	bp->b_kvabase, bp->b_kvasize);
	if (bp->b_npages) {
	int i;
	db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
	for (i = 0; i < bp->b_npages; i++) {
	vm_page_t m;
	m = bp->b_pages[i];
	if (m != NULL)
	db_printf("(%p, 0x%lx, 0x%lx)", m->object,
	(u_long)m->pindex,
	(u_long)VM_PAGE_TO_PHYS(m));
	else
	db_printf("( ??? )");
	if ((i + 1) < bp->b_npages)
	db_printf(",");
	}
	db_printf("\n");
	}
	BUF_LOCKPRINTINFO(bp);
	#if defined(FULL_BUF_TRACKING)
	db_printf("b_io_tracking: b_io_tcnt = %u\n", bp->b_io_tcnt);

	i = bp->b_io_tcnt % BUF_TRACKING_SIZE;
	for (j = 1; j <= BUF_TRACKING_SIZE; j++) {
	if (bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)] == NULL)
	continue;
	db_printf(" %2u: %s\n", j,
	bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)]);
	}
	#elif defined(BUF_TRACKING)
	db_printf("b_io_tracking: %s\n", bp->b_io_tracking);
	#endif
	db_printf(" ");
	}

	DB_SHOW_COMMAND(bufqueues, bufqueues)
	{
	struct bufdomain *bd;
	struct buf *bp;
	long total;
	int i, j, cnt;

	db_printf("bqempty: %d\n", bqempty.bq_len);

	for (i = 0; i < buf_domains; i++) {
	bd = &bdomain[i];
	db_printf("Buf domain %d\n", i);
	db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers);
	db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers);
	db_printf("\thifreebufs\t%d\n", bd->bd_hifreebuffers);
	db_printf("\n");
	db_printf("\tbufspace\t%ld\n", bd->bd_bufspace);
	db_printf("\tmaxbufspace\t%ld\n", bd->bd_maxbufspace);
	db_printf("\thibufspace\t%ld\n", bd->bd_hibufspace);
	db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace);
	db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh);
	db_printf("\n");
	db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers);
	db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers);
	db_printf("\thidirtybuffers\t%d\n", bd->bd_hidirtybuffers);
	db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh);
	db_printf("\n");
	total = 0;
	TAILQ_FOREACH(bp, &bd->bd_cleanq->bq_queue, b_freelist)
	total += bp->b_bufsize;
	db_printf("\tcleanq count\t%d (%ld)\n",
	bd->bd_cleanq->bq_len, total);
	total = 0;
	TAILQ_FOREACH(bp, &bd->bd_dirtyq.bq_queue, b_freelist)
	total += bp->b_bufsize;
	db_printf("\tdirtyq count\t%d (%ld)\n",
	bd->bd_dirtyq.bq_len, total);
	db_printf("\twakeup\t\t%d\n", bd->bd_wanted);
	db_printf("\tlim\t\t%d\n", bd->bd_lim);
	db_printf("\tCPU ");
	for (j = 0; j <= mp_maxid; j++)
	db_printf("%d, ", bd->bd_subq[j].bq_len);
	db_printf("\n");
	cnt = 0;
	total = 0;
	for (j = 0; j < nbuf; j++)
	if (buf[j].b_domain == i && BUF_ISLOCKED(&buf[j])) {
	cnt++;
	total += buf[j].b_bufsize;
	}
	db_printf("\tLocked buffers: %d space %ld\n", cnt, total);
	cnt = 0;
	total = 0;
	for (j = 0; j < nbuf; j++)
	if (buf[j].b_domain == i) {
	cnt++;
	total += buf[j].b_bufsize;
	}
	db_printf("\tTotal buffers: %d space %ld\n", cnt, total);
	}
	}

	DB_SHOW_COMMAND(lockedbufs, lockedbufs)
	{
	struct buf *bp;
	int i;

	for (i = 0; i < nbuf; i++) {
	bp = &buf[i];
	if (BUF_ISLOCKED(bp)) {
	db_show_buffer((uintptr_t)bp, 1, 0, NULL);
	db_printf("\n");
	if (db_pager_quit)
	break;
	}
	}
	}

	DB_SHOW_COMMAND(vnodebufs, db_show_vnodebufs)
	{
	struct vnode *vp;
	struct buf *bp;

	if (!have_addr) {
	db_printf("usage: show vnodebufs <addr>\n");
	return;
	}
	vp = (struct vnode *)addr;
	db_printf("Clean buffers:\n");
	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
	db_show_buffer((uintptr_t)bp, 1, 0, NULL);
	db_printf("\n");
	}
	db_printf("Dirty buffers:\n");
	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
	db_show_buffer((uintptr_t)bp, 1, 0, NULL);
	db_printf("\n");
	}
	}

	DB_COMMAND(countfreebufs, db_coundfreebufs)
	{
	struct buf *bp;
	int i, used = 0, nfree = 0;

	if (have_addr) {
	db_printf("usage: countfreebufs\n");
	return;
	}

	for (i = 0; i < nbuf; i++) {
	bp = &buf[i];
	if (bp->b_qindex == QUEUE_EMPTY)
	nfree++;
	else
	used++;
	}

	db_printf("Counted %d free, %d used (%d tot)\n", nfree, used,
	nfree + used);
	db_printf("numfreebuffers is %d\n", numfreebuffers);
	}
	#endif /* DDB */
	diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
	index 16c493a1e39a..6eff71606ac9 100644
	--- a/sys/kern/vfs_cache.c
	+++ b/sys/kern/vfs_cache.c
	@@ -1,3705 +1,3886 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993, 1995
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* Poul-Henning Kamp of the FreeBSD Project.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)vfs_cache.c 8.5 (Berkeley) 3/22/95
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_ddb.h"
	#include "opt_ktrace.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/capsicum.h>
	#include <sys/counter.h>
	#include <sys/filedesc.h>
	#include <sys/fnv_hash.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/fcntl.h>
	#include <sys/mount.h>
	#include <sys/namei.h>
	#include <sys/proc.h>
	#include <sys/rwlock.h>
	#include <sys/seqc.h>
	#include <sys/sdt.h>
	#include <sys/smr.h>
	#include <sys/smp.h>
	#include <sys/syscallsubr.h>
	#include <sys/sysctl.h>
	#include <sys/sysproto.h>
	#include <sys/vnode.h>
	#include <ck_queue.h>
	#ifdef KTRACE
	#include <sys/ktrace.h>
	#endif

	#include <sys/capsicum.h>

	#include <security/audit/audit.h>
	#include <security/mac/mac_framework.h>

	#ifdef DDB
	#include <ddb/ddb.h>
	#endif

	#include <vm/uma.h>

	SDT_PROVIDER_DECLARE(vfs);
	SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode ", "char ",
	"struct vnode *");
	SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
	"char *");
	SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
	SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
	"char ", "struct vnode ");
	SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
	SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int",
	"struct vnode ", "char ");
	SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode ", "char ",
	"struct vnode *");
	SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative,
	"struct vnode ", "char ");
	SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
	"char *");
	SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *");
	SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
	SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
	SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode ", "char ",
	"struct vnode *");
	SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *",
	"char *");
	SDT_PROBE_DEFINE2(vfs, namecache, shrink_negative, done, "struct vnode *",
	"char *");

	SDT_PROBE_DEFINE3(vfs, fplookup, lookup, done, "struct nameidata", "int", "bool");
	SDT_PROBE_DECLARE(vfs, namei, lookup, entry);
	SDT_PROBE_DECLARE(vfs, namei, lookup, return);

	/*
	* This structure describes the elements in the cache of recent
	* names looked up by namei.
	*/
	struct negstate {
	u_char neg_flag;
	};
	_Static_assert(sizeof(struct negstate) <= sizeof(struct vnode *),
	"the state must fit in a union with a pointer without growing it");

	struct namecache {
	CK_LIST_ENTRY(namecache) nc_hash;/* hash chain */
	LIST_ENTRY(namecache) nc_src; /* source vnode list */
	TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */
	struct vnode nc_dvp; / vnode of parent of name */
	union {
	struct vnode nu_vp; / vnode the name refers to */
	struct negstate nu_neg;/* negative entry state */
	} n_un;
	u_char nc_flag; /* flag bits */
	u_char nc_nlen; /* length of name */
	char nc_name[0]; /* segment name + nul */
	};

	/*
	* struct namecache_ts repeats struct namecache layout up to the
	* nc_nlen member.
	* struct namecache_ts is used in place of struct namecache when time(s) need
	* to be stored. The nc_dotdottime field is used when a cache entry is mapping
	* both a non-dotdot directory name plus dotdot for the directory's
	* parent.
	*/
	struct namecache_ts {
	struct timespec nc_time; /* timespec provided by fs */
	struct timespec nc_dotdottime; /* dotdot timespec provided by fs */
	int nc_ticks; /* ticks value when entry was added */
	struct namecache nc_nc;
	};

	#define nc_vp n_un.nu_vp
	#define nc_neg n_un.nu_neg

	/*
	* Flags in namecache.nc_flag
	*/
	#define NCF_WHITE 0x01
	#define NCF_ISDOTDOT 0x02
	#define NCF_TS 0x04
	#define NCF_DTS 0x08
	#define NCF_DVDROP 0x10
	#define NCF_NEGATIVE 0x20
	#define NCF_INVALID 0x40

	/*
	* Flags in negstate.neg_flag
	*/
	#define NEG_HOT 0x01

	/*
	* Mark an entry as invalid.
	*
	* This is called before it starts getting deconstructed.
	*/
	static void
	cache_ncp_invalidate(struct namecache *ncp)
	{

	KASSERT((ncp->nc_flag & NCF_INVALID) == 0,
	("%s: entry %p already invalid", __func__, ncp));
	ncp->nc_flag \|= NCF_INVALID;
	atomic_thread_fence_rel();
	}

	/*
	* Verify validity of an entry.
	*
	* All places which elide locks are supposed to call this after they are
	* done with reading from an entry.
	*/
	static bool
	cache_ncp_invalid(struct namecache *ncp)
	{

	atomic_thread_fence_acq();
	return ((ncp->nc_flag & NCF_INVALID) != 0);
	}

	/*
	* Name caching works as follows:
	*
	* Names found by directory scans are retained in a cache
	* for future reference. It is managed LRU, so frequently
	* used names will hang around. Cache is indexed by hash value
	* obtained from (dvp, name) where dvp refers to the directory
	* containing name.
	*
	* If it is a "negative" entry, (i.e. for a name that is known NOT to
	* exist) the vnode pointer will be NULL.
	*
	* Upon reaching the last segment of a path, if the reference
	* is for DELETE, or NOCACHE is set (rewrite), and the
	* name is located in the cache, it will be dropped.
	*
	* These locks are used (in the order in which they can be taken):
	* NAME TYPE ROLE
	* vnodelock mtx vnode lists and v_cache_dd field protection
	* bucketlock rwlock for access to given set of hash buckets
	* neglist mtx negative entry LRU management
	*
	* Additionally, ncneg_shrink_lock mtx is used to have at most one thread
	* shrinking the LRU list.
	*
	* It is legal to take multiple vnodelock and bucketlock locks. The locking
	* order is lower address first. Both are recursive.
	*
	* "." lookups are lockless.
	*
	* ".." and vnode -> name lookups require vnodelock.
	*
	* name -> vnode lookup requires the relevant bucketlock to be held for reading.
	*
	* Insertions and removals of entries require involved vnodes and bucketlocks
	* to be write-locked to prevent other threads from seeing the entry.
	*
	* Some lookups result in removal of the found entry (e.g. getting rid of a
	* negative entry with the intent to create a positive one), which poses a
	* problem when multiple threads reach the state. Similarly, two different
	* threads can purge two different vnodes and try to remove the same name.
	*
	* If the already held vnode lock is lower than the second required lock, we
	* can just take the other lock. However, in the opposite case, this could
	* deadlock. As such, this is resolved by trylocking and if that fails unlocking
	* the first node, locking everything in order and revalidating the state.
	*/

	VFS_SMR_DECLARE;

	/*
	* Structures associated with name caching.
	*/
	#define NCHHASH(hash) \
	(&nchashtbl[(hash) & nchash])
	static __read_mostly CK_LIST_HEAD(nchashhead, namecache) nchashtbl;/ Hash Table */
	static u_long __read_mostly nchash; /* size of hash table */
	SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0,
	"Size of namecache hash table");
	static u_long __read_mostly ncnegfactor = 5; /* ratio of negative entries */
	SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0,
	"Ratio of negative namecache entries");
	static u_long __exclusive_cache_line numneg; /* number of negative entries allocated */
	static u_long __exclusive_cache_line numcache;/* number of cache entries allocated */
	u_int ncsizefactor = 2;
	SYSCTL_UINT(_vfs, OID_AUTO, ncsizefactor, CTLFLAG_RW, &ncsizefactor, 0,
	"Size factor for namecache");
	static u_int __read_mostly ncpurgeminvnodes;
	SYSCTL_UINT(_vfs, OID_AUTO, ncpurgeminvnodes, CTLFLAG_RW, &ncpurgeminvnodes, 0,
	"Number of vnodes below which purgevfs ignores the request");
	static u_int __read_mostly ncsize; /* the size as computed on creation or resizing */

	struct nchstats nchstats; /* cache effectiveness statistics */

	static struct mtx __exclusive_cache_line ncneg_shrink_lock;

	struct neglist {
	struct mtx nl_lock;
	TAILQ_HEAD(, namecache) nl_list;
	} __aligned(CACHE_LINE_SIZE);

	static struct neglist __read_mostly *neglists;
	static struct neglist ncneg_hot;
	static u_long numhotneg;

	#define numneglists (ncneghash + 1)
	static u_int __read_mostly ncneghash;
	static inline struct neglist *
	NCP2NEGLIST(struct namecache *ncp)
	{

	return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]);
	}

	static inline struct negstate *
	NCP2NEGSTATE(struct namecache *ncp)
	{

	MPASS(ncp->nc_flag & NCF_NEGATIVE);
	return (&ncp->nc_neg);
	}

	#define numbucketlocks (ncbuckethash + 1)
	static u_int __read_mostly ncbuckethash;
	static struct rwlock_padalign __read_mostly *bucketlocks;
	#define HASH2BUCKETLOCK(hash) \
	((struct rwlock *)(&bucketlocks[((hash) & ncbuckethash)]))

	#define numvnodelocks (ncvnodehash + 1)
	static u_int __read_mostly ncvnodehash;
	static struct mtx __read_mostly *vnodelocks;
	static inline struct mtx *
	VP2VNODELOCK(struct vnode *vp)
	{

	return (&vnodelocks[(((uintptr_t)(vp) >> 8) & ncvnodehash)]);
	}

	/*
	* UMA zones for the VFS cache.
	*
	* The small cache is used for entries with short names, which are the
	* most common. The large cache is used for entries which are too big to
	* fit in the small cache.
	*/
	static uma_zone_t __read_mostly cache_zone_small;
	static uma_zone_t __read_mostly cache_zone_small_ts;
	static uma_zone_t __read_mostly cache_zone_large;
	static uma_zone_t __read_mostly cache_zone_large_ts;

	#define CACHE_PATH_CUTOFF 35

	static struct namecache *
	cache_alloc(int len, int ts)
	{
	struct namecache_ts *ncp_ts;
	struct namecache *ncp;

	if (__predict_false(ts)) {
	if (len <= CACHE_PATH_CUTOFF)
	ncp_ts = uma_zalloc_smr(cache_zone_small_ts, M_WAITOK);
	else
	ncp_ts = uma_zalloc_smr(cache_zone_large_ts, M_WAITOK);
	ncp = &ncp_ts->nc_nc;
	} else {
	if (len <= CACHE_PATH_CUTOFF)
	ncp = uma_zalloc_smr(cache_zone_small, M_WAITOK);
	else
	ncp = uma_zalloc_smr(cache_zone_large, M_WAITOK);
	}
	return (ncp);
	}

	static void
	cache_free(struct namecache *ncp)
	{
	struct namecache_ts *ncp_ts;

	if (ncp == NULL)
	return;
	if ((ncp->nc_flag & NCF_DVDROP) != 0)
	vdrop(ncp->nc_dvp);
	if (__predict_false(ncp->nc_flag & NCF_TS)) {
	ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
	if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
	uma_zfree_smr(cache_zone_small_ts, ncp_ts);
	else
	uma_zfree_smr(cache_zone_large_ts, ncp_ts);
	} else {
	if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
	uma_zfree_smr(cache_zone_small, ncp);
	else
	uma_zfree_smr(cache_zone_large, ncp);
	}
	}

	static void
	cache_out_ts(struct namecache ncp, struct timespec tsp, int *ticksp)
	{
	struct namecache_ts *ncp_ts;

	KASSERT((ncp->nc_flag & NCF_TS) != 0 \|\|
	(tsp == NULL && ticksp == NULL),
	("No NCF_TS"));

	if (tsp == NULL && ticksp == NULL)
	return;

	ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
	if (tsp != NULL)
	*tsp = ncp_ts->nc_time;
	if (ticksp != NULL)
	*ticksp = ncp_ts->nc_ticks;
	}

	#ifdef DEBUG_CACHE
	static int __read_mostly doingcache = 1; /* 1 => enable the cache */
	SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0,
	"VFS namecache enabled");
	#endif

	/* Export size information to userland */
	SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR,
	sizeof(struct namecache), "sizeof(struct namecache)");

	/*
	* The new name cache statistics
	*/
	static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Name cache statistics");
	#define STATNODE_ULONG(name, descr) \
	SYSCTL_ULONG(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, 0, descr);
	#define STATNODE_COUNTER(name, descr) \
	static COUNTER_U64_DEFINE_EARLY(name); \
	SYSCTL_COUNTER_U64(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, \
	descr);
	STATNODE_ULONG(numneg, "Number of negative cache entries");
	STATNODE_ULONG(numcache, "Number of cache entries");
	STATNODE_COUNTER(numcachehv, "Number of namecache entries with vnodes held");
	STATNODE_COUNTER(numdrops, "Number of dropped entries due to reaching the limit");
	STATNODE_COUNTER(dothits, "Number of '.' hits");
	STATNODE_COUNTER(dotdothits, "Number of '..' hits");
	-STATNODE_COUNTER(numchecks, "Number of checks in lookup");
	STATNODE_COUNTER(nummiss, "Number of cache misses");
	STATNODE_COUNTER(nummisszap, "Number of cache misses we do not want to cache");
	STATNODE_COUNTER(numposzaps,
	"Number of cache hits (positive) we do not want to cache");
	STATNODE_COUNTER(numposhits, "Number of cache hits (positive)");
	STATNODE_COUNTER(numnegzaps,
	"Number of cache hits (negative) we do not want to cache");
	STATNODE_COUNTER(numneghits, "Number of cache hits (negative)");
	/* These count for vn_getcwd(), too. */
	STATNODE_COUNTER(numfullpathcalls, "Number of fullpath search calls");
	STATNODE_COUNTER(numfullpathfail1, "Number of fullpath search errors (ENOTDIR)");
	STATNODE_COUNTER(numfullpathfail2,
	"Number of fullpath search errors (VOP_VPTOCNP failures)");
	STATNODE_COUNTER(numfullpathfail4, "Number of fullpath search errors (ENOMEM)");
	STATNODE_COUNTER(numfullpathfound, "Number of successful fullpath calls");
	STATNODE_COUNTER(zap_and_exit_bucket_relock_success,
	"Number of successful removals after relocking");
	static long zap_and_exit_bucket_fail; STATNODE_ULONG(zap_and_exit_bucket_fail,
	"Number of times zap_and_exit failed to lock");
	static long zap_and_exit_bucket_fail2; STATNODE_ULONG(zap_and_exit_bucket_fail2,
	"Number of times zap_and_exit failed to lock");
	static long cache_lock_vnodes_cel_3_failures;
	STATNODE_ULONG(cache_lock_vnodes_cel_3_failures,
	"Number of times 3-way vnode locking failed");
	STATNODE_ULONG(numhotneg, "Number of hot negative entries");
	STATNODE_COUNTER(numneg_evicted,
	"Number of negative entries evicted when adding a new entry");
	STATNODE_COUNTER(shrinking_skipped,
	"Number of times shrinking was already in progress");

	static void cache_zap_locked(struct namecache *ncp);
	static int vn_fullpath_hardlink(struct thread td, struct nameidata ndp, char **retbuf,
	char *freebuf, size_t buflen);
	static int vn_fullpath_any(struct thread td, struct vnode vp, struct vnode *rdir,
	char buf, char retbuf, size_t buflen);
	static int vn_fullpath_dir(struct thread td, struct vnode vp, struct vnode *rdir,
	char buf, char retbuf, size_t len, bool slash_prefixed, size_t addend);

	static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");

	static int cache_yield;
	SYSCTL_INT(_vfs_cache, OID_AUTO, yield, CTLFLAG_RD, &cache_yield, 0,
	"Number of times cache called yield");

	static void __noinline
	cache_maybe_yield(void)
	{

	if (should_yield()) {
	cache_yield++;
	kern_yield(PRI_USER);
	}
	}

	static inline void
	cache_assert_vlp_locked(struct mtx *vlp)
	{

	if (vlp != NULL)
	mtx_assert(vlp, MA_OWNED);
	}

	static inline void
	cache_assert_vnode_locked(struct vnode *vp)
	{
	struct mtx *vlp;

	vlp = VP2VNODELOCK(vp);
	cache_assert_vlp_locked(vlp);
	}

	static uint32_t
	cache_get_hash(char name, u_char len, struct vnode dvp)
	{
	uint32_t hash;

	hash = fnv_32_buf(name, len, FNV1_32_INIT);
	hash = fnv_32_buf(&dvp, sizeof(dvp), hash);
	return (hash);
	}

	static inline struct rwlock *
	NCP2BUCKETLOCK(struct namecache *ncp)
	{
	uint32_t hash;

	hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
	return (HASH2BUCKETLOCK(hash));
	}

	#ifdef INVARIANTS
	static void
	cache_assert_bucket_locked(struct namecache *ncp, int mode)
	{
	struct rwlock *blp;

	blp = NCP2BUCKETLOCK(ncp);
	rw_assert(blp, mode);
	}
	#else
	#define cache_assert_bucket_locked(x, y) do { } while (0)
	#endif

	#define cache_sort_vnodes(x, y) _cache_sort_vnodes((void )(x), (void )(y))
	static void
	_cache_sort_vnodes(void p1, void p2)
	{
	void *tmp;

	MPASS(p1 != NULL \|\| p2 != NULL);

	if (p1 > p2) {
	tmp = *p2;
	p2 = p1;
	*p1 = tmp;
	}
	}

	static void
	cache_lock_all_buckets(void)
	{
	u_int i;

	for (i = 0; i < numbucketlocks; i++)
	rw_wlock(&bucketlocks[i]);
	}

	static void
	cache_unlock_all_buckets(void)
	{
	u_int i;

	for (i = 0; i < numbucketlocks; i++)
	rw_wunlock(&bucketlocks[i]);
	}

	static void
	cache_lock_all_vnodes(void)
	{
	u_int i;

	for (i = 0; i < numvnodelocks; i++)
	mtx_lock(&vnodelocks[i]);
	}

	static void
	cache_unlock_all_vnodes(void)
	{
	u_int i;

	for (i = 0; i < numvnodelocks; i++)
	mtx_unlock(&vnodelocks[i]);
	}

	static int
	cache_trylock_vnodes(struct mtx vlp1, struct mtx vlp2)
	{

	cache_sort_vnodes(&vlp1, &vlp2);

	if (vlp1 != NULL) {
	if (!mtx_trylock(vlp1))
	return (EAGAIN);
	}
	if (!mtx_trylock(vlp2)) {
	if (vlp1 != NULL)
	mtx_unlock(vlp1);
	return (EAGAIN);
	}

	return (0);
	}

	static void
	cache_lock_vnodes(struct mtx vlp1, struct mtx vlp2)
	{

	MPASS(vlp1 != NULL \|\| vlp2 != NULL);
	MPASS(vlp1 <= vlp2);

	if (vlp1 != NULL)
	mtx_lock(vlp1);
	if (vlp2 != NULL)
	mtx_lock(vlp2);
	}

	static void
	cache_unlock_vnodes(struct mtx vlp1, struct mtx vlp2)
	{

	MPASS(vlp1 != NULL \|\| vlp2 != NULL);

	if (vlp1 != NULL)
	mtx_unlock(vlp1);
	if (vlp2 != NULL)
	mtx_unlock(vlp2);
	}

	static int
	sysctl_nchstats(SYSCTL_HANDLER_ARGS)
	{
	struct nchstats snap;

	if (req->oldptr == NULL)
	return (SYSCTL_OUT(req, 0, sizeof(snap)));

	snap = nchstats;
	snap.ncs_goodhits = counter_u64_fetch(numposhits);
	snap.ncs_neghits = counter_u64_fetch(numneghits);
	snap.ncs_badhits = counter_u64_fetch(numposzaps) +
	counter_u64_fetch(numnegzaps);
	snap.ncs_miss = counter_u64_fetch(nummisszap) +
	counter_u64_fetch(nummiss);

	return (SYSCTL_OUT(req, &snap, sizeof(snap)));
	}
	SYSCTL_PROC(_vfs_cache, OID_AUTO, nchstats, CTLTYPE_OPAQUE \| CTLFLAG_RD \|
	CTLFLAG_MPSAFE, 0, 0, sysctl_nchstats, "LU",
	"VFS cache effectiveness statistics");

	#ifdef DIAGNOSTIC
	/*
	* Grab an atomic snapshot of the name cache hash chain lengths
	*/
	static SYSCTL_NODE(_debug, OID_AUTO, hashstat,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, NULL,
	"hash table stats");

	static int
	sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
	{
	struct nchashhead *ncpp;
	struct namecache *ncp;
	int i, error, n_nchash, *cntbuf;

	retry:
	n_nchash = nchash + 1; /* nchash is max index, not count */
	if (req->oldptr == NULL)
	return SYSCTL_OUT(req, 0, n_nchash * sizeof(int));
	cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO \| M_WAITOK);
	cache_lock_all_buckets();
	if (n_nchash != nchash + 1) {
	cache_unlock_all_buckets();
	free(cntbuf, M_TEMP);
	goto retry;
	}
	/* Scan hash tables counting entries */
	for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++)
	CK_LIST_FOREACH(ncp, ncpp, nc_hash)
	cntbuf[i]++;
	cache_unlock_all_buckets();
	for (error = 0, i = 0; i < n_nchash; i++)
	if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0)
	break;
	free(cntbuf, M_TEMP);
	return (error);
	}
	SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT\|CTLFLAG_RD\|
	CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int",
	"nchash chain lengths");

	static int
	sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS)
	{
	int error;
	struct nchashhead *ncpp;
	struct namecache *ncp;
	int n_nchash;
	int count, maxlength, used, pct;

	if (!req->oldptr)
	return SYSCTL_OUT(req, 0, 4 * sizeof(int));

	cache_lock_all_buckets();
	n_nchash = nchash + 1; /* nchash is max index, not count */
	used = 0;
	maxlength = 0;

	/* Scan hash tables for applicable entries */
	for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
	count = 0;
	CK_LIST_FOREACH(ncp, ncpp, nc_hash) {
	count++;
	}
	if (count)
	used++;
	if (maxlength < count)
	maxlength = count;
	}
	n_nchash = nchash + 1;
	cache_unlock_all_buckets();
	pct = (used * 100) / (n_nchash / 100);
	error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash));
	if (error)
	return (error);
	error = SYSCTL_OUT(req, &used, sizeof(used));
	if (error)
	return (error);
	error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength));
	if (error)
	return (error);
	error = SYSCTL_OUT(req, &pct, sizeof(pct));
	if (error)
	return (error);
	return (0);
	}
	SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT\|CTLFLAG_RD\|
	CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I",
	"nchash statistics (number of total/used buckets, maximum chain length, usage percentage)");
	#endif

	/*
	* Negative entries management
	*
	* A variation of LRU scheme is used. New entries are hashed into one of
	* numneglists cold lists. Entries get promoted to the hot list on first hit.
	*
	* The shrinker will demote hot list head and evict from the cold list in a
	* round-robin manner.
	*/
	static void
	cache_negative_init(struct namecache *ncp)
	{
	struct negstate *negstate;

	ncp->nc_flag \|= NCF_NEGATIVE;
	negstate = NCP2NEGSTATE(ncp);
	negstate->neg_flag = 0;
	}

	static void
	cache_negative_hit(struct namecache *ncp)
	{
	struct neglist *neglist;
	struct negstate *negstate;

	negstate = NCP2NEGSTATE(ncp);
	if ((negstate->neg_flag & NEG_HOT) != 0)
	return;
	neglist = NCP2NEGLIST(ncp);
	mtx_lock(&ncneg_hot.nl_lock);
	mtx_lock(&neglist->nl_lock);
	if ((negstate->neg_flag & NEG_HOT) == 0) {
	numhotneg++;
	TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
	TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
	negstate->neg_flag \|= NEG_HOT;
	}
	mtx_unlock(&neglist->nl_lock);
	mtx_unlock(&ncneg_hot.nl_lock);
	}

	static void
	cache_negative_insert(struct namecache *ncp)
	{
	struct neglist *neglist;

	MPASS(ncp->nc_flag & NCF_NEGATIVE);
	cache_assert_bucket_locked(ncp, RA_WLOCKED);
	neglist = NCP2NEGLIST(ncp);
	mtx_lock(&neglist->nl_lock);
	TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
	mtx_unlock(&neglist->nl_lock);
	atomic_add_rel_long(&numneg, 1);
	}

	static void
	cache_negative_remove(struct namecache *ncp)
	{
	struct neglist *neglist;
	struct negstate *negstate;
	bool hot_locked = false;
	bool list_locked = false;

	cache_assert_bucket_locked(ncp, RA_WLOCKED);
	neglist = NCP2NEGLIST(ncp);
	negstate = NCP2NEGSTATE(ncp);
	if ((negstate->neg_flag & NEG_HOT) != 0) {
	hot_locked = true;
	mtx_lock(&ncneg_hot.nl_lock);
	if ((negstate->neg_flag & NEG_HOT) == 0) {
	list_locked = true;
	mtx_lock(&neglist->nl_lock);
	}
	} else {
	list_locked = true;
	mtx_lock(&neglist->nl_lock);
	+ /*
	+ * We may be racing against promotion in lockless lookup.
	+ */
	+ if ((negstate->neg_flag & NEG_HOT) != 0) {
	+ mtx_unlock(&neglist->nl_lock);
	+ hot_locked = true;
	+ mtx_lock(&ncneg_hot.nl_lock);
	+ mtx_lock(&neglist->nl_lock);
	+ }
	}
	if ((negstate->neg_flag & NEG_HOT) != 0) {
	mtx_assert(&ncneg_hot.nl_lock, MA_OWNED);
	TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
	numhotneg--;
	} else {
	mtx_assert(&neglist->nl_lock, MA_OWNED);
	TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
	}
	if (list_locked)
	mtx_unlock(&neglist->nl_lock);
	if (hot_locked)
	mtx_unlock(&ncneg_hot.nl_lock);
	atomic_subtract_rel_long(&numneg, 1);
	}

	static void
	cache_negative_shrink_select(struct namecache **ncpp,
	struct neglist **neglistpp)
	{
	struct neglist *neglist;
	struct namecache *ncp;
	static u_int cycle;
	u_int i;

	*ncpp = ncp = NULL;

	for (i = 0; i < numneglists; i++) {
	neglist = &neglists[(cycle + i) % numneglists];
	if (TAILQ_FIRST(&neglist->nl_list) == NULL)
	continue;
	mtx_lock(&neglist->nl_lock);
	ncp = TAILQ_FIRST(&neglist->nl_list);
	if (ncp != NULL)
	break;
	mtx_unlock(&neglist->nl_lock);
	}

	*neglistpp = neglist;
	*ncpp = ncp;
	cycle++;
	}

	static void
	cache_negative_zap_one(void)
	{
	struct namecache ncp, ncp2;
	struct neglist *neglist;
	struct negstate *negstate;
	struct mtx *dvlp;
	struct rwlock *blp;

	if (mtx_owner(&ncneg_shrink_lock) != NULL \|\|
	!mtx_trylock(&ncneg_shrink_lock)) {
	counter_u64_add(shrinking_skipped, 1);
	return;
	}

	mtx_lock(&ncneg_hot.nl_lock);
	ncp = TAILQ_FIRST(&ncneg_hot.nl_list);
	if (ncp != NULL) {
	neglist = NCP2NEGLIST(ncp);
	negstate = NCP2NEGSTATE(ncp);
	mtx_lock(&neglist->nl_lock);
	MPASS((negstate->neg_flag & NEG_HOT) != 0);
	TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
	TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
	negstate->neg_flag &= ~NEG_HOT;
	numhotneg--;
	mtx_unlock(&neglist->nl_lock);
	}
	mtx_unlock(&ncneg_hot.nl_lock);

	cache_negative_shrink_select(&ncp, &neglist);

	mtx_unlock(&ncneg_shrink_lock);
	if (ncp == NULL)
	return;

	MPASS(ncp->nc_flag & NCF_NEGATIVE);
	dvlp = VP2VNODELOCK(ncp->nc_dvp);
	blp = NCP2BUCKETLOCK(ncp);
	mtx_unlock(&neglist->nl_lock);
	mtx_lock(dvlp);
	rw_wlock(blp);
	/*
	* Enter SMR to safely check the negative list.
	* Even if the found pointer matches, the entry may now be reallocated
	* and used by a different vnode.
	*/
	vfs_smr_enter();
	ncp2 = TAILQ_FIRST(&neglist->nl_list);
	if (ncp != ncp2 \|\| dvlp != VP2VNODELOCK(ncp2->nc_dvp) \|\|
	blp != NCP2BUCKETLOCK(ncp2)) {
	vfs_smr_exit();
	ncp = NULL;
	} else {
	vfs_smr_exit();
	SDT_PROBE2(vfs, namecache, shrink_negative, done, ncp->nc_dvp,
	ncp->nc_name);
	cache_zap_locked(ncp);
	counter_u64_add(numneg_evicted, 1);
	}
	rw_wunlock(blp);
	mtx_unlock(dvlp);
	cache_free(ncp);
	}

	/*
	* cache_zap_locked():
	*
	* Removes a namecache entry from cache, whether it contains an actual
	* pointer to a vnode or if it is just a negative cache entry.
	*/
	static void
	cache_zap_locked(struct namecache *ncp)
	{

	if (!(ncp->nc_flag & NCF_NEGATIVE))
	cache_assert_vnode_locked(ncp->nc_vp);
	cache_assert_vnode_locked(ncp->nc_dvp);
	cache_assert_bucket_locked(ncp, RA_WLOCKED);

	CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp,
	(ncp->nc_flag & NCF_NEGATIVE) ? NULL : ncp->nc_vp);

	cache_ncp_invalidate(ncp);

	CK_LIST_REMOVE(ncp, nc_hash);
	if (!(ncp->nc_flag & NCF_NEGATIVE)) {
	SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp,
	ncp->nc_name, ncp->nc_vp);
	TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst);
	if (ncp == ncp->nc_vp->v_cache_dd)
	ncp->nc_vp->v_cache_dd = NULL;
	} else {
	SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp,
	ncp->nc_name);
	cache_negative_remove(ncp);
	}
	if (ncp->nc_flag & NCF_ISDOTDOT) {
	if (ncp == ncp->nc_dvp->v_cache_dd)
	ncp->nc_dvp->v_cache_dd = NULL;
	} else {
	LIST_REMOVE(ncp, nc_src);
	if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
	ncp->nc_flag \|= NCF_DVDROP;
	counter_u64_add(numcachehv, -1);
	}
	}
	atomic_subtract_rel_long(&numcache, 1);
	}

	static void
	cache_zap_negative_locked_vnode_kl(struct namecache ncp, struct vnode vp)
	{
	struct rwlock *blp;

	MPASS(ncp->nc_dvp == vp);
	MPASS(ncp->nc_flag & NCF_NEGATIVE);
	cache_assert_vnode_locked(vp);

	blp = NCP2BUCKETLOCK(ncp);
	rw_wlock(blp);
	cache_zap_locked(ncp);
	rw_wunlock(blp);
	}

	static bool
	cache_zap_locked_vnode_kl2(struct namecache ncp, struct vnode vp,
	struct mtx **vlpp)
	{
	struct mtx pvlp, vlp1, vlp2, to_unlock;
	struct rwlock *blp;

	MPASS(vp == ncp->nc_dvp \|\| vp == ncp->nc_vp);
	cache_assert_vnode_locked(vp);

	if (ncp->nc_flag & NCF_NEGATIVE) {
	if (*vlpp != NULL) {
	mtx_unlock(*vlpp);
	*vlpp = NULL;
	}
	cache_zap_negative_locked_vnode_kl(ncp, vp);
	return (true);
	}

	pvlp = VP2VNODELOCK(vp);
	blp = NCP2BUCKETLOCK(ncp);
	vlp1 = VP2VNODELOCK(ncp->nc_dvp);
	vlp2 = VP2VNODELOCK(ncp->nc_vp);

	if (vlpp == vlp1 \|\| vlpp == vlp2) {
	to_unlock = *vlpp;
	*vlpp = NULL;
	} else {
	if (*vlpp != NULL) {
	mtx_unlock(*vlpp);
	*vlpp = NULL;
	}
	cache_sort_vnodes(&vlp1, &vlp2);
	if (vlp1 == pvlp) {
	mtx_lock(vlp2);
	to_unlock = vlp2;
	} else {
	if (!mtx_trylock(vlp1))
	goto out_relock;
	to_unlock = vlp1;
	}
	}
	rw_wlock(blp);
	cache_zap_locked(ncp);
	rw_wunlock(blp);
	if (to_unlock != NULL)
	mtx_unlock(to_unlock);
	return (true);

	out_relock:
	mtx_unlock(vlp2);
	mtx_lock(vlp1);
	mtx_lock(vlp2);
	MPASS(*vlpp == NULL);
	*vlpp = vlp1;
	return (false);
	}

	static int __noinline
	cache_zap_locked_vnode(struct namecache ncp, struct vnode vp)
	{
	struct mtx pvlp, vlp1, vlp2, to_unlock;
	struct rwlock *blp;
	int error = 0;

	MPASS(vp == ncp->nc_dvp \|\| vp == ncp->nc_vp);
	cache_assert_vnode_locked(vp);

	pvlp = VP2VNODELOCK(vp);
	if (ncp->nc_flag & NCF_NEGATIVE) {
	cache_zap_negative_locked_vnode_kl(ncp, vp);
	goto out;
	}

	blp = NCP2BUCKETLOCK(ncp);
	vlp1 = VP2VNODELOCK(ncp->nc_dvp);
	vlp2 = VP2VNODELOCK(ncp->nc_vp);
	cache_sort_vnodes(&vlp1, &vlp2);
	if (vlp1 == pvlp) {
	mtx_lock(vlp2);
	to_unlock = vlp2;
	} else {
	if (!mtx_trylock(vlp1)) {
	error = EAGAIN;
	goto out;
	}
	to_unlock = vlp1;
	}
	rw_wlock(blp);
	cache_zap_locked(ncp);
	rw_wunlock(blp);
	mtx_unlock(to_unlock);
	out:
	mtx_unlock(pvlp);
	return (error);
	}

	/*
	* If trylocking failed we can get here. We know enough to take all needed locks
	* in the right order and re-lookup the entry.
	*/
	static int
	cache_zap_unlocked_bucket(struct namecache ncp, struct componentname cnp,
	struct vnode dvp, struct mtx dvlp, struct mtx *vlp, uint32_t hash,
	struct rwlock *blp)
	{
	struct namecache *rncp;

	cache_assert_bucket_locked(ncp, RA_UNLOCKED);

	cache_sort_vnodes(&dvlp, &vlp);
	cache_lock_vnodes(dvlp, vlp);
	rw_wlock(blp);
	CK_LIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) {
	if (rncp == ncp && rncp->nc_dvp == dvp &&
	rncp->nc_nlen == cnp->cn_namelen &&
	!bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen))
	break;
	}
	if (rncp != NULL) {
	cache_zap_locked(rncp);
	rw_wunlock(blp);
	cache_unlock_vnodes(dvlp, vlp);
	counter_u64_add(zap_and_exit_bucket_relock_success, 1);
	return (0);
	}

	rw_wunlock(blp);
	cache_unlock_vnodes(dvlp, vlp);
	return (EAGAIN);
	}

	static int __noinline
	cache_zap_wlocked_bucket(struct namecache ncp, struct componentname cnp,
	uint32_t hash, struct rwlock *blp)
	{
	struct mtx dvlp, vlp;
	struct vnode *dvp;

	cache_assert_bucket_locked(ncp, RA_WLOCKED);

	dvlp = VP2VNODELOCK(ncp->nc_dvp);
	vlp = NULL;
	if (!(ncp->nc_flag & NCF_NEGATIVE))
	vlp = VP2VNODELOCK(ncp->nc_vp);
	if (cache_trylock_vnodes(dvlp, vlp) == 0) {
	cache_zap_locked(ncp);
	rw_wunlock(blp);
	cache_unlock_vnodes(dvlp, vlp);
	return (0);
	}

	dvp = ncp->nc_dvp;
	rw_wunlock(blp);
	return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp));
	}

	static int __noinline
	cache_zap_rlocked_bucket(struct namecache ncp, struct componentname cnp,
	uint32_t hash, struct rwlock *blp)
	{
	struct mtx dvlp, vlp;
	struct vnode *dvp;

	cache_assert_bucket_locked(ncp, RA_RLOCKED);

	dvlp = VP2VNODELOCK(ncp->nc_dvp);
	vlp = NULL;
	if (!(ncp->nc_flag & NCF_NEGATIVE))
	vlp = VP2VNODELOCK(ncp->nc_vp);
	if (cache_trylock_vnodes(dvlp, vlp) == 0) {
	rw_runlock(blp);
	rw_wlock(blp);
	cache_zap_locked(ncp);
	rw_wunlock(blp);
	cache_unlock_vnodes(dvlp, vlp);
	return (0);
	}

	dvp = ncp->nc_dvp;
	rw_runlock(blp);
	return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp));
	}

	static int
	cache_zap_wlocked_bucket_kl(struct namecache ncp, struct rwlock blp,
	struct mtx vlpp1, struct mtx vlpp2)
	{
	struct mtx dvlp, vlp;

	cache_assert_bucket_locked(ncp, RA_WLOCKED);

	dvlp = VP2VNODELOCK(ncp->nc_dvp);
	vlp = NULL;
	if (!(ncp->nc_flag & NCF_NEGATIVE))
	vlp = VP2VNODELOCK(ncp->nc_vp);
	cache_sort_vnodes(&dvlp, &vlp);

	if (vlpp1 == dvlp && vlpp2 == vlp) {
	cache_zap_locked(ncp);
	cache_unlock_vnodes(dvlp, vlp);
	*vlpp1 = NULL;
	*vlpp2 = NULL;
	return (0);
	}

	if (*vlpp1 != NULL)
	mtx_unlock(*vlpp1);
	if (*vlpp2 != NULL)
	mtx_unlock(*vlpp2);
	*vlpp1 = NULL;
	*vlpp2 = NULL;

	if (cache_trylock_vnodes(dvlp, vlp) == 0) {
	cache_zap_locked(ncp);
	cache_unlock_vnodes(dvlp, vlp);
	return (0);
	}

	rw_wunlock(blp);
	*vlpp1 = dvlp;
	*vlpp2 = vlp;
	if (*vlpp1 != NULL)
	mtx_lock(*vlpp1);
	mtx_lock(*vlpp2);
	rw_wlock(blp);
	return (EAGAIN);
	}

	static void
	cache_lookup_unlock(struct rwlock blp, struct mtx vlp)
	{

	if (blp != NULL) {
	rw_runlock(blp);
	} else {
	mtx_unlock(vlp);
	}
	}

	static int __noinline
	cache_lookup_dot(struct vnode dvp, struct vnode vpp, struct componentname cnp,
	struct timespec tsp, int ticksp)
	{
	int ltype;

	*vpp = dvp;
	CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .",
	dvp, cnp->cn_nameptr);
	counter_u64_add(dothits, 1);
	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
	if (tsp != NULL)
	timespecclear(tsp);
	if (ticksp != NULL)
	*ticksp = ticks;
	vrefact(*vpp);
	/*
	* When we lookup "." we still can be asked to lock it
	* differently...
	*/
	ltype = cnp->cn_lkflags & LK_TYPE_MASK;
	if (ltype != VOP_ISLOCKED(*vpp)) {
	if (ltype == LK_EXCLUSIVE) {
	vn_lock(*vpp, LK_UPGRADE \| LK_RETRY);
	if (VN_IS_DOOMED((*vpp))) {
	/* forced unmount */
	vrele(*vpp);
	*vpp = NULL;
	return (ENOENT);
	}
	} else
	vn_lock(*vpp, LK_DOWNGRADE \| LK_RETRY);
	}
	return (-1);
	}

	static __noinline int
	cache_lookup_nomakeentry(struct vnode dvp, struct vnode *vpp,
	struct componentname cnp, struct timespec tsp, int *ticksp)
	{
	struct namecache *ncp;
	struct rwlock *blp;
	struct mtx dvlp, dvlp2;
	uint32_t hash;
	int error;

	if (cnp->cn_namelen == 2 &&
	cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
	counter_u64_add(dotdothits, 1);
	dvlp = VP2VNODELOCK(dvp);
	dvlp2 = NULL;
	mtx_lock(dvlp);
	retry_dotdot:
	ncp = dvp->v_cache_dd;
	if (ncp == NULL) {
	SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
	"..", NULL);
	mtx_unlock(dvlp);
	if (dvlp2 != NULL)
	mtx_unlock(dvlp2);
	return (0);
	}
	if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
	if (ncp->nc_dvp != dvp)
	panic("dvp %p v_cache_dd %p\n", dvp, ncp);
	if (!cache_zap_locked_vnode_kl2(ncp,
	dvp, &dvlp2))
	goto retry_dotdot;
	MPASS(dvp->v_cache_dd == NULL);
	mtx_unlock(dvlp);
	if (dvlp2 != NULL)
	mtx_unlock(dvlp2);
	cache_free(ncp);
	} else {
	dvp->v_cache_dd = NULL;
	mtx_unlock(dvlp);
	if (dvlp2 != NULL)
	mtx_unlock(dvlp2);
	}
	return (0);
	}

	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
	blp = HASH2BUCKETLOCK(hash);
	retry:
	if (CK_LIST_EMPTY(NCHHASH(hash)))
	goto out_no_entry;

	rw_wlock(blp);

	CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
	- counter_u64_add(numchecks, 1);
	if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
	!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
	break;
	}

	/* We failed to find an entry */
	if (ncp == NULL) {
	rw_wunlock(blp);
	goto out_no_entry;
	}

	error = cache_zap_wlocked_bucket(ncp, cnp, hash, blp);
	if (__predict_false(error != 0)) {
	zap_and_exit_bucket_fail++;
	cache_maybe_yield();
	goto retry;
	}
	counter_u64_add(numposzaps, 1);
	cache_free(ncp);
	return (0);
	out_no_entry:
	SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr, NULL);
	counter_u64_add(nummisszap, 1);
	return (0);
	}

	/**
	* Lookup a name in the name cache
	*
	* # Arguments
	*
	* - dvp: Parent directory in which to search.
	* - vpp: Return argument. Will contain desired vnode on cache hit.
	* - cnp: Parameters of the name search. The most interesting bits of
	* the cn_flags field have the following meanings:
	* - MAKEENTRY: If clear, free an entry from the cache rather than look
	* it up.
	* - ISDOTDOT: Must be set if and only if cn_nameptr == ".."
	* - tsp: Return storage for cache timestamp. On a successful (positive
	* or negative) lookup, tsp will be filled with any timespec that
	* was stored when this cache entry was created. However, it will
	* be clear for "." entries.
	* - ticks: Return storage for alternate cache timestamp. On a successful
	* (positive or negative) lookup, it will contain the ticks value
	* that was current when the cache entry was created, unless cnp
	* was ".".
	*
	* # Returns
	*
	* - -1: A positive cache hit. vpp will contain the desired vnode.
	* - ENOENT: A negative cache hit, or dvp was recycled out from under us due
	* to a forced unmount. vpp will not be modified. If the entry
	* is a whiteout, then the ISWHITEOUT flag will be set in
	* cnp->cn_flags.
	* - 0: A cache miss. vpp will not be modified.
	*
	* # Locking
	*
	* On a cache hit, vpp will be returned locked and ref'd. If we're looking up
	* .., dvp is unlocked. If we're looking up . an extra ref is taken, but the
	* lock is not recursively acquired.
	*/
	int
	cache_lookup(struct vnode dvp, struct vnode vpp, struct componentname cnp,
	struct timespec tsp, int ticksp)
	{
	struct namecache_ts *ncp_ts;
	struct namecache *ncp;
	struct negstate *negstate;
	struct rwlock *blp;
	struct mtx *dvlp;
	uint32_t hash;
	enum vgetstate vs;
	int error, ltype;
	bool try_smr, doing_smr, whiteout;

	#ifdef DEBUG_CACHE
	if (__predict_false(!doingcache)) {
	cnp->cn_flags &= ~MAKEENTRY;
	return (0);
	}
	#endif

	if (__predict_false(cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.'))
	return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp));

	if ((cnp->cn_flags & MAKEENTRY) == 0)
	return (cache_lookup_nomakeentry(dvp, vpp, cnp, tsp, ticksp));

	try_smr = true;
	if (cnp->cn_nameiop == CREATE)
	try_smr = false;
	retry:
	doing_smr = false;
	blp = NULL;
	dvlp = NULL;
	error = 0;
	if (cnp->cn_namelen == 2 &&
	cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
	counter_u64_add(dotdothits, 1);
	dvlp = VP2VNODELOCK(dvp);
	mtx_lock(dvlp);
	ncp = dvp->v_cache_dd;
	if (ncp == NULL) {
	SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
	"..", NULL);
	mtx_unlock(dvlp);
	return (0);
	}
	if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
	if (ncp->nc_flag & NCF_NEGATIVE)
	*vpp = NULL;
	else
	*vpp = ncp->nc_vp;
	} else
	*vpp = ncp->nc_dvp;
	/* Return failure if negative entry was found. */
	if (*vpp == NULL)
	goto negative_success;
	CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..",
	dvp, cnp->cn_nameptr, *vpp);
	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..",
	*vpp);
	cache_out_ts(ncp, tsp, ticksp);
	if ((ncp->nc_flag & (NCF_ISDOTDOT \| NCF_DTS)) ==
	NCF_DTS && tsp != NULL) {
	ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
	*tsp = ncp_ts->nc_dotdottime;
	}
	goto success;
	}

	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
	retry_hashed:
	if (try_smr) {
	vfs_smr_enter();
	doing_smr = true;
	try_smr = false;
	} else {
	blp = HASH2BUCKETLOCK(hash);
	rw_rlock(blp);
	}

	CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
	- counter_u64_add(numchecks, 1);
	if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
	!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
	break;
	}

	/* We failed to find an entry */
	if (__predict_false(ncp == NULL)) {
	if (doing_smr)
	vfs_smr_exit();
	else
	rw_runlock(blp);
	SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
	NULL);
	counter_u64_add(nummiss, 1);
	return (0);
	}

	if (ncp->nc_flag & NCF_NEGATIVE)
	goto negative_success;

	/* We found a "positive" match, return the vnode */
	counter_u64_add(numposhits, 1);
	*vpp = ncp->nc_vp;
	CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
	dvp, cnp->cn_nameptr, *vpp, ncp);
	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name,
	*vpp);
	cache_out_ts(ncp, tsp, ticksp);
	success:
	/*
	* On success we return a locked and ref'd vnode as per the lookup
	* protocol.
	*/
	MPASS(dvp != *vpp);
	ltype = 0; /* silence gcc warning */
	if (cnp->cn_flags & ISDOTDOT) {
	ltype = VOP_ISLOCKED(dvp);
	VOP_UNLOCK(dvp);
	}
	if (doing_smr) {
	if (cache_ncp_invalid(ncp)) {
	vfs_smr_exit();
	*vpp = NULL;
	goto retry;
	}
	vs = vget_prep_smr(*vpp);
	vfs_smr_exit();
	if (vs == VGET_NONE) {
	*vpp = NULL;
	goto retry;
	}
	} else {
	vs = vget_prep(*vpp);
	cache_lookup_unlock(blp, dvlp);
	}
	error = vget_finish(*vpp, cnp->cn_lkflags, vs);
	if (cnp->cn_flags & ISDOTDOT) {
	vn_lock(dvp, ltype \| LK_RETRY);
	if (VN_IS_DOOMED(dvp)) {
	if (error == 0)
	vput(*vpp);
	*vpp = NULL;
	return (ENOENT);
	}
	}
	if (error) {
	*vpp = NULL;
	goto retry;
	}
	if ((cnp->cn_flags & ISLASTCN) &&
	(cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) {
	ASSERT_VOP_ELOCKED(*vpp, "cache_lookup");
	}
	return (-1);

	negative_success:
	/* We found a negative match, and want to create it, so purge */
	if (cnp->cn_nameiop == CREATE) {
	MPASS(!doing_smr);
	counter_u64_add(numnegzaps, 1);
	goto zap_and_exit;
	}

	SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name);
	cache_out_ts(ncp, tsp, ticksp);
	counter_u64_add(numneghits, 1);
	whiteout = (ncp->nc_flag & NCF_WHITE);

	if (doing_smr) {
	/*
	* We need to take locks to promote an entry.
	*/
	negstate = NCP2NEGSTATE(ncp);
	if ((negstate->neg_flag & NEG_HOT) == 0 \|\|
	cache_ncp_invalid(ncp)) {
	vfs_smr_exit();
	doing_smr = false;
	goto retry_hashed;
	}
	vfs_smr_exit();
	} else {
	cache_negative_hit(ncp);
	cache_lookup_unlock(blp, dvlp);
	}
	if (whiteout)
	cnp->cn_flags \|= ISWHITEOUT;
	return (ENOENT);

	zap_and_exit:
	MPASS(!doing_smr);
	if (blp != NULL)
	error = cache_zap_rlocked_bucket(ncp, cnp, hash, blp);
	else
	error = cache_zap_locked_vnode(ncp, dvp);
	if (__predict_false(error != 0)) {
	zap_and_exit_bucket_fail2++;
	cache_maybe_yield();
	goto retry;
	}
	cache_free(ncp);
	return (0);
	}

	struct celockstate {
	struct mtx *vlp[3];
	struct rwlock *blp[2];
	};
	CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3));
	CTASSERT((nitems(((struct celockstate *)0)->blp) == 2));

	static inline void
	cache_celockstate_init(struct celockstate *cel)
	{

	bzero(cel, sizeof(*cel));
	}

	static void
	cache_lock_vnodes_cel(struct celockstate cel, struct vnode vp,
	struct vnode *dvp)
	{
	struct mtx vlp1, vlp2;

	MPASS(cel->vlp[0] == NULL);
	MPASS(cel->vlp[1] == NULL);
	MPASS(cel->vlp[2] == NULL);

	MPASS(vp != NULL \|\| dvp != NULL);

	vlp1 = VP2VNODELOCK(vp);
	vlp2 = VP2VNODELOCK(dvp);
	cache_sort_vnodes(&vlp1, &vlp2);

	if (vlp1 != NULL) {
	mtx_lock(vlp1);
	cel->vlp[0] = vlp1;
	}
	mtx_lock(vlp2);
	cel->vlp[1] = vlp2;
	}

	static void
	cache_unlock_vnodes_cel(struct celockstate *cel)
	{

	MPASS(cel->vlp[0] != NULL \|\| cel->vlp[1] != NULL);

	if (cel->vlp[0] != NULL)
	mtx_unlock(cel->vlp[0]);
	if (cel->vlp[1] != NULL)
	mtx_unlock(cel->vlp[1]);
	if (cel->vlp[2] != NULL)
	mtx_unlock(cel->vlp[2]);
	}

	static bool
	cache_lock_vnodes_cel_3(struct celockstate cel, struct vnode vp)
	{
	struct mtx *vlp;
	bool ret;

	cache_assert_vlp_locked(cel->vlp[0]);
	cache_assert_vlp_locked(cel->vlp[1]);
	MPASS(cel->vlp[2] == NULL);

	MPASS(vp != NULL);
	vlp = VP2VNODELOCK(vp);

	ret = true;
	if (vlp >= cel->vlp[1]) {
	mtx_lock(vlp);
	} else {
	if (mtx_trylock(vlp))
	goto out;
	cache_lock_vnodes_cel_3_failures++;
	cache_unlock_vnodes_cel(cel);
	if (vlp < cel->vlp[0]) {
	mtx_lock(vlp);
	mtx_lock(cel->vlp[0]);
	mtx_lock(cel->vlp[1]);
	} else {
	if (cel->vlp[0] != NULL)
	mtx_lock(cel->vlp[0]);
	mtx_lock(vlp);
	mtx_lock(cel->vlp[1]);
	}
	ret = false;
	}
	out:
	cel->vlp[2] = vlp;
	return (ret);
	}

	static void
	cache_lock_buckets_cel(struct celockstate cel, struct rwlock blp1,
	struct rwlock *blp2)
	{

	MPASS(cel->blp[0] == NULL);
	MPASS(cel->blp[1] == NULL);

	cache_sort_vnodes(&blp1, &blp2);

	if (blp1 != NULL) {
	rw_wlock(blp1);
	cel->blp[0] = blp1;
	}
	rw_wlock(blp2);
	cel->blp[1] = blp2;
	}

	static void
	cache_unlock_buckets_cel(struct celockstate *cel)
	{

	if (cel->blp[0] != NULL)
	rw_wunlock(cel->blp[0]);
	rw_wunlock(cel->blp[1]);
	}

	/*
	* Lock part of the cache affected by the insertion.
	*
	* This means vnodelocks for dvp, vp and the relevant bucketlock.
	* However, insertion can result in removal of an old entry. In this
	* case we have an additional vnode and bucketlock pair to lock. If the
	* entry is negative, ncelock is locked instead of the vnode.
	*
	* That is, in the worst case we have to lock 3 vnodes and 2 bucketlocks, while
	* preserving the locking order (smaller address first).
	*/
	static void
	cache_enter_lock(struct celockstate cel, struct vnode dvp, struct vnode *vp,
	uint32_t hash)
	{
	struct namecache *ncp;
	struct rwlock *blps[2];

	blps[0] = HASH2BUCKETLOCK(hash);
	for (;;) {
	blps[1] = NULL;
	cache_lock_vnodes_cel(cel, dvp, vp);
	if (vp == NULL \|\| vp->v_type != VDIR)
	break;
	ncp = vp->v_cache_dd;
	if (ncp == NULL)
	break;
	if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
	break;
	MPASS(ncp->nc_dvp == vp);
	blps[1] = NCP2BUCKETLOCK(ncp);
	if (ncp->nc_flag & NCF_NEGATIVE)
	break;
	if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
	break;
	/*
	* All vnodes got re-locked. Re-validate the state and if
	* nothing changed we are done. Otherwise restart.
	*/
	if (ncp == vp->v_cache_dd &&
	(ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
	blps[1] == NCP2BUCKETLOCK(ncp) &&
	VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
	break;
	cache_unlock_vnodes_cel(cel);
	cel->vlp[0] = NULL;
	cel->vlp[1] = NULL;
	cel->vlp[2] = NULL;
	}
	cache_lock_buckets_cel(cel, blps[0], blps[1]);
	}

	static void
	cache_enter_lock_dd(struct celockstate cel, struct vnode dvp, struct vnode *vp,
	uint32_t hash)
	{
	struct namecache *ncp;
	struct rwlock *blps[2];

	blps[0] = HASH2BUCKETLOCK(hash);
	for (;;) {
	blps[1] = NULL;
	cache_lock_vnodes_cel(cel, dvp, vp);
	ncp = dvp->v_cache_dd;
	if (ncp == NULL)
	break;
	if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
	break;
	MPASS(ncp->nc_dvp == dvp);
	blps[1] = NCP2BUCKETLOCK(ncp);
	if (ncp->nc_flag & NCF_NEGATIVE)
	break;
	if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
	break;
	if (ncp == dvp->v_cache_dd &&
	(ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
	blps[1] == NCP2BUCKETLOCK(ncp) &&
	VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
	break;
	cache_unlock_vnodes_cel(cel);
	cel->vlp[0] = NULL;
	cel->vlp[1] = NULL;
	cel->vlp[2] = NULL;
	}
	cache_lock_buckets_cel(cel, blps[0], blps[1]);
	}

	static void
	cache_enter_unlock(struct celockstate *cel)
	{

	cache_unlock_buckets_cel(cel);
	cache_unlock_vnodes_cel(cel);
	}

	static void __noinline
	cache_enter_dotdot_prep(struct vnode dvp, struct vnode vp,
	struct componentname *cnp)
	{
	struct celockstate cel;
	struct namecache *ncp;
	uint32_t hash;
	int len;

	if (dvp->v_cache_dd == NULL)
	return;
	len = cnp->cn_namelen;
	cache_celockstate_init(&cel);
	hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
	cache_enter_lock_dd(&cel, dvp, vp, hash);
	ncp = dvp->v_cache_dd;
	if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) {
	KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent"));
	cache_zap_locked(ncp);
	} else {
	ncp = NULL;
	}
	dvp->v_cache_dd = NULL;
	cache_enter_unlock(&cel);
	cache_free(ncp);
	}

	/*
	* Add an entry to the cache.
	*/
	void
	cache_enter_time(struct vnode dvp, struct vnode vp, struct componentname *cnp,
	struct timespec tsp, struct timespec dtsp)
	{
	struct celockstate cel;
	struct namecache ncp, n2, *ndd;
	struct namecache_ts ncp_ts, n2_ts;
	struct nchashhead *ncpp;
	uint32_t hash;
	int flag;
	int len;
	u_long lnumcache;

	CTR3(KTR_VFS, "cache_enter(%p, %p, %s)", dvp, vp, cnp->cn_nameptr);
	VNASSERT(vp == NULL \|\| !VN_IS_DOOMED(vp), vp,
	("cache_enter: Adding a doomed vnode"));
	VNASSERT(dvp == NULL \|\| !VN_IS_DOOMED(dvp), dvp,
	("cache_enter: Doomed vnode used as src"));

	#ifdef DEBUG_CACHE
	if (__predict_false(!doingcache))
	return;
	#endif

	flag = 0;
	if (__predict_false(cnp->cn_nameptr[0] == '.')) {
	if (cnp->cn_namelen == 1)
	return;
	if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
	cache_enter_dotdot_prep(dvp, vp, cnp);
	flag = NCF_ISDOTDOT;
	}
	}

	/*
	* Avoid blowout in namecache entries.
	*/
	lnumcache = atomic_fetchadd_long(&numcache, 1) + 1;
	if (__predict_false(lnumcache >= ncsize)) {
	atomic_add_long(&numcache, -1);
	counter_u64_add(numdrops, 1);
	return;
	}

	cache_celockstate_init(&cel);
	ndd = NULL;
	ncp_ts = NULL;

	/*
	* Calculate the hash key and setup as much of the new
	* namecache entry as possible before acquiring the lock.
	*/
	ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
	ncp->nc_flag = flag;
	ncp->nc_vp = vp;
	if (vp == NULL)
	cache_negative_init(ncp);
	ncp->nc_dvp = dvp;
	if (tsp != NULL) {
	ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
	ncp_ts->nc_time = *tsp;
	ncp_ts->nc_ticks = ticks;
	ncp_ts->nc_nc.nc_flag \|= NCF_TS;
	if (dtsp != NULL) {
	ncp_ts->nc_dotdottime = *dtsp;
	ncp_ts->nc_nc.nc_flag \|= NCF_DTS;
	}
	}
	len = ncp->nc_nlen = cnp->cn_namelen;
	hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
	strlcpy(ncp->nc_name, cnp->cn_nameptr, len + 1);
	cache_enter_lock(&cel, dvp, vp, hash);

	/*
	* See if this vnode or negative entry is already in the cache
	* with this name. This can happen with concurrent lookups of
	* the same path name.
	*/
	ncpp = NCHHASH(hash);
	CK_LIST_FOREACH(n2, ncpp, nc_hash) {
	if (n2->nc_dvp == dvp &&
	n2->nc_nlen == cnp->cn_namelen &&
	!bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) {
	if (tsp != NULL) {
	KASSERT((n2->nc_flag & NCF_TS) != 0,
	("no NCF_TS"));
	n2_ts = __containerof(n2, struct namecache_ts, nc_nc);
	n2_ts->nc_time = ncp_ts->nc_time;
	n2_ts->nc_ticks = ncp_ts->nc_ticks;
	if (dtsp != NULL) {
	n2_ts->nc_dotdottime = ncp_ts->nc_dotdottime;
	n2_ts->nc_nc.nc_flag \|= NCF_DTS;
	}
	}
	goto out_unlock_free;
	}
	}

	if (flag == NCF_ISDOTDOT) {
	/*
	* See if we are trying to add .. entry, but some other lookup
	* has populated v_cache_dd pointer already.
	*/
	if (dvp->v_cache_dd != NULL)
	goto out_unlock_free;
	KASSERT(vp == NULL \|\| vp->v_type == VDIR,
	("wrong vnode type %p", vp));
	dvp->v_cache_dd = ncp;
	}

	if (vp != NULL) {
	if (vp->v_type == VDIR) {
	if (flag != NCF_ISDOTDOT) {
	/*
	* For this case, the cache entry maps both the
	* directory name in it and the name ".." for the
	* directory's parent.
	*/
	if ((ndd = vp->v_cache_dd) != NULL) {
	if ((ndd->nc_flag & NCF_ISDOTDOT) != 0)
	cache_zap_locked(ndd);
	else
	ndd = NULL;
	}
	vp->v_cache_dd = ncp;
	}
	} else {
	vp->v_cache_dd = NULL;
	}
	}

	if (flag != NCF_ISDOTDOT) {
	if (LIST_EMPTY(&dvp->v_cache_src)) {
	vhold(dvp);
	counter_u64_add(numcachehv, 1);
	}
	LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
	}

	/*
	* If the entry is "negative", we place it into the
	* "negative" cache queue, otherwise, we place it into the
	* destination vnode's cache entries queue.
	*/
	if (vp != NULL) {
	TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
	SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
	vp);
	} else {
	if (cnp->cn_flags & ISWHITEOUT)
	ncp->nc_flag \|= NCF_WHITE;
	cache_negative_insert(ncp);
	SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
	ncp->nc_name);
	}

	atomic_thread_fence_rel();
	/*
	* Insert the new namecache entry into the appropriate chain
	* within the cache entries table.
	*/
	CK_LIST_INSERT_HEAD(ncpp, ncp, nc_hash);

	cache_enter_unlock(&cel);
	if (numneg * ncnegfactor > lnumcache)
	cache_negative_zap_one();
	cache_free(ndd);
	return;
	out_unlock_free:
	cache_enter_unlock(&cel);
	cache_free(ncp);
	return;
	}

	static u_int
	cache_roundup_2(u_int val)
	{
	u_int res;

	for (res = 1; res <= val; res <<= 1)
	continue;

	return (res);
	}

	/*
	* Name cache initialization, from vfs_init() when we are booting
	*/
	static void
	nchinit(void *dummy __unused)
	{
	u_int i;

	cache_zone_small = uma_zcreate("S VFS Cache",
	sizeof(struct namecache) + CACHE_PATH_CUTOFF + 1,
	NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache),
	UMA_ZONE_ZINIT);
	cache_zone_small_ts = uma_zcreate("STS VFS Cache",
	sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1,
	NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts),
	UMA_ZONE_ZINIT);
	cache_zone_large = uma_zcreate("L VFS Cache",
	sizeof(struct namecache) + NAME_MAX + 1,
	NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache),
	UMA_ZONE_ZINIT);
	cache_zone_large_ts = uma_zcreate("LTS VFS Cache",
	sizeof(struct namecache_ts) + NAME_MAX + 1,
	NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts),
	UMA_ZONE_ZINIT);

	VFS_SMR_ZONE_SET(cache_zone_small);
	VFS_SMR_ZONE_SET(cache_zone_small_ts);
	VFS_SMR_ZONE_SET(cache_zone_large);
	VFS_SMR_ZONE_SET(cache_zone_large_ts);

	ncsize = desiredvnodes * ncsizefactor;
	nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash);
	ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1;
	if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */
	ncbuckethash = 7;
	if (ncbuckethash > nchash)
	ncbuckethash = nchash;
	bucketlocks = malloc(sizeof(bucketlocks) numbucketlocks, M_VFSCACHE,
	M_WAITOK \| M_ZERO);
	for (i = 0; i < numbucketlocks; i++)
	rw_init_flags(&bucketlocks[i], "ncbuc", RW_DUPOK \| RW_RECURSE);
	ncvnodehash = ncbuckethash;
	vnodelocks = malloc(sizeof(vnodelocks) numvnodelocks, M_VFSCACHE,
	M_WAITOK \| M_ZERO);
	for (i = 0; i < numvnodelocks; i++)
	mtx_init(&vnodelocks[i], "ncvn", NULL, MTX_DUPOK \| MTX_RECURSE);
	ncpurgeminvnodes = numbucketlocks * 2;

	ncneghash = 3;
	neglists = malloc(sizeof(neglists) numneglists, M_VFSCACHE,
	M_WAITOK \| M_ZERO);
	for (i = 0; i < numneglists; i++) {
	mtx_init(&neglists[i].nl_lock, "ncnegl", NULL, MTX_DEF);
	TAILQ_INIT(&neglists[i].nl_list);
	}
	mtx_init(&ncneg_hot.nl_lock, "ncneglh", NULL, MTX_DEF);
	TAILQ_INIT(&ncneg_hot.nl_list);

	mtx_init(&ncneg_shrink_lock, "ncnegs", NULL, MTX_DEF);
	}
	SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL);

	void
	cache_changesize(u_long newmaxvnodes)
	{
	struct nchashhead new_nchashtbl, old_nchashtbl;
	u_long new_nchash, old_nchash;
	struct namecache *ncp;
	uint32_t hash;
	u_long newncsize;
	int i;

	newncsize = newmaxvnodes * ncsizefactor;
	newmaxvnodes = cache_roundup_2(newmaxvnodes * 2);
	if (newmaxvnodes < numbucketlocks)
	newmaxvnodes = numbucketlocks;

	new_nchashtbl = hashinit(newmaxvnodes, M_VFSCACHE, &new_nchash);
	/* If same hash table size, nothing to do */
	if (nchash == new_nchash) {
	free(new_nchashtbl, M_VFSCACHE);
	return;
	}
	/*
	* Move everything from the old hash table to the new table.
	* None of the namecache entries in the table can be removed
	* because to do so, they have to be removed from the hash table.
	*/
	cache_lock_all_vnodes();
	cache_lock_all_buckets();
	old_nchashtbl = nchashtbl;
	old_nchash = nchash;
	nchashtbl = new_nchashtbl;
	nchash = new_nchash;
	for (i = 0; i <= old_nchash; i++) {
	while ((ncp = CK_LIST_FIRST(&old_nchashtbl[i])) != NULL) {
	hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
	ncp->nc_dvp);
	CK_LIST_REMOVE(ncp, nc_hash);
	CK_LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash);
	}
	}
	ncsize = newncsize;
	cache_unlock_all_buckets();
	cache_unlock_all_vnodes();
	free(old_nchashtbl, M_VFSCACHE);
	}

	/*
	* Invalidate all entries from and to a particular vnode.
	*/
	void
	cache_purge(struct vnode *vp)
	{
	TAILQ_HEAD(, namecache) ncps;
	struct namecache ncp, nnp;
	struct mtx vlp, vlp2;

	CTR1(KTR_VFS, "cache_purge(%p)", vp);
	SDT_PROBE1(vfs, namecache, purge, done, vp);
	if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) &&
	vp->v_cache_dd == NULL)
	return;
	TAILQ_INIT(&ncps);
	vlp = VP2VNODELOCK(vp);
	vlp2 = NULL;
	mtx_lock(vlp);
	retry:
	while (!LIST_EMPTY(&vp->v_cache_src)) {
	ncp = LIST_FIRST(&vp->v_cache_src);
	if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
	goto retry;
	TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
	}
	while (!TAILQ_EMPTY(&vp->v_cache_dst)) {
	ncp = TAILQ_FIRST(&vp->v_cache_dst);
	if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
	goto retry;
	TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
	}
	ncp = vp->v_cache_dd;
	if (ncp != NULL) {
	KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
	("lost dotdot link"));
	if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
	goto retry;
	TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
	}
	KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
	mtx_unlock(vlp);
	if (vlp2 != NULL)
	mtx_unlock(vlp2);
	TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
	cache_free(ncp);
	}
	}

	/*
	* Invalidate all negative entries for a particular directory vnode.
	*/
	void
	cache_purge_negative(struct vnode *vp)
	{
	TAILQ_HEAD(, namecache) ncps;
	struct namecache ncp, nnp;
	struct mtx *vlp;

	CTR1(KTR_VFS, "cache_purge_negative(%p)", vp);
	SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
	if (LIST_EMPTY(&vp->v_cache_src))
	return;
	TAILQ_INIT(&ncps);
	vlp = VP2VNODELOCK(vp);
	mtx_lock(vlp);
	LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
	if (!(ncp->nc_flag & NCF_NEGATIVE))
	continue;
	cache_zap_negative_locked_vnode_kl(ncp, vp);
	TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
	}
	mtx_unlock(vlp);
	TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
	cache_free(ncp);
	}
	}

	/*
	* Flush all entries referencing a particular filesystem.
	*/
	void
	cache_purgevfs(struct mount *mp, bool force)
	{
	TAILQ_HEAD(, namecache) ncps;
	struct mtx vlp1, vlp2;
	struct rwlock *blp;
	struct nchashhead *bucket;
	struct namecache ncp, nnp;
	u_long i, j, n_nchash;
	int error;

	/* Scan hash tables for applicable entries */
	SDT_PROBE1(vfs, namecache, purgevfs, done, mp);
	if (!force && mp->mnt_nvnodelistsize <= ncpurgeminvnodes)
	return;
	TAILQ_INIT(&ncps);
	n_nchash = nchash + 1;
	vlp1 = vlp2 = NULL;
	for (i = 0; i < numbucketlocks; i++) {
	blp = (struct rwlock *)&bucketlocks[i];
	rw_wlock(blp);
	for (j = i; j < n_nchash; j += numbucketlocks) {
	retry:
	bucket = &nchashtbl[j];
	CK_LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) {
	cache_assert_bucket_locked(ncp, RA_WLOCKED);
	if (ncp->nc_dvp->v_mount != mp)
	continue;
	error = cache_zap_wlocked_bucket_kl(ncp, blp,
	&vlp1, &vlp2);
	if (error != 0)
	goto retry;
	TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst);
	}
	}
	rw_wunlock(blp);
	if (vlp1 == NULL && vlp2 == NULL)
	cache_maybe_yield();
	}
	if (vlp1 != NULL)
	mtx_unlock(vlp1);
	if (vlp2 != NULL)
	mtx_unlock(vlp2);

	TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
	cache_free(ncp);
	}
	}

	/*
	* Perform canonical checks and cache lookup and pass on to filesystem
	* through the vop_cachedlookup only if needed.
	*/

	int
	vfs_cache_lookup(struct vop_lookup_args *ap)
	{
	struct vnode *dvp;
	int error;
	struct vnode **vpp = ap->a_vpp;
	struct componentname *cnp = ap->a_cnp;
	int flags = cnp->cn_flags;

	*vpp = NULL;
	dvp = ap->a_dvp;

	if (dvp->v_type != VDIR)
	return (ENOTDIR);

	if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
	(cnp->cn_nameiop == DELETE \|\| cnp->cn_nameiop == RENAME))
	return (EROFS);

	error = vn_dir_check_exec(dvp, cnp);
	if (error != 0)
	return (error);

	error = cache_lookup(dvp, vpp, cnp, NULL, NULL);
	if (error == 0)
	return (VOP_CACHEDLOOKUP(dvp, vpp, cnp));
	if (error == -1)
	return (0);
	return (error);
	}

	/* Implementation of the getcwd syscall. */
	int
	sys___getcwd(struct thread td, struct __getcwd_args uap)
	{
	char buf, retbuf;
	size_t buflen;
	int error;

	buflen = uap->buflen;
	if (__predict_false(buflen < 2))
	return (EINVAL);
	if (buflen > MAXPATHLEN)
	buflen = MAXPATHLEN;

	buf = malloc(buflen, M_TEMP, M_WAITOK);
	error = vn_getcwd(td, buf, &retbuf, &buflen);
	if (error == 0)
	error = copyout(retbuf, uap->buf, buflen);
	free(buf, M_TEMP);
	return (error);
	}

	int
	vn_getcwd(struct thread td, char buf, char *retbuf, size_t buflen)
	{
	struct pwd *pwd;
	int error;

	pwd = pwd_hold(td);
	error = vn_fullpath_any(td, pwd->pwd_cdir, pwd->pwd_rdir, buf, retbuf, buflen);
	pwd_drop(pwd);

	#ifdef KTRACE
	if (KTRPOINT(curthread, KTR_NAMEI) && error == 0)
	ktrnamei(*retbuf);
	#endif
	return (error);
	}

	static int
	kern___realpathat(struct thread td, int fd, const char path, char *buf,
	size_t size, int flags, enum uio_seg pathseg)
	{
	struct nameidata nd;
	char retbuf, freebuf;
	int error;

	if (flags != 0)
	return (EINVAL);
	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW \| SAVENAME \| WANTPARENT \| AUDITVNODE1,
	pathseg, path, fd, &cap_fstat_rights, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	error = vn_fullpath_hardlink(td, &nd, &retbuf, &freebuf, &size);
	if (error == 0) {
	error = copyout(retbuf, buf, size);
	free(freebuf, M_TEMP);
	}
	NDFREE(&nd, 0);
	return (error);
	}

	int
	sys___realpathat(struct thread td, struct __realpathat_args uap)
	{

	return (kern___realpathat(td, uap->fd, uap->path, uap->buf, uap->size,
	uap->flags, UIO_USERSPACE));
	}

	/*
	* Retrieve the full filesystem path that correspond to a vnode from the name
	* cache (if available)
	*/
	int
	vn_fullpath(struct thread td, struct vnode vn, char retbuf, char freebuf)
	{
	struct pwd *pwd;
	char *buf;
	size_t buflen;
	int error;

	if (__predict_false(vn == NULL))
	return (EINVAL);

	buflen = MAXPATHLEN;
	buf = malloc(buflen, M_TEMP, M_WAITOK);
	pwd = pwd_hold(td);
	error = vn_fullpath_any(td, vn, pwd->pwd_rdir, buf, retbuf, &buflen);
	pwd_drop(pwd);

	if (!error)
	*freebuf = buf;
	else
	free(buf, M_TEMP);
	return (error);
	}

	/*
	* This function is similar to vn_fullpath, but it attempts to lookup the
	* pathname relative to the global root mount point. This is required for the
	* auditing sub-system, as audited pathnames must be absolute, relative to the
	* global root mount point.
	*/
	int
	vn_fullpath_global(struct thread td, struct vnode vn,
	char retbuf, char freebuf)
	{
	char *buf;
	size_t buflen;
	int error;

	if (__predict_false(vn == NULL))
	return (EINVAL);
	buflen = MAXPATHLEN;
	buf = malloc(buflen, M_TEMP, M_WAITOK);
	error = vn_fullpath_any(td, vn, rootvnode, buf, retbuf, &buflen);
	if (!error)
	*freebuf = buf;
	else
	free(buf, M_TEMP);
	return (error);
	}

	int
	vn_vptocnp(struct vnode *vp, struct ucred cred, char buf, size_t buflen)
	{
	struct vnode *dvp;
	struct namecache *ncp;
	struct mtx *vlp;
	int error;

	vlp = VP2VNODELOCK(*vp);
	mtx_lock(vlp);
	TAILQ_FOREACH(ncp, &((*vp)->v_cache_dst), nc_dst) {
	if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
	break;
	}
	if (ncp != NULL) {
	if (*buflen < ncp->nc_nlen) {
	mtx_unlock(vlp);
	vrele(*vp);
	counter_u64_add(numfullpathfail4, 1);
	error = ENOMEM;
	SDT_PROBE3(vfs, namecache, fullpath, return, error,
	vp, NULL);
	return (error);
	}
	*buflen -= ncp->nc_nlen;
	memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
	SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
	ncp->nc_name, vp);
	dvp = *vp;
	*vp = ncp->nc_dvp;
	vref(*vp);
	mtx_unlock(vlp);
	vrele(dvp);
	return (0);
	}
	SDT_PROBE1(vfs, namecache, fullpath, miss, vp);

	mtx_unlock(vlp);
	vn_lock(*vp, LK_SHARED \| LK_RETRY);
	error = VOP_VPTOCNP(*vp, &dvp, cred, buf, buflen);
	vput(*vp);
	if (error) {
	counter_u64_add(numfullpathfail2, 1);
	SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
	return (error);
	}

	*vp = dvp;
	if (VN_IS_DOOMED(dvp)) {
	/* forced unmount */
	vrele(dvp);
	error = ENOENT;
	SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
	return (error);
	}
	/*
	* *vp has its use count incremented still.
	*/

	return (0);
	}

	/*
	* Resolve a directory to a pathname.
	*
	* The name of the directory can always be found in the namecache or fetched
	* from the filesystem. There is also guaranteed to be only one parent, meaning
	* we can just follow vnodes up until we find the root.
	*
	* The vnode must be referenced.
	*/
	static int
	vn_fullpath_dir(struct thread td, struct vnode vp, struct vnode *rdir,
	char buf, char retbuf, size_t len, bool slash_prefixed, size_t addend)
	{
	#ifdef KDTRACE_HOOKS
	struct vnode *startvp = vp;
	#endif
	struct vnode *vp1;
	size_t buflen;
	int error;

	VNPASS(vp->v_type == VDIR \|\| VN_IS_DOOMED(vp), vp);
	VNPASS(vp->v_usecount > 0, vp);

	buflen = *len;

	if (!slash_prefixed) {
	MPASS(*len >= 2);
	buflen--;
	buf[buflen] = '\0';
	}

	error = 0;

	SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
	counter_u64_add(numfullpathcalls, 1);
	while (vp != rdir && vp != rootvnode) {
	/*
	* The vp vnode must be already fully constructed,
	* since it is either found in namecache or obtained
	* from VOP_VPTOCNP(). We may test for VV_ROOT safely
	* without obtaining the vnode lock.
	*/
	if ((vp->v_vflag & VV_ROOT) != 0) {
	vn_lock(vp, LK_RETRY \| LK_SHARED);

	/*
	* With the vnode locked, check for races with
	* unmount, forced or not. Note that we
	* already verified that vp is not equal to
	* the root vnode, which means that
	* mnt_vnodecovered can be NULL only for the
	* case of unmount.
	*/
	if (VN_IS_DOOMED(vp) \|\|
	(vp1 = vp->v_mount->mnt_vnodecovered) == NULL \|\|
	vp1->v_mountedhere != vp->v_mount) {
	vput(vp);
	error = ENOENT;
	SDT_PROBE3(vfs, namecache, fullpath, return,
	error, vp, NULL);
	break;
	}

	vref(vp1);
	vput(vp);
	vp = vp1;
	continue;
	}
	if (vp->v_type != VDIR) {
	vrele(vp);
	counter_u64_add(numfullpathfail1, 1);
	error = ENOTDIR;
	SDT_PROBE3(vfs, namecache, fullpath, return,
	error, vp, NULL);
	break;
	}
	error = vn_vptocnp(&vp, td->td_ucred, buf, &buflen);
	if (error)
	break;
	if (buflen == 0) {
	vrele(vp);
	error = ENOMEM;
	SDT_PROBE3(vfs, namecache, fullpath, return, error,
	startvp, NULL);
	break;
	}
	buf[--buflen] = '/';
	slash_prefixed = true;
	}
	if (error)
	return (error);
	if (!slash_prefixed) {
	if (buflen == 0) {
	vrele(vp);
	counter_u64_add(numfullpathfail4, 1);
	SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
	startvp, NULL);
	return (ENOMEM);
	}
	buf[--buflen] = '/';
	}
	counter_u64_add(numfullpathfound, 1);
	vrele(vp);

	*retbuf = buf + buflen;
	SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, *retbuf);
	*len -= buflen;
	*len += addend;
	return (0);
	}

	/*
	* Resolve an arbitrary vnode to a pathname.
	*
	* Note 2 caveats:
	* - hardlinks are not tracked, thus if the vnode is not a directory this can
	* resolve to a different path than the one used to find it
	* - namecache is not mandatory, meaning names are not guaranteed to be added
	* (in which case resolving fails)
	*/
	static int
	vn_fullpath_any(struct thread td, struct vnode vp, struct vnode *rdir,
	char buf, char retbuf, size_t buflen)
	{
	size_t orig_buflen;
	bool slash_prefixed;
	int error;

	if (*buflen < 2)
	return (EINVAL);

	orig_buflen = *buflen;

	vref(vp);
	slash_prefixed = false;
	if (vp->v_type != VDIR) {
	*buflen -= 1;
	buf[*buflen] = '\0';
	error = vn_vptocnp(&vp, td->td_ucred, buf, buflen);
	if (error)
	return (error);
	if (*buflen == 0) {
	vrele(vp);
	return (ENOMEM);
	}
	*buflen -= 1;
	buf[*buflen] = '/';
	slash_prefixed = true;
	}

	return (vn_fullpath_dir(td, vp, rdir, buf, retbuf, buflen, slash_prefixed,
	orig_buflen - *buflen));
	}

	/*
	* Resolve an arbitrary vnode to a pathname (taking care of hardlinks).
	*
	* Since the namecache does not track handlings, the caller is expected to first
	* look up the target vnode with SAVENAME \| WANTPARENT flags passed to namei.
	*
	* Then we have 2 cases:
	* - if the found vnode is a directory, the path can be constructed just by
	* fullowing names up the chain
	* - otherwise we populate the buffer with the saved name and start resolving
	* from the parent
	*/
	static int
	vn_fullpath_hardlink(struct thread td, struct nameidata ndp, char **retbuf,
	char *freebuf, size_t buflen)
	{
	char buf, tmpbuf;
	struct pwd *pwd;
	struct componentname *cnp;
	struct vnode *vp;
	size_t addend;
	int error;
	bool slash_prefixed;

	if (*buflen < 2)
	return (EINVAL);
	if (*buflen > MAXPATHLEN)
	*buflen = MAXPATHLEN;

	slash_prefixed = false;

	buf = malloc(*buflen, M_TEMP, M_WAITOK);
	pwd = pwd_hold(td);

	addend = 0;
	vp = ndp->ni_vp;
	if (vp->v_type != VDIR) {
	cnp = &ndp->ni_cnd;
	addend = cnp->cn_namelen + 2;
	if (*buflen < addend) {
	error = ENOMEM;
	goto out_bad;
	}
	*buflen -= addend;
	tmpbuf = buf + *buflen;
	tmpbuf[0] = '/';
	memcpy(&tmpbuf[1], cnp->cn_nameptr, cnp->cn_namelen);
	tmpbuf[addend - 1] = '\0';
	slash_prefixed = true;
	vp = ndp->ni_dvp;
	}

	vref(vp);
	error = vn_fullpath_dir(td, vp, pwd->pwd_rdir, buf, retbuf, buflen,
	slash_prefixed, addend);
	if (error != 0)
	goto out_bad;

	pwd_drop(pwd);
	*freebuf = buf;

	return (0);
	out_bad:
	pwd_drop(pwd);
	free(buf, M_TEMP);
	return (error);
	}

	struct vnode *
	vn_dir_dd_ino(struct vnode *vp)
	{
	struct namecache *ncp;
	struct vnode *ddvp;
	struct mtx *vlp;
	enum vgetstate vs;

	ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino");
	vlp = VP2VNODELOCK(vp);
	mtx_lock(vlp);
	TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
	if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
	continue;
	ddvp = ncp->nc_dvp;
	vs = vget_prep(ddvp);
	mtx_unlock(vlp);
	if (vget_finish(ddvp, LK_SHARED \| LK_NOWAIT, vs))
	return (NULL);
	return (ddvp);
	}
	mtx_unlock(vlp);
	return (NULL);
	}

	int
	vn_commname(struct vnode vp, char buf, u_int buflen)
	{
	struct namecache *ncp;
	struct mtx *vlp;
	int l;

	vlp = VP2VNODELOCK(vp);
	mtx_lock(vlp);
	TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
	if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
	break;
	if (ncp == NULL) {
	mtx_unlock(vlp);
	return (ENOENT);
	}
	l = min(ncp->nc_nlen, buflen - 1);
	memcpy(buf, ncp->nc_name, l);
	mtx_unlock(vlp);
	buf[l] = '\0';
	return (0);
	}

	/*
	* This function updates path string to vnode's full global path
	* and checks the size of the new path string against the pathlen argument.
	*
	* Requires a locked, referenced vnode.
	* Vnode is re-locked on success or ENODEV, otherwise unlocked.
	*
	* If vp is a directory, the call to vn_fullpath_global() always succeeds
	* because it falls back to the ".." lookup if the namecache lookup fails.
	*/
	int
	vn_path_to_global_path(struct thread td, struct vnode vp, char *path,
	u_int pathlen)
	{
	struct nameidata nd;
	struct vnode *vp1;
	char rpath, fbuf;
	int error;

	ASSERT_VOP_ELOCKED(vp, __func__);

	/* Construct global filesystem path from vp. */
	VOP_UNLOCK(vp);
	error = vn_fullpath_global(td, vp, &rpath, &fbuf);

	if (error != 0) {
	vrele(vp);
	return (error);
	}

	if (strlen(rpath) >= pathlen) {
	vrele(vp);
	error = ENAMETOOLONG;
	goto out;
	}

	/*
	* Re-lookup the vnode by path to detect a possible rename.
	* As a side effect, the vnode is relocked.
	* If vnode was renamed, return ENOENT.
	*/
	NDINIT(&nd, LOOKUP, FOLLOW \| LOCKLEAF \| AUDITVNODE1,
	UIO_SYSSPACE, path, td);
	error = namei(&nd);
	if (error != 0) {
	vrele(vp);
	goto out;
	}
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vp1 = nd.ni_vp;
	vrele(vp);
	if (vp1 == vp)
	strcpy(path, rpath);
	else {
	vput(vp1);
	error = ENOENT;
	}

	out:
	free(fbuf, M_TEMP);
	return (error);
	}

	#ifdef DDB
	static void
	db_print_vpath(struct vnode *vp)
	{

	while (vp != NULL) {
	db_printf("%p: ", vp);
	if (vp == rootvnode) {
	db_printf("/");
	vp = NULL;
	} else {
	if (vp->v_vflag & VV_ROOT) {
	db_printf("<mount point>");
	vp = vp->v_mount->mnt_vnodecovered;
	} else {
	struct namecache *ncp;
	char *ncn;
	int i;

	ncp = TAILQ_FIRST(&vp->v_cache_dst);
	if (ncp != NULL) {
	ncn = ncp->nc_name;
	for (i = 0; i < ncp->nc_nlen; i++)
	db_printf("%c", *ncn++);
	vp = ncp->nc_dvp;
	} else {
	vp = NULL;
	}
	}
	}
	db_printf("\n");
	}

	return;
	}

	DB_SHOW_COMMAND(vpath, db_show_vpath)
	{
	struct vnode *vp;

	if (!have_addr) {
	db_printf("usage: show vpath <struct vnode *>\n");
	return;
	}

	vp = (struct vnode *)addr;
	db_print_vpath(vp);
	}

	#endif

	extern uma_zone_t namei_zone;

	static bool __read_frequently cache_fast_lookup = true;
	SYSCTL_BOOL(_vfs, OID_AUTO, cache_fast_lookup, CTLFLAG_RW,
	&cache_fast_lookup, 0, "");

	#define CACHE_FPL_FAILED -2020

	static void
	cache_fpl_cleanup_cnp(struct componentname *cnp)
	{

	uma_zfree(namei_zone, cnp->cn_pnbuf);
	#ifdef DIAGNOSTIC
	cnp->cn_pnbuf = NULL;
	cnp->cn_nameptr = NULL;
	#endif
	}

	static void
	cache_fpl_handle_root(struct nameidata ndp, struct vnode *dpp)
	{
	struct componentname *cnp;

	cnp = &ndp->ni_cnd;
	while (*(cnp->cn_nameptr) == '/') {
	cnp->cn_nameptr++;
	ndp->ni_pathlen--;
	}

	*dpp = ndp->ni_rootdir;
	}

	/*
	* Components of nameidata (or objects it can point to) which may
	* need restoring in case fast path lookup fails.
	*/
	struct nameidata_saved {
	int cn_flags;
	long cn_namelen;
	char *cn_nameptr;
	size_t ni_pathlen;
	};

	struct cache_fpl {
	int line;
	enum cache_fpl_status status;
	bool in_smr;
	struct nameidata *ndp;
	struct nameidata_saved snd;
	struct componentname *cnp;
	struct vnode *dvp;
	seqc_t dvp_seqc;
	struct vnode *tvp;
	seqc_t tvp_seqc;
	struct pwd *pwd;
	};

	static void
	cache_fpl_checkpoint(struct cache_fpl fpl, struct nameidata_saved snd)
	{

	snd->cn_flags = fpl->ndp->ni_cnd.cn_flags;
	snd->cn_namelen = fpl->ndp->ni_cnd.cn_namelen;
	snd->cn_nameptr = fpl->ndp->ni_cnd.cn_nameptr;
	snd->ni_pathlen = fpl->ndp->ni_pathlen;
	}

	static void
	cache_fpl_restore(struct cache_fpl fpl, struct nameidata_saved snd)
	{

	fpl->ndp->ni_cnd.cn_flags = snd->cn_flags;
	fpl->ndp->ni_cnd.cn_namelen = snd->cn_namelen;
	fpl->ndp->ni_cnd.cn_nameptr = snd->cn_nameptr;
	fpl->ndp->ni_pathlen = snd->ni_pathlen;
	}

	#ifdef INVARIANTS
	#define cache_fpl_smr_assert_entered(fpl) ({ \
	struct cache_fpl *_fpl = (fpl); \
	MPASS(_fpl->in_smr == true); \
	VFS_SMR_ASSERT_ENTERED(); \
	})
	#define cache_fpl_smr_assert_not_entered(fpl) ({ \
	struct cache_fpl *_fpl = (fpl); \
	MPASS(_fpl->in_smr == false); \
	VFS_SMR_ASSERT_NOT_ENTERED(); \
	})
	#else
	#define cache_fpl_smr_assert_entered(fpl) do { } while (0)
	#define cache_fpl_smr_assert_not_entered(fpl) do { } while (0)
	#endif

	#define cache_fpl_smr_enter(fpl) ({ \
	struct cache_fpl *_fpl = (fpl); \
	MPASS(_fpl->in_smr == false); \
	vfs_smr_enter(); \
	_fpl->in_smr = true; \
	})

	#define cache_fpl_smr_exit(fpl) ({ \
	struct cache_fpl *_fpl = (fpl); \
	MPASS(_fpl->in_smr == true); \
	vfs_smr_exit(); \
	_fpl->in_smr = false; \
	})

	static int
	cache_fpl_aborted_impl(struct cache_fpl *fpl, int line)
	{

	if (fpl->status != CACHE_FPL_STATUS_UNSET) {
	KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
	("%s: converting to abort from %d at %d, set at %d\n",
	__func__, fpl->status, line, fpl->line));
	}
	fpl->status = CACHE_FPL_STATUS_ABORTED;
	fpl->line = line;
	return (CACHE_FPL_FAILED);
	}

	#define cache_fpl_aborted(x) cache_fpl_aborted_impl((x), __LINE__)

	static int
	cache_fpl_partial_impl(struct cache_fpl *fpl, int line)
	{

	KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
	("%s: setting to partial at %d, but already set to %d at %d\n",
	__func__, line, fpl->status, fpl->line));
	cache_fpl_smr_assert_entered(fpl);
	fpl->status = CACHE_FPL_STATUS_PARTIAL;
	fpl->line = line;
	return (CACHE_FPL_FAILED);
	}

	#define cache_fpl_partial(x) cache_fpl_partial_impl((x), __LINE__)

	static int
	cache_fpl_handled_impl(struct cache_fpl *fpl, int error, int line)
	{

	KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
	("%s: setting to handled at %d, but already set to %d at %d\n",
	__func__, line, fpl->status, fpl->line));
	cache_fpl_smr_assert_not_entered(fpl);
	MPASS(error != CACHE_FPL_FAILED);
	fpl->status = CACHE_FPL_STATUS_HANDLED;
	fpl->line = line;
	return (error);
	}

	#define cache_fpl_handled(x, e) cache_fpl_handled_impl((x), (e), __LINE__)

	#define CACHE_FPL_SUPPORTED_CN_FLAGS \
	- (LOCKLEAF \| FOLLOW \| LOCKSHARED \| SAVENAME \| ISOPEN \| AUDITVNODE1)
	+ (LOCKLEAF \| LOCKPARENT \| WANTPARENT \| FOLLOW \| LOCKSHARED \| SAVENAME \| \
	+ ISOPEN \| NOMACCHECK \| AUDITVNODE1 \| AUDITVNODE2)

	static bool
	cache_can_fplookup(struct cache_fpl *fpl)
	{
	struct nameidata *ndp;
	struct componentname *cnp;
	struct thread *td;

	ndp = fpl->ndp;
	cnp = fpl->cnp;
	td = cnp->cn_thread;

	if (!cache_fast_lookup) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	#ifdef MAC
	if (mac_vnode_check_lookup_enabled()) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	#endif
	if ((cnp->cn_flags & ~CACHE_FPL_SUPPORTED_CN_FLAGS) != 0) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	if (cnp->cn_nameiop != LOOKUP) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	if (ndp->ni_dirfd != AT_FDCWD) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	if (IN_CAPABILITY_MODE(td)) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	if (AUDITING_TD(td)) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	if (ndp->ni_startdir != NULL) {
	cache_fpl_aborted(fpl);
	return (false);
	}
	return (true);
	}

	static bool
	cache_fplookup_vnode_supported(struct vnode *vp)
	{

	return (vp->v_type != VLNK);
	}

	+/*
	+ * Move a negative entry to the hot list.
	+ *
	+ * We have to take locks, but they may be contended and in the worst
	+ * case we may need to go off CPU. We don't want to spin within the
	+ * smr section and we can't block with it. Instead we are going to
	+ * look up the entry again.
	+ */
	+static int __noinline
	+cache_fplookup_negative_promote(struct cache_fpl fpl, struct namecache oncp,
	+ uint32_t hash)
	+{
	+ struct componentname *cnp;
	+ struct namecache *ncp;
	+ struct neglist *neglist;
	+ struct negstate *negstate;
	+ struct vnode *dvp;
	+ u_char nc_flag;
	+
	+ cnp = fpl->cnp;
	+ dvp = fpl->dvp;
	+
	+ if (!vhold_smr(dvp))
	+ return (cache_fpl_aborted(fpl));
	+
	+ neglist = NCP2NEGLIST(oncp);
	+ cache_fpl_smr_exit(fpl);
	+
	+ mtx_lock(&ncneg_hot.nl_lock);
	+ mtx_lock(&neglist->nl_lock);
	+ /*
	+ * For hash iteration.
	+ */
	+ cache_fpl_smr_enter(fpl);
	+
	+ /*
	+ * Avoid all surprises by only succeeding if we got the same entry and
	+ * bailing completely otherwise.
	+ *
	+ * In particular at this point there can be a new ncp which matches the
	+ * search but hashes to a different neglist.
	+ */
	+ CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
	+ if (ncp == oncp)
	+ break;
	+ }
	+
	+ /*
	+ * No match to begin with.
	+ */
	+ if (__predict_false(ncp == NULL)) {
	+ goto out_abort;
	+ }
	+
	+ /*
	+ * The newly found entry may be something different...
	+ */
	+ if (!(ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
	+ !bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))) {
	+ goto out_abort;
	+ }
	+
	+ /*
	+ * ... and not even negative.
	+ */
	+ nc_flag = atomic_load_char(&ncp->nc_flag);
	+ if ((nc_flag & NCF_NEGATIVE) == 0) {
	+ goto out_abort;
	+ }
	+
	+ if (__predict_false(cache_ncp_invalid(ncp))) {
	+ goto out_abort;
	+ }
	+
	+ negstate = NCP2NEGSTATE(ncp);
	+ if ((negstate->neg_flag & NEG_HOT) == 0) {
	+ numhotneg++;
	+ TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
	+ TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
	+ negstate->neg_flag \|= NEG_HOT;
	+ }
	+
	+ SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp, ncp->nc_name);
	+ counter_u64_add(numneghits, 1);
	+ cache_fpl_smr_exit(fpl);
	+ mtx_unlock(&neglist->nl_lock);
	+ mtx_unlock(&ncneg_hot.nl_lock);
	+ vdrop(dvp);
	+ return (cache_fpl_handled(fpl, ENOENT));
	+out_abort:
	+ cache_fpl_smr_exit(fpl);
	+ mtx_unlock(&neglist->nl_lock);
	+ mtx_unlock(&ncneg_hot.nl_lock);
	+ vdrop(dvp);
	+ return (cache_fpl_aborted(fpl));
	+}
	+
	/*
	* The target vnode is not supported, prepare for the slow path to take over.
	*/
	static int
	cache_fplookup_partial_setup(struct cache_fpl *fpl)
	{
	struct componentname *cnp;
	enum vgetstate dvs;
	struct vnode *dvp;
	struct pwd *pwd;
	seqc_t dvp_seqc;

	cnp = fpl->cnp;
	dvp = fpl->dvp;
	dvp_seqc = fpl->dvp_seqc;

	dvs = vget_prep_smr(dvp);
	if (dvs == VGET_NONE) {
	cache_fpl_smr_exit(fpl);
	return (cache_fpl_aborted(fpl));
	}

	cache_fpl_smr_exit(fpl);

	vget_finish_ref(dvp, dvs);
	if (!vn_seqc_consistent(dvp, dvp_seqc)) {
	vrele(dvp);
	return (cache_fpl_aborted(fpl));
	}

	pwd = pwd_hold(curthread);
	if (fpl->pwd != pwd) {
	vrele(dvp);
	pwd_drop(pwd);
	return (cache_fpl_aborted(fpl));
	}

	fpl->ndp->ni_startdir = dvp;
	return (0);
	}

	static int
	-cache_fplookup_final(struct cache_fpl *fpl)
	+cache_fplookup_final_child(struct cache_fpl *fpl, enum vgetstate tvs)
	{
	struct componentname *cnp;
	- enum vgetstate tvs;
	+ struct vnode *tvp;
	+ seqc_t tvp_seqc;
	+ int error;
	+
	+ cnp = fpl->cnp;
	+ tvp = fpl->tvp;
	+ tvp_seqc = fpl->tvp_seqc;
	+
	+ if ((cnp->cn_flags & LOCKLEAF) != 0) {
	+ error = vget_finish(tvp, cnp->cn_lkflags, tvs);
	+ if (error != 0) {
	+ return (cache_fpl_aborted(fpl));
	+ }
	+ } else {
	+ vget_finish_ref(tvp, tvs);
	+ }
	+
	+ if (!vn_seqc_consistent(tvp, tvp_seqc)) {
	+ if ((cnp->cn_flags & LOCKLEAF) != 0)
	+ vput(tvp);
	+ else
	+ vrele(tvp);
	+ return (cache_fpl_aborted(fpl));
	+ }
	+
	+ return (cache_fpl_handled(fpl, 0));
	+}
	+
	+static int __noinline
	+cache_fplookup_final_withparent(struct cache_fpl *fpl)
	+{
	+ enum vgetstate dvs, tvs;
	+ struct componentname *cnp;
	struct vnode dvp, tvp;
	seqc_t dvp_seqc, tvp_seqc;
	int error;

	cnp = fpl->cnp;
	dvp = fpl->dvp;
	dvp_seqc = fpl->dvp_seqc;
	tvp = fpl->tvp;
	tvp_seqc = fpl->tvp_seqc;

	- VNPASS(cache_fplookup_vnode_supported(dvp), dvp);
	+ MPASS((cnp->cn_flags & (LOCKPARENT\|WANTPARENT)) != 0);

	+ /*
	+ * This is less efficient than it can be for simplicity.
	+ */
	+ dvs = vget_prep_smr(dvp);
	+ if (dvs == VGET_NONE) {
	+ return (cache_fpl_aborted(fpl));
	+ }
	tvs = vget_prep_smr(tvp);
	if (tvs == VGET_NONE) {
	- return (cache_fpl_partial(fpl));
	- }
	-
	- if (!vn_seqc_consistent(dvp, dvp_seqc)) {
	cache_fpl_smr_exit(fpl);
	- vget_abort(tvp, tvs);
	+ vget_abort(dvp, dvs);
	return (cache_fpl_aborted(fpl));
	}

	cache_fpl_smr_exit(fpl);

	- if ((cnp->cn_flags & LOCKLEAF) != 0) {
	- error = vget_finish(tvp, cnp->cn_lkflags, tvs);
	+ if ((cnp->cn_flags & LOCKPARENT) != 0) {
	+ error = vget_finish(dvp, LK_EXCLUSIVE, dvs);
	if (error != 0) {
	+ vget_abort(tvp, tvs);
	return (cache_fpl_aborted(fpl));
	}
	} else {
	- vget_finish_ref(tvp, tvs);
	+ vget_finish_ref(dvp, dvs);
	}

	- if (!vn_seqc_consistent(tvp, tvp_seqc)) {
	- if ((cnp->cn_flags & LOCKLEAF) != 0)
	- vput(tvp);
	+ if (!vn_seqc_consistent(dvp, dvp_seqc)) {
	+ vget_abort(tvp, tvs);
	+ if ((cnp->cn_flags & LOCKPARENT) != 0)
	+ vput(dvp);
	else
	- vrele(tvp);
	+ vrele(dvp);
	+ cache_fpl_aborted(fpl);
	+ return (error);
	+ }
	+
	+ error = cache_fplookup_final_child(fpl, tvs);
	+ if (error != 0) {
	+ MPASS(fpl->status == CACHE_FPL_STATUS_ABORTED);
	+ if ((cnp->cn_flags & LOCKPARENT) != 0)
	+ vput(dvp);
	+ else
	+ vrele(dvp);
	+ return (error);
	+ }
	+
	+ MPASS(fpl->status == CACHE_FPL_STATUS_HANDLED);
	+ return (0);
	+}
	+
	+static int
	+cache_fplookup_final(struct cache_fpl *fpl)
	+{
	+ struct componentname *cnp;
	+ enum vgetstate tvs;
	+ struct vnode dvp, tvp;
	+ seqc_t dvp_seqc, tvp_seqc;
	+
	+ cnp = fpl->cnp;
	+ dvp = fpl->dvp;
	+ dvp_seqc = fpl->dvp_seqc;
	+ tvp = fpl->tvp;
	+ tvp_seqc = fpl->tvp_seqc;
	+
	+ VNPASS(cache_fplookup_vnode_supported(dvp), dvp);
	+
	+ if ((cnp->cn_flags & (LOCKPARENT\|WANTPARENT)) != 0)
	+ return (cache_fplookup_final_withparent(fpl));
	+
	+ tvs = vget_prep_smr(tvp);
	+ if (tvs == VGET_NONE) {
	+ return (cache_fpl_partial(fpl));
	+ }
	+
	+ if (!vn_seqc_consistent(dvp, dvp_seqc)) {
	+ cache_fpl_smr_exit(fpl);
	+ vget_abort(tvp, tvs);
	return (cache_fpl_aborted(fpl));
	}

	- return (cache_fpl_handled(fpl, 0));
	+ cache_fpl_smr_exit(fpl);
	+ return (cache_fplookup_final_child(fpl, tvs));
	}

	static int
	cache_fplookup_next(struct cache_fpl *fpl)
	{
	struct componentname *cnp;
	struct namecache *ncp;
	struct negstate *negstate;
	struct vnode dvp, tvp;
	u_char nc_flag;
	uint32_t hash;
	bool neg_hot;

	cnp = fpl->cnp;
	dvp = fpl->dvp;

	if (__predict_false(cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.')) {
	fpl->tvp = dvp;
	fpl->tvp_seqc = vn_seqc_read_any(dvp);
	if (seqc_in_modify(fpl->tvp_seqc)) {
	return (cache_fpl_aborted(fpl));
	}
	return (0);
	}

	hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);

	CK_LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
	- counter_u64_add(numchecks, 1);
	if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
	!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
	break;
	}

	/*
	* If there is no entry we have to punt to the slow path to perform
	* actual lookup. Should there be nothing with this name a negative
	* entry will be created.
	*/
	if (__predict_false(ncp == NULL)) {
	return (cache_fpl_partial(fpl));
	}

	tvp = atomic_load_ptr(&ncp->nc_vp);
	nc_flag = atomic_load_char(&ncp->nc_flag);
	if ((nc_flag & NCF_NEGATIVE) != 0) {
	negstate = NCP2NEGSTATE(ncp);
	neg_hot = ((negstate->neg_flag & NEG_HOT) != 0);
	if (__predict_false(cache_ncp_invalid(ncp))) {
	return (cache_fpl_partial(fpl));
	}
	if (__predict_false((nc_flag & NCF_WHITE) != 0)) {
	return (cache_fpl_partial(fpl));
	}
	if (!neg_hot) {
	- /*
	- * TODO
	- * Promoting to hot negative requires locks, thus is
	- * left not yet supported for simplicity.
	- */
	- return (cache_fpl_partial(fpl));
	+ return (cache_fplookup_negative_promote(fpl, ncp, hash));
	}
	SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp,
	ncp->nc_name);
	counter_u64_add(numneghits, 1);
	cache_fpl_smr_exit(fpl);
	return (cache_fpl_handled(fpl, ENOENT));
	}

	if (__predict_false(cache_ncp_invalid(ncp))) {
	return (cache_fpl_partial(fpl));
	}

	fpl->tvp = tvp;
	fpl->tvp_seqc = vn_seqc_read_any(tvp);
	if (seqc_in_modify(fpl->tvp_seqc)) {
	return (cache_fpl_partial(fpl));
	}

	if (!cache_fplookup_vnode_supported(tvp)) {
	return (cache_fpl_partial(fpl));
	}

	counter_u64_add(numposhits, 1);
	SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp);
	return (0);
	}

	static bool
	cache_fplookup_mp_supported(struct mount *mp)
	{

	if (mp == NULL)
	return (false);
	if ((mp->mnt_kern_flag & MNTK_FPLOOKUP) == 0)
	return (false);
	if ((mp->mnt_flag & MNT_UNION) != 0)
	return (false);
	return (true);
	}

	/*
	* Walk up the mount stack (if any).
	*
	* Correctness is provided in the following ways:
	* - all vnodes are protected from freeing with SMR
	* - struct mount objects are type stable making them always safe to access
	* - stability of the particular mount is provided by busying it
	* - relationship between the vnode which is mounted on and the mount is
	* verified with the vnode sequence counter after busying
	* - association between root vnode of the mount and the mount is protected
	* by busy
	*
	* From that point on we can read the sequence counter of the root vnode
	* and get the next mount on the stack (if any) using the same protection.
	*
	* By the end of successful walk we are guaranteed the reached state was
	* indeed present at least at some point which matches the regular lookup.
	*/
	static int
	cache_fplookup_climb_mount(struct cache_fpl *fpl)
	{
	struct mount mp, prev_mp;
	struct vnode *vp;
	seqc_t vp_seqc;

	vp = fpl->tvp;
	vp_seqc = fpl->tvp_seqc;
	if (vp->v_type != VDIR)
	return (0);

	mp = atomic_load_ptr(&vp->v_mountedhere);
	if (mp == NULL)
	return (0);

	prev_mp = NULL;
	for (;;) {
	if (!vfs_op_thread_enter(mp)) {
	if (prev_mp != NULL)
	vfs_op_thread_exit(prev_mp);
	return (cache_fpl_partial(fpl));
	}
	if (prev_mp != NULL)
	vfs_op_thread_exit(prev_mp);
	if (!vn_seqc_consistent(vp, vp_seqc)) {
	vfs_op_thread_exit(mp);
	return (cache_fpl_partial(fpl));
	}
	if (!cache_fplookup_mp_supported(mp)) {
	vfs_op_thread_exit(mp);
	return (cache_fpl_partial(fpl));
	}
	vp = atomic_load_ptr(&mp->mnt_rootvnode);
	if (vp == NULL \|\| VN_IS_DOOMED(vp)) {
	vfs_op_thread_exit(mp);
	return (cache_fpl_partial(fpl));
	}
	vp_seqc = vn_seqc_read_any(vp);
	if (seqc_in_modify(vp_seqc)) {
	vfs_op_thread_exit(mp);
	return (cache_fpl_partial(fpl));
	}
	prev_mp = mp;
	mp = atomic_load_ptr(&vp->v_mountedhere);
	if (mp == NULL)
	break;
	}

	vfs_op_thread_exit(prev_mp);
	fpl->tvp = vp;
	fpl->tvp_seqc = vp_seqc;
	return (0);
	}

	/*
	* Parse the path.
	*
	* The code is mostly copy-pasted from regular lookup, see lookup().
	* The structure is maintained along with comments for easier maintenance.
	* Deduplicating the code will become feasible after fast path lookup
	* becomes more feature-complete.
	*/
	static int
	cache_fplookup_parse(struct cache_fpl *fpl)
	{
	struct nameidata *ndp;
	struct componentname *cnp;
	char *cp;
	char prev_ni_next; / saved ndp->ni_next */
	size_t prev_ni_pathlen; /* saved ndp->ni_pathlen */

	ndp = fpl->ndp;
	cnp = fpl->cnp;

	/*
	* Search a new directory.
	*
	* The last component of the filename is left accessible via
	* cnp->cn_nameptr for callers that need the name. Callers needing
	* the name set the SAVENAME flag. When done, they assume
	* responsibility for freeing the pathname buffer.
	*/
	for (cp = cnp->cn_nameptr; cp != 0 && cp != '/'; cp++)
	continue;
	cnp->cn_namelen = cp - cnp->cn_nameptr;
	if (cnp->cn_namelen > NAME_MAX) {
	cache_fpl_smr_exit(fpl);
	return (cache_fpl_handled(fpl, ENAMETOOLONG));
	}
	prev_ni_pathlen = ndp->ni_pathlen;
	ndp->ni_pathlen -= cnp->cn_namelen;
	KASSERT(ndp->ni_pathlen <= PATH_MAX,
	("%s: ni_pathlen underflow to %zd\n", __func__, ndp->ni_pathlen));
	prev_ni_next = ndp->ni_next;
	ndp->ni_next = cp;

	/*
	* Replace multiple slashes by a single slash and trailing slashes
	* by a null. This must be done before VOP_LOOKUP() because some
	* fs's don't know about trailing slashes. Remember if there were
	* trailing slashes to handle symlinks, existing non-directories
	* and non-existing files that won't be directories specially later.
	*/
	while (*cp == '/' && (cp[1] == '/' \|\| cp[1] == '\0')) {
	cp++;
	ndp->ni_pathlen--;
	if (*cp == '\0') {
	/*
	* TODO
	* Regular lookup performs the following:
	* *ndp->ni_next = '\0';
	* cnp->cn_flags \|= TRAILINGSLASH;
	*
	* Which is problematic since it modifies data read
	* from userspace. Then if fast path lookup was to
	* abort we would have to either restore it or convey
	* the flag. Since this is a corner case just ignore
	* it for simplicity.
	*/
	return (cache_fpl_partial(fpl));
	}
	}
	ndp->ni_next = cp;

	cnp->cn_flags \|= MAKEENTRY;

	if (cnp->cn_namelen == 2 &&
	cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
	cnp->cn_flags \|= ISDOTDOT;
	else
	cnp->cn_flags &= ~ISDOTDOT;
	if (*ndp->ni_next == 0)
	cnp->cn_flags \|= ISLASTCN;
	else
	cnp->cn_flags &= ~ISLASTCN;

	/*
	* Check for degenerate name (e.g. / or "")
	* which is a way of talking about a directory,
	* e.g. like "/." or ".".
	*
	* TODO
	* Another corner case handled by the regular lookup
	*/
	if (__predict_false(cnp->cn_nameptr[0] == '\0')) {
	return (cache_fpl_partial(fpl));
	}
	return (0);
	}

	static void
	cache_fplookup_parse_advance(struct cache_fpl *fpl)
	{
	struct nameidata *ndp;
	struct componentname *cnp;

	ndp = fpl->ndp;
	cnp = fpl->cnp;

	cnp->cn_nameptr = ndp->ni_next;
	while (*cnp->cn_nameptr == '/') {
	cnp->cn_nameptr++;
	ndp->ni_pathlen--;
	}
	}

	static int
	cache_fplookup_impl(struct vnode dvp, struct cache_fpl fpl)
	{
	struct nameidata *ndp;
	struct componentname *cnp;
	struct mount *mp;
	int error;

	error = CACHE_FPL_FAILED;
	ndp = fpl->ndp;
	ndp->ni_lcf = 0;
	cnp = fpl->cnp;
	cnp->cn_lkflags = LK_SHARED;
	if ((cnp->cn_flags & LOCKSHARED) == 0)
	cnp->cn_lkflags = LK_EXCLUSIVE;

	cache_fpl_checkpoint(fpl, &fpl->snd);

	fpl->dvp = dvp;
	fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
	if (seqc_in_modify(fpl->dvp_seqc)) {
	cache_fpl_aborted(fpl);
	goto out;
	}
	mp = atomic_load_ptr(&fpl->dvp->v_mount);
	if (!cache_fplookup_mp_supported(mp)) {
	cache_fpl_aborted(fpl);
	goto out;
	}

	VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp);

	for (;;) {
	error = cache_fplookup_parse(fpl);
	if (__predict_false(error != 0)) {
	break;
	}

	if (cnp->cn_flags & ISDOTDOT) {
	error = cache_fpl_partial(fpl);
	break;
	}

	VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp);

	error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred, cnp->cn_thread);
	if (__predict_false(error != 0)) {
	switch (error) {
	case EAGAIN:
	case EOPNOTSUPP: /* can happen when racing against vgone */
	cache_fpl_partial(fpl);
	break;
	default:
	/*
	* See the API contract for VOP_FPLOOKUP_VEXEC.
	*/
	if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) {
	error = cache_fpl_aborted(fpl);
	} else {
	cache_fpl_smr_exit(fpl);
	cache_fpl_handled(fpl, error);
	}
	break;
	}
	break;
	}

	error = cache_fplookup_next(fpl);
	if (__predict_false(error != 0)) {
	break;
	}

	VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp);

	error = cache_fplookup_climb_mount(fpl);
	if (__predict_false(error != 0)) {
	break;
	}

	VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp);

	if (cnp->cn_flags & ISLASTCN) {
	error = cache_fplookup_final(fpl);
	break;
	}

	if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) {
	error = cache_fpl_aborted(fpl);
	break;
	}

	fpl->dvp = fpl->tvp;
	fpl->dvp_seqc = fpl->tvp_seqc;

	cache_fplookup_parse_advance(fpl);
	cache_fpl_checkpoint(fpl, &fpl->snd);
	}
	out:
	switch (fpl->status) {
	case CACHE_FPL_STATUS_UNSET:
	__assert_unreachable();
	break;
	case CACHE_FPL_STATUS_PARTIAL:
	cache_fpl_smr_assert_entered(fpl);
	return (cache_fplookup_partial_setup(fpl));
	case CACHE_FPL_STATUS_ABORTED:
	if (fpl->in_smr)
	cache_fpl_smr_exit(fpl);
	return (CACHE_FPL_FAILED);
	case CACHE_FPL_STATUS_HANDLED:
	cache_fpl_smr_assert_not_entered(fpl);
	if (__predict_false(error != 0)) {
	ndp->ni_dvp = NULL;
	ndp->ni_vp = NULL;
	cache_fpl_cleanup_cnp(cnp);
	return (error);
	}
	ndp->ni_dvp = fpl->dvp;
	ndp->ni_vp = fpl->tvp;
	if (cnp->cn_flags & SAVENAME)
	cnp->cn_flags \|= HASBUF;
	else
	cache_fpl_cleanup_cnp(cnp);
	return (error);
	}
	}

	/*
	* Fast path lookup protected with SMR and sequence counters.
	*
	* Note: all VOP_FPLOOKUP_VEXEC routines have a comment referencing this one.
	*
	* Filesystems can opt in by setting the MNTK_FPLOOKUP flag and meeting criteria
	* outlined below.
	*
	* Traditional vnode lookup conceptually looks like this:
	*
	* vn_lock(current);
	* for (;;) {
	* next = find();
	* vn_lock(next);
	* vn_unlock(current);
	* current = next;
	* if (last)
	* break;
	* }
	* return (current);
	*
	* Each jump to the next vnode is safe memory-wise and atomic with respect to
	* any modifications thanks to holding respective locks.
	*
	* The same guarantee can be provided with a combination of safe memory
	* reclamation and sequence counters instead. If all operations which affect
	* the relationship between the current vnode and the one we are looking for
	* also modify the counter, we can verify whether all the conditions held as
	* we made the jump. This includes things like permissions, mount points etc.
	* Counter modification is provided by enclosing relevant places in
	* vn_seqc_write_begin()/end() calls.
	*
	* Thus this translates to:
	*
	* vfs_smr_enter();
	* dvp_seqc = seqc_read_any(dvp);
	* if (seqc_in_modify(dvp_seqc)) // someone is altering the vnode
	* abort();
	* for (;;) {
	* tvp = find();
	* tvp_seqc = seqc_read_any(tvp);
	* if (seqc_in_modify(tvp_seqc)) // someone is altering the target vnode
	* abort();
	* if (!seqc_consistent(dvp, dvp_seqc) // someone is altering the vnode
	* abort();
	* dvp = tvp; // we know nothing of importance has changed
	* dvp_seqc = tvp_seqc; // store the counter for the tvp iteration
	* if (last)
	* break;
	* }
	* vget(); // secure the vnode
	* if (!seqc_consistent(tvp, tvp_seqc) // final check
	* abort();
	* // at this point we know nothing has changed for any parent<->child pair
	* // as they were crossed during the lookup, meaning we matched the guarantee
	* // of the locked variant
	* return (tvp);
	*
	* The API contract for VOP_FPLOOKUP_VEXEC routines is as follows:
	* - they are called while within vfs_smr protection which they must never exit
	* - EAGAIN can be returned to denote checking could not be performed, it is
	* always valid to return it
	* - if the sequence counter has not changed the result must be valid
	* - if the sequence counter has changed both false positives and false negatives
	* are permitted (since the result will be rejected later)
	* - for simple cases of unix permission checks vaccess_vexec_smr can be used
	*
	* Caveats to watch out for:
	* - vnodes are passed unlocked and unreferenced with nothing stopping
	* VOP_RECLAIM, in turn meaning that ->v_data can become NULL. It is advised
	* to use atomic_load_ptr to fetch it.
	* - the aforementioned object can also get freed, meaning absent other means it
	* should be protected with vfs_smr
	* - either safely checking permissions as they are modified or guaranteeing
	* their stability is left to the routine
	*/
	int
	cache_fplookup(struct nameidata ndp, enum cache_fpl_status status,
	struct pwd **pwdp)
	{
	struct cache_fpl fpl;
	struct pwd *pwd;
	struct vnode *dvp;
	struct componentname *cnp;
	struct nameidata_saved orig;
	int error;

	*status = CACHE_FPL_STATUS_UNSET;
	bzero(&fpl, sizeof(fpl));
	fpl.status = CACHE_FPL_STATUS_UNSET;
	fpl.ndp = ndp;
	fpl.cnp = &ndp->ni_cnd;
	MPASS(curthread == fpl.cnp->cn_thread);

	if (!cache_can_fplookup(&fpl)) {
	SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
	*status = fpl.status;
	return (EOPNOTSUPP);
	}

	cache_fpl_checkpoint(&fpl, &orig);

	cache_fpl_smr_enter(&fpl);
	pwd = pwd_get_smr();
	fpl.pwd = pwd;
	ndp->ni_rootdir = pwd->pwd_rdir;
	ndp->ni_topdir = pwd->pwd_jdir;

	cnp = fpl.cnp;
	cnp->cn_nameptr = cnp->cn_pnbuf;
	if (cnp->cn_pnbuf[0] == '/') {
	cache_fpl_handle_root(ndp, &dvp);
	} else {
	MPASS(ndp->ni_dirfd == AT_FDCWD);
	dvp = pwd->pwd_cdir;
	}

	SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true);

	error = cache_fplookup_impl(dvp, &fpl);
	cache_fpl_smr_assert_not_entered(&fpl);
	SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);

	*status = fpl.status;
	switch (fpl.status) {
	case CACHE_FPL_STATUS_UNSET:
	__assert_unreachable();
	break;
	case CACHE_FPL_STATUS_HANDLED:
	SDT_PROBE3(vfs, namei, lookup, return, error,
	(error == 0 ? ndp->ni_vp : NULL), true);
	break;
	case CACHE_FPL_STATUS_PARTIAL:
	*pwdp = fpl.pwd;
	cache_fpl_restore(&fpl, &fpl.snd);
	break;
	case CACHE_FPL_STATUS_ABORTED:
	cache_fpl_restore(&fpl, &orig);
	break;
	}
	return (error);
	}
	diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
	index 9dc4e714ed79..f67bc9bf3ef0 100644
	--- a/sys/kern/vfs_default.c
	+++ b/sys/kern/vfs_default.c
	@@ -1,1461 +1,1463 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed
	* to Berkeley by John Heidemann of the UCLA Ficus project.
	*
	* Source: * @(#)i405_init.c 2.10 92/04/27 UCLA Ficus project
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bio.h>
	#include <sys/buf.h>
	#include <sys/conf.h>
	#include <sys/event.h>
	#include <sys/filio.h>
	#include <sys/kernel.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/lockf.h>
	#include <sys/malloc.h>
	#include <sys/mount.h>
	#include <sys/namei.h>
	#include <sys/rwlock.h>
	#include <sys/fcntl.h>
	#include <sys/unistd.h>
	#include <sys/vnode.h>
	#include <sys/dirent.h>
	#include <sys/poll.h>

	#include <security/mac/mac_framework.h>

	#include <vm/vm.h>
	#include <vm/vm_object.h>
	#include <vm/vm_extern.h>
	#include <vm/pmap.h>
	#include <vm/vm_map.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pager.h>
	#include <vm/vnode_pager.h>

	static int vop_nolookup(struct vop_lookup_args *);
	static int vop_norename(struct vop_rename_args *);
	static int vop_nostrategy(struct vop_strategy_args *);
	static int get_next_dirent(struct vnode vp, struct dirent *dpp,
	char dirbuf, int dirbuflen, off_t off,
	char *cpos, int len, int *eofflag,
	struct thread *td);
	static int dirent_exists(struct vnode vp, const char dirname,
	struct thread *td);

	#define DIRENT_MINSIZE (sizeof(struct dirent) - (MAXNAMLEN+1) + 4)

	static int vop_stdis_text(struct vop_is_text_args *ap);
	static int vop_stdunset_text(struct vop_unset_text_args *ap);
	static int vop_stdadd_writecount(struct vop_add_writecount_args *ap);
	static int vop_stdcopy_file_range(struct vop_copy_file_range_args *ap);
	static int vop_stdfdatasync(struct vop_fdatasync_args *ap);
	static int vop_stdgetpages_async(struct vop_getpages_async_args *ap);

	/*
	* This vnode table stores what we want to do if the filesystem doesn't
	* implement a particular VOP.
	*
	* If there is no specific entry here, we will return EOPNOTSUPP.
	*
	* Note that every filesystem has to implement either vop_access
	* or vop_accessx; failing to do so will result in immediate crash
	* due to stack overflow, as vop_stdaccess() calls vop_stdaccessx(),
	* which calls vop_stdaccess() etc.
	*/

	struct vop_vector default_vnodeops = {
	.vop_default = NULL,
	.vop_bypass = VOP_EOPNOTSUPP,

	.vop_access = vop_stdaccess,
	.vop_accessx = vop_stdaccessx,
	.vop_advise = vop_stdadvise,
	.vop_advlock = vop_stdadvlock,
	.vop_advlockasync = vop_stdadvlockasync,
	.vop_advlockpurge = vop_stdadvlockpurge,
	.vop_allocate = vop_stdallocate,
	.vop_bmap = vop_stdbmap,
	.vop_close = VOP_NULL,
	.vop_fsync = VOP_NULL,
	.vop_fdatasync = vop_stdfdatasync,
	.vop_getpages = vop_stdgetpages,
	.vop_getpages_async = vop_stdgetpages_async,
	.vop_getwritemount = vop_stdgetwritemount,
	.vop_inactive = VOP_NULL,
	.vop_need_inactive = vop_stdneed_inactive,
	.vop_ioctl = vop_stdioctl,
	.vop_kqfilter = vop_stdkqfilter,
	.vop_islocked = vop_stdislocked,
	.vop_lock1 = vop_stdlock,
	.vop_lookup = vop_nolookup,
	.vop_open = VOP_NULL,
	.vop_pathconf = VOP_EINVAL,
	.vop_poll = vop_nopoll,
	.vop_putpages = vop_stdputpages,
	.vop_readlink = VOP_EINVAL,
	.vop_rename = vop_norename,
	.vop_revoke = VOP_PANIC,
	.vop_strategy = vop_nostrategy,
	.vop_unlock = vop_stdunlock,
	.vop_vptocnp = vop_stdvptocnp,
	.vop_vptofh = vop_stdvptofh,
	.vop_unp_bind = vop_stdunp_bind,
	.vop_unp_connect = vop_stdunp_connect,
	.vop_unp_detach = vop_stdunp_detach,
	.vop_is_text = vop_stdis_text,
	.vop_set_text = vop_stdset_text,
	.vop_unset_text = vop_stdunset_text,
	.vop_add_writecount = vop_stdadd_writecount,
	.vop_copy_file_range = vop_stdcopy_file_range,
	};
	VFS_VOP_VECTOR_REGISTER(default_vnodeops);

	/*
	* Series of placeholder functions for various error returns for
	* VOPs.
	*/

	int
	vop_eopnotsupp(struct vop_generic_args *ap)
	{
	/*
	printf("vop_notsupp[%s]\n", ap->a_desc->vdesc_name);
	*/

	return (EOPNOTSUPP);
	}

	int
	vop_ebadf(struct vop_generic_args *ap)
	{

	return (EBADF);
	}

	int
	vop_enotty(struct vop_generic_args *ap)
	{

	return (ENOTTY);
	}

	int
	vop_einval(struct vop_generic_args *ap)
	{

	return (EINVAL);
	}

	int
	vop_enoent(struct vop_generic_args *ap)
	{

	return (ENOENT);
	}

	int
	vop_null(struct vop_generic_args *ap)
	{

	return (0);
	}

	/*
	* Helper function to panic on some bad VOPs in some filesystems.
	*/
	int
	vop_panic(struct vop_generic_args *ap)
	{

	panic("filesystem goof: vop_panic[%s]", ap->a_desc->vdesc_name);
	}

	/*
	* vop_std<something> and vop_no<something> are default functions for use by
	* filesystems that need the "default reasonable" implementation for a
	* particular operation.
	*
	* The documentation for the operations they implement exists (if it exists)
	* in the VOP_<SOMETHING>(9) manpage (all uppercase).
	*/

	/*
	* Default vop for filesystems that do not support name lookup
	*/
	static int
	vop_nolookup(ap)
	struct vop_lookup_args /* {
	struct vnode *a_dvp;
	struct vnode **a_vpp;
	struct componentname *a_cnp;
	} / ap;
	{

	*ap->a_vpp = NULL;
	return (ENOTDIR);
	}

	/*
	* vop_norename:
	*
	* Handle unlock and reference counting for arguments of vop_rename
	* for filesystems that do not implement rename operation.
	*/
	static int
	vop_norename(struct vop_rename_args *ap)
	{

	vop_rename_fail(ap);
	return (EOPNOTSUPP);
	}

	/*
	* vop_nostrategy:
	*
	* Strategy routine for VFS devices that have none.
	*
	* BIO_ERROR and B_INVAL must be cleared prior to calling any strategy
	* routine. Typically this is done for a BIO_READ strategy call.
	* Typically B_INVAL is assumed to already be clear prior to a write
	* and should not be cleared manually unless you just made the buffer
	* invalid. BIO_ERROR should be cleared either way.
	*/

	static int
	vop_nostrategy (struct vop_strategy_args *ap)
	{
	printf("No strategy for buffer at %p\n", ap->a_bp);
	vn_printf(ap->a_vp, "vnode ");
	ap->a_bp->b_ioflags \|= BIO_ERROR;
	ap->a_bp->b_error = EOPNOTSUPP;
	bufdone(ap->a_bp);
	return (EOPNOTSUPP);
	}

	static int
	get_next_dirent(struct vnode vp, struct dirent dpp, char dirbuf,
	int dirbuflen, off_t off, char cpos, int len,
	int eofflag, struct thread td)
	{
	int error, reclen;
	struct uio uio;
	struct iovec iov;
	struct dirent *dp;

	KASSERT(VOP_ISLOCKED(vp), ("vp %p is not locked", vp));
	KASSERT(vp->v_type == VDIR, ("vp %p is not a directory", vp));

	if (*len == 0) {
	iov.iov_base = dirbuf;
	iov.iov_len = dirbuflen;

	uio.uio_iov = &iov;
	uio.uio_iovcnt = 1;
	uio.uio_offset = *off;
	uio.uio_resid = dirbuflen;
	uio.uio_segflg = UIO_SYSSPACE;
	uio.uio_rw = UIO_READ;
	uio.uio_td = td;

	*eofflag = 0;

	#ifdef MAC
	error = mac_vnode_check_readdir(td->td_ucred, vp);
	if (error == 0)
	#endif
	error = VOP_READDIR(vp, &uio, td->td_ucred, eofflag,
	NULL, NULL);
	if (error)
	return (error);

	*off = uio.uio_offset;

	*cpos = dirbuf;
	*len = (dirbuflen - uio.uio_resid);

	if (*len == 0)
	return (ENOENT);
	}

	dp = (struct dirent )(cpos);
	reclen = dp->d_reclen;
	*dpp = dp;

	/* check for malformed directory.. */
	if (reclen < DIRENT_MINSIZE)
	return (EINVAL);

	*cpos += reclen;
	*len -= reclen;

	return (0);
	}

	/*
	* Check if a named file exists in a given directory vnode.
	*/
	static int
	dirent_exists(struct vnode vp, const char dirname, struct thread *td)
	{
	char dirbuf, cpos;
	int error, eofflag, dirbuflen, len, found;
	off_t off;
	struct dirent *dp;
	struct vattr va;

	KASSERT(VOP_ISLOCKED(vp), ("vp %p is not locked", vp));
	KASSERT(vp->v_type == VDIR, ("vp %p is not a directory", vp));

	found = 0;

	error = VOP_GETATTR(vp, &va, td->td_ucred);
	if (error)
	return (found);

	dirbuflen = DEV_BSIZE;
	if (dirbuflen < va.va_blocksize)
	dirbuflen = va.va_blocksize;
	dirbuf = (char *)malloc(dirbuflen, M_TEMP, M_WAITOK);

	off = 0;
	len = 0;
	do {
	error = get_next_dirent(vp, &dp, dirbuf, dirbuflen, &off,
	&cpos, &len, &eofflag, td);
	if (error)
	goto out;

	if (dp->d_type != DT_WHT && dp->d_fileno != 0 &&
	strcmp(dp->d_name, dirname) == 0) {
	found = 1;
	goto out;
	}
	} while (len > 0 \|\| !eofflag);

	out:
	free(dirbuf, M_TEMP);
	return (found);
	}

	int
	vop_stdaccess(struct vop_access_args *ap)
	{

	KASSERT((ap->a_accmode & ~(VEXEC \| VWRITE \| VREAD \| VADMIN \|
	VAPPEND)) == 0, ("invalid bit in accmode"));

	return (VOP_ACCESSX(ap->a_vp, ap->a_accmode, ap->a_cred, ap->a_td));
	}

	int
	vop_stdaccessx(struct vop_accessx_args *ap)
	{
	int error;
	accmode_t accmode = ap->a_accmode;

	error = vfs_unixify_accmode(&accmode);
	if (error != 0)
	return (error);

	if (accmode == 0)
	return (0);

	return (VOP_ACCESS(ap->a_vp, accmode, ap->a_cred, ap->a_td));
	}

	/*
	* Advisory record locking support
	*/
	int
	vop_stdadvlock(struct vop_advlock_args *ap)
	{
	struct vnode *vp;
	struct vattr vattr;
	int error;

	vp = ap->a_vp;
	if (ap->a_fl->l_whence == SEEK_END) {
	/*
	* The NFSv4 server must avoid doing a vn_lock() here, since it
	* can deadlock the nfsd threads, due to a LOR. Fortunately
	* the NFSv4 server always uses SEEK_SET and this code is
	* only required for the SEEK_END case.
	*/
	vn_lock(vp, LK_SHARED \| LK_RETRY);
	error = VOP_GETATTR(vp, &vattr, curthread->td_ucred);
	VOP_UNLOCK(vp);
	if (error)
	return (error);
	} else
	vattr.va_size = 0;

	return (lf_advlock(ap, &(vp->v_lockf), vattr.va_size));
	}

	int
	vop_stdadvlockasync(struct vop_advlockasync_args *ap)
	{
	struct vnode *vp;
	struct vattr vattr;
	int error;

	vp = ap->a_vp;
	if (ap->a_fl->l_whence == SEEK_END) {
	/* The size argument is only needed for SEEK_END. */
	vn_lock(vp, LK_SHARED \| LK_RETRY);
	error = VOP_GETATTR(vp, &vattr, curthread->td_ucred);
	VOP_UNLOCK(vp);
	if (error)
	return (error);
	} else
	vattr.va_size = 0;

	return (lf_advlockasync(ap, &(vp->v_lockf), vattr.va_size));
	}

	int
	vop_stdadvlockpurge(struct vop_advlockpurge_args *ap)
	{
	struct vnode *vp;

	vp = ap->a_vp;
	lf_purgelocks(vp, &vp->v_lockf);
	return (0);
	}

	/*
	* vop_stdpathconf:
	*
	* Standard implementation of POSIX pathconf, to get information about limits
	* for a filesystem.
	* Override per filesystem for the case where the filesystem has smaller
	* limits.
	*/
	int
	vop_stdpathconf(ap)
	struct vop_pathconf_args /* {
	struct vnode *a_vp;
	int a_name;
	int *a_retval;
	} / ap;
	{

	switch (ap->a_name) {
	case _PC_ASYNC_IO:
	*ap->a_retval = _POSIX_ASYNCHRONOUS_IO;
	return (0);
	case _PC_PATH_MAX:
	*ap->a_retval = PATH_MAX;
	return (0);
	case _PC_ACL_EXTENDED:
	case _PC_ACL_NFS4:
	case _PC_CAP_PRESENT:
	case _PC_INF_PRESENT:
	case _PC_MAC_PRESENT:
	*ap->a_retval = 0;
	return (0);
	default:
	return (EINVAL);
	}
	/* NOTREACHED */
	}

	/*
	* Standard lock, unlock and islocked functions.
	*/
	int
	vop_stdlock(ap)
	struct vop_lock1_args /* {
	struct vnode *a_vp;
	int a_flags;
	char *file;
	int line;
	} / ap;
	{
	struct vnode *vp = ap->a_vp;
	struct mtx *ilk;

	ilk = VI_MTX(vp);
	return (lockmgr_lock_flags(vp->v_vnlock, ap->a_flags,
	&ilk->lock_object, ap->a_file, ap->a_line));
	}

	/* See above. */
	int
	vop_stdunlock(ap)
	struct vop_unlock_args /* {
	struct vnode *a_vp;
	} / ap;
	{
	struct vnode *vp = ap->a_vp;

	return (lockmgr_unlock(vp->v_vnlock));
	}

	/* See above. */
	int
	vop_stdislocked(ap)
	struct vop_islocked_args /* {
	struct vnode *a_vp;
	} / ap;
	{

	return (lockstatus(ap->a_vp->v_vnlock));
	}

	/*
	* Variants of the above set.
	*
	* Differences are:
	* - shared locking disablement is not supported
	* - v_vnlock pointer is not honored
	*/
	int
	vop_lock(ap)
	struct vop_lock1_args /* {
	struct vnode *a_vp;
	int a_flags;
	char *file;
	int line;
	} / ap;
	{
	struct vnode *vp = ap->a_vp;
	int flags = ap->a_flags;
	struct mtx *ilk;

	MPASS(vp->v_vnlock == &vp->v_lock);

	if (__predict_false((flags & ~(LK_TYPE_MASK \| LK_NODDLKTREAT \| LK_RETRY)) != 0))
	goto other;

	switch (flags & LK_TYPE_MASK) {
	case LK_SHARED:
	return (lockmgr_slock(&vp->v_lock, flags, ap->a_file, ap->a_line));
	case LK_EXCLUSIVE:
	return (lockmgr_xlock(&vp->v_lock, flags, ap->a_file, ap->a_line));
	}
	other:
	ilk = VI_MTX(vp);
	return (lockmgr_lock_flags(&vp->v_lock, flags,
	&ilk->lock_object, ap->a_file, ap->a_line));
	}

	int
	vop_unlock(ap)
	struct vop_unlock_args /* {
	struct vnode *a_vp;
	} / ap;
	{
	struct vnode *vp = ap->a_vp;

	MPASS(vp->v_vnlock == &vp->v_lock);

	return (lockmgr_unlock(&vp->v_lock));
	}

	int
	vop_islocked(ap)
	struct vop_islocked_args /* {
	struct vnode *a_vp;
	} / ap;
	{
	struct vnode *vp = ap->a_vp;

	MPASS(vp->v_vnlock == &vp->v_lock);

	return (lockstatus(&vp->v_lock));
	}

	/*
	* Return true for select/poll.
	*/
	int
	vop_nopoll(ap)
	struct vop_poll_args /* {
	struct vnode *a_vp;
	int a_events;
	struct ucred *a_cred;
	struct thread *a_td;
	} / ap;
	{

	- return (poll_no_poll(ap->a_events));
	+ if (ap->a_events & ~POLLSTANDARD)
	+ return (POLLNVAL);
	+ return (ap->a_events & (POLLIN \| POLLOUT \| POLLRDNORM \| POLLWRNORM));
	}

	/*
	* Implement poll for local filesystems that support it.
	*/
	int
	vop_stdpoll(ap)
	struct vop_poll_args /* {
	struct vnode *a_vp;
	int a_events;
	struct ucred *a_cred;
	struct thread *a_td;
	} / ap;
	{
	if (ap->a_events & ~POLLSTANDARD)
	return (vn_pollrecord(ap->a_vp, ap->a_td, ap->a_events));
	return (ap->a_events & (POLLIN \| POLLOUT \| POLLRDNORM \| POLLWRNORM));
	}

	/*
	* Return our mount point, as we will take charge of the writes.
	*/
	int
	vop_stdgetwritemount(ap)
	struct vop_getwritemount_args /* {
	struct vnode *a_vp;
	struct mount **a_mpp;
	} / ap;
	{
	struct mount *mp;
	struct vnode *vp;

	/*
	* Note that having a reference does not prevent forced unmount from
	* setting ->v_mount to NULL after the lock gets released. This is of
	* no consequence for typical consumers (most notably vn_start_write)
	* since in this case the vnode is VIRF_DOOMED. Unmount might have
	* progressed far enough that its completion is only delayed by the
	* reference obtained here. The consumer only needs to concern itself
	* with releasing it.
	*/
	vp = ap->a_vp;
	mp = vp->v_mount;
	if (mp == NULL) {
	*(ap->a_mpp) = NULL;
	return (0);
	}
	if (vfs_op_thread_enter(mp)) {
	if (mp == vp->v_mount) {
	vfs_mp_count_add_pcpu(mp, ref, 1);
	vfs_op_thread_exit(mp);
	} else {
	vfs_op_thread_exit(mp);
	mp = NULL;
	}
	} else {
	MNT_ILOCK(mp);
	if (mp == vp->v_mount) {
	MNT_REF(mp);
	MNT_IUNLOCK(mp);
	} else {
	MNT_IUNLOCK(mp);
	mp = NULL;
	}
	}
	*(ap->a_mpp) = mp;
	return (0);
	}

	/*
	* If the file system doesn't implement VOP_BMAP, then return sensible defaults:
	* - Return the vnode's bufobj instead of any underlying device's bufobj
	* - Calculate the physical block number as if there were equal size
	* consecutive blocks, but
	* - Report no contiguous runs of blocks.
	*/
	int
	vop_stdbmap(ap)
	struct vop_bmap_args /* {
	struct vnode *a_vp;
	daddr_t a_bn;
	struct bufobj **a_bop;
	daddr_t *a_bnp;
	int *a_runp;
	int *a_runb;
	} / ap;
	{

	if (ap->a_bop != NULL)
	*ap->a_bop = &ap->a_vp->v_bufobj;
	if (ap->a_bnp != NULL)
	ap->a_bnp = ap->a_bn btodb(ap->a_vp->v_mount->mnt_stat.f_iosize);
	if (ap->a_runp != NULL)
	*ap->a_runp = 0;
	if (ap->a_runb != NULL)
	*ap->a_runb = 0;
	return (0);
	}

	int
	vop_stdfsync(ap)
	struct vop_fsync_args /* {
	struct vnode *a_vp;
	int a_waitfor;
	struct thread *a_td;
	} / ap;
	{

	return (vn_fsync_buf(ap->a_vp, ap->a_waitfor));
	}

	static int
	vop_stdfdatasync(struct vop_fdatasync_args *ap)
	{

	return (VOP_FSYNC(ap->a_vp, MNT_WAIT, ap->a_td));
	}

	int
	vop_stdfdatasync_buf(struct vop_fdatasync_args *ap)
	{

	return (vn_fsync_buf(ap->a_vp, MNT_WAIT));
	}

	/* XXX Needs good comment and more info in the manpage (VOP_GETPAGES(9)). */
	int
	vop_stdgetpages(ap)
	struct vop_getpages_args /* {
	struct vnode *a_vp;
	vm_page_t *a_m;
	int a_count;
	int *a_rbehind;
	int *a_rahead;
	} / ap;
	{

	return vnode_pager_generic_getpages(ap->a_vp, ap->a_m,
	ap->a_count, ap->a_rbehind, ap->a_rahead, NULL, NULL);
	}

	static int
	vop_stdgetpages_async(struct vop_getpages_async_args *ap)
	{
	int error;

	error = VOP_GETPAGES(ap->a_vp, ap->a_m, ap->a_count, ap->a_rbehind,
	ap->a_rahead);
	if (ap->a_iodone != NULL)
	ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error);
	return (error);
	}

	int
	vop_stdkqfilter(struct vop_kqfilter_args *ap)
	{
	return vfs_kqfilter(ap);
	}

	/* XXX Needs good comment and more info in the manpage (VOP_PUTPAGES(9)). */
	int
	vop_stdputpages(ap)
	struct vop_putpages_args /* {
	struct vnode *a_vp;
	vm_page_t *a_m;
	int a_count;
	int a_sync;
	int *a_rtvals;
	} / ap;
	{

	return vnode_pager_generic_putpages(ap->a_vp, ap->a_m, ap->a_count,
	ap->a_sync, ap->a_rtvals);
	}

	int
	vop_stdvptofh(struct vop_vptofh_args *ap)
	{
	return (EOPNOTSUPP);
	}

	int
	vop_stdvptocnp(struct vop_vptocnp_args *ap)
	{
	struct vnode *vp = ap->a_vp;
	struct vnode **dvp = ap->a_vpp;
	struct ucred *cred = ap->a_cred;
	char *buf = ap->a_buf;
	size_t *buflen = ap->a_buflen;
	char dirbuf, cpos;
	int i, error, eofflag, dirbuflen, flags, locked, len, covered;
	off_t off;
	ino_t fileno;
	struct vattr va;
	struct nameidata nd;
	struct thread *td;
	struct dirent *dp;
	struct vnode *mvp;

	i = *buflen;
	error = 0;
	covered = 0;
	td = curthread;

	if (vp->v_type != VDIR)
	return (ENOENT);

	error = VOP_GETATTR(vp, &va, cred);
	if (error)
	return (error);

	VREF(vp);
	locked = VOP_ISLOCKED(vp);
	VOP_UNLOCK(vp);
	NDINIT_ATVP(&nd, LOOKUP, FOLLOW \| LOCKSHARED \| LOCKLEAF, UIO_SYSSPACE,
	"..", vp, td);
	flags = FREAD;
	error = vn_open_cred(&nd, &flags, 0, VN_OPEN_NOAUDIT, cred, NULL);
	if (error) {
	vn_lock(vp, locked \| LK_RETRY);
	return (error);
	}
	NDFREE(&nd, NDF_ONLY_PNBUF);

	mvp = *dvp = nd.ni_vp;

	if (vp->v_mount != (*dvp)->v_mount &&
	((*dvp)->v_vflag & VV_ROOT) &&
	((*dvp)->v_mount->mnt_flag & MNT_UNION)) {
	dvp = (dvp)->v_mount->mnt_vnodecovered;
	VREF(mvp);
	VOP_UNLOCK(mvp);
	vn_close(mvp, FREAD, cred, td);
	VREF(*dvp);
	vn_lock(*dvp, LK_SHARED \| LK_RETRY);
	covered = 1;
	}

	fileno = va.va_fileid;

	dirbuflen = DEV_BSIZE;
	if (dirbuflen < va.va_blocksize)
	dirbuflen = va.va_blocksize;
	dirbuf = (char *)malloc(dirbuflen, M_TEMP, M_WAITOK);

	if ((*dvp)->v_type != VDIR) {
	error = ENOENT;
	goto out;
	}

	off = 0;
	len = 0;
	do {
	/* call VOP_READDIR of parent */
	error = get_next_dirent(*dvp, &dp, dirbuf, dirbuflen, &off,
	&cpos, &len, &eofflag, td);
	if (error)
	goto out;

	if ((dp->d_type != DT_WHT) &&
	(dp->d_fileno == fileno)) {
	if (covered) {
	VOP_UNLOCK(*dvp);
	vn_lock(mvp, LK_SHARED \| LK_RETRY);
	if (dirent_exists(mvp, dp->d_name, td)) {
	error = ENOENT;
	VOP_UNLOCK(mvp);
	vn_lock(*dvp, LK_SHARED \| LK_RETRY);
	goto out;
	}
	VOP_UNLOCK(mvp);
	vn_lock(*dvp, LK_SHARED \| LK_RETRY);
	}
	i -= dp->d_namlen;

	if (i < 0) {
	error = ENOMEM;
	goto out;
	}
	if (dp->d_namlen == 1 && dp->d_name[0] == '.') {
	error = ENOENT;
	} else {
	bcopy(dp->d_name, buf + i, dp->d_namlen);
	error = 0;
	}
	goto out;
	}
	} while (len > 0 \|\| !eofflag);
	error = ENOENT;

	out:
	free(dirbuf, M_TEMP);
	if (!error) {
	*buflen = i;
	vref(*dvp);
	}
	if (covered) {
	vput(*dvp);
	vrele(mvp);
	} else {
	VOP_UNLOCK(mvp);
	vn_close(mvp, FREAD, cred, td);
	}
	vn_lock(vp, locked \| LK_RETRY);
	return (error);
	}

	int
	vop_stdallocate(struct vop_allocate_args *ap)
	{
	#ifdef __notyet__
	struct statfs *sfs;
	off_t maxfilesize = 0;
	#endif
	struct iovec aiov;
	struct vattr vattr, *vap;
	struct uio auio;
	off_t fsize, len, cur, offset;
	uint8_t *buf;
	struct thread *td;
	struct vnode *vp;
	size_t iosize;
	int error;

	buf = NULL;
	error = 0;
	td = curthread;
	vap = &vattr;
	vp = ap->a_vp;
	len = *ap->a_len;
	offset = *ap->a_offset;

	error = VOP_GETATTR(vp, vap, td->td_ucred);
	if (error != 0)
	goto out;
	fsize = vap->va_size;
	iosize = vap->va_blocksize;
	if (iosize == 0)
	iosize = BLKDEV_IOSIZE;
	if (iosize > MAXPHYS)
	iosize = MAXPHYS;
	buf = malloc(iosize, M_TEMP, M_WAITOK);

	#ifdef __notyet__
	/*
	* Check if the filesystem sets f_maxfilesize; if not use
	* VOP_SETATTR to perform the check.
	*/
	sfs = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = VFS_STATFS(vp->v_mount, sfs, td);
	if (error == 0)
	maxfilesize = sfs->f_maxfilesize;
	free(sfs, M_STATFS);
	if (error != 0)
	goto out;
	if (maxfilesize) {
	if (offset > maxfilesize \|\| len > maxfilesize \|\|
	offset + len > maxfilesize) {
	error = EFBIG;
	goto out;
	}
	} else
	#endif
	if (offset + len > vap->va_size) {
	/*
	* Test offset + len against the filesystem's maxfilesize.
	*/
	VATTR_NULL(vap);
	vap->va_size = offset + len;
	error = VOP_SETATTR(vp, vap, td->td_ucred);
	if (error != 0)
	goto out;
	VATTR_NULL(vap);
	vap->va_size = fsize;
	error = VOP_SETATTR(vp, vap, td->td_ucred);
	if (error != 0)
	goto out;
	}

	for (;;) {
	/*
	* Read and write back anything below the nominal file
	* size. There's currently no way outside the filesystem
	* to know whether this area is sparse or not.
	*/
	cur = iosize;
	if ((offset % iosize) != 0)
	cur -= (offset % iosize);
	if (cur > len)
	cur = len;
	if (offset < fsize) {
	aiov.iov_base = buf;
	aiov.iov_len = cur;
	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_offset = offset;
	auio.uio_resid = cur;
	auio.uio_segflg = UIO_SYSSPACE;
	auio.uio_rw = UIO_READ;
	auio.uio_td = td;
	error = VOP_READ(vp, &auio, 0, td->td_ucred);
	if (error != 0)
	break;
	if (auio.uio_resid > 0) {
	bzero(buf + cur - auio.uio_resid,
	auio.uio_resid);
	}
	} else {
	bzero(buf, cur);
	}

	aiov.iov_base = buf;
	aiov.iov_len = cur;
	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_offset = offset;
	auio.uio_resid = cur;
	auio.uio_segflg = UIO_SYSSPACE;
	auio.uio_rw = UIO_WRITE;
	auio.uio_td = td;

	error = VOP_WRITE(vp, &auio, 0, td->td_ucred);
	if (error != 0)
	break;

	len -= cur;
	offset += cur;
	if (len == 0)
	break;
	if (should_yield())
	break;
	}

	out:
	*ap->a_len = len;
	*ap->a_offset = offset;
	free(buf, M_TEMP);
	return (error);
	}

	int
	vop_stdadvise(struct vop_advise_args *ap)
	{
	struct vnode *vp;
	struct bufobj *bo;
	daddr_t startn, endn;
	off_t bstart, bend, start, end;
	int bsize, error;

	vp = ap->a_vp;
	switch (ap->a_advice) {
	case POSIX_FADV_WILLNEED:
	/*
	* Do nothing for now. Filesystems should provide a
	* custom method which starts an asynchronous read of
	* the requested region.
	*/
	error = 0;
	break;
	case POSIX_FADV_DONTNEED:
	error = 0;
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	if (VN_IS_DOOMED(vp)) {
	VOP_UNLOCK(vp);
	break;
	}

	/*
	* Round to block boundaries (and later possibly further to
	* page boundaries). Applications cannot reasonably be aware
	* of the boundaries, and the rounding must be to expand at
	* both extremities to cover enough. It still doesn't cover
	* read-ahead. For partial blocks, this gives unnecessary
	* discarding of buffers but is efficient enough since the
	* pages usually remain in VMIO for some time.
	*/
	bsize = vp->v_bufobj.bo_bsize;
	bstart = rounddown(ap->a_start, bsize);
	bend = roundup(ap->a_end, bsize);

	/*
	* Deactivate pages in the specified range from the backing VM
	* object. Pages that are resident in the buffer cache will
	* remain wired until their corresponding buffers are released
	* below.
	*/
	if (vp->v_object != NULL) {
	start = trunc_page(bstart);
	end = round_page(bend);
	VM_OBJECT_RLOCK(vp->v_object);
	vm_object_page_noreuse(vp->v_object, OFF_TO_IDX(start),
	OFF_TO_IDX(end));
	VM_OBJECT_RUNLOCK(vp->v_object);
	}

	bo = &vp->v_bufobj;
	BO_RLOCK(bo);
	startn = bstart / bsize;
	endn = bend / bsize;
	error = bnoreuselist(&bo->bo_clean, bo, startn, endn);
	if (error == 0)
	error = bnoreuselist(&bo->bo_dirty, bo, startn, endn);
	BO_RUNLOCK(bo);
	VOP_UNLOCK(vp);
	break;
	default:
	error = EINVAL;
	break;
	}
	return (error);
	}

	int
	vop_stdunp_bind(struct vop_unp_bind_args *ap)
	{

	ap->a_vp->v_unpcb = ap->a_unpcb;
	return (0);
	}

	int
	vop_stdunp_connect(struct vop_unp_connect_args *ap)
	{

	*ap->a_unpcb = ap->a_vp->v_unpcb;
	return (0);
	}

	int
	vop_stdunp_detach(struct vop_unp_detach_args *ap)
	{

	ap->a_vp->v_unpcb = NULL;
	return (0);
	}

	static int
	vop_stdis_text(struct vop_is_text_args *ap)
	{

	return (ap->a_vp->v_writecount < 0);
	}

	int
	vop_stdset_text(struct vop_set_text_args *ap)
	{
	struct vnode *vp;
	struct mount *mp;
	int error;

	vp = ap->a_vp;
	VI_LOCK(vp);
	if (vp->v_writecount > 0) {
	error = ETXTBSY;
	} else {
	/*
	* If requested by fs, keep a use reference to the
	* vnode until the last text reference is released.
	*/
	mp = vp->v_mount;
	if (mp != NULL && (mp->mnt_kern_flag & MNTK_TEXT_REFS) != 0 &&
	vp->v_writecount == 0) {
	vp->v_iflag \|= VI_TEXT_REF;
	vrefl(vp);
	}

	vp->v_writecount--;
	error = 0;
	}
	VI_UNLOCK(vp);
	return (error);
	}

	static int
	vop_stdunset_text(struct vop_unset_text_args *ap)
	{
	struct vnode *vp;
	int error;
	bool last;

	vp = ap->a_vp;
	last = false;
	VI_LOCK(vp);
	if (vp->v_writecount < 0) {
	if ((vp->v_iflag & VI_TEXT_REF) != 0 &&
	vp->v_writecount == -1) {
	last = true;
	vp->v_iflag &= ~VI_TEXT_REF;
	}
	vp->v_writecount++;
	error = 0;
	} else {
	error = EINVAL;
	}
	VI_UNLOCK(vp);
	if (last)
	vunref(vp);
	return (error);
	}

	static int
	vop_stdadd_writecount(struct vop_add_writecount_args *ap)
	{
	struct vnode *vp;
	struct mount *mp;
	int error;

	vp = ap->a_vp;
	VI_LOCK_FLAGS(vp, MTX_DUPOK);
	if (vp->v_writecount < 0) {
	error = ETXTBSY;
	} else {
	VNASSERT(vp->v_writecount + ap->a_inc >= 0, vp,
	("neg writecount increment %d", ap->a_inc));
	if (vp->v_writecount == 0) {
	mp = vp->v_mount;
	if (mp != NULL && (mp->mnt_kern_flag & MNTK_NOMSYNC) == 0)
	vlazy(vp);
	}
	vp->v_writecount += ap->a_inc;
	error = 0;
	}
	VI_UNLOCK(vp);
	return (error);
	}

	int
	vop_stdneed_inactive(struct vop_need_inactive_args *ap)
	{

	return (1);
	}

	int
	vop_stdioctl(struct vop_ioctl_args *ap)
	{
	struct vnode *vp;
	struct vattr va;
	off_t *offp;
	int error;

	switch (ap->a_command) {
	case FIOSEEKDATA:
	case FIOSEEKHOLE:
	vp = ap->a_vp;
	error = vn_lock(vp, LK_SHARED);
	if (error != 0)
	return (EBADF);
	if (vp->v_type == VREG)
	error = VOP_GETATTR(vp, &va, ap->a_cred);
	else
	error = ENOTTY;
	if (error == 0) {
	offp = ap->a_data;
	if (offp < 0 \|\| offp >= va.va_size)
	error = ENXIO;
	else if (ap->a_command == FIOSEEKHOLE)
	*offp = va.va_size;
	}
	VOP_UNLOCK(vp);
	break;
	default:
	error = ENOTTY;
	break;
	}
	return (error);
	}

	/*
	* vfs default ops
	* used to fill the vfs function table to get reasonable default return values.
	*/
	int
	vfs_stdroot (mp, flags, vpp)
	struct mount *mp;
	int flags;
	struct vnode **vpp;
	{

	return (EOPNOTSUPP);
	}

	int
	vfs_stdstatfs (mp, sbp)
	struct mount *mp;
	struct statfs *sbp;
	{

	return (EOPNOTSUPP);
	}

	int
	vfs_stdquotactl (mp, cmds, uid, arg)
	struct mount *mp;
	int cmds;
	uid_t uid;
	void *arg;
	{

	return (EOPNOTSUPP);
	}

	int
	vfs_stdsync(mp, waitfor)
	struct mount *mp;
	int waitfor;
	{
	struct vnode vp, mvp;
	struct thread *td;
	int error, lockreq, allerror = 0;

	td = curthread;
	lockreq = LK_EXCLUSIVE \| LK_INTERLOCK;
	if (waitfor != MNT_WAIT)
	lockreq \|= LK_NOWAIT;
	/*
	* Force stale buffer cache information to be flushed.
	*/
	loop:
	MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
	if (vp->v_bufobj.bo_dirty.bv_cnt == 0) {
	VI_UNLOCK(vp);
	continue;
	}
	if ((error = vget(vp, lockreq, td)) != 0) {
	if (error == ENOENT) {
	MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
	goto loop;
	}
	continue;
	}
	error = VOP_FSYNC(vp, waitfor, td);
	if (error)
	allerror = error;
	vput(vp);
	}
	return (allerror);
	}

	int
	vfs_stdnosync (mp, waitfor)
	struct mount *mp;
	int waitfor;
	{

	return (0);
	}

	static int
	vop_stdcopy_file_range(struct vop_copy_file_range_args *ap)
	{
	int error;

	error = vn_generic_copy_file_range(ap->a_invp, ap->a_inoffp,
	ap->a_outvp, ap->a_outoffp, ap->a_lenp, ap->a_flags, ap->a_incred,
	ap->a_outcred, ap->a_fsizetd);
	return (error);
	}

	int
	vfs_stdvget (mp, ino, flags, vpp)
	struct mount *mp;
	ino_t ino;
	int flags;
	struct vnode **vpp;
	{

	return (EOPNOTSUPP);
	}

	int
	vfs_stdfhtovp (mp, fhp, flags, vpp)
	struct mount *mp;
	struct fid *fhp;
	int flags;
	struct vnode **vpp;
	{

	return (EOPNOTSUPP);
	}

	int
	vfs_stdinit (vfsp)
	struct vfsconf *vfsp;
	{

	return (0);
	}

	int
	vfs_stduninit (vfsp)
	struct vfsconf *vfsp;
	{

	return(0);
	}

	int
	vfs_stdextattrctl(mp, cmd, filename_vp, attrnamespace, attrname)
	struct mount *mp;
	int cmd;
	struct vnode *filename_vp;
	int attrnamespace;
	const char *attrname;
	{

	if (filename_vp != NULL)
	VOP_UNLOCK(filename_vp);
	return (EOPNOTSUPP);
	}

	int
	vfs_stdsysctl(mp, op, req)
	struct mount *mp;
	fsctlop_t op;
	struct sysctl_req *req;
	{

	return (EOPNOTSUPP);
	}

	static vop_bypass_t *
	bp_by_off(struct vop_vector vop, struct vop_generic_args a)
	{

	return ((vop_bypass_t )((char )vop + a->a_desc->vdesc_vop_offset));
	}

	int
	vop_sigdefer(struct vop_vector vop, struct vop_generic_args a)
	{
	vop_bypass_t *bp;
	int prev_stops, rc;

	bp = bp_by_off(vop, a);
	MPASS(bp != NULL);

	prev_stops = sigdeferstop(SIGDEFERSTOP_SILENT);
	rc = bp(a);
	sigallowstop(prev_stops);
	return (rc);
	}
	diff --git a/sys/kern/vfs_lookup.c b/sys/kern/vfs_lookup.c
	index 13e1d6ddce99..7e90d86bb6a1 100644
	--- a/sys/kern/vfs_lookup.c
	+++ b/sys/kern/vfs_lookup.c
	@@ -1,1552 +1,1562 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1982, 1986, 1989, 1993
	* The Regents of the University of California. All rights reserved.
	* (c) UNIX System Laboratories, Inc.
	* All or some portions of this file are derived from material licensed
	* to the University of California by American Telephone and Telegraph
	* Co. or Unix System Laboratories, Inc. and are reproduced herein with
	* the permission of UNIX System Laboratories, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)vfs_lookup.c 8.4 (Berkeley) 2/16/94
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_capsicum.h"
	#include "opt_ktrace.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/capsicum.h>
	#include <sys/fcntl.h>
	#include <sys/jail.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/namei.h>
	#include <sys/vnode.h>
	#include <sys/mount.h>
	#include <sys/filedesc.h>
	#include <sys/proc.h>
	#include <sys/sdt.h>
	#include <sys/syscallsubr.h>
	#include <sys/sysctl.h>
	#ifdef KTRACE
	#include <sys/ktrace.h>
	#endif

	#include <security/audit/audit.h>
	#include <security/mac/mac_framework.h>

	#include <vm/uma.h>

	#define NAMEI_DIAGNOSTIC 1
	#undef NAMEI_DIAGNOSTIC

	SDT_PROVIDER_DECLARE(vfs);
	SDT_PROBE_DEFINE4(vfs, namei, lookup, entry, "struct vnode ", "char ",
	"unsigned long", "bool");
	SDT_PROBE_DEFINE3(vfs, namei, lookup, return, "int", "struct vnode *", "bool");

	/* Allocation zone for namei. */
	uma_zone_t namei_zone;

	/* Placeholder vnode for mp traversal. */
	static struct vnode *vp_crossmp;

	static int
	crossmp_vop_islocked(struct vop_islocked_args *ap)
	{

	return (LK_SHARED);
	}

	static int
	crossmp_vop_lock1(struct vop_lock1_args *ap)
	{
	struct vnode *vp;
	struct lock *lk __unused;
	const char *file __unused;
	int flags, line __unused;

	vp = ap->a_vp;
	lk = vp->v_vnlock;
	flags = ap->a_flags;
	file = ap->a_file;
	line = ap->a_line;

	if ((flags & LK_SHARED) == 0)
	panic("invalid lock request for crossmp");

	WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, file, line,
	flags & LK_INTERLOCK ? &VI_MTX(vp)->lock_object : NULL);
	WITNESS_LOCK(&lk->lock_object, 0, file, line);
	if ((flags & LK_INTERLOCK) != 0)
	VI_UNLOCK(vp);
	LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, ap->a_file, line);
	return (0);
	}

	static int
	crossmp_vop_unlock(struct vop_unlock_args *ap)
	{
	struct vnode *vp;
	struct lock *lk __unused;

	vp = ap->a_vp;
	lk = vp->v_vnlock;

	WITNESS_UNLOCK(&lk->lock_object, 0, LOCK_FILE, LOCK_LINE);
	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, LOCK_FILE,
	LOCK_LINE);
	return (0);
	}

	static struct vop_vector crossmp_vnodeops = {
	.vop_default = &default_vnodeops,
	.vop_islocked = crossmp_vop_islocked,
	.vop_lock1 = crossmp_vop_lock1,
	.vop_unlock = crossmp_vop_unlock,
	};
	/*
	* VFS_VOP_VECTOR_REGISTER(crossmp_vnodeops) is not used here since the vnode
	* gets allocated early. See nameiinit for the direct call below.
	*/

	struct nameicap_tracker {
	struct vnode *dp;
	TAILQ_ENTRY(nameicap_tracker) nm_link;
	};

	/* Zone for cap mode tracker elements used for dotdot capability checks. */
	static uma_zone_t nt_zone;

	static void
	nameiinit(void *dummy __unused)
	{

	namei_zone = uma_zcreate("NAMEI", MAXPATHLEN, NULL, NULL, NULL, NULL,
	UMA_ALIGN_PTR, 0);
	nt_zone = uma_zcreate("rentr", sizeof(struct nameicap_tracker),
	NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
	vfs_vector_op_register(&crossmp_vnodeops);
	getnewvnode("crossmp", NULL, &crossmp_vnodeops, &vp_crossmp);
	}
	SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nameiinit, NULL);

	static int lookup_cap_dotdot = 1;
	SYSCTL_INT(_vfs, OID_AUTO, lookup_cap_dotdot, CTLFLAG_RWTUN,
	&lookup_cap_dotdot, 0,
	"enables \"..\" components in path lookup in capability mode");
	static int lookup_cap_dotdot_nonlocal = 1;
	SYSCTL_INT(_vfs, OID_AUTO, lookup_cap_dotdot_nonlocal, CTLFLAG_RWTUN,
	&lookup_cap_dotdot_nonlocal, 0,
	"enables \"..\" components in path lookup in capability mode "
	"on non-local mount");

	static void
	nameicap_tracker_add(struct nameidata ndp, struct vnode dp)
	{
	struct nameicap_tracker *nt;

	if ((ndp->ni_lcf & NI_LCF_CAP_DOTDOT) == 0 \|\| dp->v_type != VDIR)
	return;
	if ((ndp->ni_lcf & (NI_LCF_BENEATH_ABS \| NI_LCF_BENEATH_LATCHED)) ==
	NI_LCF_BENEATH_ABS) {
	MPASS((ndp->ni_lcf & NI_LCF_LATCH) != 0);
	if (dp != ndp->ni_beneath_latch)
	return;
	ndp->ni_lcf \|= NI_LCF_BENEATH_LATCHED;
	}
	nt = uma_zalloc(nt_zone, M_WAITOK);
	vhold(dp);
	nt->dp = dp;
	TAILQ_INSERT_TAIL(&ndp->ni_cap_tracker, nt, nm_link);
	}

	static void
	nameicap_cleanup(struct nameidata *ndp, bool clean_latch)
	{
	struct nameicap_tracker nt, nt1;

	KASSERT(TAILQ_EMPTY(&ndp->ni_cap_tracker) \|\|
	(ndp->ni_lcf & NI_LCF_CAP_DOTDOT) != 0, ("not strictrelative"));
	TAILQ_FOREACH_SAFE(nt, &ndp->ni_cap_tracker, nm_link, nt1) {
	TAILQ_REMOVE(&ndp->ni_cap_tracker, nt, nm_link);
	vdrop(nt->dp);
	uma_zfree(nt_zone, nt);
	}
	if (clean_latch && (ndp->ni_lcf & NI_LCF_LATCH) != 0) {
	ndp->ni_lcf &= ~NI_LCF_LATCH;
	vrele(ndp->ni_beneath_latch);
	}
	}

	/*
	* For dotdot lookups in capability mode, only allow the component
	* lookup to succeed if the resulting directory was already traversed
	* during the operation. Also fail dotdot lookups for non-local
	* filesystems, where external agents might assist local lookups to
	* escape the compartment.
	*/
	static int
	nameicap_check_dotdot(struct nameidata ndp, struct vnode dp)
	{
	struct nameicap_tracker *nt;
	struct mount *mp;

	if ((ndp->ni_lcf & NI_LCF_CAP_DOTDOT) == 0 \|\| dp == NULL \|\|
	dp->v_type != VDIR)
	return (0);
	mp = dp->v_mount;
	if (lookup_cap_dotdot_nonlocal == 0 && mp != NULL &&
	(mp->mnt_flag & MNT_LOCAL) == 0)
	return (ENOTCAPABLE);
	TAILQ_FOREACH_REVERSE(nt, &ndp->ni_cap_tracker, nameicap_tracker_head,
	nm_link) {
	if (dp == nt->dp)
	return (0);
	}
	if ((ndp->ni_lcf & NI_LCF_BENEATH_ABS) != 0) {
	ndp->ni_lcf &= ~NI_LCF_BENEATH_LATCHED;
	nameicap_cleanup(ndp, false);
	return (0);
	}
	return (ENOTCAPABLE);
	}

	static void
	namei_cleanup_cnp(struct componentname *cnp)
	{

	uma_zfree(namei_zone, cnp->cn_pnbuf);
	#ifdef DIAGNOSTIC
	cnp->cn_pnbuf = NULL;
	cnp->cn_nameptr = NULL;
	#endif
	}

	static int
	namei_handle_root(struct nameidata ndp, struct vnode *dpp)
	{
	struct componentname *cnp;

	cnp = &ndp->ni_cnd;
	if ((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) != 0) {
	#ifdef KTRACE
	if (KTRPOINT(curthread, KTR_CAPFAIL))
	ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
	#endif
	return (ENOTCAPABLE);
	}
	if ((cnp->cn_flags & BENEATH) != 0) {
	ndp->ni_lcf \|= NI_LCF_BENEATH_ABS;
	ndp->ni_lcf &= ~NI_LCF_BENEATH_LATCHED;
	nameicap_cleanup(ndp, false);
	}
	while (*(cnp->cn_nameptr) == '/') {
	cnp->cn_nameptr++;
	ndp->ni_pathlen--;
	}
	*dpp = ndp->ni_rootdir;
	vrefact(*dpp);
	return (0);
	}

	static int
	namei_setup(struct nameidata ndp, struct vnode dpp, struct pwd *pwdp)
	{
	struct componentname *cnp;
	struct file *dfp;
	struct thread *td;
	struct pwd *pwd;
	cap_rights_t rights;
	struct filecaps dirfd_caps;
	- int error, startdir_used;
	+ int error;
	+ bool startdir_used;

	cnp = &ndp->ni_cnd;
	td = cnp->cn_thread;

	+ startdir_used = false;
	*pwdp = NULL;

	#ifdef CAPABILITY_MODE
	/*
	* In capability mode, lookups must be restricted to happen in
	* the subtree with the root specified by the file descriptor:
	* - The root must be real file descriptor, not the pseudo-descriptor
	* AT_FDCWD.
	* - The passed path must be relative and not absolute.
	* - If lookup_cap_dotdot is disabled, path must not contain the
	* '..' components.
	* - If lookup_cap_dotdot is enabled, we verify that all '..'
	* components lookups result in the directories which were
	* previously walked by us, which prevents an escape from
	* the relative root.
	*/
	if (IN_CAPABILITY_MODE(td) && (cnp->cn_flags & NOCAPCHECK) == 0) {
	ndp->ni_lcf \|= NI_LCF_STRICTRELATIVE;
	if (ndp->ni_dirfd == AT_FDCWD) {
	#ifdef KTRACE
	if (KTRPOINT(td, KTR_CAPFAIL))
	ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
	#endif
	return (ECAPMODE);
	}
	}
	#endif
	error = 0;

	/*
	* Get starting point for the translation.
	*/
	pwd = pwd_hold(td);
	/*
	* The reference on ni_rootdir is acquired in the block below to avoid
	* back-to-back atomics for absolute lookups.
	*/
	ndp->ni_rootdir = pwd->pwd_rdir;
	ndp->ni_topdir = pwd->pwd_jdir;

	if (cnp->cn_pnbuf[0] == '/') {
	ndp->ni_resflags \|= NIRES_ABS;
	error = namei_handle_root(ndp, dpp);
	} else {
	if (ndp->ni_startdir != NULL) {
	*dpp = ndp->ni_startdir;
	- startdir_used = 1;
	+ startdir_used = true;
	} else if (ndp->ni_dirfd == AT_FDCWD) {
	*dpp = pwd->pwd_cdir;
	vrefact(*dpp);
	} else {
	rights = ndp->ni_rightsneeded;
	cap_rights_set_one(&rights, CAP_LOOKUP);

	if (cnp->cn_flags & AUDITVNODE1)
	AUDIT_ARG_ATFD1(ndp->ni_dirfd);
	if (cnp->cn_flags & AUDITVNODE2)
	AUDIT_ARG_ATFD2(ndp->ni_dirfd);
	/*
	* Effectively inlined fgetvp_rights, because we need to
	* inspect the file as well as grabbing the vnode.
	*/
	error = fget_cap(td, ndp->ni_dirfd, &rights,
	&dfp, &ndp->ni_filecaps);
	if (error != 0) {
	/*
	* Preserve the error; it should either be EBADF
	* or capability-related, both of which can be
	* safely returned to the caller.
	*/
	} else {
	if (dfp->f_ops == &badfileops) {
	error = EBADF;
	} else if (dfp->f_vnode == NULL) {
	error = ENOTDIR;
	} else {
	*dpp = dfp->f_vnode;
	vrefact(*dpp);

	if ((dfp->f_flag & FSEARCH) != 0)
	cnp->cn_flags \|= NOEXECCHECK;
	}
	fdrop(dfp, td);
	}
	#ifdef CAPABILITIES
	/*
	* If file descriptor doesn't have all rights,
	* all lookups relative to it must also be
	* strictly relative.
	*/
	CAP_ALL(&rights);
	if (!cap_rights_contains(&ndp->ni_filecaps.fc_rights,
	&rights) \|\|
	ndp->ni_filecaps.fc_fcntls != CAP_FCNTL_ALL \|\|
	ndp->ni_filecaps.fc_nioctls != -1) {
	ndp->ni_lcf \|= NI_LCF_STRICTRELATIVE;
	}
	#endif
	}
	if (error == 0 && (*dpp)->v_type != VDIR)
	error = ENOTDIR;
	}
	if (error == 0 && (cnp->cn_flags & BENEATH) != 0) {
	if (ndp->ni_dirfd == AT_FDCWD) {
	ndp->ni_beneath_latch = pwd->pwd_cdir;
	vrefact(ndp->ni_beneath_latch);
	} else {
	rights = ndp->ni_rightsneeded;
	cap_rights_set_one(&rights, CAP_LOOKUP);
	error = fgetvp_rights(td, ndp->ni_dirfd, &rights,
	&dirfd_caps, &ndp->ni_beneath_latch);
	if (error == 0 && (*dpp)->v_type != VDIR) {
	vrele(ndp->ni_beneath_latch);
	error = ENOTDIR;
	}
	}
	if (error == 0)
	ndp->ni_lcf \|= NI_LCF_LATCH;
	}
	/*
	* If we are auditing the kernel pathname, save the user pathname.
	*/
	if (cnp->cn_flags & AUDITVNODE1)
	AUDIT_ARG_UPATH1_VP(td, ndp->ni_rootdir, *dpp, cnp->cn_pnbuf);
	if (cnp->cn_flags & AUDITVNODE2)
	AUDIT_ARG_UPATH2_VP(td, ndp->ni_rootdir, *dpp, cnp->cn_pnbuf);
	if (ndp->ni_startdir != NULL && !startdir_used)
	vrele(ndp->ni_startdir);
	if (error != 0) {
	if (*dpp != NULL)
	vrele(*dpp);
	return (error);
	}
	MPASS((ndp->ni_lcf & (NI_LCF_BENEATH_ABS \| NI_LCF_LATCH)) !=
	NI_LCF_BENEATH_ABS);
	if (((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) != 0 &&
	lookup_cap_dotdot != 0) \|\|
	((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) == 0 &&
	(cnp->cn_flags & BENEATH) != 0))
	ndp->ni_lcf \|= NI_LCF_CAP_DOTDOT;
	SDT_PROBE4(vfs, namei, lookup, entry, *dpp, cnp->cn_pnbuf,
	cnp->cn_flags, false);
	*pwdp = pwd;
	return (0);
	}

	/*
	* Convert a pathname into a pointer to a locked vnode.
	*
	* The FOLLOW flag is set when symbolic links are to be followed
	* when they occur at the end of the name translation process.
	* Symbolic links are always followed for all other pathname
	* components other than the last.
	*
	* The segflg defines whether the name is to be copied from user
	* space or kernel space.
	*
	* Overall outline of namei:
	*
	* copy in name
	* get starting directory
	* while (!done && !error) {
	* call lookup to search path.
	* if symbolic link, massage name in buffer and continue
	* }
	*/
	int
	namei(struct nameidata *ndp)
	{
	char cp; / pointer into pathname argument */
	struct vnode dp; / the directory we are searching */
	struct iovec aiov; /* uio for reading symbolic links */
	struct componentname *cnp;
	struct thread *td;
	struct proc *p;
	struct pwd *pwd;
	struct uio auio;
	int error, linklen;
	enum cache_fpl_status status;

	cnp = &ndp->ni_cnd;
	td = cnp->cn_thread;
	p = td->td_proc;
	ndp->ni_cnd.cn_cred = ndp->ni_cnd.cn_thread->td_ucred;
	KASSERT(cnp->cn_cred && p, ("namei: bad cred/proc"));
	KASSERT((cnp->cn_nameiop & (~OPMASK)) == 0,
	("namei: nameiop contaminated with flags"));
	KASSERT((cnp->cn_flags & OPMASK) == 0,
	("namei: flags contaminated with nameiops"));
	MPASS(ndp->ni_startdir == NULL \|\| ndp->ni_startdir->v_type == VDIR \|\|
	ndp->ni_startdir->v_type == VBAD);
	TAILQ_INIT(&ndp->ni_cap_tracker);
	ndp->ni_lcf = 0;
	ndp->ni_loopcnt = 0;
	dp = NULL;

	/* We will set this ourselves if we need it. */
	cnp->cn_flags &= ~TRAILINGSLASH;

	ndp->ni_vp = NULL;

	/*
	* Get a buffer for the name to be translated, and copy the
	* name into the buffer.
	*/
	if ((cnp->cn_flags & HASBUF) == 0)
	cnp->cn_pnbuf = uma_zalloc(namei_zone, M_WAITOK);
	if (ndp->ni_segflg == UIO_SYSSPACE)
	error = copystr(ndp->ni_dirp, cnp->cn_pnbuf, MAXPATHLEN,
	&ndp->ni_pathlen);
	else
	error = copyinstr(ndp->ni_dirp, cnp->cn_pnbuf, MAXPATHLEN,
	&ndp->ni_pathlen);

	if (error != 0) {
	namei_cleanup_cnp(cnp);
	return (error);
	}

	cnp->cn_nameptr = cnp->cn_pnbuf;

	/*
	* Don't allow empty pathnames.
	*/
	if (*cnp->cn_pnbuf == '\0') {
	namei_cleanup_cnp(cnp);
	return (ENOENT);
	}

	#ifdef KTRACE
	if (KTRPOINT(td, KTR_NAMEI)) {
	KASSERT(cnp->cn_thread == curthread,
	("namei not using curthread"));
	ktrnamei(cnp->cn_pnbuf);
	}
	#endif

	/*
	* First try looking up the target without locking any vnodes.
	*
	* We may need to start from scratch or pick up where it left off.
	*/
	error = cache_fplookup(ndp, &status, &pwd);
	switch (status) {
	case CACHE_FPL_STATUS_UNSET:
	__assert_unreachable();
	break;
	case CACHE_FPL_STATUS_HANDLED:
	return (error);
	case CACHE_FPL_STATUS_PARTIAL:
	dp = ndp->ni_startdir;
	break;
	case CACHE_FPL_STATUS_ABORTED:
	error = namei_setup(ndp, &dp, &pwd);
	if (error != 0) {
	namei_cleanup_cnp(cnp);
	return (error);
	}
	break;
	}

	/*
	* Locked lookup.
	*/
	for (;;) {
	ndp->ni_startdir = dp;
	error = lookup(ndp);
	if (error != 0)
	goto out;
	/*
	* If not a symbolic link, we're done.
	*/
	if ((cnp->cn_flags & ISSYMLINK) == 0) {
	if ((cnp->cn_flags & (SAVENAME \| SAVESTART)) == 0) {
	namei_cleanup_cnp(cnp);
	} else
	cnp->cn_flags \|= HASBUF;
	if ((ndp->ni_lcf & (NI_LCF_BENEATH_ABS \|
	NI_LCF_BENEATH_LATCHED)) == NI_LCF_BENEATH_ABS) {
	NDFREE(ndp, 0);
	error = ENOTCAPABLE;
	}
	nameicap_cleanup(ndp, true);
	SDT_PROBE3(vfs, namei, lookup, return, error,
	(error == 0 ? ndp->ni_vp : NULL), false);
	pwd_drop(pwd);
	return (error);
	}
	if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
	error = ELOOP;
	break;
	}
	#ifdef MAC
	if ((cnp->cn_flags & NOMACCHECK) == 0) {
	error = mac_vnode_check_readlink(td->td_ucred,
	ndp->ni_vp);
	if (error != 0)
	break;
	}
	#endif
	if (ndp->ni_pathlen > 1)
	cp = uma_zalloc(namei_zone, M_WAITOK);
	else
	cp = cnp->cn_pnbuf;
	aiov.iov_base = cp;
	aiov.iov_len = MAXPATHLEN;
	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_offset = 0;
	auio.uio_rw = UIO_READ;
	auio.uio_segflg = UIO_SYSSPACE;
	auio.uio_td = td;
	auio.uio_resid = MAXPATHLEN;
	error = VOP_READLINK(ndp->ni_vp, &auio, cnp->cn_cred);
	if (error != 0) {
	if (ndp->ni_pathlen > 1)
	uma_zfree(namei_zone, cp);
	break;
	}
	linklen = MAXPATHLEN - auio.uio_resid;
	if (linklen == 0) {
	if (ndp->ni_pathlen > 1)
	uma_zfree(namei_zone, cp);
	error = ENOENT;
	break;
	}
	if (linklen + ndp->ni_pathlen > MAXPATHLEN) {
	if (ndp->ni_pathlen > 1)
	uma_zfree(namei_zone, cp);
	error = ENAMETOOLONG;
	break;
	}
	if (ndp->ni_pathlen > 1) {
	bcopy(ndp->ni_next, cp + linklen, ndp->ni_pathlen);
	uma_zfree(namei_zone, cnp->cn_pnbuf);
	cnp->cn_pnbuf = cp;
	} else
	cnp->cn_pnbuf[linklen] = '\0';
	ndp->ni_pathlen += linklen;
	vput(ndp->ni_vp);
	dp = ndp->ni_dvp;
	/*
	* Check if root directory should replace current directory.
	*/
	cnp->cn_nameptr = cnp->cn_pnbuf;
	if (*(cnp->cn_nameptr) == '/') {
	vrele(dp);
	error = namei_handle_root(ndp, &dp);
	if (error != 0)
	goto out;
	}
	}
	vput(ndp->ni_vp);
	ndp->ni_vp = NULL;
	vrele(ndp->ni_dvp);
	out:
	MPASS(error != 0);
	namei_cleanup_cnp(cnp);
	nameicap_cleanup(ndp, true);
	SDT_PROBE3(vfs, namei, lookup, return, error, NULL, false);
	pwd_drop(pwd);
	return (error);
	}

	static int
	compute_cn_lkflags(struct mount *mp, int lkflags, int cnflags)
	{

	if (mp == NULL \|\| ((lkflags & LK_SHARED) &&
	(!(mp->mnt_kern_flag & MNTK_LOOKUP_SHARED) \|\|
	((cnflags & ISDOTDOT) &&
	(mp->mnt_kern_flag & MNTK_LOOKUP_EXCL_DOTDOT))))) {
	lkflags &= ~LK_SHARED;
	lkflags \|= LK_EXCLUSIVE;
	}
	lkflags \|= LK_NODDLKTREAT;
	return (lkflags);
	}

	static __inline int
	needs_exclusive_leaf(struct mount *mp, int flags)
	{

	/*
	* Intermediate nodes can use shared locks, we only need to
	* force an exclusive lock for leaf nodes.
	*/
	if ((flags & (ISLASTCN \| LOCKLEAF)) != (ISLASTCN \| LOCKLEAF))
	return (0);

	/* Always use exclusive locks if LOCKSHARED isn't set. */
	if (!(flags & LOCKSHARED))
	return (1);

	/*
	* For lookups during open(), if the mount point supports
	* extended shared operations, then use a shared lock for the
	* leaf node, otherwise use an exclusive lock.
	*/
	if ((flags & ISOPEN) != 0)
	return (!MNT_EXTENDED_SHARED(mp));

	/*
	* Lookup requests outside of open() that specify LOCKSHARED
	* only need a shared lock on the leaf vnode.
	*/
	return (0);
	}

	/*
	* Search a pathname.
	* This is a very central and rather complicated routine.
	*
	* The pathname is pointed to by ni_ptr and is of length ni_pathlen.
	* The starting directory is taken from ni_startdir. The pathname is
	* descended until done, or a symbolic link is encountered. The variable
	* ni_more is clear if the path is completed; it is set to one if a
	* symbolic link needing interpretation is encountered.
	*
	* The flag argument is LOOKUP, CREATE, RENAME, or DELETE depending on
	* whether the name is to be looked up, created, renamed, or deleted.
	* When CREATE, RENAME, or DELETE is specified, information usable in
	* creating, renaming, or deleting a directory entry may be calculated.
	* If flag has LOCKPARENT or'ed into it, the parent directory is returned
	* locked. If flag has WANTPARENT or'ed into it, the parent directory is
	* returned unlocked. Otherwise the parent directory is not returned. If
	* the target of the pathname exists and LOCKLEAF is or'ed into the flag
	* the target is returned locked, otherwise it is returned unlocked.
	* When creating or renaming and LOCKPARENT is specified, the target may not
	* be ".". When deleting and LOCKPARENT is specified, the target may be ".".
	*
	* Overall outline of lookup:
	*
	* dirloop:
	* identify next component of name at ndp->ni_ptr
	* handle degenerate case where name is null string
	* if .. and crossing mount points and on mounted filesys, find parent
	* call VOP_LOOKUP routine for next component name
	* directory vnode returned in ni_dvp, unlocked unless LOCKPARENT set
	* component vnode returned in ni_vp (if it exists), locked.
	* if result vnode is mounted on and crossing mount points,
	* find mounted on vnode
	* if more components of name, do next level at dirloop
	* return the answer in ni_vp, locked if LOCKLEAF set
	* if LOCKPARENT set, return locked parent in ni_dvp
	* if WANTPARENT set, return unlocked parent in ni_dvp
	*/
	int
	lookup(struct nameidata *ndp)
	{
	char cp; / pointer into pathname argument */
	char prev_ni_next; / saved ndp->ni_next */
	struct vnode dp = NULL; / the directory we are searching */
	struct vnode tdp; / saved dp */
	struct mount mp; / mount table entry */
	struct prison *pr;
	size_t prev_ni_pathlen; /* saved ndp->ni_pathlen */
	int docache; /* == 0 do not cache last component */
	int wantparent; /* 1 => wantparent or lockparent flag */
	int rdonly; /* lookup read-only flag bit */
	int error = 0;
	int dpunlocked = 0; /* dp has already been unlocked */
	int relookup = 0; /* do not consume the path component */
	struct componentname *cnp = &ndp->ni_cnd;
	int lkflags_save;
	int ni_dvp_unlocked;

	/*
	* Setup: break out flag bits into variables.
	*/
	ni_dvp_unlocked = 0;
	wantparent = cnp->cn_flags & (LOCKPARENT \| WANTPARENT);
	KASSERT(cnp->cn_nameiop == LOOKUP \|\| wantparent,
	("CREATE, DELETE, RENAME require LOCKPARENT or WANTPARENT."));
	docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
	if (cnp->cn_nameiop == DELETE \|\|
	(wantparent && cnp->cn_nameiop != CREATE &&
	cnp->cn_nameiop != LOOKUP))
	docache = 0;
	rdonly = cnp->cn_flags & RDONLY;
	cnp->cn_flags &= ~ISSYMLINK;
	ndp->ni_dvp = NULL;
	/*
	* We use shared locks until we hit the parent of the last cn then
	* we adjust based on the requesting flags.
	*/
	cnp->cn_lkflags = LK_SHARED;
	dp = ndp->ni_startdir;
	ndp->ni_startdir = NULLVP;
	vn_lock(dp,
	compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags \| LK_RETRY,
	cnp->cn_flags));

	dirloop:
	/*
	* Search a new directory.
	*
	* The last component of the filename is left accessible via
	* cnp->cn_nameptr for callers that need the name. Callers needing
	* the name set the SAVENAME flag. When done, they assume
	* responsibility for freeing the pathname buffer.
	*/
	for (cp = cnp->cn_nameptr; cp != 0 && cp != '/'; cp++)
	continue;
	cnp->cn_namelen = cp - cnp->cn_nameptr;
	if (cnp->cn_namelen > NAME_MAX) {
	error = ENAMETOOLONG;
	goto bad;
	}
	#ifdef NAMEI_DIAGNOSTIC
	{ char c = *cp;
	*cp = '\0';
	printf("{%s}: ", cnp->cn_nameptr);
	*cp = c; }
	#endif
	prev_ni_pathlen = ndp->ni_pathlen;
	ndp->ni_pathlen -= cnp->cn_namelen;
	KASSERT(ndp->ni_pathlen <= PATH_MAX,
	("%s: ni_pathlen underflow to %zd\n", __func__, ndp->ni_pathlen));
	prev_ni_next = ndp->ni_next;
	ndp->ni_next = cp;

	/*
	* Replace multiple slashes by a single slash and trailing slashes
	* by a null. This must be done before VOP_LOOKUP() because some
	* fs's don't know about trailing slashes. Remember if there were
	* trailing slashes to handle symlinks, existing non-directories
	* and non-existing files that won't be directories specially later.
	*/
	while (*cp == '/' && (cp[1] == '/' \|\| cp[1] == '\0')) {
	cp++;
	ndp->ni_pathlen--;
	if (*cp == '\0') {
	*ndp->ni_next = '\0';
	cnp->cn_flags \|= TRAILINGSLASH;
	}
	}
	ndp->ni_next = cp;

	cnp->cn_flags \|= MAKEENTRY;
	if (*cp == '\0' && docache == 0)
	cnp->cn_flags &= ~MAKEENTRY;
	if (cnp->cn_namelen == 2 &&
	cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
	cnp->cn_flags \|= ISDOTDOT;
	else
	cnp->cn_flags &= ~ISDOTDOT;
	if (*ndp->ni_next == 0)
	cnp->cn_flags \|= ISLASTCN;
	else
	cnp->cn_flags &= ~ISLASTCN;

	if ((cnp->cn_flags & ISLASTCN) != 0 &&
	cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.' &&
	(cnp->cn_nameiop == DELETE \|\| cnp->cn_nameiop == RENAME)) {
	error = EINVAL;
	goto bad;
	}

	nameicap_tracker_add(ndp, dp);

	/*
	* Check for degenerate name (e.g. / or "")
	* which is a way of talking about a directory,
	* e.g. like "/." or ".".
	*/
	if (cnp->cn_nameptr[0] == '\0') {
	if (dp->v_type != VDIR) {
	error = ENOTDIR;
	goto bad;
	}
	if (cnp->cn_nameiop != LOOKUP) {
	error = EISDIR;
	goto bad;
	}
	if (wantparent) {
	ndp->ni_dvp = dp;
	VREF(dp);
	}
	ndp->ni_vp = dp;

	if (cnp->cn_flags & AUDITVNODE1)
	AUDIT_ARG_VNODE1(dp);
	else if (cnp->cn_flags & AUDITVNODE2)
	AUDIT_ARG_VNODE2(dp);

	if (!(cnp->cn_flags & (LOCKPARENT \| LOCKLEAF)))
	VOP_UNLOCK(dp);
	/* XXX This should probably move to the top of function. */
	if (cnp->cn_flags & SAVESTART)
	panic("lookup: SAVESTART");
	goto success;
	}

	/*
	* Handle "..": five special cases.
	* 0. If doing a capability lookup and lookup_cap_dotdot is
	* disabled, return ENOTCAPABLE.
	* 1. Return an error if this is the last component of
	* the name and the operation is DELETE or RENAME.
	* 2. If at root directory (e.g. after chroot)
	* or at absolute root directory
	* then ignore it so can't get out.
	* 3. If this vnode is the root of a mounted
	* filesystem, then replace it with the
	* vnode which was mounted on so we take the
	* .. in the other filesystem.
	* 4. If the vnode is the top directory of
	* the jail or chroot, don't let them out.
	* 5. If doing a capability lookup and lookup_cap_dotdot is
	* enabled, return ENOTCAPABLE if the lookup would escape
	* from the initial file descriptor directory. Checks are
	* done by ensuring that namei() already traversed the
	* result of dotdot lookup.
	*/
	if (cnp->cn_flags & ISDOTDOT) {
	if ((ndp->ni_lcf & (NI_LCF_STRICTRELATIVE \| NI_LCF_CAP_DOTDOT))
	== NI_LCF_STRICTRELATIVE) {
	#ifdef KTRACE
	if (KTRPOINT(curthread, KTR_CAPFAIL))
	ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
	#endif
	error = ENOTCAPABLE;
	goto bad;
	}
	if ((cnp->cn_flags & ISLASTCN) != 0 &&
	(cnp->cn_nameiop == DELETE \|\| cnp->cn_nameiop == RENAME)) {
	error = EINVAL;
	goto bad;
	}
	for (;;) {
	for (pr = cnp->cn_cred->cr_prison; pr != NULL;
	pr = pr->pr_parent)
	if (dp == pr->pr_root)
	break;
	if (dp == ndp->ni_rootdir \|\|
	dp == ndp->ni_topdir \|\|
	dp == rootvnode \|\|
	pr != NULL \|\|
	((dp->v_vflag & VV_ROOT) != 0 &&
	(cnp->cn_flags & NOCROSSMOUNT) != 0)) {
	ndp->ni_dvp = dp;
	ndp->ni_vp = dp;
	VREF(dp);
	goto nextname;
	}
	if ((dp->v_vflag & VV_ROOT) == 0)
	break;
	if (VN_IS_DOOMED(dp)) { /* forced unmount */
	error = ENOENT;
	goto bad;
	}
	tdp = dp;
	dp = dp->v_mount->mnt_vnodecovered;
	VREF(dp);
	vput(tdp);
	vn_lock(dp,
	compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags \|
	LK_RETRY, ISDOTDOT));
	error = nameicap_check_dotdot(ndp, dp);
	if (error != 0) {
	#ifdef KTRACE
	if (KTRPOINT(curthread, KTR_CAPFAIL))
	ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
	#endif
	goto bad;
	}
	}
	}

	/*
	* We now have a segment name to search for, and a directory to search.
	*/
	unionlookup:
	#ifdef MAC
	error = mac_vnode_check_lookup(cnp->cn_thread->td_ucred, dp, cnp);
	if (error)
	goto bad;
	#endif
	ndp->ni_dvp = dp;
	ndp->ni_vp = NULL;
	ASSERT_VOP_LOCKED(dp, "lookup");
	/*
	* If we have a shared lock we may need to upgrade the lock for the
	* last operation.
	*/
	if ((cnp->cn_flags & LOCKPARENT) && (cnp->cn_flags & ISLASTCN) &&
	dp != vp_crossmp && VOP_ISLOCKED(dp) == LK_SHARED)
	vn_lock(dp, LK_UPGRADE\|LK_RETRY);
	if (VN_IS_DOOMED(dp)) {
	error = ENOENT;
	goto bad;
	}
	/*
	* If we're looking up the last component and we need an exclusive
	* lock, adjust our lkflags.
	*/
	if (needs_exclusive_leaf(dp->v_mount, cnp->cn_flags))
	cnp->cn_lkflags = LK_EXCLUSIVE;
	#ifdef NAMEI_DIAGNOSTIC
	vn_printf(dp, "lookup in ");
	#endif
	lkflags_save = cnp->cn_lkflags;
	cnp->cn_lkflags = compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags,
	cnp->cn_flags);
	error = VOP_LOOKUP(dp, &ndp->ni_vp, cnp);
	cnp->cn_lkflags = lkflags_save;
	if (error != 0) {
	KASSERT(ndp->ni_vp == NULL, ("leaf should be empty"));
	#ifdef NAMEI_DIAGNOSTIC
	printf("not found\n");
	#endif
	if ((error == ENOENT) &&
	(dp->v_vflag & VV_ROOT) && (dp->v_mount != NULL) &&
	(dp->v_mount->mnt_flag & MNT_UNION)) {
	tdp = dp;
	dp = dp->v_mount->mnt_vnodecovered;
	VREF(dp);
	vput(tdp);
	vn_lock(dp,
	compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags \|
	LK_RETRY, cnp->cn_flags));
	nameicap_tracker_add(ndp, dp);
	goto unionlookup;
	}

	if (error == ERELOOKUP) {
	vref(dp);
	ndp->ni_vp = dp;
	error = 0;
	relookup = 1;
	goto good;
	}

	if (error != EJUSTRETURN)
	goto bad;
	/*
	* At this point, we know we're at the end of the
	* pathname. If creating / renaming, we can consider
	* allowing the file or directory to be created / renamed,
	* provided we're not on a read-only filesystem.
	*/
	if (rdonly) {
	error = EROFS;
	goto bad;
	}
	/* trailing slash only allowed for directories */
	if ((cnp->cn_flags & TRAILINGSLASH) &&
	!(cnp->cn_flags & WILLBEDIR)) {
	error = ENOENT;
	goto bad;
	}
	if ((cnp->cn_flags & LOCKPARENT) == 0)
	VOP_UNLOCK(dp);
	/*
	* We return with ni_vp NULL to indicate that the entry
	* doesn't currently exist, leaving a pointer to the
	* (possibly locked) directory vnode in ndp->ni_dvp.
	*/
	if (cnp->cn_flags & SAVESTART) {
	ndp->ni_startdir = ndp->ni_dvp;
	VREF(ndp->ni_startdir);
	}
	goto success;
	}

	good:
	#ifdef NAMEI_DIAGNOSTIC
	printf("found\n");
	#endif
	dp = ndp->ni_vp;

	/*
	* Check to see if the vnode has been mounted on;
	* if so find the root of the mounted filesystem.
	*/
	while (dp->v_type == VDIR && (mp = dp->v_mountedhere) &&
	(cnp->cn_flags & NOCROSSMOUNT) == 0) {
	if (vfs_busy(mp, 0))
	continue;
	vput(dp);
	if (dp != ndp->ni_dvp)
	vput(ndp->ni_dvp);
	else
	vrele(ndp->ni_dvp);
	vrefact(vp_crossmp);
	ndp->ni_dvp = vp_crossmp;
	error = VFS_ROOT(mp, compute_cn_lkflags(mp, cnp->cn_lkflags,
	cnp->cn_flags), &tdp);
	vfs_unbusy(mp);
	if (vn_lock(vp_crossmp, LK_SHARED \| LK_NOWAIT))
	panic("vp_crossmp exclusively locked or reclaimed");
	if (error) {
	dpunlocked = 1;
	goto bad2;
	}
	ndp->ni_vp = dp = tdp;
	}

	/*
	* Check for symbolic link
	*/
	if ((dp->v_type == VLNK) &&
	((cnp->cn_flags & FOLLOW) \|\| (cnp->cn_flags & TRAILINGSLASH) \|\|
	*ndp->ni_next == '/')) {
	cnp->cn_flags \|= ISSYMLINK;
	if (VN_IS_DOOMED(dp)) {
	/*
	* We can't know whether the directory was mounted with
	* NOSYMFOLLOW, so we can't follow safely.
	*/
	error = ENOENT;
	goto bad2;
	}
	if (dp->v_mount->mnt_flag & MNT_NOSYMFOLLOW) {
	error = EACCES;
	goto bad2;
	}
	/*
	* Symlink code always expects an unlocked dvp.
	*/
	if (ndp->ni_dvp != ndp->ni_vp) {
	VOP_UNLOCK(ndp->ni_dvp);
	ni_dvp_unlocked = 1;
	}
	goto success;
	}

	nextname:
	/*
	* Not a symbolic link that we will follow. Continue with the
	* next component if there is any; otherwise, we're done.
	*/
	KASSERT((cnp->cn_flags & ISLASTCN) \|\| *ndp->ni_next == '/',
	("lookup: invalid path state."));
	if (relookup) {
	relookup = 0;
	ndp->ni_pathlen = prev_ni_pathlen;
	ndp->ni_next = prev_ni_next;
	if (ndp->ni_dvp != dp)
	vput(ndp->ni_dvp);
	else
	vrele(ndp->ni_dvp);
	goto dirloop;
	}
	if (cnp->cn_flags & ISDOTDOT) {
	error = nameicap_check_dotdot(ndp, ndp->ni_vp);
	if (error != 0) {
	#ifdef KTRACE
	if (KTRPOINT(curthread, KTR_CAPFAIL))
	ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
	#endif
	goto bad2;
	}
	}
	if (*ndp->ni_next == '/') {
	cnp->cn_nameptr = ndp->ni_next;
	while (*cnp->cn_nameptr == '/') {
	cnp->cn_nameptr++;
	ndp->ni_pathlen--;
	}
	if (ndp->ni_dvp != dp)
	vput(ndp->ni_dvp);
	else
	vrele(ndp->ni_dvp);
	goto dirloop;
	}
	/*
	* If we're processing a path with a trailing slash,
	* check that the end result is a directory.
	*/
	if ((cnp->cn_flags & TRAILINGSLASH) && dp->v_type != VDIR) {
	error = ENOTDIR;
	goto bad2;
	}
	/*
	* Disallow directory write attempts on read-only filesystems.
	*/
	if (rdonly &&
	(cnp->cn_nameiop == DELETE \|\| cnp->cn_nameiop == RENAME)) {
	error = EROFS;
	goto bad2;
	}
	if (cnp->cn_flags & SAVESTART) {
	ndp->ni_startdir = ndp->ni_dvp;
	VREF(ndp->ni_startdir);
	}
	if (!wantparent) {
	ni_dvp_unlocked = 2;
	if (ndp->ni_dvp != dp)
	vput(ndp->ni_dvp);
	else
	vrele(ndp->ni_dvp);
	} else if ((cnp->cn_flags & LOCKPARENT) == 0 && ndp->ni_dvp != dp) {
	VOP_UNLOCK(ndp->ni_dvp);
	ni_dvp_unlocked = 1;
	}

	if (cnp->cn_flags & AUDITVNODE1)
	AUDIT_ARG_VNODE1(dp);
	else if (cnp->cn_flags & AUDITVNODE2)
	AUDIT_ARG_VNODE2(dp);

	if ((cnp->cn_flags & LOCKLEAF) == 0)
	VOP_UNLOCK(dp);
	success:
	/*
	* Because of shared lookup we may have the vnode shared locked, but
	* the caller may want it to be exclusively locked.
	*/
	if (needs_exclusive_leaf(dp->v_mount, cnp->cn_flags) &&
	VOP_ISLOCKED(dp) != LK_EXCLUSIVE) {
	vn_lock(dp, LK_UPGRADE \| LK_RETRY);
	if (VN_IS_DOOMED(dp)) {
	error = ENOENT;
	goto bad2;
	}
	}
	return (0);

	bad2:
	if (ni_dvp_unlocked != 2) {
	if (dp != ndp->ni_dvp && !ni_dvp_unlocked)
	vput(ndp->ni_dvp);
	else
	vrele(ndp->ni_dvp);
	}
	bad:
	if (!dpunlocked)
	vput(dp);
	ndp->ni_vp = NULL;
	return (error);
	}

	/*
	* relookup - lookup a path name component
	* Used by lookup to re-acquire things.
	*/
	int
	relookup(struct vnode dvp, struct vnode vpp, struct componentname cnp)
	{
	struct vnode dp = NULL; / the directory we are searching */
	int wantparent; /* 1 => wantparent or lockparent flag */
	int rdonly; /* lookup read-only flag bit */
	int error = 0;

	KASSERT(cnp->cn_flags & ISLASTCN,
	("relookup: Not given last component."));
	/*
	* Setup: break out flag bits into variables.
	*/
	wantparent = cnp->cn_flags & (LOCKPARENT\|WANTPARENT);
	KASSERT(wantparent, ("relookup: parent not wanted."));
	rdonly = cnp->cn_flags & RDONLY;
	cnp->cn_flags &= ~ISSYMLINK;
	dp = dvp;
	cnp->cn_lkflags = LK_EXCLUSIVE;
	vn_lock(dp, LK_EXCLUSIVE \| LK_RETRY);

	/*
	* Search a new directory.
	*
	* The last component of the filename is left accessible via
	* cnp->cn_nameptr for callers that need the name. Callers needing
	* the name set the SAVENAME flag. When done, they assume
	* responsibility for freeing the pathname buffer.
	*/
	#ifdef NAMEI_DIAGNOSTIC
	printf("{%s}: ", cnp->cn_nameptr);
	#endif

	/*
	* Check for "" which represents the root directory after slash
	* removal.
	*/
	if (cnp->cn_nameptr[0] == '\0') {
	/*
	* Support only LOOKUP for "/" because lookup()
	* can't succeed for CREATE, DELETE and RENAME.
	*/
	KASSERT(cnp->cn_nameiop == LOOKUP, ("nameiop must be LOOKUP"));
	KASSERT(dp->v_type == VDIR, ("dp is not a directory"));

	if (!(cnp->cn_flags & LOCKLEAF))
	VOP_UNLOCK(dp);
	*vpp = dp;
	/* XXX This should probably move to the top of function. */
	if (cnp->cn_flags & SAVESTART)
	panic("lookup: SAVESTART");
	return (0);
	}

	if (cnp->cn_flags & ISDOTDOT)
	panic ("relookup: lookup on dot-dot");

	/*
	* We now have a segment name to search for, and a directory to search.
	*/
	#ifdef NAMEI_DIAGNOSTIC
	vn_printf(dp, "search in ");
	#endif
	if ((error = VOP_LOOKUP(dp, vpp, cnp)) != 0) {
	KASSERT(*vpp == NULL, ("leaf should be empty"));
	if (error != EJUSTRETURN)
	goto bad;
	/*
	* If creating and at end of pathname, then can consider
	* allowing file to be created.
	*/
	if (rdonly) {
	error = EROFS;
	goto bad;
	}
	/* ASSERT(dvp == ndp->ni_startdir) */
	if (cnp->cn_flags & SAVESTART)
	VREF(dvp);
	if ((cnp->cn_flags & LOCKPARENT) == 0)
	VOP_UNLOCK(dp);
	/*
	* We return with ni_vp NULL to indicate that the entry
	* doesn't currently exist, leaving a pointer to the
	* (possibly locked) directory vnode in ndp->ni_dvp.
	*/
	return (0);
	}

	dp = *vpp;

	/*
	* Disallow directory write attempts on read-only filesystems.
	*/
	if (rdonly &&
	(cnp->cn_nameiop == DELETE \|\| cnp->cn_nameiop == RENAME)) {
	if (dvp == dp)
	vrele(dvp);
	else
	vput(dvp);
	error = EROFS;
	goto bad;
	}
	/*
	* Set the parent lock/ref state to the requested state.
	*/
	if ((cnp->cn_flags & LOCKPARENT) == 0 && dvp != dp) {
	if (wantparent)
	VOP_UNLOCK(dvp);
	else
	vput(dvp);
	} else if (!wantparent)
	vrele(dvp);
	/*
	* Check for symbolic link
	*/
	KASSERT(dp->v_type != VLNK \|\| !(cnp->cn_flags & FOLLOW),
	("relookup: symlink found.\n"));

	/* ASSERT(dvp == ndp->ni_startdir) */
	if (cnp->cn_flags & SAVESTART)
	VREF(dvp);

	if ((cnp->cn_flags & LOCKLEAF) == 0)
	VOP_UNLOCK(dp);
	return (0);
	bad:
	vput(dp);
	*vpp = NULL;
	return (error);
	}

	void
	NDINIT_ALL(struct nameidata *ndp, u_long op, u_long flags, enum uio_seg segflg,
	const char namep, int dirfd, struct vnode startdir, cap_rights_t *rightsp,
	struct thread *td)
	{

	ndp->ni_cnd.cn_nameiop = op;
	ndp->ni_cnd.cn_flags = flags;
	ndp->ni_segflg = segflg;
	ndp->ni_dirp = namep;
	ndp->ni_dirfd = dirfd;
	ndp->ni_startdir = startdir;
	ndp->ni_resflags = 0;
	filecaps_init(&ndp->ni_filecaps);
	ndp->ni_cnd.cn_thread = td;
	if (rightsp != NULL)
	ndp->ni_rightsneeded = *rightsp;
	else
	cap_rights_init_zero(&ndp->ni_rightsneeded);
	}

	/*
	* Free data allocated by namei(); see namei(9) for details.
	*/
	void
	-NDFREE(struct nameidata *ndp, const u_int flags)
	+NDFREE_PNBUF(struct nameidata *ndp)
	+{
	+
	+ if ((ndp->ni_cnd.cn_flags & HASBUF) != 0) {
	+ uma_zfree(namei_zone, ndp->ni_cnd.cn_pnbuf);
	+ ndp->ni_cnd.cn_flags &= ~HASBUF;
	+ }
	+}
	+
	+void
	+(NDFREE)(struct nameidata *ndp, const u_int flags)
	{
	int unlock_dvp;
	int unlock_vp;

	unlock_dvp = 0;
	unlock_vp = 0;

	- if (!(flags & NDF_NO_FREE_PNBUF) &&
	- (ndp->ni_cnd.cn_flags & HASBUF)) {
	- uma_zfree(namei_zone, ndp->ni_cnd.cn_pnbuf);
	- ndp->ni_cnd.cn_flags &= ~HASBUF;
	+ if (!(flags & NDF_NO_FREE_PNBUF)) {
	+ NDFREE_PNBUF(ndp);
	}
	if (!(flags & NDF_NO_VP_UNLOCK) &&
	(ndp->ni_cnd.cn_flags & LOCKLEAF) && ndp->ni_vp)
	unlock_vp = 1;
	if (!(flags & NDF_NO_DVP_UNLOCK) &&
	(ndp->ni_cnd.cn_flags & LOCKPARENT) &&
	ndp->ni_dvp != ndp->ni_vp)
	unlock_dvp = 1;
	if (!(flags & NDF_NO_VP_RELE) && ndp->ni_vp) {
	if (unlock_vp) {
	vput(ndp->ni_vp);
	unlock_vp = 0;
	} else
	vrele(ndp->ni_vp);
	ndp->ni_vp = NULL;
	}
	if (unlock_vp)
	VOP_UNLOCK(ndp->ni_vp);
	if (!(flags & NDF_NO_DVP_RELE) &&
	(ndp->ni_cnd.cn_flags & (LOCKPARENT\|WANTPARENT))) {
	if (unlock_dvp) {
	vput(ndp->ni_dvp);
	unlock_dvp = 0;
	} else
	vrele(ndp->ni_dvp);
	ndp->ni_dvp = NULL;
	}
	if (unlock_dvp)
	VOP_UNLOCK(ndp->ni_dvp);
	if (!(flags & NDF_NO_STARTDIR_RELE) &&
	(ndp->ni_cnd.cn_flags & SAVESTART)) {
	vrele(ndp->ni_startdir);
	ndp->ni_startdir = NULL;
	}
	}

	/*
	* Determine if there is a suitable alternate filename under the specified
	* prefix for the specified path. If the create flag is set, then the
	* alternate prefix will be used so long as the parent directory exists.
	* This is used by the various compatibility ABIs so that Linux binaries prefer
	* files under /compat/linux for example. The chosen path (whether under
	* the prefix or under /) is returned in a kernel malloc'd buffer pointed
	* to by pathbuf. The caller is responsible for free'ing the buffer from
	* the M_TEMP bucket if one is returned.
	*/
	int
	kern_alternate_path(struct thread td, const char prefix, const char *path,
	enum uio_seg pathseg, char **pathbuf, int create, int dirfd)
	{
	struct nameidata nd, ndroot;
	char ptr, buf, *cp;
	size_t len, sz;
	int error;

	buf = (char *) malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
	*pathbuf = buf;

	/* Copy the prefix into the new pathname as a starting point. */
	len = strlcpy(buf, prefix, MAXPATHLEN);
	if (len >= MAXPATHLEN) {
	*pathbuf = NULL;
	free(buf, M_TEMP);
	return (EINVAL);
	}
	sz = MAXPATHLEN - len;
	ptr = buf + len;

	/* Append the filename to the prefix. */
	if (pathseg == UIO_SYSSPACE)
	error = copystr(path, ptr, sz, &len);
	else
	error = copyinstr(path, ptr, sz, &len);

	if (error) {
	*pathbuf = NULL;
	free(buf, M_TEMP);
	return (error);
	}

	/* Only use a prefix with absolute pathnames. */
	if (*ptr != '/') {
	error = EINVAL;
	goto keeporig;
	}

	if (dirfd != AT_FDCWD) {
	/*
	* We want the original because the "prefix" is
	* included in the already opened dirfd.
	*/
	bcopy(ptr, buf, len);
	return (0);
	}

	/*
	* We know that there is a / somewhere in this pathname.
	* Search backwards for it, to find the file's parent dir
	* to see if it exists in the alternate tree. If it does,
	* and we want to create a file (cflag is set). We don't
	* need to worry about the root comparison in this case.
	*/

	if (create) {
	for (cp = &ptr[len] - 1; *cp != '/'; cp--);
	*cp = '\0';

	NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, buf, td);
	error = namei(&nd);
	*cp = '/';
	if (error != 0)
	goto keeporig;
	} else {
	NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, buf, td);

	error = namei(&nd);
	if (error != 0)
	goto keeporig;

	/*
	* We now compare the vnode of the prefix to the one
	* vnode asked. If they resolve to be the same, then we
	* ignore the match so that the real root gets used.
	* This avoids the problem of traversing "../.." to find the
	* root directory and never finding it, because "/" resolves
	* to the emulation root directory. This is expensive :-(
	*/
	NDINIT(&ndroot, LOOKUP, FOLLOW, UIO_SYSSPACE, prefix,
	td);

	/* We shouldn't ever get an error from this namei(). */
	error = namei(&ndroot);
	if (error == 0) {
	if (nd.ni_vp == ndroot.ni_vp)
	error = ENOENT;

	NDFREE(&ndroot, NDF_ONLY_PNBUF);
	vrele(ndroot.ni_vp);
	}
	}

	NDFREE(&nd, NDF_ONLY_PNBUF);
	vrele(nd.ni_vp);

	keeporig:
	/* If there was an error, use the original path name. */
	if (error)
	bcopy(ptr, buf, len);
	return (error);
	}
	diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
	index 2eec46774327..d7a69b49d7f0 100644
	--- a/sys/kern/vfs_subr.c
	+++ b/sys/kern/vfs_subr.c
	@@ -1,6924 +1,6924 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	* (c) UNIX System Laboratories, Inc.
	* All or some portions of this file are derived from material licensed
	* to the University of California by American Telephone and Telegraph
	* Co. or Unix System Laboratories, Inc. and are reproduced herein with
	* the permission of UNIX System Laboratories, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95
	*/

	/*
	* External virtual filesystem routines
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_ddb.h"
	#include "opt_watchdog.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bio.h>
	#include <sys/buf.h>
	#include <sys/capsicum.h>
	#include <sys/condvar.h>
	#include <sys/conf.h>
	#include <sys/counter.h>
	#include <sys/dirent.h>
	#include <sys/event.h>
	#include <sys/eventhandler.h>
	#include <sys/extattr.h>
	#include <sys/file.h>
	#include <sys/fcntl.h>
	#include <sys/jail.h>
	#include <sys/kdb.h>
	#include <sys/kernel.h>
	#include <sys/kthread.h>
	#include <sys/ktr.h>
	#include <sys/lockf.h>
	#include <sys/malloc.h>
	#include <sys/mount.h>
	#include <sys/namei.h>
	#include <sys/pctrie.h>
	#include <sys/priv.h>
	#include <sys/reboot.h>
	#include <sys/refcount.h>
	#include <sys/rwlock.h>
	#include <sys/sched.h>
	#include <sys/sleepqueue.h>
	#include <sys/smr.h>
	#include <sys/smp.h>
	#include <sys/stat.h>
	#include <sys/sysctl.h>
	#include <sys/syslog.h>
	#include <sys/vmmeter.h>
	#include <sys/vnode.h>
	#include <sys/watchdog.h>

	#include <machine/stdarg.h>

	#include <security/mac/mac_framework.h>

	#include <vm/vm.h>
	#include <vm/vm_object.h>
	#include <vm/vm_extern.h>
	#include <vm/pmap.h>
	#include <vm/vm_map.h>
	#include <vm/vm_page.h>
	#include <vm/vm_kern.h>
	#include <vm/uma.h>

	#ifdef DDB
	#include <ddb/ddb.h>
	#endif

	static void delmntque(struct vnode *vp);
	static int flushbuflist(struct bufv bufv, int flags, struct bufobj bo,
	int slpflag, int slptimeo);
	static void syncer_shutdown(void *arg, int howto);
	static int vtryrecycle(struct vnode *vp);
	static void v_init_counters(struct vnode *);
	static void v_incr_devcount(struct vnode *);
	static void v_decr_devcount(struct vnode *);
	static void vgonel(struct vnode *);
	static void vfs_knllock(void *arg);
	static void vfs_knlunlock(void *arg);
	static void vfs_knl_assert_locked(void *arg);
	static void vfs_knl_assert_unlocked(void *arg);
	static void destroy_vpollinfo(struct vpollinfo *vi);
	static int v_inval_buf_range_locked(struct vnode vp, struct bufobj bo,
	daddr_t startlbn, daddr_t endlbn);
	static void vnlru_recalc(void);

	/*
	* These fences are intended for cases where some synchronization is
	* needed between access of v_iflags and lockless vnode refcount (v_holdcnt
	* and v_usecount) updates. Access to v_iflags is generally synchronized
	* by the interlock, but we have some internal assertions that check vnode
	* flags without acquiring the lock. Thus, these fences are INVARIANTS-only
	* for now.
	*/
	#ifdef INVARIANTS
	#define VNODE_REFCOUNT_FENCE_ACQ() atomic_thread_fence_acq()
	#define VNODE_REFCOUNT_FENCE_REL() atomic_thread_fence_rel()
	#else
	#define VNODE_REFCOUNT_FENCE_ACQ()
	#define VNODE_REFCOUNT_FENCE_REL()
	#endif

	/*
	* Number of vnodes in existence. Increased whenever getnewvnode()
	* allocates a new vnode, decreased in vdropl() for VIRF_DOOMED vnode.
	*/
	static u_long __exclusive_cache_line numvnodes;

	SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0,
	"Number of vnodes in existence");

	static counter_u64_t vnodes_created;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created,
	"Number of vnodes created by getnewvnode");

	/*
	* Conversion tables for conversion from vnode types to inode formats
	* and back.
	*/
	enum vtype iftovt_tab[16] = {
	VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
	VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON
	};
	int vttoif_tab[10] = {
	0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
	S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT
	};

	/*
	* List of allocates vnodes in the system.
	*/
	static TAILQ_HEAD(freelst, vnode) vnode_list;
	static struct vnode *vnode_list_free_marker;
	static struct vnode *vnode_list_reclaim_marker;

	/*
	* "Free" vnode target. Free vnodes are rarely completely free, but are
	* just ones that are cheap to recycle. Usually they are for files which
	* have been stat'd but not read; these usually have inode and namecache
	* data attached to them. This target is the preferred minimum size of a
	* sub-cache consisting mostly of such files. The system balances the size
	* of this sub-cache with its complement to try to prevent either from
	* thrashing while the other is relatively inactive. The targets express
	* a preference for the best balance.
	*
	* "Above" this target there are 2 further targets (watermarks) related
	* to recyling of free vnodes. In the best-operating case, the cache is
	* exactly full, the free list has size between vlowat and vhiwat above the
	* free target, and recycling from it and normal use maintains this state.
	* Sometimes the free list is below vlowat or even empty, but this state
	* is even better for immediate use provided the cache is not full.
	* Otherwise, vnlru_proc() runs to reclaim enough vnodes (usually non-free
	* ones) to reach one of these states. The watermarks are currently hard-
	* coded as 4% and 9% of the available space higher. These and the default
	* of 25% for wantfreevnodes are too large if the memory size is large.
	* E.g., 9% of 75% of MAXVNODES is more than 566000 vnodes to reclaim
	* whenever vnlru_proc() becomes active.
	*/
	static long wantfreevnodes;
	static long __exclusive_cache_line freevnodes;
	SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD,
	&freevnodes, 0, "Number of \"free\" vnodes");
	static long freevnodes_old;

	static counter_u64_t recycles_count;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles, CTLFLAG_RD, &recycles_count,
	"Number of vnodes recycled to meet vnode cache targets");

	static counter_u64_t recycles_free_count;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, recycles_free, CTLFLAG_RD, &recycles_free_count,
	"Number of free vnodes recycled to meet vnode cache targets");

	/*
	* Various variables used for debugging the new implementation of
	* reassignbuf().
	* XXX these are probably of (very) limited utility now.
	*/
	static int reassignbufcalls;
	SYSCTL_INT(_vfs, OID_AUTO, reassignbufcalls, CTLFLAG_RW \| CTLFLAG_STATS,
	&reassignbufcalls, 0, "Number of calls to reassignbuf");

	static counter_u64_t deferred_inact;
	SYSCTL_COUNTER_U64(_vfs, OID_AUTO, deferred_inact, CTLFLAG_RD, &deferred_inact,
	"Number of times inactive processing was deferred");

	/* To keep more than one thread at a time from running vfs_getnewfsid */
	static struct mtx mntid_mtx;

	/*
	* Lock for any access to the following:
	* vnode_list
	* numvnodes
	* freevnodes
	*/
	static struct mtx __exclusive_cache_line vnode_list_mtx;

	/* Publicly exported FS */
	struct nfs_public nfs_pub;

	static uma_zone_t buf_trie_zone;
	static smr_t buf_trie_smr;

	/* Zone for allocation of new vnodes - used exclusively by getnewvnode() */
	static uma_zone_t vnode_zone;
	static uma_zone_t vnodepoll_zone;

	__read_frequently smr_t vfs_smr;

	/*
	* The workitem queue.
	*
	* It is useful to delay writes of file data and filesystem metadata
	* for tens of seconds so that quickly created and deleted files need
	* not waste disk bandwidth being created and removed. To realize this,
	* we append vnodes to a "workitem" queue. When running with a soft
	* updates implementation, most pending metadata dependencies should
	* not wait for more than a few seconds. Thus, mounted on block devices
	* are delayed only about a half the time that file data is delayed.
	* Similarly, directory updates are more critical, so are only delayed
	* about a third the time that file data is delayed. Thus, there are
	* SYNCER_MAXDELAY queues that are processed round-robin at a rate of
	* one each second (driven off the filesystem syncer process). The
	* syncer_delayno variable indicates the next queue that is to be processed.
	* Items that need to be processed soon are placed in this queue:
	*
	* syncer_workitem_pending[syncer_delayno]
	*
	* A delay of fifteen seconds is done by placing the request fifteen
	* entries later in the queue:
	*
	* syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask]
	*
	*/
	static int syncer_delayno;
	static long syncer_mask;
	LIST_HEAD(synclist, bufobj);
	static struct synclist *syncer_workitem_pending;
	/*
	* The sync_mtx protects:
	* bo->bo_synclist
	* sync_vnode_count
	* syncer_delayno
	* syncer_state
	* syncer_workitem_pending
	* syncer_worklist_len
	* rushjob
	*/
	static struct mtx sync_mtx;
	static struct cv sync_wakeup;

	#define SYNCER_MAXDELAY 32
	static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */
	static int syncdelay = 30; /* max time to delay syncing data */
	static int filedelay = 30; /* time to delay syncing files */
	SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0,
	"Time to delay syncing files (in seconds)");
	static int dirdelay = 29; /* time to delay syncing directories */
	SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0,
	"Time to delay syncing directories (in seconds)");
	static int metadelay = 28; /* time to delay syncing metadata */
	SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0,
	"Time to delay syncing metadata (in seconds)");
	static int rushjob; /* number of slots to run ASAP */
	static int stat_rush_requests; /* number of times I/O speeded up */
	SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0,
	"Number of times I/O speeded up (rush requests)");

	#define VDBATCH_SIZE 8
	struct vdbatch {
	u_int index;
	long freevnodes;
	struct mtx lock;
	struct vnode *tab[VDBATCH_SIZE];
	};
	DPCPU_DEFINE_STATIC(struct vdbatch, vd);

	static void vdbatch_dequeue(struct vnode *vp);

	/*
	* When shutting down the syncer, run it at four times normal speed.
	*/
	#define SYNCER_SHUTDOWN_SPEEDUP 4
	static int sync_vnode_count;
	static int syncer_worklist_len;
	static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY }
	syncer_state;

	/* Target for maximum number of vnodes. */
	u_long desiredvnodes;
	static u_long gapvnodes; /* gap between wanted and desired */
	static u_long vhiwat; /* enough extras after expansion */
	static u_long vlowat; /* minimal extras before expansion */
	static u_long vstir; /* nonzero to stir non-free vnodes */
	static volatile int vsmalltrigger = 8; /* pref to keep if > this many pages */

	static u_long vnlru_read_freevnodes(void);

	/*
	* Note that no attempt is made to sanitize these parameters.
	*/
	static int
	sysctl_maxvnodes(SYSCTL_HANDLER_ARGS)
	{
	u_long val;
	int error;

	val = desiredvnodes;
	error = sysctl_handle_long(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	if (val == desiredvnodes)
	return (0);
	mtx_lock(&vnode_list_mtx);
	desiredvnodes = val;
	wantfreevnodes = desiredvnodes / 4;
	vnlru_recalc();
	mtx_unlock(&vnode_list_mtx);
	/*
	* XXX There is no protection against multiple threads changing
	* desiredvnodes at the same time. Locking above only helps vnlru and
	* getnewvnode.
	*/
	vfs_hash_changesize(desiredvnodes);
	cache_changesize(desiredvnodes);
	return (0);
	}

	SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes,
	CTLTYPE_ULONG \| CTLFLAG_MPSAFE \| CTLFLAG_RW, NULL, 0, sysctl_maxvnodes,
	"LU", "Target for maximum number of vnodes");

	static int
	sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS)
	{
	u_long val;
	int error;

	val = wantfreevnodes;
	error = sysctl_handle_long(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);

	if (val == wantfreevnodes)
	return (0);
	mtx_lock(&vnode_list_mtx);
	wantfreevnodes = val;
	vnlru_recalc();
	mtx_unlock(&vnode_list_mtx);
	return (0);
	}

	SYSCTL_PROC(_vfs, OID_AUTO, wantfreevnodes,
	CTLTYPE_ULONG \| CTLFLAG_MPSAFE \| CTLFLAG_RW, NULL, 0, sysctl_wantfreevnodes,
	"LU", "Target for minimum number of \"free\" vnodes");

	SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW,
	&wantfreevnodes, 0, "Old name for vfs.wantfreevnodes (legacy)");
	static int vnlru_nowhere;
	SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
	&vnlru_nowhere, 0, "Number of times the vnlru process ran without success");

	static int
	sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS)
	{
	struct vnode *vp;
	struct nameidata nd;
	char *buf;
	unsigned long ndflags;
	int error;

	if (req->newptr == NULL)
	return (EINVAL);
	if (req->newlen >= PATH_MAX)
	return (E2BIG);

	buf = malloc(PATH_MAX, M_TEMP, M_WAITOK);
	error = SYSCTL_IN(req, buf, req->newlen);
	if (error != 0)
	goto out;

	buf[req->newlen] = '\0';

	ndflags = LOCKLEAF \| NOFOLLOW \| AUDITVNODE1 \| NOCACHE \| SAVENAME;
	NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE, buf, curthread);
	if ((error = namei(&nd)) != 0)
	goto out;
	vp = nd.ni_vp;

	if (VN_IS_DOOMED(vp)) {
	/*
	* This vnode is being recycled. Return != 0 to let the caller
	* know that the sysctl had no effect. Return EAGAIN because a
	* subsequent call will likely succeed (since namei will create
	* a new vnode if necessary)
	*/
	error = EAGAIN;
	goto putvnode;
	}

	counter_u64_add(recycles_count, 1);
	vgone(vp);
	putvnode:
	NDFREE(&nd, 0);
	out:
	free(buf, M_TEMP);
	return (error);
	}

	static int
	sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS)
	{
	struct thread *td = curthread;
	struct vnode *vp;
	struct file *fp;
	int error;
	int fd;

	if (req->newptr == NULL)
	return (EBADF);

	error = sysctl_handle_int(oidp, &fd, 0, req);
	if (error != 0)
	return (error);
	error = getvnode(curthread, fd, &cap_fcntl_rights, &fp);
	if (error != 0)
	return (error);
	vp = fp->f_vnode;

	error = vn_lock(vp, LK_EXCLUSIVE);
	if (error != 0)
	goto drop;

	counter_u64_add(recycles_count, 1);
	vgone(vp);
	VOP_UNLOCK(vp);
	drop:
	fdrop(fp, td);
	return (error);
	}

	SYSCTL_PROC(_debug, OID_AUTO, try_reclaim_vnode,
	CTLTYPE_STRING \| CTLFLAG_MPSAFE \| CTLFLAG_WR, NULL, 0,
	sysctl_try_reclaim_vnode, "A", "Try to reclaim a vnode by its pathname");
	SYSCTL_PROC(_debug, OID_AUTO, ftry_reclaim_vnode,
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_WR, NULL, 0,
	sysctl_ftry_reclaim_vnode, "I",
	"Try to reclaim a vnode by its file descriptor");

	/* Shift count for (uintptr_t)vp to initialize vp->v_hash. */
	static int vnsz2log;

	/*
	* Support for the bufobj clean & dirty pctrie.
	*/
	static void *
	buf_trie_alloc(struct pctrie *ptree)
	{
	return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT));
	}

	static void
	buf_trie_free(struct pctrie ptree, void node)
	{
	uma_zfree_smr(buf_trie_zone, node);
	}
	PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free,
	buf_trie_smr);

	/*
	* Initialize the vnode management data structures.
	*
	* Reevaluate the following cap on the number of vnodes after the physical
	* memory size exceeds 512GB. In the limit, as the physical memory size
	* grows, the ratio of the memory size in KB to vnodes approaches 64:1.
	*/
	#ifndef MAXVNODES_MAX
	#define MAXVNODES_MAX (512UL * 1024 * 1024 / 64) /* 8M */
	#endif

	static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker");

	static struct vnode *
	vn_alloc_marker(struct mount *mp)
	{
	struct vnode *vp;

	vp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK \| M_ZERO);
	vp->v_type = VMARKER;
	vp->v_mount = mp;

	return (vp);
	}

	static void
	vn_free_marker(struct vnode *vp)
	{

	MPASS(vp->v_type == VMARKER);
	free(vp, M_VNODE_MARKER);
	}

	/*
	* Initialize a vnode as it first enters the zone.
	*/
	static int
	vnode_init(void *mem, int size, int flags)
	{
	struct vnode *vp;

	vp = mem;
	bzero(vp, size);
	/*
	* Setup locks.
	*/
	vp->v_vnlock = &vp->v_lock;
	mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF);
	/*
	* By default, don't allow shared locks unless filesystems opt-in.
	*/
	lockinit(vp->v_vnlock, PVFS, "vnode", VLKTIMEOUT,
	LK_NOSHARE \| LK_IS_VNODE);
	/*
	* Initialize bufobj.
	*/
	bufobj_init(&vp->v_bufobj, vp);
	/*
	* Initialize namecache.
	*/
	LIST_INIT(&vp->v_cache_src);
	TAILQ_INIT(&vp->v_cache_dst);
	/*
	* Initialize rangelocks.
	*/
	rangelock_init(&vp->v_rl);

	vp->v_dbatchcpu = NOCPU;

	mtx_lock(&vnode_list_mtx);
	TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist);
	mtx_unlock(&vnode_list_mtx);
	return (0);
	}

	/*
	* Free a vnode when it is cleared from the zone.
	*/
	static void
	vnode_fini(void *mem, int size)
	{
	struct vnode *vp;
	struct bufobj *bo;

	vp = mem;
	vdbatch_dequeue(vp);
	mtx_lock(&vnode_list_mtx);
	TAILQ_REMOVE(&vnode_list, vp, v_vnodelist);
	mtx_unlock(&vnode_list_mtx);
	rangelock_destroy(&vp->v_rl);
	lockdestroy(vp->v_vnlock);
	mtx_destroy(&vp->v_interlock);
	bo = &vp->v_bufobj;
	rw_destroy(BO_LOCKPTR(bo));
	}

	/*
	* Provide the size of NFS nclnode and NFS fh for calculation of the
	* vnode memory consumption. The size is specified directly to
	* eliminate dependency on NFS-private header.
	*
	* Other filesystems may use bigger or smaller (like UFS and ZFS)
	* private inode data, but the NFS-based estimation is ample enough.
	* Still, we care about differences in the size between 64- and 32-bit
	* platforms.
	*
	* Namecache structure size is heuristically
	* sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1.
	*/
	#ifdef _LP64
	#define NFS_NCLNODE_SZ (528 + 64)
	#define NC_SZ 148
	#else
	#define NFS_NCLNODE_SZ (360 + 32)
	#define NC_SZ 92
	#endif

	static void
	vntblinit(void *dummy __unused)
	{
	struct vdbatch *vd;
	int cpu, physvnodes, virtvnodes;
	u_int i;

	/*
	* Desiredvnodes is a function of the physical memory size and the
	* kernel's heap size. Generally speaking, it scales with the
	* physical memory size. The ratio of desiredvnodes to the physical
	* memory size is 1:16 until desiredvnodes exceeds 98,304.
	* Thereafter, the
	* marginal ratio of desiredvnodes to the physical memory size is
	* 1:64. However, desiredvnodes is limited by the kernel's heap
	* size. The memory required by desiredvnodes vnodes and vm objects
	* must not exceed 1/10th of the kernel's heap size.
	*/
	physvnodes = maxproc + pgtok(vm_cnt.v_page_count) / 64 +
	3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64;
	virtvnodes = vm_kmem_size / (10 * (sizeof(struct vm_object) +
	sizeof(struct vnode) + NC_SZ * ncsizefactor + NFS_NCLNODE_SZ));
	desiredvnodes = min(physvnodes, virtvnodes);
	if (desiredvnodes > MAXVNODES_MAX) {
	if (bootverbose)
	printf("Reducing kern.maxvnodes %lu -> %lu\n",
	desiredvnodes, MAXVNODES_MAX);
	desiredvnodes = MAXVNODES_MAX;
	}
	wantfreevnodes = desiredvnodes / 4;
	mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF);
	TAILQ_INIT(&vnode_list);
	mtx_init(&vnode_list_mtx, "vnode_list", NULL, MTX_DEF);
	/*
	* The lock is taken to appease WITNESS.
	*/
	mtx_lock(&vnode_list_mtx);
	vnlru_recalc();
	mtx_unlock(&vnode_list_mtx);
	vnode_list_free_marker = vn_alloc_marker(NULL);
	TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist);
	vnode_list_reclaim_marker = vn_alloc_marker(NULL);
	TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist);
	vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL,
	vnode_init, vnode_fini, UMA_ALIGN_PTR, 0);
	uma_zone_set_smr(vnode_zone, vfs_smr);
	vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo),
	NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
	/*
	* Preallocate enough nodes to support one-per buf so that
	* we can not fail an insert. reassignbuf() callers can not
	* tolerate the insertion failure.
	*/
	buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(),
	NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR,
	UMA_ZONE_NOFREE \| UMA_ZONE_SMR);
	buf_trie_smr = uma_zone_get_smr(buf_trie_zone);
	uma_prealloc(buf_trie_zone, nbuf);

	vnodes_created = counter_u64_alloc(M_WAITOK);
	recycles_count = counter_u64_alloc(M_WAITOK);
	recycles_free_count = counter_u64_alloc(M_WAITOK);
	deferred_inact = counter_u64_alloc(M_WAITOK);

	/*
	* Initialize the filesystem syncer.
	*/
	syncer_workitem_pending = hashinit(syncer_maxdelay, M_VNODE,
	&syncer_mask);
	syncer_maxdelay = syncer_mask + 1;
	mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF);
	cv_init(&sync_wakeup, "syncer");
	for (i = 1; i <= sizeof(struct vnode); i <<= 1)
	vnsz2log++;
	vnsz2log--;

	CPU_FOREACH(cpu) {
	vd = DPCPU_ID_PTR((cpu), vd);
	bzero(vd, sizeof(*vd));
	mtx_init(&vd->lock, "vdbatch", NULL, MTX_DEF);
	}
	}
	SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);

	/*
	* Mark a mount point as busy. Used to synchronize access and to delay
	* unmounting. Eventually, mountlist_mtx is not released on failure.
	*
	* vfs_busy() is a custom lock, it can block the caller.
	* vfs_busy() only sleeps if the unmount is active on the mount point.
	* For a mountpoint mp, vfs_busy-enforced lock is before lock of any
	* vnode belonging to mp.
	*
	* Lookup uses vfs_busy() to traverse mount points.
	* root fs var fs
	* / vnode lock A / vnode lock (/var) D
	* /var vnode lock B /log vnode lock(/var/log) E
	* vfs_busy lock C vfs_busy lock F
	*
	* Within each file system, the lock order is C->A->B and F->D->E.
	*
	* When traversing across mounts, the system follows that lock order:
	*
	* C->A->B
	* \|
	* +->F->D->E
	*
	* The lookup() process for namei("/var") illustrates the process:
	* VOP_LOOKUP() obtains B while A is held
	* vfs_busy() obtains a shared lock on F while A and B are held
	* vput() releases lock on B
	* vput() releases lock on A
	* VFS_ROOT() obtains lock on D while shared lock on F is held
	* vfs_unbusy() releases shared lock on F
	* vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A.
	* Attempt to lock A (instead of vp_crossmp) while D is held would
	* violate the global order, causing deadlocks.
	*
	* dounmount() locks B while F is drained.
	*/
	int
	vfs_busy(struct mount *mp, int flags)
	{

	MPASS((flags & ~MBF_MASK) == 0);
	CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags);

	if (vfs_op_thread_enter(mp)) {
	MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
	MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0);
	MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0);
	vfs_mp_count_add_pcpu(mp, ref, 1);
	vfs_mp_count_add_pcpu(mp, lockref, 1);
	vfs_op_thread_exit(mp);
	if (flags & MBF_MNTLSTLOCK)
	mtx_unlock(&mountlist_mtx);
	return (0);
	}

	MNT_ILOCK(mp);
	vfs_assert_mount_counters(mp);
	MNT_REF(mp);
	/*
	* If mount point is currently being unmounted, sleep until the
	* mount point fate is decided. If thread doing the unmounting fails,
	* it will clear MNTK_UNMOUNT flag before waking us up, indicating
	* that this mount point has survived the unmount attempt and vfs_busy
	* should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE
	* flag in addition to MNTK_UNMOUNT, indicating that mount point is
	* about to be really destroyed. vfs_busy needs to release its
	* reference on the mount point in this case and return with ENOENT,
	* telling the caller that mount mount it tried to busy is no longer
	* valid.
	*/
	while (mp->mnt_kern_flag & MNTK_UNMOUNT) {
	if (flags & MBF_NOWAIT \|\| mp->mnt_kern_flag & MNTK_REFEXPIRE) {
	MNT_REL(mp);
	MNT_IUNLOCK(mp);
	CTR1(KTR_VFS, "%s: failed busying before sleeping",
	__func__);
	return (ENOENT);
	}
	if (flags & MBF_MNTLSTLOCK)
	mtx_unlock(&mountlist_mtx);
	mp->mnt_kern_flag \|= MNTK_MWAIT;
	msleep(mp, MNT_MTX(mp), PVFS \| PDROP, "vfs_busy", 0);
	if (flags & MBF_MNTLSTLOCK)
	mtx_lock(&mountlist_mtx);
	MNT_ILOCK(mp);
	}
	if (flags & MBF_MNTLSTLOCK)
	mtx_unlock(&mountlist_mtx);
	mp->mnt_lockref++;
	MNT_IUNLOCK(mp);
	return (0);
	}

	/*
	* Free a busy filesystem.
	*/
	void
	vfs_unbusy(struct mount *mp)
	{
	int c;

	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);

	if (vfs_op_thread_enter(mp)) {
	MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
	vfs_mp_count_sub_pcpu(mp, lockref, 1);
	vfs_mp_count_sub_pcpu(mp, ref, 1);
	vfs_op_thread_exit(mp);
	return;
	}

	MNT_ILOCK(mp);
	vfs_assert_mount_counters(mp);
	MNT_REL(mp);
	c = --mp->mnt_lockref;
	if (mp->mnt_vfs_ops == 0) {
	MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
	MNT_IUNLOCK(mp);
	return;
	}
	if (c < 0)
	vfs_dump_mount_counters(mp);
	if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) {
	MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT);
	CTR1(KTR_VFS, "%s: waking up waiters", __func__);
	mp->mnt_kern_flag &= ~MNTK_DRAINING;
	wakeup(&mp->mnt_lockref);
	}
	MNT_IUNLOCK(mp);
	}

	/*
	* Lookup a mount point by filesystem identifier.
	*/
	struct mount *
	vfs_getvfs(fsid_t *fsid)
	{
	struct mount *mp;

	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
	mtx_lock(&mountlist_mtx);
	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
	if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) {
	vfs_ref(mp);
	mtx_unlock(&mountlist_mtx);
	return (mp);
	}
	}
	mtx_unlock(&mountlist_mtx);
	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
	return ((struct mount *) 0);
	}

	/*
	* Lookup a mount point by filesystem identifier, busying it before
	* returning.
	*
	* To avoid congestion on mountlist_mtx, implement simple direct-mapped
	* cache for popular filesystem identifiers. The cache is lockess, using
	* the fact that struct mount's are never freed. In worst case we may
	* get pointer to unmounted or even different filesystem, so we have to
	* check what we got, and go slow way if so.
	*/
	struct mount *
	vfs_busyfs(fsid_t *fsid)
	{
	#define FSID_CACHE_SIZE 256
	typedef struct mount * volatile vmp_t;
	static vmp_t cache[FSID_CACHE_SIZE];
	struct mount *mp;
	int error;
	uint32_t hash;

	CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
	hash = fsid->val[0] ^ fsid->val[1];
	hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1);
	mp = cache[hash];
	if (mp == NULL \|\| fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0)
	goto slow;
	if (vfs_busy(mp, 0) != 0) {
	cache[hash] = NULL;
	goto slow;
	}
	if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0)
	return (mp);
	else
	vfs_unbusy(mp);

	slow:
	mtx_lock(&mountlist_mtx);
	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
	if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) {
	error = vfs_busy(mp, MBF_MNTLSTLOCK);
	if (error) {
	cache[hash] = NULL;
	mtx_unlock(&mountlist_mtx);
	return (NULL);
	}
	cache[hash] = mp;
	return (mp);
	}
	}
	CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
	mtx_unlock(&mountlist_mtx);
	return ((struct mount *) 0);
	}

	/*
	* Check if a user can access privileged mount options.
	*/
	int
	vfs_suser(struct mount mp, struct thread td)
	{
	int error;

	if (jailed(td->td_ucred)) {
	/*
	* If the jail of the calling thread lacks permission for
	* this type of file system, deny immediately.
	*/
	if (!prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag))
	return (EPERM);

	/*
	* If the file system was mounted outside the jail of the
	* calling thread, deny immediately.
	*/
	if (prison_check(td->td_ucred, mp->mnt_cred) != 0)
	return (EPERM);
	}

	/*
	* If file system supports delegated administration, we don't check
	* for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified
	* by the file system itself.
	* If this is not the user that did original mount, we check for
	* the PRIV_VFS_MOUNT_OWNER privilege.
	*/
	if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) &&
	mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) {
	if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0)
	return (error);
	}
	return (0);
	}

	/*
	* Get a new unique fsid. Try to make its val[0] unique, since this value
	* will be used to create fake device numbers for stat(). Also try (but
	* not so hard) make its val[0] unique mod 2^16, since some emulators only
	* support 16-bit device numbers. We end up with unique val[0]'s for the
	* first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls.
	*
	* Keep in mind that several mounts may be running in parallel. Starting
	* the search one past where the previous search terminated is both a
	* micro-optimization and a defense against returning the same fsid to
	* different mounts.
	*/
	void
	vfs_getnewfsid(struct mount *mp)
	{
	static uint16_t mntid_base;
	struct mount *nmp;
	fsid_t tfsid;
	int mtype;

	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
	mtx_lock(&mntid_mtx);
	mtype = mp->mnt_vfc->vfc_typenum;
	tfsid.val[1] = mtype;
	mtype = (mtype & 0xFF) << 24;
	for (;;) {
	tfsid.val[0] = makedev(255,
	mtype \| ((mntid_base & 0xFF00) << 8) \| (mntid_base & 0xFF));
	mntid_base++;
	if ((nmp = vfs_getvfs(&tfsid)) == NULL)
	break;
	vfs_rel(nmp);
	}
	mp->mnt_stat.f_fsid.val[0] = tfsid.val[0];
	mp->mnt_stat.f_fsid.val[1] = tfsid.val[1];
	mtx_unlock(&mntid_mtx);
	}

	/*
	* Knob to control the precision of file timestamps:
	*
	* 0 = seconds only; nanoseconds zeroed.
	* 1 = seconds and nanoseconds, accurate within 1/HZ.
	* 2 = seconds and nanoseconds, truncated to microseconds.
	* >=3 = seconds and nanoseconds, maximum precision.
	*/
	enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC };

	static int timestamp_precision = TSP_USEC;
	SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW,
	&timestamp_precision, 0, "File timestamp precision (0: seconds, "
	"1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, "
	"3+: sec + ns (max. precision))");

	/*
	* Get a current timestamp.
	*/
	void
	vfs_timestamp(struct timespec *tsp)
	{
	struct timeval tv;

	switch (timestamp_precision) {
	case TSP_SEC:
	tsp->tv_sec = time_second;
	tsp->tv_nsec = 0;
	break;
	case TSP_HZ:
	getnanotime(tsp);
	break;
	case TSP_USEC:
	microtime(&tv);
	TIMEVAL_TO_TIMESPEC(&tv, tsp);
	break;
	case TSP_NSEC:
	default:
	nanotime(tsp);
	break;
	}
	}

	/*
	* Set vnode attributes to VNOVAL
	*/
	void
	vattr_null(struct vattr *vap)
	{

	vap->va_type = VNON;
	vap->va_size = VNOVAL;
	vap->va_bytes = VNOVAL;
	vap->va_mode = VNOVAL;
	vap->va_nlink = VNOVAL;
	vap->va_uid = VNOVAL;
	vap->va_gid = VNOVAL;
	vap->va_fsid = VNOVAL;
	vap->va_fileid = VNOVAL;
	vap->va_blocksize = VNOVAL;
	vap->va_rdev = VNOVAL;
	vap->va_atime.tv_sec = VNOVAL;
	vap->va_atime.tv_nsec = VNOVAL;
	vap->va_mtime.tv_sec = VNOVAL;
	vap->va_mtime.tv_nsec = VNOVAL;
	vap->va_ctime.tv_sec = VNOVAL;
	vap->va_ctime.tv_nsec = VNOVAL;
	vap->va_birthtime.tv_sec = VNOVAL;
	vap->va_birthtime.tv_nsec = VNOVAL;
	vap->va_flags = VNOVAL;
	vap->va_gen = VNOVAL;
	vap->va_vaflags = 0;
	}

	/*
	* Try to reduce the total number of vnodes.
	*
	* This routine (and its user) are buggy in at least the following ways:
	* - all parameters were picked years ago when RAM sizes were significantly
	* smaller
	* - it can pick vnodes based on pages used by the vm object, but filesystems
	* like ZFS don't use it making the pick broken
	* - since ZFS has its own aging policy it gets partially combated by this one
	* - a dedicated method should be provided for filesystems to let them decide
	* whether the vnode should be recycled
	*
	* This routine is called when we have too many vnodes. It attempts
	* to free <count> vnodes and will potentially free vnodes that still
	* have VM backing store (VM backing store is typically the cause
	* of a vnode blowout so we want to do this). Therefore, this operation
	* is not considered cheap.
	*
	* A number of conditions may prevent a vnode from being reclaimed.
	* the buffer cache may have references on the vnode, a directory
	* vnode may still have references due to the namei cache representing
	* underlying files, or the vnode may be in active use. It is not
	* desirable to reuse such vnodes. These conditions may cause the
	* number of vnodes to reach some minimum value regardless of what
	* you set kern.maxvnodes to. Do not set kern.maxvnodes too low.
	*
	* @param reclaim_nc_src Only reclaim directories with outgoing namecache
	* entries if this argument is strue
	* @param trigger Only reclaim vnodes with fewer than this many resident
	* pages.
	* @param target How many vnodes to reclaim.
	* @return The number of vnodes that were reclaimed.
	*/
	static int
	vlrureclaim(bool reclaim_nc_src, int trigger, u_long target)
	{
	struct vnode vp, mvp;
	struct mount *mp;
	struct vm_object *object;
	u_long done;
	bool retried;

	mtx_assert(&vnode_list_mtx, MA_OWNED);

	retried = false;
	done = 0;

	mvp = vnode_list_reclaim_marker;
	restart:
	vp = mvp;
	while (done < target) {
	vp = TAILQ_NEXT(vp, v_vnodelist);
	if (__predict_false(vp == NULL))
	break;

	if (__predict_false(vp->v_type == VMARKER))
	continue;

	/*
	* If it's been deconstructed already, it's still
	* referenced, or it exceeds the trigger, skip it.
	* Also skip free vnodes. We are trying to make space
	* to expand the free list, not reduce it.
	*/
	if (vp->v_usecount > 0 \|\| vp->v_holdcnt == 0 \|\|
	(!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)))
	goto next_iter;

	if (vp->v_type == VBAD \|\| vp->v_type == VNON)
	goto next_iter;

	if (!VI_TRYLOCK(vp))
	goto next_iter;

	if (vp->v_usecount > 0 \|\| vp->v_holdcnt == 0 \|\|
	(!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) \|\|
	VN_IS_DOOMED(vp) \|\| vp->v_type == VNON) {
	VI_UNLOCK(vp);
	goto next_iter;
	}

	object = atomic_load_ptr(&vp->v_object);
	if (object == NULL \|\| object->resident_page_count > trigger) {
	VI_UNLOCK(vp);
	goto next_iter;
	}

	vholdl(vp);
	VI_UNLOCK(vp);
	TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
	TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist);
	mtx_unlock(&vnode_list_mtx);

	if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
	vdrop(vp);
	goto next_iter_unlocked;
	}
	if (VOP_LOCK(vp, LK_EXCLUSIVE\|LK_NOWAIT) != 0) {
	vdrop(vp);
	vn_finished_write(mp);
	goto next_iter_unlocked;
	}

	VI_LOCK(vp);
	if (vp->v_usecount > 0 \|\|
	(!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) \|\|
	(vp->v_object != NULL &&
	vp->v_object->resident_page_count > trigger)) {
	VOP_UNLOCK(vp);
	vdropl(vp);
	vn_finished_write(mp);
	goto next_iter_unlocked;
	}
	counter_u64_add(recycles_count, 1);
	vgonel(vp);
	VOP_UNLOCK(vp);
	vdropl(vp);
	vn_finished_write(mp);
	done++;
	next_iter_unlocked:
	if (should_yield())
	kern_yield(PRI_USER);
	mtx_lock(&vnode_list_mtx);
	goto restart;
	next_iter:
	MPASS(vp->v_type != VMARKER);
	if (!should_yield())
	continue;
	TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
	TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist);
	mtx_unlock(&vnode_list_mtx);
	kern_yield(PRI_USER);
	mtx_lock(&vnode_list_mtx);
	goto restart;
	}
	if (done == 0 && !retried) {
	TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
	TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist);
	retried = true;
	goto restart;
	}
	return (done);
	}

	static int max_vnlru_free = 10000; /* limit on vnode free requests per call */
	SYSCTL_INT(_debug, OID_AUTO, max_vnlru_free, CTLFLAG_RW, &max_vnlru_free,
	0,
	"limit on vnode free requests per call to the vnlru_free routine");

	/*
	* Attempt to reduce the free list by the requested amount.
	*/
	static int
	vnlru_free_locked(int count, struct vfsops *mnt_op)
	{
	struct vnode vp, mvp;
	struct mount *mp;
	int ocount;

	mtx_assert(&vnode_list_mtx, MA_OWNED);
	if (count > max_vnlru_free)
	count = max_vnlru_free;
	ocount = count;
	mvp = vnode_list_free_marker;
	restart:
	vp = mvp;
	while (count > 0) {
	vp = TAILQ_NEXT(vp, v_vnodelist);
	if (__predict_false(vp == NULL)) {
	TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
	TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist);
	break;
	}
	if (__predict_false(vp->v_type == VMARKER))
	continue;

	/*
	* Don't recycle if our vnode is from different type
	* of mount point. Note that mp is type-safe, the
	* check does not reach unmapped address even if
	* vnode is reclaimed.
	* Don't recycle if we can't get the interlock without
	* blocking.
	*/
	if (vp->v_holdcnt > 0 \|\| (mnt_op != NULL && (mp = vp->v_mount) != NULL &&
	mp->mnt_op != mnt_op) \|\| !VI_TRYLOCK(vp)) {
	continue;
	}
	TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
	TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist);
	if (__predict_false(vp->v_type == VBAD \|\| vp->v_type == VNON)) {
	VI_UNLOCK(vp);
	continue;
	}
	vholdl(vp);
	count--;
	mtx_unlock(&vnode_list_mtx);
	VI_UNLOCK(vp);
	vtryrecycle(vp);
	vdrop(vp);
	mtx_lock(&vnode_list_mtx);
	goto restart;
	}
	return (ocount - count);
	}

	void
	vnlru_free(int count, struct vfsops *mnt_op)
	{

	mtx_lock(&vnode_list_mtx);
	vnlru_free_locked(count, mnt_op);
	mtx_unlock(&vnode_list_mtx);
	}

	static void
	vnlru_recalc(void)
	{

	mtx_assert(&vnode_list_mtx, MA_OWNED);
	gapvnodes = imax(desiredvnodes - wantfreevnodes, 100);
	vhiwat = gapvnodes / 11; /* 9% -- just under the 10% in vlrureclaim() */
	vlowat = vhiwat / 2;
	}

	/*
	* Attempt to recycle vnodes in a context that is always safe to block.
	* Calling vlrurecycle() from the bowels of filesystem code has some
	* interesting deadlock problems.
	*/
	static struct proc *vnlruproc;
	static int vnlruproc_sig;

	/*
	* The main freevnodes counter is only updated when threads requeue their vnode
	* batches. CPUs are conditionally walked to compute a more accurate total.
	*
	* Limit how much of a slop are we willing to tolerate. Note: the actual value
	* at any given moment can still exceed slop, but it should not be by significant
	* margin in practice.
	*/
	#define VNLRU_FREEVNODES_SLOP 128

	static u_long
	vnlru_read_freevnodes(void)
	{
	struct vdbatch *vd;
	long slop;
	int cpu;

	mtx_assert(&vnode_list_mtx, MA_OWNED);
	if (freevnodes > freevnodes_old)
	slop = freevnodes - freevnodes_old;
	else
	slop = freevnodes_old - freevnodes;
	if (slop < VNLRU_FREEVNODES_SLOP)
	return (freevnodes >= 0 ? freevnodes : 0);
	freevnodes_old = freevnodes;
	CPU_FOREACH(cpu) {
	vd = DPCPU_ID_PTR((cpu), vd);
	freevnodes_old += vd->freevnodes;
	}
	return (freevnodes_old >= 0 ? freevnodes_old : 0);
	}

	static bool
	vnlru_under(u_long rnumvnodes, u_long limit)
	{
	u_long rfreevnodes, space;

	if (__predict_false(rnumvnodes > desiredvnodes))
	return (true);

	space = desiredvnodes - rnumvnodes;
	if (space < limit) {
	rfreevnodes = vnlru_read_freevnodes();
	if (rfreevnodes > wantfreevnodes)
	space += rfreevnodes - wantfreevnodes;
	}
	return (space < limit);
	}

	static bool
	vnlru_under_unlocked(u_long rnumvnodes, u_long limit)
	{
	long rfreevnodes, space;

	if (__predict_false(rnumvnodes > desiredvnodes))
	return (true);

	space = desiredvnodes - rnumvnodes;
	if (space < limit) {
	rfreevnodes = atomic_load_long(&freevnodes);
	if (rfreevnodes > wantfreevnodes)
	space += rfreevnodes - wantfreevnodes;
	}
	return (space < limit);
	}

	static void
	vnlru_kick(void)
	{

	mtx_assert(&vnode_list_mtx, MA_OWNED);
	if (vnlruproc_sig == 0) {
	vnlruproc_sig = 1;
	wakeup(vnlruproc);
	}
	}

	static void
	vnlru_proc(void)
	{
	u_long rnumvnodes, rfreevnodes, target;
	unsigned long onumvnodes;
	int done, force, trigger, usevnodes;
	bool reclaim_nc_src, want_reread;

	EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, vnlruproc,
	SHUTDOWN_PRI_FIRST);

	force = 0;
	want_reread = false;
	for (;;) {
	kproc_suspend_check(vnlruproc);
	mtx_lock(&vnode_list_mtx);
	rnumvnodes = atomic_load_long(&numvnodes);

	if (want_reread) {
	force = vnlru_under(numvnodes, vhiwat) ? 1 : 0;
	want_reread = false;
	}

	/*
	* If numvnodes is too large (due to desiredvnodes being
	* adjusted using its sysctl, or emergency growth), first
	* try to reduce it by discarding from the free list.
	*/
	if (rnumvnodes > desiredvnodes) {
	vnlru_free_locked(rnumvnodes - desiredvnodes, NULL);
	rnumvnodes = atomic_load_long(&numvnodes);
	}
	/*
	* Sleep if the vnode cache is in a good state. This is
	* when it is not over-full and has space for about a 4%
	* or 9% expansion (by growing its size or inexcessively
	* reducing its free list). Otherwise, try to reclaim
	* space for a 10% expansion.
	*/
	if (vstir && force == 0) {
	force = 1;
	vstir = 0;
	}
	if (force == 0 && !vnlru_under(rnumvnodes, vlowat)) {
	vnlruproc_sig = 0;
	wakeup(&vnlruproc_sig);
	msleep(vnlruproc, &vnode_list_mtx,
	PVFS\|PDROP, "vlruwt", hz);
	continue;
	}
	rfreevnodes = vnlru_read_freevnodes();

	onumvnodes = rnumvnodes;
	/*
	* Calculate parameters for recycling. These are the same
	* throughout the loop to give some semblance of fairness.
	* The trigger point is to avoid recycling vnodes with lots
	* of resident pages. We aren't trying to free memory; we
	* are trying to recycle or at least free vnodes.
	*/
	if (rnumvnodes <= desiredvnodes)
	usevnodes = rnumvnodes - rfreevnodes;
	else
	usevnodes = rnumvnodes;
	if (usevnodes <= 0)
	usevnodes = 1;
	/*
	* The trigger value is is chosen to give a conservatively
	* large value to ensure that it alone doesn't prevent
	* making progress. The value can easily be so large that
	* it is effectively infinite in some congested and
	* misconfigured cases, and this is necessary. Normally
	* it is about 8 to 100 (pages), which is quite large.
	*/
	trigger = vm_cnt.v_page_count * 2 / usevnodes;
	if (force < 2)
	trigger = vsmalltrigger;
	reclaim_nc_src = force >= 3;
	target = rnumvnodes * (int64_t)gapvnodes / imax(desiredvnodes, 1);
	target = target / 10 + 1;
	done = vlrureclaim(reclaim_nc_src, trigger, target);
	mtx_unlock(&vnode_list_mtx);
	if (onumvnodes > desiredvnodes && numvnodes <= desiredvnodes)
	uma_reclaim(UMA_RECLAIM_DRAIN);
	if (done == 0) {
	if (force == 0 \|\| force == 1) {
	force = 2;
	continue;
	}
	if (force == 2) {
	force = 3;
	continue;
	}
	want_reread = true;
	force = 0;
	vnlru_nowhere++;
	tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3);
	} else {
	want_reread = true;
	kern_yield(PRI_USER);
	}
	}
	}

	static struct kproc_desc vnlru_kp = {
	"vnlru",
	vnlru_proc,
	&vnlruproc
	};
	SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start,
	&vnlru_kp);

	/*
	* Routines having to do with the management of the vnode table.
	*/

	/*
	* Try to recycle a freed vnode. We abort if anyone picks up a reference
	* before we actually vgone(). This function must be called with the vnode
	* held to prevent the vnode from being returned to the free list midway
	* through vgone().
	*/
	static int
	vtryrecycle(struct vnode *vp)
	{
	struct mount *vnmp;

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	VNASSERT(vp->v_holdcnt, vp,
	("vtryrecycle: Recycling vp %p without a reference.", vp));
	/*
	* This vnode may found and locked via some other list, if so we
	* can't recycle it yet.
	*/
	if (VOP_LOCK(vp, LK_EXCLUSIVE \| LK_NOWAIT) != 0) {
	CTR2(KTR_VFS,
	"%s: impossible to recycle, vp %p lock is already held",
	__func__, vp);
	return (EWOULDBLOCK);
	}
	/*
	* Don't recycle if its filesystem is being suspended.
	*/
	if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) {
	VOP_UNLOCK(vp);
	CTR2(KTR_VFS,
	"%s: impossible to recycle, cannot start the write for %p",
	__func__, vp);
	return (EBUSY);
	}
	/*
	* If we got this far, we need to acquire the interlock and see if
	* anyone picked up this vnode from another list. If not, we will
	* mark it with DOOMED via vgonel() so that anyone who does find it
	* will skip over it.
	*/
	VI_LOCK(vp);
	if (vp->v_usecount) {
	VOP_UNLOCK(vp);
	VI_UNLOCK(vp);
	vn_finished_write(vnmp);
	CTR2(KTR_VFS,
	"%s: impossible to recycle, %p is already referenced",
	__func__, vp);
	return (EBUSY);
	}
	if (!VN_IS_DOOMED(vp)) {
	counter_u64_add(recycles_free_count, 1);
	vgonel(vp);
	}
	VOP_UNLOCK(vp);
	VI_UNLOCK(vp);
	vn_finished_write(vnmp);
	return (0);
	}

	/*
	* Allocate a new vnode.
	*
	* The operation never returns an error. Returning an error was disabled
	* in r145385 (dated 2005) with the following comment:
	*
	* XXX Not all VFS_VGET/ffs_vget callers check returns.
	*
	* Given the age of this commit (almost 15 years at the time of writing this
	* comment) restoring the ability to fail requires a significant audit of
	* all codepaths.
	*
	* The routine can try to free a vnode or stall for up to 1 second waiting for
	* vnlru to clear things up, but ultimately always performs a M_WAITOK allocation.
	*/
	static u_long vn_alloc_cyclecount;

	static struct vnode * __noinline
	vn_alloc_hard(struct mount *mp)
	{
	u_long rnumvnodes, rfreevnodes;

	mtx_lock(&vnode_list_mtx);
	rnumvnodes = atomic_load_long(&numvnodes);
	if (rnumvnodes + 1 < desiredvnodes) {
	vn_alloc_cyclecount = 0;
	goto alloc;
	}
	rfreevnodes = vnlru_read_freevnodes();
	if (vn_alloc_cyclecount++ >= rfreevnodes) {
	vn_alloc_cyclecount = 0;
	vstir = 1;
	}
	/*
	* Grow the vnode cache if it will not be above its target max
	* after growing. Otherwise, if the free list is nonempty, try
	* to reclaim 1 item from it before growing the cache (possibly
	* above its target max if the reclamation failed or is delayed).
	* Otherwise, wait for some space. In all cases, schedule
	* vnlru_proc() if we are getting short of space. The watermarks
	* should be chosen so that we never wait or even reclaim from
	* the free list to below its target minimum.
	*/
	if (vnlru_free_locked(1, NULL) > 0)
	goto alloc;
	if (mp == NULL \|\| (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) {
	/*
	* Wait for space for a new vnode.
	*/
	vnlru_kick();
	msleep(&vnlruproc_sig, &vnode_list_mtx, PVFS, "vlruwk", hz);
	if (atomic_load_long(&numvnodes) + 1 > desiredvnodes &&
	vnlru_read_freevnodes() > 1)
	vnlru_free_locked(1, NULL);
	}
	alloc:
	rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1;
	if (vnlru_under(rnumvnodes, vlowat))
	vnlru_kick();
	mtx_unlock(&vnode_list_mtx);
	return (uma_zalloc_smr(vnode_zone, M_WAITOK));
	}

	static struct vnode *
	vn_alloc(struct mount *mp)
	{
	u_long rnumvnodes;

	if (__predict_false(vn_alloc_cyclecount != 0))
	return (vn_alloc_hard(mp));
	rnumvnodes = atomic_fetchadd_long(&numvnodes, 1) + 1;
	if (__predict_false(vnlru_under_unlocked(rnumvnodes, vlowat))) {
	atomic_subtract_long(&numvnodes, 1);
	return (vn_alloc_hard(mp));
	}

	return (uma_zalloc_smr(vnode_zone, M_WAITOK));
	}

	static void
	vn_free(struct vnode *vp)
	{

	atomic_subtract_long(&numvnodes, 1);
	uma_zfree_smr(vnode_zone, vp);
	}

	/*
	* Return the next vnode from the free list.
	*/
	int
	getnewvnode(const char tag, struct mount mp, struct vop_vector *vops,
	struct vnode **vpp)
	{
	struct vnode *vp;
	struct thread *td;
	struct lock_object *lo;

	CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag);

	KASSERT(vops->registered,
	("%s: not registered vector op %p\n", __func__, vops));

	td = curthread;
	if (td->td_vp_reserved != NULL) {
	vp = td->td_vp_reserved;
	td->td_vp_reserved = NULL;
	} else {
	vp = vn_alloc(mp);
	}
	counter_u64_add(vnodes_created, 1);
	/*
	* Locks are given the generic name "vnode" when created.
	* Follow the historic practice of using the filesystem
	* name when they allocated, e.g., "zfs", "ufs", "nfs, etc.
	*
	* Locks live in a witness group keyed on their name. Thus,
	* when a lock is renamed, it must also move from the witness
	* group of its old name to the witness group of its new name.
	*
	* The change only needs to be made when the vnode moves
	* from one filesystem type to another. We ensure that each
	* filesystem use a single static name pointer for its tag so
	* that we can compare pointers rather than doing a strcmp().
	*/
	lo = &vp->v_vnlock->lock_object;
	#ifdef WITNESS
	if (lo->lo_name != tag) {
	#endif
	lo->lo_name = tag;
	#ifdef WITNESS
	WITNESS_DESTROY(lo);
	WITNESS_INIT(lo, tag);
	}
	#endif
	/*
	* By default, don't allow shared locks unless filesystems opt-in.
	*/
	vp->v_vnlock->lock_object.lo_flags \|= LK_NOSHARE;
	/*
	* Finalize various vnode identity bits.
	*/
	KASSERT(vp->v_object == NULL, ("stale v_object %p", vp));
	KASSERT(vp->v_lockf == NULL, ("stale v_lockf %p", vp));
	KASSERT(vp->v_pollinfo == NULL, ("stale v_pollinfo %p", vp));
	vp->v_type = VNON;
	vp->v_op = vops;
	v_init_counters(vp);
	vp->v_bufobj.bo_ops = &buf_ops_bio;
	#ifdef DIAGNOSTIC
	if (mp == NULL && vops != &dead_vnodeops)
	printf("NULL mp in getnewvnode(9), tag %s\n", tag);
	#endif
	#ifdef MAC
	mac_vnode_init(vp);
	if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
	mac_vnode_associate_singlelabel(mp, vp);
	#endif
	if (mp != NULL) {
	vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize;
	if ((mp->mnt_kern_flag & MNTK_NOKNOTE) != 0)
	vp->v_vflag \|= VV_NOKNOTE;
	}

	/*
	* For the filesystems which do not use vfs_hash_insert(),
	* still initialize v_hash to have vfs_hash_index() useful.
	* E.g., nullfs uses vfs_hash_index() on the lower vnode for
	* its own hashing.
	*/
	vp->v_hash = (uintptr_t)vp >> vnsz2log;

	*vpp = vp;
	return (0);
	}

	void
	getnewvnode_reserve(void)
	{
	struct thread *td;

	td = curthread;
	MPASS(td->td_vp_reserved == NULL);
	td->td_vp_reserved = vn_alloc(NULL);
	}

	void
	getnewvnode_drop_reserve(void)
	{
	struct thread *td;

	td = curthread;
	if (td->td_vp_reserved != NULL) {
	vn_free(td->td_vp_reserved);
	td->td_vp_reserved = NULL;
	}
	}

	static void
	freevnode(struct vnode *vp)
	{
	struct bufobj *bo;

	/*
	* The vnode has been marked for destruction, so free it.
	*
	* The vnode will be returned to the zone where it will
	* normally remain until it is needed for another vnode. We
	* need to cleanup (or verify that the cleanup has already
	* been done) any residual data left from its current use
	* so as not to contaminate the freshly allocated vnode.
	*/
	CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp);
	/*
	* Paired with vgone.
	*/
	vn_seqc_write_end_locked(vp);
	VNPASS(vp->v_seqc_users == 0, vp);

	bo = &vp->v_bufobj;
	VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't"));
	VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp);
	VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count"));
	VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count"));
	VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's"));
	VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0"));
	VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp,
	("clean blk trie not empty"));
	VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0"));
	VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp,
	("dirty blk trie not empty"));
	VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst"));
	VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src"));
	VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for .."));
	VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp,
	("Dangling rangelock waiters"));
	VI_UNLOCK(vp);
	#ifdef MAC
	mac_vnode_destroy(vp);
	#endif
	if (vp->v_pollinfo != NULL) {
	destroy_vpollinfo(vp->v_pollinfo);
	vp->v_pollinfo = NULL;
	}
	#ifdef INVARIANTS
	/* XXX Elsewhere we detect an already freed vnode via NULL v_op. */
	vp->v_op = NULL;
	#endif
	vp->v_mountedhere = NULL;
	vp->v_unpcb = NULL;
	vp->v_rdev = NULL;
	vp->v_fifoinfo = NULL;
	vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
	vp->v_irflag = 0;
	vp->v_iflag = 0;
	vp->v_vflag = 0;
	bo->bo_flag = 0;
	vn_free(vp);
	}

	/*
	* Delete from old mount point vnode list, if on one.
	*/
	static void
	delmntque(struct vnode *vp)
	{
	struct mount *mp;

	VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp);

	mp = vp->v_mount;
	if (mp == NULL)
	return;
	MNT_ILOCK(mp);
	VI_LOCK(vp);
	vp->v_mount = NULL;
	VI_UNLOCK(vp);
	VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
	("bad mount point vnode list size"));
	TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
	mp->mnt_nvnodelistsize--;
	MNT_REL(mp);
	MNT_IUNLOCK(mp);
	}

	static void
	insmntque_stddtr(struct vnode vp, void dtr_arg)
	{

	vp->v_data = NULL;
	vp->v_op = &dead_vnodeops;
	vgone(vp);
	vput(vp);
	}

	/*
	* Insert into list of vnodes for the new mount point, if available.
	*/
	int
	insmntque1(struct vnode vp, struct mount mp,
	void (dtr)(struct vnode , void ), void dtr_arg)
	{

	KASSERT(vp->v_mount == NULL,
	("insmntque: vnode already on per mount vnode list"));
	VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)"));
	ASSERT_VOP_ELOCKED(vp, "insmntque: non-locked vp");

	/*
	* We acquire the vnode interlock early to ensure that the
	* vnode cannot be recycled by another process releasing a
	* holdcnt on it before we get it on both the vnode list
	* and the active vnode list. The mount mutex protects only
	* manipulation of the vnode list and the vnode freelist
	* mutex protects only manipulation of the active vnode list.
	* Hence the need to hold the vnode interlock throughout.
	*/
	MNT_ILOCK(mp);
	VI_LOCK(vp);
	if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 &&
	((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 \|\|
	mp->mnt_nvnodelistsize == 0)) &&
	(vp->v_vflag & VV_FORCEINSMQ) == 0) {
	VI_UNLOCK(vp);
	MNT_IUNLOCK(mp);
	if (dtr != NULL)
	dtr(vp, dtr_arg);
	return (EBUSY);
	}
	vp->v_mount = mp;
	MNT_REF(mp);
	TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
	VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
	("neg mount point vnode list size"));
	mp->mnt_nvnodelistsize++;
	VI_UNLOCK(vp);
	MNT_IUNLOCK(mp);
	return (0);
	}

	int
	insmntque(struct vnode vp, struct mount mp)
	{

	return (insmntque1(vp, mp, insmntque_stddtr, NULL));
	}

	/*
	* Flush out and invalidate all buffers associated with a bufobj
	* Called with the underlying object locked.
	*/
	int
	bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
	{
	int error;

	BO_LOCK(bo);
	if (flags & V_SAVE) {
	error = bufobj_wwait(bo, slpflag, slptimeo);
	if (error) {
	BO_UNLOCK(bo);
	return (error);
	}
	if (bo->bo_dirty.bv_cnt > 0) {
	BO_UNLOCK(bo);
	if ((error = BO_SYNC(bo, MNT_WAIT)) != 0)
	return (error);
	/*
	* XXX We could save a lock/unlock if this was only
	* enabled under INVARIANTS
	*/
	BO_LOCK(bo);
	if (bo->bo_numoutput > 0 \|\| bo->bo_dirty.bv_cnt > 0)
	panic("vinvalbuf: dirty bufs");
	}
	}
	/*
	* If you alter this loop please notice that interlock is dropped and
	* reacquired in flushbuflist. Special care is needed to ensure that
	* no race conditions occur from this.
	*/
	do {
	error = flushbuflist(&bo->bo_clean,
	flags, bo, slpflag, slptimeo);
	if (error == 0 && !(flags & V_CLEANONLY))
	error = flushbuflist(&bo->bo_dirty,
	flags, bo, slpflag, slptimeo);
	if (error != 0 && error != EAGAIN) {
	BO_UNLOCK(bo);
	return (error);
	}
	} while (error != 0);

	/*
	* Wait for I/O to complete. XXX needs cleaning up. The vnode can
	* have write I/O in-progress but if there is a VM object then the
	* VM object can also have read-I/O in-progress.
	*/
	do {
	bufobj_wwait(bo, 0, 0);
	if ((flags & V_VMIO) == 0 && bo->bo_object != NULL) {
	BO_UNLOCK(bo);
	vm_object_pip_wait_unlocked(bo->bo_object, "bovlbx");
	BO_LOCK(bo);
	}
	} while (bo->bo_numoutput > 0);
	BO_UNLOCK(bo);

	/*
	* Destroy the copy in the VM cache, too.
	*/
	if (bo->bo_object != NULL &&
	(flags & (V_ALT \| V_NORMAL \| V_CLEANONLY \| V_VMIO)) == 0) {
	VM_OBJECT_WLOCK(bo->bo_object);
	vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
	OBJPR_CLEANONLY : 0);
	VM_OBJECT_WUNLOCK(bo->bo_object);
	}

	#ifdef INVARIANTS
	BO_LOCK(bo);
	if ((flags & (V_ALT \| V_NORMAL \| V_CLEANONLY \| V_VMIO \|
	V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 \|\|
	bo->bo_clean.bv_cnt > 0))
	panic("vinvalbuf: flush failed");
	if ((flags & (V_ALT \| V_NORMAL \| V_CLEANONLY \| V_VMIO)) == 0 &&
	bo->bo_dirty.bv_cnt > 0)
	panic("vinvalbuf: flush dirty failed");
	BO_UNLOCK(bo);
	#endif
	return (0);
	}

	/*
	* Flush out and invalidate all buffers associated with a vnode.
	* Called with the underlying object locked.
	*/
	int
	vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
	{

	CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
	ASSERT_VOP_LOCKED(vp, "vinvalbuf");
	if (vp->v_object != NULL && vp->v_object->handle != vp)
	return (0);
	return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo));
	}

	/*
	* Flush out buffers on the specified list.
	*
	*/
	static int
	flushbuflist(struct bufv bufv, int flags, struct bufobj bo, int slpflag,
	int slptimeo)
	{
	struct buf bp, nbp;
	int retval, error;
	daddr_t lblkno;
	b_xflags_t xflags;

	ASSERT_BO_WLOCKED(bo);

	retval = 0;
	TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) {
	/*
	* If we are flushing both V_NORMAL and V_ALT buffers then
	* do not skip any buffers. If we are flushing only V_NORMAL
	* buffers then skip buffers marked as BX_ALTDATA. If we are
	* flushing only V_ALT buffers then skip buffers not marked
	* as BX_ALTDATA.
	*/
	if (((flags & (V_NORMAL \| V_ALT)) != (V_NORMAL \| V_ALT)) &&
	(((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) \|\|
	((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) {
	continue;
	}
	if (nbp != NULL) {
	lblkno = nbp->b_lblkno;
	xflags = nbp->b_xflags & (BX_VNDIRTY \| BX_VNCLEAN);
	}
	retval = EAGAIN;
	error = BUF_TIMELOCK(bp,
	LK_EXCLUSIVE \| LK_SLEEPFAIL \| LK_INTERLOCK, BO_LOCKPTR(bo),
	"flushbuf", slpflag, slptimeo);
	if (error) {
	BO_LOCK(bo);
	return (error != ENOLCK ? error : EAGAIN);
	}
	KASSERT(bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p",
	bp, bp->b_bufobj, bo));
	/*
	* XXX Since there are no node locks for NFS, I
	* believe there is a slight chance that a delayed
	* write will occur while sleeping just above, so
	* check for it.
	*/
	if (((bp->b_flags & (B_DELWRI \| B_INVAL)) == B_DELWRI) &&
	(flags & V_SAVE)) {
	bremfree(bp);
	bp->b_flags \|= B_ASYNC;
	bwrite(bp);
	BO_LOCK(bo);
	return (EAGAIN); /* XXX: why not loop ? */
	}
	bremfree(bp);
	bp->b_flags \|= (B_INVAL \| B_RELBUF);
	bp->b_flags &= ~B_ASYNC;
	brelse(bp);
	BO_LOCK(bo);
	if (nbp == NULL)
	break;
	nbp = gbincore(bo, lblkno);
	if (nbp == NULL \|\| (nbp->b_xflags & (BX_VNDIRTY \| BX_VNCLEAN))
	!= xflags)
	break; /* nbp invalid */
	}
	return (retval);
	}

	int
	bnoreuselist(struct bufv bufv, struct bufobj bo, daddr_t startn, daddr_t endn)
	{
	struct buf *bp;
	int error;
	daddr_t lblkno;

	ASSERT_BO_LOCKED(bo);

	for (lblkno = startn;;) {
	again:
	bp = BUF_PCTRIE_LOOKUP_GE(&bufv->bv_root, lblkno);
	if (bp == NULL \|\| bp->b_lblkno >= endn \|\|
	bp->b_lblkno < startn)
	break;
	error = BUF_TIMELOCK(bp, LK_EXCLUSIVE \| LK_SLEEPFAIL \|
	LK_INTERLOCK, BO_LOCKPTR(bo), "brlsfl", 0, 0);
	if (error != 0) {
	BO_RLOCK(bo);
	if (error == ENOLCK)
	goto again;
	return (error);
	}
	KASSERT(bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p",
	bp, bp->b_bufobj, bo));
	lblkno = bp->b_lblkno + 1;
	if ((bp->b_flags & B_MANAGED) == 0)
	bremfree(bp);
	bp->b_flags \|= B_RELBUF;
	/*
	* In the VMIO case, use the B_NOREUSE flag to hint that the
	* pages backing each buffer in the range are unlikely to be
	* reused. Dirty buffers will have the hint applied once
	* they've been written.
	*/
	if ((bp->b_flags & B_VMIO) != 0)
	bp->b_flags \|= B_NOREUSE;
	brelse(bp);
	BO_RLOCK(bo);
	}
	return (0);
	}

	/*
	* Truncate a file's buffer and pages to a specified length. This
	* is in lieu of the old vinvalbuf mechanism, which performed unneeded
	* sync activity.
	*/
	int
	vtruncbuf(struct vnode *vp, off_t length, int blksize)
	{
	struct buf bp, nbp;
	struct bufobj *bo;
	daddr_t startlbn;

	CTR4(KTR_VFS, "%s: vp %p with block %d:%ju", __func__,
	vp, blksize, (uintmax_t)length);

	/*
	* Round up to the next lbn.
	*/
	startlbn = howmany(length, blksize);

	ASSERT_VOP_LOCKED(vp, "vtruncbuf");

	bo = &vp->v_bufobj;
	restart_unlocked:
	BO_LOCK(bo);

	while (v_inval_buf_range_locked(vp, bo, startlbn, INT64_MAX) == EAGAIN)
	;

	if (length > 0) {
	restartsync:
	TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
	if (bp->b_lblkno > 0)
	continue;
	/*
	* Since we hold the vnode lock this should only
	* fail if we're racing with the buf daemon.
	*/
	if (BUF_LOCK(bp,
	LK_EXCLUSIVE \| LK_SLEEPFAIL \| LK_INTERLOCK,
	BO_LOCKPTR(bo)) == ENOLCK)
	goto restart_unlocked;

	VNASSERT((bp->b_flags & B_DELWRI), vp,
	("buf(%p) on dirty queue without DELWRI", bp));

	bremfree(bp);
	bawrite(bp);
	BO_LOCK(bo);
	goto restartsync;
	}
	}

	bufobj_wwait(bo, 0, 0);
	BO_UNLOCK(bo);
	vnode_pager_setsize(vp, length);

	return (0);
	}

	/*
	* Invalidate the cached pages of a file's buffer within the range of block
	* numbers [startlbn, endlbn).
	*/
	void
	v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn,
	int blksize)
	{
	struct bufobj *bo;
	off_t start, end;

	ASSERT_VOP_LOCKED(vp, "v_inval_buf_range");

	start = blksize * startlbn;
	end = blksize * endlbn;

	bo = &vp->v_bufobj;
	BO_LOCK(bo);
	MPASS(blksize == bo->bo_bsize);

	while (v_inval_buf_range_locked(vp, bo, startlbn, endlbn) == EAGAIN)
	;

	BO_UNLOCK(bo);
	vn_pages_remove(vp, OFF_TO_IDX(start), OFF_TO_IDX(end + PAGE_SIZE - 1));
	}

	static int
	v_inval_buf_range_locked(struct vnode vp, struct bufobj bo,
	daddr_t startlbn, daddr_t endlbn)
	{
	struct buf bp, nbp;
	bool anyfreed;

	ASSERT_VOP_LOCKED(vp, "v_inval_buf_range_locked");
	ASSERT_BO_LOCKED(bo);

	do {
	anyfreed = false;
	TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) {
	if (bp->b_lblkno < startlbn \|\| bp->b_lblkno >= endlbn)
	continue;
	if (BUF_LOCK(bp,
	LK_EXCLUSIVE \| LK_SLEEPFAIL \| LK_INTERLOCK,
	BO_LOCKPTR(bo)) == ENOLCK) {
	BO_LOCK(bo);
	return (EAGAIN);
	}

	bremfree(bp);
	bp->b_flags \|= B_INVAL \| B_RELBUF;
	bp->b_flags &= ~B_ASYNC;
	brelse(bp);
	anyfreed = true;

	BO_LOCK(bo);
	if (nbp != NULL &&
	(((nbp->b_xflags & BX_VNCLEAN) == 0) \|\|
	nbp->b_vp != vp \|\|
	(nbp->b_flags & B_DELWRI) != 0))
	return (EAGAIN);
	}

	TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
	if (bp->b_lblkno < startlbn \|\| bp->b_lblkno >= endlbn)
	continue;
	if (BUF_LOCK(bp,
	LK_EXCLUSIVE \| LK_SLEEPFAIL \| LK_INTERLOCK,
	BO_LOCKPTR(bo)) == ENOLCK) {
	BO_LOCK(bo);
	return (EAGAIN);
	}
	bremfree(bp);
	bp->b_flags \|= B_INVAL \| B_RELBUF;
	bp->b_flags &= ~B_ASYNC;
	brelse(bp);
	anyfreed = true;

	BO_LOCK(bo);
	if (nbp != NULL &&
	(((nbp->b_xflags & BX_VNDIRTY) == 0) \|\|
	(nbp->b_vp != vp) \|\|
	(nbp->b_flags & B_DELWRI) == 0))
	return (EAGAIN);
	}
	} while (anyfreed);
	return (0);
	}

	static void
	buf_vlist_remove(struct buf *bp)
	{
	struct bufv *bv;

	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
	ASSERT_BO_WLOCKED(bp->b_bufobj);
	KASSERT((bp->b_xflags & (BX_VNDIRTY\|BX_VNCLEAN)) !=
	(BX_VNDIRTY\|BX_VNCLEAN),
	("buf_vlist_remove: Buf %p is on two lists", bp));
	if (bp->b_xflags & BX_VNDIRTY)
	bv = &bp->b_bufobj->bo_dirty;
	else
	bv = &bp->b_bufobj->bo_clean;
	BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno);
	TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs);
	bv->bv_cnt--;
	bp->b_xflags &= ~(BX_VNDIRTY \| BX_VNCLEAN);
	}

	/*
	* Add the buffer to the sorted clean or dirty block list.
	*
	* NOTE: xflags is passed as a constant, optimizing this inline function!
	*/
	static void
	buf_vlist_add(struct buf bp, struct bufobj bo, b_xflags_t xflags)
	{
	struct bufv *bv;
	struct buf *n;
	int error;

	ASSERT_BO_WLOCKED(bo);
	KASSERT((bo->bo_flag & BO_NOBUFS) == 0,
	("buf_vlist_add: bo %p does not allow bufs", bo));
	KASSERT((xflags & BX_VNDIRTY) == 0 \|\| (bo->bo_flag & BO_DEAD) == 0,
	("dead bo %p", bo));
	KASSERT((bp->b_xflags & (BX_VNDIRTY\|BX_VNCLEAN)) == 0,
	("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags));
	bp->b_xflags \|= xflags;
	if (xflags & BX_VNDIRTY)
	bv = &bo->bo_dirty;
	else
	bv = &bo->bo_clean;

	/*
	* Keep the list ordered. Optimize empty list insertion. Assume
	* we tend to grow at the tail so lookup_le should usually be cheaper
	* than _ge.
	*/
	if (bv->bv_cnt == 0 \|\|
	bp->b_lblkno > TAILQ_LAST(&bv->bv_hd, buflists)->b_lblkno)
	TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs);
	else if ((n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, bp->b_lblkno)) == NULL)
	TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs);
	else
	TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs);
	error = BUF_PCTRIE_INSERT(&bv->bv_root, bp);
	if (error)
	panic("buf_vlist_add: Preallocated nodes insufficient.");
	bv->bv_cnt++;
	}

	/*
	* Look up a buffer using the buffer tries.
	*/
	struct buf *
	gbincore(struct bufobj *bo, daddr_t lblkno)
	{
	struct buf *bp;

	ASSERT_BO_LOCKED(bo);
	bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno);
	if (bp != NULL)
	return (bp);
	return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno));
	}

	/*
	* Look up a buf using the buffer tries, without the bufobj lock. This relies
	* on SMR for safe lookup, and bufs being in a no-free zone to provide type
	* stability of the result. Like other lockless lookups, the found buf may
	* already be invalid by the time this function returns.
	*/
	struct buf *
	gbincore_unlocked(struct bufobj *bo, daddr_t lblkno)
	{
	struct buf *bp;

	ASSERT_BO_UNLOCKED(bo);
	bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno);
	if (bp != NULL)
	return (bp);
	return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno));
	}

	/*
	* Associate a buffer with a vnode.
	*/
	void
	bgetvp(struct vnode vp, struct buf bp)
	{
	struct bufobj *bo;

	bo = &vp->v_bufobj;
	ASSERT_BO_WLOCKED(bo);
	VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free"));

	CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags);
	VNASSERT((bp->b_xflags & (BX_VNDIRTY\|BX_VNCLEAN)) == 0, vp,
	("bgetvp: bp already attached! %p", bp));

	vhold(vp);
	bp->b_vp = vp;
	bp->b_bufobj = bo;
	/*
	* Insert onto list for new vnode.
	*/
	buf_vlist_add(bp, bo, BX_VNCLEAN);
	}

	/*
	* Disassociate a buffer from a vnode.
	*/
	void
	brelvp(struct buf *bp)
	{
	struct bufobj *bo;
	struct vnode *vp;

	CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
	KASSERT(bp->b_vp != NULL, ("brelvp: NULL"));

	/*
	* Delete from old vnode list, if on one.
	*/
	vp = bp->b_vp; /* XXX */
	bo = bp->b_bufobj;
	BO_LOCK(bo);
	if (bp->b_xflags & (BX_VNDIRTY \| BX_VNCLEAN))
	buf_vlist_remove(bp);
	else
	panic("brelvp: Buffer %p not on queue.", bp);
	if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
	bo->bo_flag &= ~BO_ONWORKLST;
	mtx_lock(&sync_mtx);
	LIST_REMOVE(bo, bo_synclist);
	syncer_worklist_len--;
	mtx_unlock(&sync_mtx);
	}
	bp->b_vp = NULL;
	bp->b_bufobj = NULL;
	BO_UNLOCK(bo);
	vdrop(vp);
	}

	/*
	* Add an item to the syncer work queue.
	*/
	static void
	vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
	{
	int slot;

	ASSERT_BO_WLOCKED(bo);

	mtx_lock(&sync_mtx);
	if (bo->bo_flag & BO_ONWORKLST)
	LIST_REMOVE(bo, bo_synclist);
	else {
	bo->bo_flag \|= BO_ONWORKLST;
	syncer_worklist_len++;
	}

	if (delay > syncer_maxdelay - 2)
	delay = syncer_maxdelay - 2;
	slot = (syncer_delayno + delay) & syncer_mask;

	LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist);
	mtx_unlock(&sync_mtx);
	}

	static int
	sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
	{
	int error, len;

	mtx_lock(&sync_mtx);
	len = syncer_worklist_len - sync_vnode_count;
	mtx_unlock(&sync_mtx);
	error = SYSCTL_OUT(req, &len, sizeof(len));
	return (error);
	}

	SYSCTL_PROC(_vfs, OID_AUTO, worklist_len,
	CTLTYPE_INT \| CTLFLAG_MPSAFE\| CTLFLAG_RD, NULL, 0,
	sysctl_vfs_worklist_len, "I", "Syncer thread worklist length");

	static struct proc *updateproc;
	static void sched_sync(void);
	static struct kproc_desc up_kp = {
	"syncer",
	sched_sync,
	&updateproc
	};
	SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp);

	static int
	sync_vnode(struct synclist slp, struct bufobj bo, struct thread td)
	{
	struct vnode *vp;
	struct mount *mp;

	*bo = LIST_FIRST(slp);
	if (*bo == NULL)
	return (0);
	vp = bo2vnode(*bo);
	if (VOP_ISLOCKED(vp) != 0 \|\| VI_TRYLOCK(vp) == 0)
	return (1);
	/*
	* We use vhold in case the vnode does not
	* successfully sync. vhold prevents the vnode from
	* going away when we unlock the sync_mtx so that
	* we can acquire the vnode interlock.
	*/
	vholdl(vp);
	mtx_unlock(&sync_mtx);
	VI_UNLOCK(vp);
	if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
	vdrop(vp);
	mtx_lock(&sync_mtx);
	return (*bo == LIST_FIRST(slp));
	}
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	(void) VOP_FSYNC(vp, MNT_LAZY, td);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	BO_LOCK(*bo);
	if (((*bo)->bo_flag & BO_ONWORKLST) != 0) {
	/*
	* Put us back on the worklist. The worklist
	* routine will remove us from our current
	* position and then add us back in at a later
	* position.
	*/
	vn_syncer_add_to_worklist(*bo, syncdelay);
	}
	BO_UNLOCK(*bo);
	vdrop(vp);
	mtx_lock(&sync_mtx);
	return (0);
	}

	static int first_printf = 1;

	/*
	* System filesystem synchronizer daemon.
	*/
	static void
	sched_sync(void)
	{
	struct synclist next, slp;
	struct bufobj *bo;
	long starttime;
	struct thread *td = curthread;
	int last_work_seen;
	int net_worklist_len;
	int syncer_final_iter;
	int error;

	last_work_seen = 0;
	syncer_final_iter = 0;
	syncer_state = SYNCER_RUNNING;
	starttime = time_uptime;
	td->td_pflags \|= TDP_NORUNNINGBUF;

	EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc,
	SHUTDOWN_PRI_LAST);

	mtx_lock(&sync_mtx);
	for (;;) {
	if (syncer_state == SYNCER_FINAL_DELAY &&
	syncer_final_iter == 0) {
	mtx_unlock(&sync_mtx);
	kproc_suspend_check(td->td_proc);
	mtx_lock(&sync_mtx);
	}
	net_worklist_len = syncer_worklist_len - sync_vnode_count;
	if (syncer_state != SYNCER_RUNNING &&
	starttime != time_uptime) {
	if (first_printf) {
	printf("\nSyncing disks, vnodes remaining... ");
	first_printf = 0;
	}
	printf("%d ", net_worklist_len);
	}
	starttime = time_uptime;

	/*
	* Push files whose dirty time has expired. Be careful
	* of interrupt race on slp queue.
	*
	* Skip over empty worklist slots when shutting down.
	*/
	do {
	slp = &syncer_workitem_pending[syncer_delayno];
	syncer_delayno += 1;
	if (syncer_delayno == syncer_maxdelay)
	syncer_delayno = 0;
	next = &syncer_workitem_pending[syncer_delayno];
	/*
	* If the worklist has wrapped since the
	* it was emptied of all but syncer vnodes,
	* switch to the FINAL_DELAY state and run
	* for one more second.
	*/
	if (syncer_state == SYNCER_SHUTTING_DOWN &&
	net_worklist_len == 0 &&
	last_work_seen == syncer_delayno) {
	syncer_state = SYNCER_FINAL_DELAY;
	syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP;
	}
	} while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) &&
	syncer_worklist_len > 0);

	/*
	* Keep track of the last time there was anything
	* on the worklist other than syncer vnodes.
	* Return to the SHUTTING_DOWN state if any
	* new work appears.
	*/
	if (net_worklist_len > 0 \|\| syncer_state == SYNCER_RUNNING)
	last_work_seen = syncer_delayno;
	if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY)
	syncer_state = SYNCER_SHUTTING_DOWN;
	while (!LIST_EMPTY(slp)) {
	error = sync_vnode(slp, &bo, td);
	if (error == 1) {
	LIST_REMOVE(bo, bo_synclist);
	LIST_INSERT_HEAD(next, bo, bo_synclist);
	continue;
	}

	if (first_printf == 0) {
	/*
	* Drop the sync mutex, because some watchdog
	* drivers need to sleep while patting
	*/
	mtx_unlock(&sync_mtx);
	wdog_kern_pat(WD_LASTVAL);
	mtx_lock(&sync_mtx);
	}

	}
	if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0)
	syncer_final_iter--;
	/*
	* The variable rushjob allows the kernel to speed up the
	* processing of the filesystem syncer process. A rushjob
	* value of N tells the filesystem syncer to process the next
	* N seconds worth of work on its queue ASAP. Currently rushjob
	* is used by the soft update code to speed up the filesystem
	* syncer process when the incore state is getting so far
	* ahead of the disk that the kernel memory pool is being
	* threatened with exhaustion.
	*/
	if (rushjob > 0) {
	rushjob -= 1;
	continue;
	}
	/*
	* Just sleep for a short period of time between
	* iterations when shutting down to allow some I/O
	* to happen.
	*
	* If it has taken us less than a second to process the
	* current work, then wait. Otherwise start right over
	* again. We can still lose time if any single round
	* takes more than two seconds, but it does not really
	* matter as we are just trying to generally pace the
	* filesystem activity.
	*/
	if (syncer_state != SYNCER_RUNNING \|\|
	time_uptime == starttime) {
	thread_lock(td);
	sched_prio(td, PPAUSE);
	thread_unlock(td);
	}
	if (syncer_state != SYNCER_RUNNING)
	cv_timedwait(&sync_wakeup, &sync_mtx,
	hz / SYNCER_SHUTDOWN_SPEEDUP);
	else if (time_uptime == starttime)
	cv_timedwait(&sync_wakeup, &sync_mtx, hz);
	}
	}

	/*
	* Request the syncer daemon to speed up its work.
	* We never push it to speed up more than half of its
	* normal turn time, otherwise it could take over the cpu.
	*/
	int
	speedup_syncer(void)
	{
	int ret = 0;

	mtx_lock(&sync_mtx);
	if (rushjob < syncdelay / 2) {
	rushjob += 1;
	stat_rush_requests += 1;
	ret = 1;
	}
	mtx_unlock(&sync_mtx);
	cv_broadcast(&sync_wakeup);
	return (ret);
	}

	/*
	* Tell the syncer to speed up its work and run though its work
	* list several times, then tell it to shut down.
	*/
	static void
	syncer_shutdown(void *arg, int howto)
	{

	if (howto & RB_NOSYNC)
	return;
	mtx_lock(&sync_mtx);
	syncer_state = SYNCER_SHUTTING_DOWN;
	rushjob = 0;
	mtx_unlock(&sync_mtx);
	cv_broadcast(&sync_wakeup);
	kproc_shutdown(arg, howto);
	}

	void
	syncer_suspend(void)
	{

	syncer_shutdown(updateproc, 0);
	}

	void
	syncer_resume(void)
	{

	mtx_lock(&sync_mtx);
	first_printf = 1;
	syncer_state = SYNCER_RUNNING;
	mtx_unlock(&sync_mtx);
	cv_broadcast(&sync_wakeup);
	kproc_resume(updateproc);
	}

	/*
	* Reassign a buffer from one vnode to another.
	* Used to assign file specific control information
	* (indirect blocks) to the vnode to which they belong.
	*/
	void
	reassignbuf(struct buf *bp)
	{
	struct vnode *vp;
	struct bufobj *bo;
	int delay;
	#ifdef INVARIANTS
	struct bufv *bv;
	#endif

	vp = bp->b_vp;
	bo = bp->b_bufobj;
	++reassignbufcalls;

	CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X",
	bp, bp->b_vp, bp->b_flags);
	/*
	* B_PAGING flagged buffers cannot be reassigned because their vp
	* is not fully linked in.
	*/
	if (bp->b_flags & B_PAGING)
	panic("cannot reassign paging buffer");

	/*
	* Delete from old vnode list, if on one.
	*/
	BO_LOCK(bo);
	if (bp->b_xflags & (BX_VNDIRTY \| BX_VNCLEAN))
	buf_vlist_remove(bp);
	else
	panic("reassignbuf: Buffer %p not on queue.", bp);
	/*
	* If dirty, put on list of dirty buffers; otherwise insert onto list
	* of clean buffers.
	*/
	if (bp->b_flags & B_DELWRI) {
	if ((bo->bo_flag & BO_ONWORKLST) == 0) {
	switch (vp->v_type) {
	case VDIR:
	delay = dirdelay;
	break;
	case VCHR:
	delay = metadelay;
	break;
	default:
	delay = filedelay;
	}
	vn_syncer_add_to_worklist(bo, delay);
	}
	buf_vlist_add(bp, bo, BX_VNDIRTY);
	} else {
	buf_vlist_add(bp, bo, BX_VNCLEAN);

	if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
	mtx_lock(&sync_mtx);
	LIST_REMOVE(bo, bo_synclist);
	syncer_worklist_len--;
	mtx_unlock(&sync_mtx);
	bo->bo_flag &= ~BO_ONWORKLST;
	}
	}
	#ifdef INVARIANTS
	bv = &bo->bo_clean;
	bp = TAILQ_FIRST(&bv->bv_hd);
	KASSERT(bp == NULL \|\| bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
	bp = TAILQ_LAST(&bv->bv_hd, buflists);
	KASSERT(bp == NULL \|\| bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
	bv = &bo->bo_dirty;
	bp = TAILQ_FIRST(&bv->bv_hd);
	KASSERT(bp == NULL \|\| bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
	bp = TAILQ_LAST(&bv->bv_hd, buflists);
	KASSERT(bp == NULL \|\| bp->b_bufobj == bo,
	("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
	#endif
	BO_UNLOCK(bo);
	}

	static void
	v_init_counters(struct vnode *vp)
	{

	VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0,
	vp, ("%s called for an initialized vnode", __FUNCTION__));
	ASSERT_VI_UNLOCKED(vp, __FUNCTION__);

	refcount_init(&vp->v_holdcnt, 1);
	refcount_init(&vp->v_usecount, 1);
	}

	/*
	* Increment si_usecount of the associated device, if any.
	*/
	static void
	v_incr_devcount(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __FUNCTION__);
	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
	dev_lock();
	vp->v_rdev->si_usecount++;
	dev_unlock();
	}
	}

	/*
	* Decrement si_usecount of the associated device, if any.
	*
	* The caller is required to hold the interlock when transitioning a VCHR use
	* count to zero. This prevents a race with devfs_reclaim_vchr() that would
	* leak a si_usecount reference. The vnode lock will also prevent this race
	* if it is held while dropping the last ref.
	*
	* The race is:
	*
	* CPU1 CPU2
	* devfs_reclaim_vchr
	* make v_usecount == 0
	* VI_LOCK
	* sees v_usecount == 0, no updates
	* vp->v_rdev = NULL;
	* ...
	* VI_UNLOCK
	* VI_LOCK
	* v_decr_devcount
	* sees v_rdev == NULL, no updates
	*
	* In this scenario si_devcount decrement is not performed.
	*/
	static void
	v_decr_devcount(struct vnode *vp)
	{

	ASSERT_VOP_LOCKED(vp, __func__);
	ASSERT_VI_LOCKED(vp, __FUNCTION__);
	if (vp->v_type == VCHR && vp->v_rdev != NULL) {
	dev_lock();
	VNPASS(vp->v_rdev->si_usecount > 0, vp);
	vp->v_rdev->si_usecount--;
	dev_unlock();
	}
	}

	/*
	* Grab a particular vnode from the free list, increment its
	* reference count and lock it. VIRF_DOOMED is set if the vnode
	* is being destroyed. Only callers who specify LK_RETRY will
	* see doomed vnodes. If inactive processing was delayed in
	* vput try to do it here.
	*
	* usecount is manipulated using atomics without holding any locks.
	*
	* holdcnt can be manipulated using atomics without holding any locks,
	* except when transitioning 1<->0, in which case the interlock is held.
	*
	* Consumers which don't guarantee liveness of the vnode can use SMR to
	* try to get a reference. Note this operation can fail since the vnode
	* may be awaiting getting freed by the time they get to it.
	*/
	enum vgetstate
	vget_prep_smr(struct vnode *vp)
	{
	enum vgetstate vs;

	VFS_SMR_ASSERT_ENTERED();

	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	vs = VGET_USECOUNT;
	} else {
	if (vhold_smr(vp))
	vs = VGET_HOLDCNT;
	else
	vs = VGET_NONE;
	}
	return (vs);
	}

	enum vgetstate
	vget_prep(struct vnode *vp)
	{
	enum vgetstate vs;

	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	vs = VGET_USECOUNT;
	} else {
	vhold(vp);
	vs = VGET_HOLDCNT;
	}
	return (vs);
	}

	void
	vget_abort(struct vnode *vp, enum vgetstate vs)
	{

	switch (vs) {
	case VGET_USECOUNT:
	vrele(vp);
	break;
	case VGET_HOLDCNT:
	vdrop(vp);
	break;
	default:
	__assert_unreachable();
	}
	}

	int
	vget(struct vnode vp, int flags, struct thread td)
	{
	enum vgetstate vs;

	MPASS(td == curthread);

	vs = vget_prep(vp);
	return (vget_finish(vp, flags, vs));
	}

	static void __noinline
	vget_finish_vchr(struct vnode *vp)
	{

	VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)"));

	/*
	* See the comment in vget_finish before usecount bump.
	*/
	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	#ifdef INVARIANTS
	int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
	VNASSERT(old > 0, vp, ("%s: wrong hold count %d", __func__, old));
	#else
	refcount_release(&vp->v_holdcnt);
	#endif
	return;
	}

	VI_LOCK(vp);
	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	#ifdef INVARIANTS
	int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
	VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
	#else
	refcount_release(&vp->v_holdcnt);
	#endif
	VI_UNLOCK(vp);
	return;
	}
	v_incr_devcount(vp);
	refcount_acquire(&vp->v_usecount);
	VI_UNLOCK(vp);
	}

	int
	vget_finish(struct vnode *vp, int flags, enum vgetstate vs)
	{
	int error;

	if ((flags & LK_INTERLOCK) != 0)
	ASSERT_VI_LOCKED(vp, __func__);
	else
	ASSERT_VI_UNLOCKED(vp, __func__);
	VNPASS(vs == VGET_HOLDCNT \|\| vs == VGET_USECOUNT, vp);
	VNPASS(vp->v_holdcnt > 0, vp);
	VNPASS(vs == VGET_HOLDCNT \|\| vp->v_usecount > 0, vp);

	error = vn_lock(vp, flags);
	if (__predict_false(error != 0)) {
	vget_abort(vp, vs);
	CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__,
	vp);
	return (error);
	}

	vget_finish_ref(vp, vs);
	return (0);
	}

	void
	vget_finish_ref(struct vnode *vp, enum vgetstate vs)
	{
	int old;

	VNPASS(vs == VGET_HOLDCNT \|\| vs == VGET_USECOUNT, vp);
	VNPASS(vp->v_holdcnt > 0, vp);
	VNPASS(vs == VGET_HOLDCNT \|\| vp->v_usecount > 0, vp);

	if (vs == VGET_USECOUNT)
	return;

	if (__predict_false(vp->v_type == VCHR)) {
	vget_finish_vchr(vp);
	return;
	}

	/*
	* We hold the vnode. If the usecount is 0 it will be utilized to keep
	* the vnode around. Otherwise someone else lended their hold count and
	* we have to drop ours.
	*/
	old = atomic_fetchadd_int(&vp->v_usecount, 1);
	VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old));
	if (old != 0) {
	#ifdef INVARIANTS
	old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
	VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
	#else
	refcount_release(&vp->v_holdcnt);
	#endif
	}
	}

	/*
	* Increase the reference (use) and hold count of a vnode.
	* This will also remove the vnode from the free list if it is presently free.
	*/
	static void __noinline
	vref_vchr(struct vnode *vp, bool interlock)
	{

	/*
	* See the comment in vget_finish before usecount bump.
	*/
	if (!interlock) {
	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	VNODE_REFCOUNT_FENCE_ACQ();
	VNASSERT(vp->v_holdcnt > 0, vp,
	("%s: active vnode not held", __func__));
	return;
	}
	VI_LOCK(vp);
	/*
	* By the time we get here the vnode might have been doomed, at
	* which point the 0->1 use count transition is no longer
	* protected by the interlock. Since it can't bounce back to
	* VCHR and requires vref semantics, punt it back
	*/
	if (__predict_false(vp->v_type == VBAD)) {
	VI_UNLOCK(vp);
	vref(vp);
	return;
	}
	}
	VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)"));
	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	VNODE_REFCOUNT_FENCE_ACQ();
	VNASSERT(vp->v_holdcnt > 0, vp,
	("%s: active vnode not held", __func__));
	if (!interlock)
	VI_UNLOCK(vp);
	return;
	}
	vhold(vp);
	v_incr_devcount(vp);
	refcount_acquire(&vp->v_usecount);
	if (!interlock)
	VI_UNLOCK(vp);
	return;
	}

	void
	vref(struct vnode *vp)
	{
	int old;

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	if (__predict_false(vp->v_type == VCHR)) {
	vref_vchr(vp, false);
	return;
	}

	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
	VNODE_REFCOUNT_FENCE_ACQ();
	VNASSERT(vp->v_holdcnt > 0, vp,
	("%s: active vnode not held", __func__));
	return;
	}
	vhold(vp);
	/*
	* See the comment in vget_finish.
	*/
	old = atomic_fetchadd_int(&vp->v_usecount, 1);
	VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old));
	if (old != 0) {
	#ifdef INVARIANTS
	old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
	VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
	#else
	refcount_release(&vp->v_holdcnt);
	#endif
	}
	}

	void
	vrefl(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	if (__predict_false(vp->v_type == VCHR)) {
	vref_vchr(vp, true);
	return;
	}
	vref(vp);
	}

	void
	vrefact(struct vnode *vp)
	{

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	#ifdef INVARIANTS
	int old = atomic_fetchadd_int(&vp->v_usecount, 1);
	VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old));
	#else
	refcount_acquire(&vp->v_usecount);
	#endif
	}

	void
	vrefactn(struct vnode *vp, u_int n)
	{

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	#ifdef INVARIANTS
	int old = atomic_fetchadd_int(&vp->v_usecount, n);
	VNASSERT(old > 0, vp, ("%s: wrong use count %d", __func__, old));
	#else
	atomic_add_int(&vp->v_usecount, n);
	#endif
	}

	/*
	* Return reference count of a vnode.
	*
	* The results of this call are only guaranteed when some mechanism is used to
	* stop other processes from gaining references to the vnode. This may be the
	* case if the caller holds the only reference. This is also useful when stale
	* data is acceptable as race conditions may be accounted for by some other
	* means.
	*/
	int
	vrefcnt(struct vnode *vp)
	{

	return (vp->v_usecount);
	}

	void
	vlazy(struct vnode *vp)
	{
	struct mount *mp;

	VNASSERT(vp->v_holdcnt > 0, vp, ("%s: vnode not held", __func__));

	if ((vp->v_mflag & VMP_LAZYLIST) != 0)
	return;
	/*
	* We may get here for inactive routines after the vnode got doomed.
	*/
	if (VN_IS_DOOMED(vp))
	return;
	mp = vp->v_mount;
	mtx_lock(&mp->mnt_listmtx);
	if ((vp->v_mflag & VMP_LAZYLIST) == 0) {
	vp->v_mflag \|= VMP_LAZYLIST;
	TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist);
	mp->mnt_lazyvnodelistsize++;
	}
	mtx_unlock(&mp->mnt_listmtx);
	}

	/*
	* This routine is only meant to be called from vgonel prior to dooming
	* the vnode.
	*/
	static void
	vunlazy_gone(struct vnode *vp)
	{
	struct mount *mp;

	ASSERT_VOP_ELOCKED(vp, __func__);
	ASSERT_VI_LOCKED(vp, __func__);
	VNPASS(!VN_IS_DOOMED(vp), vp);

	if (vp->v_mflag & VMP_LAZYLIST) {
	mp = vp->v_mount;
	mtx_lock(&mp->mnt_listmtx);
	VNPASS(vp->v_mflag & VMP_LAZYLIST, vp);
	vp->v_mflag &= ~VMP_LAZYLIST;
	TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist);
	mp->mnt_lazyvnodelistsize--;
	mtx_unlock(&mp->mnt_listmtx);
	}
	}

	static void
	vdefer_inactive(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	VNASSERT(vp->v_holdcnt > 0, vp,
	("%s: vnode without hold count", __func__));
	if (VN_IS_DOOMED(vp)) {
	vdropl(vp);
	return;
	}
	if (vp->v_iflag & VI_DEFINACT) {
	VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count"));
	vdropl(vp);
	return;
	}
	if (vp->v_usecount > 0) {
	vp->v_iflag &= ~VI_OWEINACT;
	vdropl(vp);
	return;
	}
	vlazy(vp);
	vp->v_iflag \|= VI_DEFINACT;
	VI_UNLOCK(vp);
	counter_u64_add(deferred_inact, 1);
	}

	static void
	vdefer_inactive_unlocked(struct vnode *vp)
	{

	VI_LOCK(vp);
	if ((vp->v_iflag & VI_OWEINACT) == 0) {
	vdropl(vp);
	return;
	}
	vdefer_inactive(vp);
	}

	enum vput_op { VRELE, VPUT, VUNREF };

	/*
	* Handle ->v_usecount transitioning to 0.
	*
	* By releasing the last usecount we take ownership of the hold count which
	* provides liveness of the vnode, meaning we have to vdrop.
	*
	* If the vnode is of type VCHR we may need to decrement si_usecount, see
	* v_decr_devcount for details.
	*
	* For all vnodes we may need to perform inactive processing. It requires an
	* exclusive lock on the vnode, while it is legal to call here with only a
	* shared lock (or no locks). If locking the vnode in an expected manner fails,
	* inactive processing gets deferred to the syncer.
	*
	* XXX Some filesystems pass in an exclusively locked vnode and strongly depend
	* on the lock being held all the way until VOP_INACTIVE. This in particular
	* happens with UFS which adds half-constructed vnodes to the hash, where they
	* can be found by other code.
	*/
	static void
	vput_final(struct vnode *vp, enum vput_op func)
	{
	int error;
	bool want_unlock;

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	VNPASS(vp->v_holdcnt > 0, vp);

	VI_LOCK(vp);
	if (__predict_false(vp->v_type == VCHR && func != VRELE))
	v_decr_devcount(vp);

	/*
	* By the time we got here someone else might have transitioned
	* the count back to > 0.
	*/
	if (vp->v_usecount > 0)
	goto out;

	/*
	* If the vnode is doomed vgone already performed inactive processing
	* (if needed).
	*/
	if (VN_IS_DOOMED(vp))
	goto out;

	if (__predict_true(VOP_NEED_INACTIVE(vp) == 0))
	goto out;

	if (vp->v_iflag & VI_DOINGINACT)
	goto out;

	/*
	* Locking operations here will drop the interlock and possibly the
	* vnode lock, opening a window where the vnode can get doomed all the
	* while ->v_usecount is 0. Set VI_OWEINACT to let vgone know to
	* perform inactive.
	*/
	vp->v_iflag \|= VI_OWEINACT;
	want_unlock = false;
	error = 0;
	switch (func) {
	case VRELE:
	switch (VOP_ISLOCKED(vp)) {
	case LK_EXCLUSIVE:
	break;
	case LK_EXCLOTHER:
	case 0:
	want_unlock = true;
	error = vn_lock(vp, LK_EXCLUSIVE \| LK_INTERLOCK);
	VI_LOCK(vp);
	break;
	default:
	/*
	* The lock has at least one sharer, but we have no way
	* to conclude whether this is us. Play it safe and
	* defer processing.
	*/
	error = EAGAIN;
	break;
	}
	break;
	case VPUT:
	want_unlock = true;
	if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
	error = VOP_LOCK(vp, LK_UPGRADE \| LK_INTERLOCK \|
	LK_NOWAIT);
	VI_LOCK(vp);
	}
	break;
	case VUNREF:
	if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
	error = VOP_LOCK(vp, LK_TRYUPGRADE \| LK_INTERLOCK);
	VI_LOCK(vp);
	}
	break;
	}
	if (error == 0) {
	vinactive(vp);
	if (want_unlock)
	VOP_UNLOCK(vp);
	vdropl(vp);
	} else {
	vdefer_inactive(vp);
	}
	return;
	out:
	if (func == VPUT)
	VOP_UNLOCK(vp);
	vdropl(vp);
	}

	/*
	* Decrement ->v_usecount for a vnode.
	*
	* Releasing the last use count requires additional processing, see vput_final
	* above for details.
	*
	* Note that releasing use count without the vnode lock requires special casing
	* for VCHR, see v_decr_devcount for details.
	*
	* Comment above each variant denotes lock state on entry and exit.
	*/

	static void __noinline
	vrele_vchr(struct vnode *vp)
	{

	if (refcount_release_if_not_last(&vp->v_usecount))
	return;
	VI_LOCK(vp);
	if (!refcount_release(&vp->v_usecount)) {
	VI_UNLOCK(vp);
	return;
	}
	v_decr_devcount(vp);
	VI_UNLOCK(vp);
	vput_final(vp, VRELE);
	}

	/*
	* in: any
	* out: same as passed in
	*/
	void
	vrele(struct vnode *vp)
	{

	ASSERT_VI_UNLOCKED(vp, __func__);
	if (__predict_false(vp->v_type == VCHR)) {
	vrele_vchr(vp);
	return;
	}
	if (!refcount_release(&vp->v_usecount))
	return;
	vput_final(vp, VRELE);
	}

	/*
	* in: locked
	* out: unlocked
	*/
	void
	vput(struct vnode *vp)
	{

	ASSERT_VOP_LOCKED(vp, __func__);
	ASSERT_VI_UNLOCKED(vp, __func__);
	if (!refcount_release(&vp->v_usecount)) {
	VOP_UNLOCK(vp);
	return;
	}
	vput_final(vp, VPUT);
	}

	/*
	* in: locked
	* out: locked
	*/
	void
	vunref(struct vnode *vp)
	{

	ASSERT_VOP_LOCKED(vp, __func__);
	ASSERT_VI_UNLOCKED(vp, __func__);
	if (!refcount_release(&vp->v_usecount))
	return;
	vput_final(vp, VUNREF);
	}

	void
	vhold(struct vnode *vp)
	{
	struct vdbatch *vd;
	int old;

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	old = atomic_fetchadd_int(&vp->v_holdcnt, 1);
	VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp,
	("%s: wrong hold count %d", __func__, old));
	if (old != 0)
	return;
	critical_enter();
	vd = DPCPU_PTR(vd);
	vd->freevnodes--;
	critical_exit();
	}

	void
	vholdl(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	vhold(vp);
	}

	void
	vholdnz(struct vnode *vp)
	{

	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	#ifdef INVARIANTS
	int old = atomic_fetchadd_int(&vp->v_holdcnt, 1);
	VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp,
	("%s: wrong hold count %d", __func__, old));
	#else
	atomic_add_int(&vp->v_holdcnt, 1);
	#endif
	}

	/*
	* Grab a hold count unless the vnode is freed.
	*
	* Only use this routine if vfs smr is the only protection you have against
	* freeing the vnode.
	*
	* The code loops trying to add a hold count as long as the VHOLD_NO_SMR flag
	* is not set. After the flag is set the vnode becomes immutable to anyone but
	* the thread which managed to set the flag.
	*
	* It may be tempting to replace the loop with:
	* count = atomic_fetchadd_int(&vp->v_holdcnt, 1);
	* if (count & VHOLD_NO_SMR) {
	* backpedal and error out;
	* }
	*
	* However, while this is more performant, it hinders debugging by eliminating
	* the previously mentioned invariant.
	*/
	bool
	vhold_smr(struct vnode *vp)
	{
	int count;

	VFS_SMR_ASSERT_ENTERED();

	count = atomic_load_int(&vp->v_holdcnt);
	for (;;) {
	if (count & VHOLD_NO_SMR) {
	VNASSERT((count & ~VHOLD_NO_SMR) == 0, vp,
	("non-zero hold count with flags %d\n", count));
	return (false);
	}

	VNASSERT(count >= 0, vp, ("invalid hold count %d\n", count));
	if (atomic_fcmpset_int(&vp->v_holdcnt, &count, count + 1))
	return (true);
	}
	}

	static void __noinline
	vdbatch_process(struct vdbatch *vd)
	{
	struct vnode *vp;
	int i;

	mtx_assert(&vd->lock, MA_OWNED);
	MPASS(curthread->td_pinned > 0);
	MPASS(vd->index == VDBATCH_SIZE);

	mtx_lock(&vnode_list_mtx);
	critical_enter();
	freevnodes += vd->freevnodes;
	for (i = 0; i < VDBATCH_SIZE; i++) {
	vp = vd->tab[i];
	TAILQ_REMOVE(&vnode_list, vp, v_vnodelist);
	TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist);
	MPASS(vp->v_dbatchcpu != NOCPU);
	vp->v_dbatchcpu = NOCPU;
	}
	mtx_unlock(&vnode_list_mtx);
	vd->freevnodes = 0;
	bzero(vd->tab, sizeof(vd->tab));
	vd->index = 0;
	critical_exit();
	}

	static void
	vdbatch_enqueue(struct vnode *vp)
	{
	struct vdbatch *vd;

	ASSERT_VI_LOCKED(vp, __func__);
	VNASSERT(!VN_IS_DOOMED(vp), vp,
	("%s: deferring requeue of a doomed vnode", __func__));

	critical_enter();
	vd = DPCPU_PTR(vd);
	vd->freevnodes++;
	if (vp->v_dbatchcpu != NOCPU) {
	VI_UNLOCK(vp);
	critical_exit();
	return;
	}

	sched_pin();
	critical_exit();
	mtx_lock(&vd->lock);
	MPASS(vd->index < VDBATCH_SIZE);
	MPASS(vd->tab[vd->index] == NULL);
	/*
	* A hack: we depend on being pinned so that we know what to put in
	* ->v_dbatchcpu.
	*/
	vp->v_dbatchcpu = curcpu;
	vd->tab[vd->index] = vp;
	vd->index++;
	VI_UNLOCK(vp);
	if (vd->index == VDBATCH_SIZE)
	vdbatch_process(vd);
	mtx_unlock(&vd->lock);
	sched_unpin();
	}

	/*
	* This routine must only be called for vnodes which are about to be
	* deallocated. Supporting dequeue for arbitrary vndoes would require
	* validating that the locked batch matches.
	*/
	static void
	vdbatch_dequeue(struct vnode *vp)
	{
	struct vdbatch *vd;
	int i;
	short cpu;

	VNASSERT(vp->v_type == VBAD \|\| vp->v_type == VNON, vp,
	("%s: called for a used vnode\n", __func__));

	cpu = vp->v_dbatchcpu;
	if (cpu == NOCPU)
	return;

	vd = DPCPU_ID_PTR(cpu, vd);
	mtx_lock(&vd->lock);
	for (i = 0; i < vd->index; i++) {
	if (vd->tab[i] != vp)
	continue;
	vp->v_dbatchcpu = NOCPU;
	vd->index--;
	vd->tab[i] = vd->tab[vd->index];
	vd->tab[vd->index] = NULL;
	break;
	}
	mtx_unlock(&vd->lock);
	/*
	* Either we dequeued the vnode above or the target CPU beat us to it.
	*/
	MPASS(vp->v_dbatchcpu == NOCPU);
	}

	/*
	* Drop the hold count of the vnode. If this is the last reference to
	* the vnode we place it on the free list unless it has been vgone'd
	* (marked VIRF_DOOMED) in which case we will free it.
	*
	* Because the vnode vm object keeps a hold reference on the vnode if
	* there is at least one resident non-cached page, the vnode cannot
	* leave the active list without the page cleanup done.
	*/
	static void
	vdrop_deactivate(struct vnode *vp)
	{
	struct mount *mp;

	ASSERT_VI_LOCKED(vp, __func__);
	/*
	* Mark a vnode as free: remove it from its active list
	* and put it up for recycling on the freelist.
	*/
	VNASSERT(!VN_IS_DOOMED(vp), vp,
	("vdrop: returning doomed vnode"));
	VNASSERT(vp->v_op != NULL, vp,
	("vdrop: vnode already reclaimed."));
	VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
	("vnode with VI_OWEINACT set"));
	VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp,
	("vnode with VI_DEFINACT set"));
	if (vp->v_mflag & VMP_LAZYLIST) {
	mp = vp->v_mount;
	mtx_lock(&mp->mnt_listmtx);
	VNASSERT(vp->v_mflag & VMP_LAZYLIST, vp, ("lost VMP_LAZYLIST"));
	/*
	* Don't remove the vnode from the lazy list if another thread
	* has increased the hold count. It may have re-enqueued the
	* vnode to the lazy list and is now responsible for its
	* removal.
	*/
	if (vp->v_holdcnt == 0) {
	vp->v_mflag &= ~VMP_LAZYLIST;
	TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist);
	mp->mnt_lazyvnodelistsize--;
	}
	mtx_unlock(&mp->mnt_listmtx);
	}
	vdbatch_enqueue(vp);
	}

	void
	vdrop(struct vnode *vp)
	{

	ASSERT_VI_UNLOCKED(vp, __func__);
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	if (refcount_release_if_not_last(&vp->v_holdcnt))
	return;
	VI_LOCK(vp);
	vdropl(vp);
	}

	void
	vdropl(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	if (!refcount_release(&vp->v_holdcnt)) {
	VI_UNLOCK(vp);
	return;
	}
	if (!VN_IS_DOOMED(vp)) {
	vdrop_deactivate(vp);
	return;
	}
	/*
	* We may be racing against vhold_smr.
	*
	* If they win we can just pretend we never got this far, they will
	* vdrop later.
	*/
	if (!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR)) {
	/*
	* We lost the aforementioned race. Note that any subsequent
	* access is invalid as they might have managed to vdropl on
	* their own.
	*/
	return;
	}
	freevnode(vp);
	}

	/*
	* Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT
	* flags. DOINGINACT prevents us from recursing in calls to vinactive.
	*/
	static void
	vinactivef(struct vnode *vp)
	{
	struct vm_object *obj;

	ASSERT_VOP_ELOCKED(vp, "vinactive");
	ASSERT_VI_LOCKED(vp, "vinactive");
	VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
	("vinactive: recursed on VI_DOINGINACT"));
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	vp->v_iflag \|= VI_DOINGINACT;
	vp->v_iflag &= ~VI_OWEINACT;
	VI_UNLOCK(vp);
	/*
	* Before moving off the active list, we must be sure that any
	* modified pages are converted into the vnode's dirty
	* buffers, since these will no longer be checked once the
	* vnode is on the inactive list.
	*
	* The write-out of the dirty pages is asynchronous. At the
	* point that VOP_INACTIVE() is called, there could still be
	* pending I/O and dirty pages in the object.
	*/
	if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
	vm_object_mightbedirty(obj)) {
	VM_OBJECT_WLOCK(obj);
	vm_object_page_clean(obj, 0, 0, 0);
	VM_OBJECT_WUNLOCK(obj);
	}
	VOP_INACTIVE(vp, curthread);
	VI_LOCK(vp);
	VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
	("vinactive: lost VI_DOINGINACT"));
	vp->v_iflag &= ~VI_DOINGINACT;
	}

	void
	vinactive(struct vnode *vp)
	{

	ASSERT_VOP_ELOCKED(vp, "vinactive");
	ASSERT_VI_LOCKED(vp, "vinactive");
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);

	if ((vp->v_iflag & VI_OWEINACT) == 0)
	return;
	if (vp->v_iflag & VI_DOINGINACT)
	return;
	if (vp->v_usecount > 0) {
	vp->v_iflag &= ~VI_OWEINACT;
	return;
	}
	vinactivef(vp);
	}

	/*
	* Remove any vnodes in the vnode table belonging to mount point mp.
	*
	* If FORCECLOSE is not specified, there should not be any active ones,
	* return error if any are found (nb: this is a user error, not a
	* system error). If FORCECLOSE is specified, detach any active vnodes
	* that are found.
	*
	* If WRITECLOSE is set, only flush out regular file vnodes open for
	* writing.
	*
	* SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
	*
	* `rootrefs' specifies the base reference count for the root vnode
	* of this filesystem. The root vnode is considered busy if its
	* v_usecount exceeds this value. On a successful return, vflush(, td)
	* will call vrele() on the root vnode exactly rootrefs times.
	* If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must
	* be zero.
	*/
	#ifdef DIAGNOSTIC
	static int busyprt = 0; /* print out busy vnodes */
	SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes");
	#endif

	int
	vflush(struct mount mp, int rootrefs, int flags, struct thread td)
	{
	struct vnode vp, mvp, *rootvp = NULL;
	struct vattr vattr;
	int busy = 0, error;

	CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp,
	rootrefs, flags);
	if (rootrefs > 0) {
	KASSERT((flags & (SKIPSYSTEM \| WRITECLOSE)) == 0,
	("vflush: bad args"));
	/*
	* Get the filesystem root vnode. We can vput() it
	* immediately, since with rootrefs > 0, it won't go away.
	*/
	if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) {
	CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d",
	__func__, error);
	return (error);
	}
	vput(rootvp);
	}
	loop:
	MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
	vholdl(vp);
	error = vn_lock(vp, LK_INTERLOCK \| LK_EXCLUSIVE);
	if (error) {
	vdrop(vp);
	MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
	goto loop;
	}
	/*
	* Skip over a vnodes marked VV_SYSTEM.
	*/
	if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
	VOP_UNLOCK(vp);
	vdrop(vp);
	continue;
	}
	/*
	* If WRITECLOSE is set, flush out unlinked but still open
	* files (even if open only for reading) and regular file
	* vnodes open for writing.
	*/
	if (flags & WRITECLOSE) {
	if (vp->v_object != NULL) {
	VM_OBJECT_WLOCK(vp->v_object);
	vm_object_page_clean(vp->v_object, 0, 0, 0);
	VM_OBJECT_WUNLOCK(vp->v_object);
	}
	error = VOP_FSYNC(vp, MNT_WAIT, td);
	if (error != 0) {
	VOP_UNLOCK(vp);
	vdrop(vp);
	MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
	return (error);
	}
	error = VOP_GETATTR(vp, &vattr, td->td_ucred);
	VI_LOCK(vp);

	if ((vp->v_type == VNON \|\|
	(error == 0 && vattr.va_nlink > 0)) &&
	(vp->v_writecount <= 0 \|\| vp->v_type != VREG)) {
	VOP_UNLOCK(vp);
	vdropl(vp);
	continue;
	}
	} else
	VI_LOCK(vp);
	/*
	* With v_usecount == 0, all we need to do is clear out the
	* vnode data structures and we are done.
	*
	* If FORCECLOSE is set, forcibly close the vnode.
	*/
	if (vp->v_usecount == 0 \|\| (flags & FORCECLOSE)) {
	vgonel(vp);
	} else {
	busy++;
	#ifdef DIAGNOSTIC
	if (busyprt)
	vn_printf(vp, "vflush: busy vnode ");
	#endif
	}
	VOP_UNLOCK(vp);
	vdropl(vp);
	}
	if (rootrefs > 0 && (flags & FORCECLOSE) == 0) {
	/*
	* If just the root vnode is busy, and if its refcount
	* is equal to `rootrefs', then go ahead and kill it.
	*/
	VI_LOCK(rootvp);
	KASSERT(busy > 0, ("vflush: not busy"));
	VNASSERT(rootvp->v_usecount >= rootrefs, rootvp,
	("vflush: usecount %d < rootrefs %d",
	rootvp->v_usecount, rootrefs));
	if (busy == 1 && rootvp->v_usecount == rootrefs) {
	VOP_LOCK(rootvp, LK_EXCLUSIVE\|LK_INTERLOCK);
	vgone(rootvp);
	VOP_UNLOCK(rootvp);
	busy = 0;
	} else
	VI_UNLOCK(rootvp);
	}
	if (busy) {
	CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__,
	busy);
	return (EBUSY);
	}
	for (; rootrefs > 0; rootrefs--)
	vrele(rootvp);
	return (0);
	}

	/*
	* Recycle an unused vnode to the front of the free list.
	*/
	int
	vrecycle(struct vnode *vp)
	{
	int recycled;

	VI_LOCK(vp);
	recycled = vrecyclel(vp);
	VI_UNLOCK(vp);
	return (recycled);
	}

	/*
	* vrecycle, with the vp interlock held.
	*/
	int
	vrecyclel(struct vnode *vp)
	{
	int recycled;

	ASSERT_VOP_ELOCKED(vp, __func__);
	ASSERT_VI_LOCKED(vp, __func__);
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	recycled = 0;
	if (vp->v_usecount == 0) {
	recycled = 1;
	vgonel(vp);
	}
	return (recycled);
	}

	/*
	* Eliminate all activity associated with a vnode
	* in preparation for reuse.
	*/
	void
	vgone(struct vnode *vp)
	{
	VI_LOCK(vp);
	vgonel(vp);
	VI_UNLOCK(vp);
	}

	static void
	notify_lowervp_vfs_dummy(struct mount *mp __unused,
	struct vnode *lowervp __unused)
	{
	}

	/*
	* Notify upper mounts about reclaimed or unlinked vnode.
	*/
	void
	vfs_notify_upper(struct vnode *vp, int event)
	{
	static struct vfsops vgonel_vfsops = {
	.vfs_reclaim_lowervp = notify_lowervp_vfs_dummy,
	.vfs_unlink_lowervp = notify_lowervp_vfs_dummy,
	};
	struct mount mp, ump, *mmp;

	mp = vp->v_mount;
	if (mp == NULL)
	return;
	if (TAILQ_EMPTY(&mp->mnt_uppers))
	return;

	mmp = malloc(sizeof(struct mount), M_TEMP, M_WAITOK \| M_ZERO);
	mmp->mnt_op = &vgonel_vfsops;
	mmp->mnt_kern_flag \|= MNTK_MARKER;
	MNT_ILOCK(mp);
	mp->mnt_kern_flag \|= MNTK_VGONE_UPPER;
	for (ump = TAILQ_FIRST(&mp->mnt_uppers); ump != NULL;) {
	if ((ump->mnt_kern_flag & MNTK_MARKER) != 0) {
	ump = TAILQ_NEXT(ump, mnt_upper_link);
	continue;
	}
	TAILQ_INSERT_AFTER(&mp->mnt_uppers, ump, mmp, mnt_upper_link);
	MNT_IUNLOCK(mp);
	switch (event) {
	case VFS_NOTIFY_UPPER_RECLAIM:
	VFS_RECLAIM_LOWERVP(ump, vp);
	break;
	case VFS_NOTIFY_UPPER_UNLINK:
	VFS_UNLINK_LOWERVP(ump, vp);
	break;
	default:
	KASSERT(0, ("invalid event %d", event));
	break;
	}
	MNT_ILOCK(mp);
	ump = TAILQ_NEXT(mmp, mnt_upper_link);
	TAILQ_REMOVE(&mp->mnt_uppers, mmp, mnt_upper_link);
	}
	free(mmp, M_TEMP);
	mp->mnt_kern_flag &= ~MNTK_VGONE_UPPER;
	if ((mp->mnt_kern_flag & MNTK_VGONE_WAITER) != 0) {
	mp->mnt_kern_flag &= ~MNTK_VGONE_WAITER;
	wakeup(&mp->mnt_uppers);
	}
	MNT_IUNLOCK(mp);
	}

	/*
	* vgone, with the vp interlock held.
	*/
	static void
	vgonel(struct vnode *vp)
	{
	struct thread *td;
	struct mount *mp;
	vm_object_t object;
	bool active, oweinact;

	ASSERT_VOP_ELOCKED(vp, "vgonel");
	ASSERT_VI_LOCKED(vp, "vgonel");
	VNASSERT(vp->v_holdcnt, vp,
	("vgonel: vp %p has no reference.", vp));
	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
	td = curthread;

	/*
	* Don't vgonel if we're already doomed.
	*/
	if (vp->v_irflag & VIRF_DOOMED)
	return;
	/*
	* Paired with freevnode.
	*/
	vn_seqc_write_begin_locked(vp);
	vunlazy_gone(vp);
	vp->v_irflag \|= VIRF_DOOMED;

	/*
	* Check to see if the vnode is in use. If so, we have to call
	* VOP_CLOSE() and VOP_INACTIVE().
	*/
	active = vp->v_usecount > 0;
	oweinact = (vp->v_iflag & VI_OWEINACT) != 0;
	/*
	* If we need to do inactive VI_OWEINACT will be set.
	*/
	if (vp->v_iflag & VI_DEFINACT) {
	VNASSERT(vp->v_holdcnt > 1, vp, ("lost hold count"));
	vp->v_iflag &= ~VI_DEFINACT;
	vdropl(vp);
	} else {
	VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count"));
	VI_UNLOCK(vp);
	}
	vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM);

	/*
	* If purging an active vnode, it must be closed and
	* deactivated before being reclaimed.
	*/
	if (active)
	VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
	if (oweinact \|\| active) {
	VI_LOCK(vp);
	vinactivef(vp);
	VI_UNLOCK(vp);
	}
	if (vp->v_type == VSOCK)
	vfs_unp_reclaim(vp);

	/*
	* Clean out any buffers associated with the vnode.
	* If the flush fails, just toss the buffers.
	*/
	mp = NULL;
	if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
	(void) vn_start_secondary_write(vp, &mp, V_WAIT);
	if (vinvalbuf(vp, V_SAVE, 0, 0) != 0) {
	while (vinvalbuf(vp, 0, 0, 0) != 0)
	;
	}

	BO_LOCK(&vp->v_bufobj);
	KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) &&
	vp->v_bufobj.bo_dirty.bv_cnt == 0 &&
	TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) &&
	vp->v_bufobj.bo_clean.bv_cnt == 0,
	("vp %p bufobj not invalidated", vp));

	/*
	* For VMIO bufobj, BO_DEAD is set later, or in
	* vm_object_terminate() after the object's page queue is
	* flushed.
	*/
	object = vp->v_bufobj.bo_object;
	if (object == NULL)
	vp->v_bufobj.bo_flag \|= BO_DEAD;
	BO_UNLOCK(&vp->v_bufobj);

	/*
	* Handle the VM part. Tmpfs handles v_object on its own (the
	* OBJT_VNODE check). Nullfs or other bypassing filesystems
	* should not touch the object borrowed from the lower vnode
	* (the handle check).
	*/
	if (object != NULL && object->type == OBJT_VNODE &&
	object->handle == vp)
	vnode_destroy_vobject(vp);

	/*
	* Reclaim the vnode.
	*/
	if (VOP_RECLAIM(vp, td))
	panic("vgone: cannot reclaim");
	if (mp != NULL)
	vn_finished_secondary_write(mp);
	VNASSERT(vp->v_object == NULL, vp,
	("vop_reclaim left v_object vp=%p", vp));
	/*
	* Clear the advisory locks and wake up waiting threads.
	*/
	(void)VOP_ADVLOCKPURGE(vp);
	vp->v_lockf = NULL;
	/*
	* Delete from old mount point vnode list.
	*/
	delmntque(vp);
	cache_purge(vp);
	/*
	* Done with purge, reset to the standard lock and invalidate
	* the vnode.
	*/
	VI_LOCK(vp);
	vp->v_vnlock = &vp->v_lock;
	vp->v_op = &dead_vnodeops;
	vp->v_type = VBAD;
	}

	/*
	* Calculate the total number of references to a special device.
	*/
	int
	vcount(struct vnode *vp)
	{
	int count;

	dev_lock();
	count = vp->v_rdev->si_usecount;
	dev_unlock();
	return (count);
	}

	/*
	* Print out a description of a vnode.
	*/
	static const char * const typename[] =
	{"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",
	"VMARKER"};

	_Static_assert((VHOLD_ALL_FLAGS & ~VHOLD_NO_SMR) == 0,
	"new hold count flag not added to vn_printf");

	void
	vn_printf(struct vnode vp, const char fmt, ...)
	{
	va_list ap;
	char buf[256], buf2[16];
	u_long flags;
	u_int holdcnt;

	va_start(ap, fmt);
	vprintf(fmt, ap);
	va_end(ap);
	printf("%p: ", (void *)vp);
	printf("type %s\n", typename[vp->v_type]);
	holdcnt = atomic_load_int(&vp->v_holdcnt);
	printf(" usecount %d, writecount %d, refcount %d seqc users %d",
	vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS,
	vp->v_seqc_users);
	switch (vp->v_type) {
	case VDIR:
	printf(" mountedhere %p\n", vp->v_mountedhere);
	break;
	case VCHR:
	printf(" rdev %p\n", vp->v_rdev);
	break;
	case VSOCK:
	printf(" socket %p\n", vp->v_unpcb);
	break;
	case VFIFO:
	printf(" fifoinfo %p\n", vp->v_fifoinfo);
	break;
	default:
	printf("\n");
	break;
	}
	buf[0] = '\0';
	buf[1] = '\0';
	if (holdcnt & VHOLD_NO_SMR)
	strlcat(buf, "\|VHOLD_NO_SMR", sizeof(buf));
	printf(" hold count flags (%s)\n", buf + 1);

	buf[0] = '\0';
	buf[1] = '\0';
	if (vp->v_irflag & VIRF_DOOMED)
	strlcat(buf, "\|VIRF_DOOMED", sizeof(buf));
	flags = vp->v_irflag & ~(VIRF_DOOMED);
	if (flags != 0) {
	snprintf(buf2, sizeof(buf2), "\|VIRF(0x%lx)", flags);
	strlcat(buf, buf2, sizeof(buf));
	}
	if (vp->v_vflag & VV_ROOT)
	strlcat(buf, "\|VV_ROOT", sizeof(buf));
	if (vp->v_vflag & VV_ISTTY)
	strlcat(buf, "\|VV_ISTTY", sizeof(buf));
	if (vp->v_vflag & VV_NOSYNC)
	strlcat(buf, "\|VV_NOSYNC", sizeof(buf));
	if (vp->v_vflag & VV_ETERNALDEV)
	strlcat(buf, "\|VV_ETERNALDEV", sizeof(buf));
	if (vp->v_vflag & VV_CACHEDLABEL)
	strlcat(buf, "\|VV_CACHEDLABEL", sizeof(buf));
	if (vp->v_vflag & VV_VMSIZEVNLOCK)
	strlcat(buf, "\|VV_VMSIZEVNLOCK", sizeof(buf));
	if (vp->v_vflag & VV_COPYONWRITE)
	strlcat(buf, "\|VV_COPYONWRITE", sizeof(buf));
	if (vp->v_vflag & VV_SYSTEM)
	strlcat(buf, "\|VV_SYSTEM", sizeof(buf));
	if (vp->v_vflag & VV_PROCDEP)
	strlcat(buf, "\|VV_PROCDEP", sizeof(buf));
	if (vp->v_vflag & VV_NOKNOTE)
	strlcat(buf, "\|VV_NOKNOTE", sizeof(buf));
	if (vp->v_vflag & VV_DELETED)
	strlcat(buf, "\|VV_DELETED", sizeof(buf));
	if (vp->v_vflag & VV_MD)
	strlcat(buf, "\|VV_MD", sizeof(buf));
	if (vp->v_vflag & VV_FORCEINSMQ)
	strlcat(buf, "\|VV_FORCEINSMQ", sizeof(buf));
	if (vp->v_vflag & VV_READLINK)
	strlcat(buf, "\|VV_READLINK", sizeof(buf));
	flags = vp->v_vflag & ~(VV_ROOT \| VV_ISTTY \| VV_NOSYNC \| VV_ETERNALDEV \|
	VV_CACHEDLABEL \| VV_COPYONWRITE \| VV_SYSTEM \| VV_PROCDEP \|
	VV_NOKNOTE \| VV_DELETED \| VV_MD \| VV_FORCEINSMQ);
	if (flags != 0) {
	snprintf(buf2, sizeof(buf2), "\|VV(0x%lx)", flags);
	strlcat(buf, buf2, sizeof(buf));
	}
	if (vp->v_iflag & VI_TEXT_REF)
	strlcat(buf, "\|VI_TEXT_REF", sizeof(buf));
	if (vp->v_iflag & VI_MOUNT)
	strlcat(buf, "\|VI_MOUNT", sizeof(buf));
	if (vp->v_iflag & VI_DOINGINACT)
	strlcat(buf, "\|VI_DOINGINACT", sizeof(buf));
	if (vp->v_iflag & VI_OWEINACT)
	strlcat(buf, "\|VI_OWEINACT", sizeof(buf));
	if (vp->v_iflag & VI_DEFINACT)
	strlcat(buf, "\|VI_DEFINACT", sizeof(buf));
	flags = vp->v_iflag & ~(VI_TEXT_REF \| VI_MOUNT \| VI_DOINGINACT \|
	VI_OWEINACT \| VI_DEFINACT);
	if (flags != 0) {
	snprintf(buf2, sizeof(buf2), "\|VI(0x%lx)", flags);
	strlcat(buf, buf2, sizeof(buf));
	}
	if (vp->v_mflag & VMP_LAZYLIST)
	strlcat(buf, "\|VMP_LAZYLIST", sizeof(buf));
	flags = vp->v_mflag & ~(VMP_LAZYLIST);
	if (flags != 0) {
	snprintf(buf2, sizeof(buf2), "\|VMP(0x%lx)", flags);
	strlcat(buf, buf2, sizeof(buf));
	}
	printf(" flags (%s)\n", buf + 1);
	if (mtx_owned(VI_MTX(vp)))
	printf(" VI_LOCKed");
	if (vp->v_object != NULL)
	printf(" v_object %p ref %d pages %d "
	"cleanbuf %d dirtybuf %d\n",
	vp->v_object, vp->v_object->ref_count,
	vp->v_object->resident_page_count,
	vp->v_bufobj.bo_clean.bv_cnt,
	vp->v_bufobj.bo_dirty.bv_cnt);
	printf(" ");
	lockmgr_printinfo(vp->v_vnlock);
	if (vp->v_data != NULL)
	VOP_PRINT(vp);
	}

	#ifdef DDB
	/*
	* List all of the locked vnodes in the system.
	* Called when debugging the kernel.
	*/
	DB_SHOW_COMMAND(lockedvnods, lockedvnodes)
	{
	struct mount *mp;
	struct vnode *vp;

	/*
	* Note: because this is DDB, we can't obey the locking semantics
	* for these structures, which means we could catch an inconsistent
	* state and dereference a nasty pointer. Not much to be done
	* about that.
	*/
	db_printf("Locked vnodes\n");
	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
	if (vp->v_type != VMARKER && VOP_ISLOCKED(vp))
	vn_printf(vp, "vnode ");
	}
	}
	}

	/*
	* Show details about the given vnode.
	*/
	DB_SHOW_COMMAND(vnode, db_show_vnode)
	{
	struct vnode *vp;

	if (!have_addr)
	return;
	vp = (struct vnode *)addr;
	vn_printf(vp, "vnode ");
	}

	/*
	* Show details about the given mount point.
	*/
	DB_SHOW_COMMAND(mount, db_show_mount)
	{
	struct mount *mp;
	struct vfsopt *opt;
	struct statfs *sp;
	struct vnode *vp;
	char buf[512];
	uint64_t mflags;
	u_int flags;

	if (!have_addr) {
	/* No address given, print short info about all mount points. */
	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
	db_printf("%p %s on %s (%s)\n", mp,
	mp->mnt_stat.f_mntfromname,
	mp->mnt_stat.f_mntonname,
	mp->mnt_stat.f_fstypename);
	if (db_pager_quit)
	break;
	}
	db_printf("\nMore info: show mount <addr>\n");
	return;
	}

	mp = (struct mount *)addr;
	db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname,
	mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename);

	buf[0] = '\0';
	mflags = mp->mnt_flag;
	#define MNT_FLAG(flag) do { \
	if (mflags & (flag)) { \
	if (buf[0] != '\0') \
	strlcat(buf, ", ", sizeof(buf)); \
	strlcat(buf, (#flag) + 4, sizeof(buf)); \
	mflags &= ~(flag); \
	} \
	} while (0)
	MNT_FLAG(MNT_RDONLY);
	MNT_FLAG(MNT_SYNCHRONOUS);
	MNT_FLAG(MNT_NOEXEC);
	MNT_FLAG(MNT_NOSUID);
	MNT_FLAG(MNT_NFS4ACLS);
	MNT_FLAG(MNT_UNION);
	MNT_FLAG(MNT_ASYNC);
	MNT_FLAG(MNT_SUIDDIR);
	MNT_FLAG(MNT_SOFTDEP);
	MNT_FLAG(MNT_NOSYMFOLLOW);
	MNT_FLAG(MNT_GJOURNAL);
	MNT_FLAG(MNT_MULTILABEL);
	MNT_FLAG(MNT_ACLS);
	MNT_FLAG(MNT_NOATIME);
	MNT_FLAG(MNT_NOCLUSTERR);
	MNT_FLAG(MNT_NOCLUSTERW);
	MNT_FLAG(MNT_SUJ);
	MNT_FLAG(MNT_EXRDONLY);
	MNT_FLAG(MNT_EXPORTED);
	MNT_FLAG(MNT_DEFEXPORTED);
	MNT_FLAG(MNT_EXPORTANON);
	MNT_FLAG(MNT_EXKERB);
	MNT_FLAG(MNT_EXPUBLIC);
	MNT_FLAG(MNT_LOCAL);
	MNT_FLAG(MNT_QUOTA);
	MNT_FLAG(MNT_ROOTFS);
	MNT_FLAG(MNT_USER);
	MNT_FLAG(MNT_IGNORE);
	MNT_FLAG(MNT_UPDATE);
	MNT_FLAG(MNT_DELEXPORT);
	MNT_FLAG(MNT_RELOAD);
	MNT_FLAG(MNT_FORCE);
	MNT_FLAG(MNT_SNAPSHOT);
	MNT_FLAG(MNT_BYFSID);
	#undef MNT_FLAG
	if (mflags != 0) {
	if (buf[0] != '\0')
	strlcat(buf, ", ", sizeof(buf));
	snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
	"0x%016jx", mflags);
	}
	db_printf(" mnt_flag = %s\n", buf);

	buf[0] = '\0';
	flags = mp->mnt_kern_flag;
	#define MNT_KERN_FLAG(flag) do { \
	if (flags & (flag)) { \
	if (buf[0] != '\0') \
	strlcat(buf, ", ", sizeof(buf)); \
	strlcat(buf, (#flag) + 5, sizeof(buf)); \
	flags &= ~(flag); \
	} \
	} while (0)
	MNT_KERN_FLAG(MNTK_UNMOUNTF);
	MNT_KERN_FLAG(MNTK_ASYNC);
	MNT_KERN_FLAG(MNTK_SOFTDEP);
	MNT_KERN_FLAG(MNTK_DRAINING);
	MNT_KERN_FLAG(MNTK_REFEXPIRE);
	MNT_KERN_FLAG(MNTK_EXTENDED_SHARED);
	MNT_KERN_FLAG(MNTK_SHARED_WRITES);
	MNT_KERN_FLAG(MNTK_NO_IOPF);
	MNT_KERN_FLAG(MNTK_VGONE_UPPER);
	MNT_KERN_FLAG(MNTK_VGONE_WAITER);
	MNT_KERN_FLAG(MNTK_LOOKUP_EXCL_DOTDOT);
	MNT_KERN_FLAG(MNTK_MARKER);
	MNT_KERN_FLAG(MNTK_USES_BCACHE);
	MNT_KERN_FLAG(MNTK_FPLOOKUP);
	MNT_KERN_FLAG(MNTK_NOASYNC);
	MNT_KERN_FLAG(MNTK_UNMOUNT);
	MNT_KERN_FLAG(MNTK_MWAIT);
	MNT_KERN_FLAG(MNTK_SUSPEND);
	MNT_KERN_FLAG(MNTK_SUSPEND2);
	MNT_KERN_FLAG(MNTK_SUSPENDED);
	MNT_KERN_FLAG(MNTK_LOOKUP_SHARED);
	MNT_KERN_FLAG(MNTK_NOKNOTE);
	#undef MNT_KERN_FLAG
	if (flags != 0) {
	if (buf[0] != '\0')
	strlcat(buf, ", ", sizeof(buf));
	snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
	"0x%08x", flags);
	}
	db_printf(" mnt_kern_flag = %s\n", buf);

	db_printf(" mnt_opt = ");
	opt = TAILQ_FIRST(mp->mnt_opt);
	if (opt != NULL) {
	db_printf("%s", opt->name);
	opt = TAILQ_NEXT(opt, link);
	while (opt != NULL) {
	db_printf(", %s", opt->name);
	opt = TAILQ_NEXT(opt, link);
	}
	}
	db_printf("\n");

	sp = &mp->mnt_stat;
	db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx "
	"bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju "
	"ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju "
	"asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n",
	(u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags,
	(uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize,
	(uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree,
	(intmax_t)sp->f_bavail, (uintmax_t)sp->f_files,
	(intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites,
	(uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads,
	(uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax,
	(u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]);

	db_printf(" mnt_cred = { uid=%u ruid=%u",
	(u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid);
	if (jailed(mp->mnt_cred))
	db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id);
	db_printf(" }\n");
	db_printf(" mnt_ref = %d (with %d in the struct)\n",
	vfs_mount_fetch_counter(mp, MNT_COUNT_REF), mp->mnt_ref);
	db_printf(" mnt_gen = %d\n", mp->mnt_gen);
	db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize);
	db_printf(" mnt_lazyvnodelistsize = %d\n",
	mp->mnt_lazyvnodelistsize);
	db_printf(" mnt_writeopcount = %d (with %d in the struct)\n",
	vfs_mount_fetch_counter(mp, MNT_COUNT_WRITEOPCOUNT), mp->mnt_writeopcount);
	db_printf(" mnt_maxsymlinklen = %d\n", mp->mnt_maxsymlinklen);
	db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max);
	db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed);
	db_printf(" mnt_lockref = %d (with %d in the struct)\n",
	vfs_mount_fetch_counter(mp, MNT_COUNT_LOCKREF), mp->mnt_lockref);
	db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes);
	db_printf(" mnt_secondary_accwrites = %d\n",
	mp->mnt_secondary_accwrites);
	db_printf(" mnt_gjprovider = %s\n",
	mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL");
	db_printf(" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops);

	db_printf("\n\nList of active vnodes\n");
	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
	if (vp->v_type != VMARKER && vp->v_holdcnt > 0) {
	vn_printf(vp, "vnode ");
	if (db_pager_quit)
	break;
	}
	}
	db_printf("\n\nList of inactive vnodes\n");
	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
	if (vp->v_type != VMARKER && vp->v_holdcnt == 0) {
	vn_printf(vp, "vnode ");
	if (db_pager_quit)
	break;
	}
	}
	}
	#endif /* DDB */

	/*
	* Fill in a struct xvfsconf based on a struct vfsconf.
	*/
	static int
	vfsconf2x(struct sysctl_req req, struct vfsconf vfsp)
	{
	struct xvfsconf xvfsp;

	bzero(&xvfsp, sizeof(xvfsp));
	strcpy(xvfsp.vfc_name, vfsp->vfc_name);
	xvfsp.vfc_typenum = vfsp->vfc_typenum;
	xvfsp.vfc_refcount = vfsp->vfc_refcount;
	xvfsp.vfc_flags = vfsp->vfc_flags;
	/*
	* These are unused in userland, we keep them
	* to not break binary compatibility.
	*/
	xvfsp.vfc_vfsops = NULL;
	xvfsp.vfc_next = NULL;
	return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
	}

	#ifdef COMPAT_FREEBSD32
	struct xvfsconf32 {
	uint32_t vfc_vfsops;
	char vfc_name[MFSNAMELEN];
	int32_t vfc_typenum;
	int32_t vfc_refcount;
	int32_t vfc_flags;
	uint32_t vfc_next;
	};

	static int
	vfsconf2x32(struct sysctl_req req, struct vfsconf vfsp)
	{
	struct xvfsconf32 xvfsp;

	bzero(&xvfsp, sizeof(xvfsp));
	strcpy(xvfsp.vfc_name, vfsp->vfc_name);
	xvfsp.vfc_typenum = vfsp->vfc_typenum;
	xvfsp.vfc_refcount = vfsp->vfc_refcount;
	xvfsp.vfc_flags = vfsp->vfc_flags;
	return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
	}
	#endif

	/*
	* Top level filesystem related information gathering.
	*/
	static int
	sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
	{
	struct vfsconf *vfsp;
	int error;

	error = 0;
	vfsconf_slock();
	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
	#ifdef COMPAT_FREEBSD32
	if (req->flags & SCTL_MASK32)
	error = vfsconf2x32(req, vfsp);
	else
	#endif
	error = vfsconf2x(req, vfsp);
	if (error)
	break;
	}
	vfsconf_sunlock();
	return (error);
	}

	SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE \| CTLFLAG_RD \|
	CTLFLAG_MPSAFE, NULL, 0, sysctl_vfs_conflist,
	"S,xvfsconf", "List of all configured filesystems");

	#ifndef BURN_BRIDGES
	static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS);

	static int
	vfs_sysctl(SYSCTL_HANDLER_ARGS)
	{
	int name = (int )arg1 - 1; /* XXX */
	u_int namelen = arg2 + 1; /* XXX */
	struct vfsconf *vfsp;

	log(LOG_WARNING, "userland calling deprecated sysctl, "
	"please rebuild world\n");

	#if 1 \|\| defined(COMPAT_PRELITE2)
	/* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */
	if (namelen == 1)
	return (sysctl_ovfs_conf(oidp, arg1, arg2, req));
	#endif

	switch (name[1]) {
	case VFS_MAXTYPENUM:
	if (namelen != 2)
	return (ENOTDIR);
	return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int)));
	case VFS_CONF:
	if (namelen != 3)
	return (ENOTDIR); /* overloaded */
	vfsconf_slock();
	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
	if (vfsp->vfc_typenum == name[2])
	break;
	}
	vfsconf_sunlock();
	if (vfsp == NULL)
	return (EOPNOTSUPP);
	#ifdef COMPAT_FREEBSD32
	if (req->flags & SCTL_MASK32)
	return (vfsconf2x32(req, vfsp));
	else
	#endif
	return (vfsconf2x(req, vfsp));
	}
	return (EOPNOTSUPP);
	}

	static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD \| CTLFLAG_SKIP \|
	CTLFLAG_MPSAFE, vfs_sysctl,
	"Generic filesystem");

	#if 1 \|\| defined(COMPAT_PRELITE2)

	static int
	sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
	{
	int error;
	struct vfsconf *vfsp;
	struct ovfsconf ovfs;

	vfsconf_slock();
	TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
	bzero(&ovfs, sizeof(ovfs));
	ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */
	strcpy(ovfs.vfc_name, vfsp->vfc_name);
	ovfs.vfc_index = vfsp->vfc_typenum;
	ovfs.vfc_refcount = vfsp->vfc_refcount;
	ovfs.vfc_flags = vfsp->vfc_flags;
	error = SYSCTL_OUT(req, &ovfs, sizeof ovfs);
	if (error != 0) {
	vfsconf_sunlock();
	return (error);
	}
	}
	vfsconf_sunlock();
	return (0);
	}

	#endif /* 1 \|\| COMPAT_PRELITE2 */
	#endif /* !BURN_BRIDGES */

	#define KINFO_VNODESLOP 10
	#ifdef notyet
	/*
	* Dump vnode list (via sysctl).
	*/
	/* ARGSUSED */
	static int
	sysctl_vnode(SYSCTL_HANDLER_ARGS)
	{
	struct xvnode *xvn;
	struct mount *mp;
	struct vnode *vp;
	int error, len, n;

	/*
	* Stale numvnodes access is not fatal here.
	*/
	req->lock = 0;
	len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn;
	if (!req->oldptr)
	/* Make an estimate */
	return (SYSCTL_OUT(req, 0, len));

	error = sysctl_wire_old_buffer(req, 0);
	if (error != 0)
	return (error);
	xvn = malloc(len, M_TEMP, M_ZERO \| M_WAITOK);
	n = 0;
	mtx_lock(&mountlist_mtx);
	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
	if (vfs_busy(mp, MBF_NOWAIT \| MBF_MNTLSTLOCK))
	continue;
	MNT_ILOCK(mp);
	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
	if (n == len)
	break;
	vref(vp);
	xvn[n].xv_size = sizeof *xvn;
	xvn[n].xv_vnode = vp;
	xvn[n].xv_id = 0; /* XXX compat */
	#define XV_COPY(field) xvn[n].xv_##field = vp->v_##field
	XV_COPY(usecount);
	XV_COPY(writecount);
	XV_COPY(holdcnt);
	XV_COPY(mount);
	XV_COPY(numoutput);
	XV_COPY(type);
	#undef XV_COPY
	xvn[n].xv_flag = vp->v_vflag;

	switch (vp->v_type) {
	case VREG:
	case VDIR:
	case VLNK:
	break;
	case VBLK:
	case VCHR:
	if (vp->v_rdev == NULL) {
	vrele(vp);
	continue;
	}
	xvn[n].xv_dev = dev2udev(vp->v_rdev);
	break;
	case VSOCK:
	xvn[n].xv_socket = vp->v_socket;
	break;
	case VFIFO:
	xvn[n].xv_fifo = vp->v_fifoinfo;
	break;
	case VNON:
	case VBAD:
	default:
	/* shouldn't happen? */
	vrele(vp);
	continue;
	}
	vrele(vp);
	++n;
	}
	MNT_IUNLOCK(mp);
	mtx_lock(&mountlist_mtx);
	vfs_unbusy(mp);
	if (n == len)
	break;
	}
	mtx_unlock(&mountlist_mtx);

	error = SYSCTL_OUT(req, xvn, n * sizeof *xvn);
	free(xvn, M_TEMP);
	return (error);
	}

	SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE \| CTLFLAG_RD \|
	CTLFLAG_MPSAFE, 0, 0, sysctl_vnode, "S,xvnode",
	"");
	#endif

	static void
	unmount_or_warn(struct mount *mp)
	{
	int error;

	error = dounmount(mp, MNT_FORCE, curthread);
	if (error != 0) {
	printf("unmount of %s failed (", mp->mnt_stat.f_mntonname);
	if (error == EBUSY)
	printf("BUSY)\n");
	else
	printf("%d)\n", error);
	}
	}

	/*
	* Unmount all filesystems. The list is traversed in reverse order
	* of mounting to avoid dependencies.
	*/
	void
	vfs_unmountall(void)
	{
	struct mount mp, tmp;

	CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__);

	/*
	* Since this only runs when rebooting, it is not interlocked.
	*/
	TAILQ_FOREACH_REVERSE_SAFE(mp, &mountlist, mntlist, mnt_list, tmp) {
	vfs_ref(mp);

	/*
	* Forcibly unmounting "/dev" before "/" would prevent clean
	* unmount of the latter.
	*/
	if (mp == rootdevmp)
	continue;

	unmount_or_warn(mp);
	}

	if (rootdevmp != NULL)
	unmount_or_warn(rootdevmp);
	}

	static void
	vfs_deferred_inactive(struct vnode *vp, int lkflags)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp, ("VI_DEFINACT still set"));
	if ((vp->v_iflag & VI_OWEINACT) == 0) {
	vdropl(vp);
	return;
	}
	if (vn_lock(vp, lkflags) == 0) {
	VI_LOCK(vp);
	vinactive(vp);
	VOP_UNLOCK(vp);
	vdropl(vp);
	return;
	}
	vdefer_inactive_unlocked(vp);
	}

	static int
	vfs_periodic_inactive_filter(struct vnode vp, void arg)
	{

	return (vp->v_iflag & VI_DEFINACT);
	}

	static void __noinline
	vfs_periodic_inactive(struct mount *mp, int flags)
	{
	struct vnode vp, mvp;
	int lkflags;

	lkflags = LK_EXCLUSIVE \| LK_INTERLOCK;
	if (flags != MNT_WAIT)
	lkflags \|= LK_NOWAIT;

	MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_inactive_filter, NULL) {
	if ((vp->v_iflag & VI_DEFINACT) == 0) {
	VI_UNLOCK(vp);
	continue;
	}
	vp->v_iflag &= ~VI_DEFINACT;
	vfs_deferred_inactive(vp, lkflags);
	}
	}

	static inline bool
	vfs_want_msync(struct vnode *vp)
	{
	struct vm_object *obj;

	/*
	* This test may be performed without any locks held.
	* We rely on vm_object's type stability.
	*/
	if (vp->v_vflag & VV_NOSYNC)
	return (false);
	obj = vp->v_object;
	return (obj != NULL && vm_object_mightbedirty(obj));
	}

	static int
	vfs_periodic_msync_inactive_filter(struct vnode vp, void arg __unused)
	{

	if (vp->v_vflag & VV_NOSYNC)
	return (false);
	if (vp->v_iflag & VI_DEFINACT)
	return (true);
	return (vfs_want_msync(vp));
	}

	static void __noinline
	vfs_periodic_msync_inactive(struct mount *mp, int flags)
	{
	struct vnode vp, mvp;
	struct vm_object *obj;
	struct thread *td;
	int lkflags, objflags;
	bool seen_defer;

	td = curthread;

	lkflags = LK_EXCLUSIVE \| LK_INTERLOCK;
	if (flags != MNT_WAIT) {
	lkflags \|= LK_NOWAIT;
	objflags = OBJPC_NOSYNC;
	} else {
	objflags = OBJPC_SYNC;
	}

	MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, vfs_periodic_msync_inactive_filter, NULL) {
	seen_defer = false;
	if (vp->v_iflag & VI_DEFINACT) {
	vp->v_iflag &= ~VI_DEFINACT;
	seen_defer = true;
	}
	if (!vfs_want_msync(vp)) {
	if (seen_defer)
	vfs_deferred_inactive(vp, lkflags);
	else
	VI_UNLOCK(vp);
	continue;
	}
	if (vget(vp, lkflags, td) == 0) {
	obj = vp->v_object;
	if (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0) {
	VM_OBJECT_WLOCK(obj);
	vm_object_page_clean(obj, 0, 0, objflags);
	VM_OBJECT_WUNLOCK(obj);
	}
	vput(vp);
	if (seen_defer)
	vdrop(vp);
	} else {
	if (seen_defer)
	vdefer_inactive_unlocked(vp);
	}
	}
	}

	void
	vfs_periodic(struct mount *mp, int flags)
	{

	CTR2(KTR_VFS, "%s: mp %p", __func__, mp);

	if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0)
	vfs_periodic_inactive(mp, flags);
	else
	vfs_periodic_msync_inactive(mp, flags);
	}

	static void
	destroy_vpollinfo_free(struct vpollinfo *vi)
	{

	knlist_destroy(&vi->vpi_selinfo.si_note);
	mtx_destroy(&vi->vpi_lock);
	uma_zfree(vnodepoll_zone, vi);
	}

	static void
	destroy_vpollinfo(struct vpollinfo *vi)
	{

	knlist_clear(&vi->vpi_selinfo.si_note, 1);
	seldrain(&vi->vpi_selinfo);
	destroy_vpollinfo_free(vi);
	}

	/*
	* Initialize per-vnode helper structure to hold poll-related state.
	*/
	void
	v_addpollinfo(struct vnode *vp)
	{
	struct vpollinfo *vi;

	if (vp->v_pollinfo != NULL)
	return;
	vi = uma_zalloc(vnodepoll_zone, M_WAITOK \| M_ZERO);
	mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF);
	knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock,
	vfs_knlunlock, vfs_knl_assert_locked, vfs_knl_assert_unlocked);
	VI_LOCK(vp);
	if (vp->v_pollinfo != NULL) {
	VI_UNLOCK(vp);
	destroy_vpollinfo_free(vi);
	return;
	}
	vp->v_pollinfo = vi;
	VI_UNLOCK(vp);
	}

	/*
	* Record a process's interest in events which might happen to
	* a vnode. Because poll uses the historic select-style interface
	* internally, this routine serves as both the ``check for any
	* pending events'' and the ``record my interest in future events''
	* functions. (These are done together, while the lock is held,
	* to avoid race conditions.)
	*/
	int
	vn_pollrecord(struct vnode vp, struct thread td, int events)
	{

	v_addpollinfo(vp);
	mtx_lock(&vp->v_pollinfo->vpi_lock);
	if (vp->v_pollinfo->vpi_revents & events) {
	/*
	* This leaves events we are not interested
	* in available for the other process which
	* which presumably had requested them
	* (otherwise they would never have been
	* recorded).
	*/
	events &= vp->v_pollinfo->vpi_revents;
	vp->v_pollinfo->vpi_revents &= ~events;

	mtx_unlock(&vp->v_pollinfo->vpi_lock);
	return (events);
	}
	vp->v_pollinfo->vpi_events \|= events;
	selrecord(td, &vp->v_pollinfo->vpi_selinfo);
	mtx_unlock(&vp->v_pollinfo->vpi_lock);
	return (0);
	}

	/*
	* Routine to create and manage a filesystem syncer vnode.
	*/
	#define sync_close ((int ()(struct vop_close_args ))nullop)
	static int sync_fsync(struct vop_fsync_args *);
	static int sync_inactive(struct vop_inactive_args *);
	static int sync_reclaim(struct vop_reclaim_args *);

	static struct vop_vector sync_vnodeops = {
	.vop_bypass = VOP_EOPNOTSUPP,
	.vop_close = sync_close, /* close */
	.vop_fsync = sync_fsync, /* fsync */
	.vop_inactive = sync_inactive, /* inactive */
	.vop_need_inactive = vop_stdneed_inactive, /* need_inactive */
	.vop_reclaim = sync_reclaim, /* reclaim */
	.vop_lock1 = vop_stdlock, /* lock */
	.vop_unlock = vop_stdunlock, /* unlock */
	.vop_islocked = vop_stdislocked, /* islocked */
	};
	VFS_VOP_VECTOR_REGISTER(sync_vnodeops);

	/*
	* Create a new filesystem syncer vnode for the specified mount point.
	*/
	void
	vfs_allocate_syncvnode(struct mount *mp)
	{
	struct vnode *vp;
	struct bufobj *bo;
	static long start, incr, next;
	int error;

	/* Allocate a new vnode */
	error = getnewvnode("syncer", mp, &sync_vnodeops, &vp);
	if (error != 0)
	panic("vfs_allocate_syncvnode: getnewvnode() failed");
	vp->v_type = VNON;
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	vp->v_vflag \|= VV_FORCEINSMQ;
	error = insmntque(vp, mp);
	if (error != 0)
	panic("vfs_allocate_syncvnode: insmntque() failed");
	vp->v_vflag &= ~VV_FORCEINSMQ;
	VOP_UNLOCK(vp);
	/*
	* Place the vnode onto the syncer worklist. We attempt to
	* scatter them about on the list so that they will go off
	* at evenly distributed times even if all the filesystems
	* are mounted at once.
	*/
	next += incr;
	if (next == 0 \|\| next > syncer_maxdelay) {
	start /= 2;
	incr /= 2;
	if (start == 0) {
	start = syncer_maxdelay / 2;
	incr = syncer_maxdelay;
	}
	next = start;
	}
	bo = &vp->v_bufobj;
	BO_LOCK(bo);
	vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0);
	/* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */
	mtx_lock(&sync_mtx);
	sync_vnode_count++;
	if (mp->mnt_syncer == NULL) {
	mp->mnt_syncer = vp;
	vp = NULL;
	}
	mtx_unlock(&sync_mtx);
	BO_UNLOCK(bo);
	if (vp != NULL) {
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	vgone(vp);
	vput(vp);
	}
	}

	void
	vfs_deallocate_syncvnode(struct mount *mp)
	{
	struct vnode *vp;

	mtx_lock(&sync_mtx);
	vp = mp->mnt_syncer;
	if (vp != NULL)
	mp->mnt_syncer = NULL;
	mtx_unlock(&sync_mtx);
	if (vp != NULL)
	vrele(vp);
	}

	/*
	* Do a lazy sync of the filesystem.
	*/
	static int
	sync_fsync(struct vop_fsync_args *ap)
	{
	struct vnode *syncvp = ap->a_vp;
	struct mount *mp = syncvp->v_mount;
	int error, save;
	struct bufobj *bo;

	/*
	* We only need to do something if this is a lazy evaluation.
	*/
	if (ap->a_waitfor != MNT_LAZY)
	return (0);

	/*
	* Move ourselves to the back of the sync list.
	*/
	bo = &syncvp->v_bufobj;
	BO_LOCK(bo);
	vn_syncer_add_to_worklist(bo, syncdelay);
	BO_UNLOCK(bo);

	/*
	* Walk the list of vnodes pushing all that are dirty and
	* not already on the sync list.
	*/
	if (vfs_busy(mp, MBF_NOWAIT) != 0)
	return (0);
	if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) {
	vfs_unbusy(mp);
	return (0);
	}
	save = curthread_pflags_set(TDP_SYNCIO);
	/*
	* The filesystem at hand may be idle with free vnodes stored in the
	* batch. Return them instead of letting them stay there indefinitely.
	*/
	vfs_periodic(mp, MNT_NOWAIT);
	error = VFS_SYNC(mp, MNT_LAZY);
	curthread_pflags_restore(save);
	vn_finished_write(mp);
	vfs_unbusy(mp);
	return (error);
	}

	/*
	* The syncer vnode is no referenced.
	*/
	static int
	sync_inactive(struct vop_inactive_args *ap)
	{

	vgone(ap->a_vp);
	return (0);
	}

	/*
	* The syncer vnode is no longer needed and is being decommissioned.
	*
	* Modifications to the worklist must be protected by sync_mtx.
	*/
	static int
	sync_reclaim(struct vop_reclaim_args *ap)
	{
	struct vnode *vp = ap->a_vp;
	struct bufobj *bo;

	bo = &vp->v_bufobj;
	BO_LOCK(bo);
	mtx_lock(&sync_mtx);
	if (vp->v_mount->mnt_syncer == vp)
	vp->v_mount->mnt_syncer = NULL;
	if (bo->bo_flag & BO_ONWORKLST) {
	LIST_REMOVE(bo, bo_synclist);
	syncer_worklist_len--;
	sync_vnode_count--;
	bo->bo_flag &= ~BO_ONWORKLST;
	}
	mtx_unlock(&sync_mtx);
	BO_UNLOCK(bo);

	return (0);
	}

	int
	vn_need_pageq_flush(struct vnode *vp)
	{
	struct vm_object *obj;
	int need;

	MPASS(mtx_owned(VI_MTX(vp)));
	need = 0;
	if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
	vm_object_mightbedirty(obj))
	need = 1;
	return (need);
	}

	/*
	* Check if vnode represents a disk device
	*/
	int
	vn_isdisk(struct vnode vp, int errp)
	{
	int error;

	if (vp->v_type != VCHR) {
	error = ENOTBLK;
	goto out;
	}
	error = 0;
	dev_lock();
	if (vp->v_rdev == NULL)
	error = ENXIO;
	else if (vp->v_rdev->si_devsw == NULL)
	error = ENXIO;
	else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK))
	error = ENOTBLK;
	dev_unlock();
	out:
	if (errp != NULL)
	*errp = error;
	return (error == 0);
	}

	/*
	* VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see
	* the comment above cache_fplookup for details.
	*
	* We never deny as priv_check_cred calls are not yet supported, see vaccess.
	*/
	int
	vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred)
	{

	VFS_SMR_ASSERT_ENTERED();

	/* Check the owner. */
	if (cred->cr_uid == file_uid) {
	if (file_mode & S_IXUSR)
	return (0);
	return (EAGAIN);
	}

	/* Otherwise, check the groups (first match) */
	if (groupmember(file_gid, cred)) {
	if (file_mode & S_IXGRP)
	return (0);
	return (EAGAIN);
	}

	/* Otherwise, check everyone else. */
	if (file_mode & S_IXOTH)
	return (0);
	return (EAGAIN);
	}

	/*
	* Common filesystem object access control check routine. Accepts a
	* vnode's type, "mode", uid and gid, requested access mode, credentials,
	* and optional call-by-reference privused argument allowing vaccess()
	* to indicate to the caller whether privilege was used to satisfy the
	* request (obsoleted). Returns 0 on success, or an errno on failure.
	*/
	int
	vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid,
	accmode_t accmode, struct ucred cred, int privused)
	{
	accmode_t dac_granted;
	accmode_t priv_granted;

	KASSERT((accmode & ~(VEXEC \| VWRITE \| VREAD \| VADMIN \| VAPPEND)) == 0,
	("invalid bit in accmode"));
	KASSERT((accmode & VAPPEND) == 0 \|\| (accmode & VWRITE),
	("VAPPEND without VWRITE"));

	/*
	* Look for a normal, non-privileged way to access the file/directory
	* as requested. If it exists, go with that.
	*/

	if (privused != NULL)
	*privused = 0;

	dac_granted = 0;

	/* Check the owner. */
	if (cred->cr_uid == file_uid) {
	dac_granted \|= VADMIN;
	if (file_mode & S_IXUSR)
	dac_granted \|= VEXEC;
	if (file_mode & S_IRUSR)
	dac_granted \|= VREAD;
	if (file_mode & S_IWUSR)
	dac_granted \|= (VWRITE \| VAPPEND);

	if ((accmode & dac_granted) == accmode)
	return (0);

	goto privcheck;
	}

	/* Otherwise, check the groups (first match) */
	if (groupmember(file_gid, cred)) {
	if (file_mode & S_IXGRP)
	dac_granted \|= VEXEC;
	if (file_mode & S_IRGRP)
	dac_granted \|= VREAD;
	if (file_mode & S_IWGRP)
	dac_granted \|= (VWRITE \| VAPPEND);

	if ((accmode & dac_granted) == accmode)
	return (0);

	goto privcheck;
	}

	/* Otherwise, check everyone else. */
	if (file_mode & S_IXOTH)
	dac_granted \|= VEXEC;
	if (file_mode & S_IROTH)
	dac_granted \|= VREAD;
	if (file_mode & S_IWOTH)
	dac_granted \|= (VWRITE \| VAPPEND);
	if ((accmode & dac_granted) == accmode)
	return (0);

	privcheck:
	/*
	* Build a privilege mask to determine if the set of privileges
	* satisfies the requirements when combined with the granted mask
	* from above. For each privilege, if the privilege is required,
	* bitwise or the request type onto the priv_granted mask.
	*/
	priv_granted = 0;

	if (type == VDIR) {
	/*
	* For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC
	* requests, instead of PRIV_VFS_EXEC.
	*/
	if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
	!priv_check_cred(cred, PRIV_VFS_LOOKUP))
	priv_granted \|= VEXEC;
	} else {
	/*
	* Ensure that at least one execute bit is on. Otherwise,
	* a privileged user will always succeed, and we don't want
	* this to happen unless the file really is executable.
	*/
	if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
	(file_mode & (S_IXUSR \| S_IXGRP \| S_IXOTH)) != 0 &&
	!priv_check_cred(cred, PRIV_VFS_EXEC))
	priv_granted \|= VEXEC;
	}

	if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) &&
	!priv_check_cred(cred, PRIV_VFS_READ))
	priv_granted \|= VREAD;

	if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) &&
	!priv_check_cred(cred, PRIV_VFS_WRITE))
	priv_granted \|= (VWRITE \| VAPPEND);

	if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) &&
	!priv_check_cred(cred, PRIV_VFS_ADMIN))
	priv_granted \|= VADMIN;

	if ((accmode & (priv_granted \| dac_granted)) == accmode) {
	/* XXX audit: privilege used */
	if (privused != NULL)
	*privused = 1;
	return (0);
	}

	return ((accmode & VADMIN) ? EPERM : EACCES);
	}

	/*
	* Credential check based on process requesting service, and per-attribute
	* permissions.
	*/
	int
	extattr_check_cred(struct vnode vp, int attrnamespace, struct ucred cred,
	struct thread *td, accmode_t accmode)
	{

	/*
	* Kernel-invoked always succeeds.
	*/
	if (cred == NOCRED)
	return (0);

	/*
	* Do not allow privileged processes in jail to directly manipulate
	* system attributes.
	*/
	switch (attrnamespace) {
	case EXTATTR_NAMESPACE_SYSTEM:
	/* Potentially should be: return (EPERM); */
	return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM));
	case EXTATTR_NAMESPACE_USER:
	return (VOP_ACCESS(vp, accmode, cred, td));
	default:
	return (EPERM);
	}
	}

	#ifdef DEBUG_VFS_LOCKS
	/*
	* This only exists to suppress warnings from unlocked specfs accesses. It is
	* no longer ok to have an unlocked VFS.
	*/
	#define IGNORE_LOCK(vp) (KERNEL_PANICKED() \|\| (vp) == NULL \|\| \
	(vp)->v_type == VCHR \|\| (vp)->v_type == VBAD)

	int vfs_badlock_ddb = 1; /* Drop into debugger on violation. */
	SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0,
	"Drop into debugger on lock violation");

	int vfs_badlock_mutex = 1; /* Check for interlock across VOPs. */
	SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex,
	0, "Check for interlock across VOPs");

	int vfs_badlock_print = 1; /* Print lock violations. */
	SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print,
	0, "Print lock violations");

	int vfs_badlock_vnode = 1; /* Print vnode details on lock violations. */
	SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode,
	0, "Print vnode details on lock violations");

	#ifdef KDB
	int vfs_badlock_backtrace = 1; /* Print backtrace at lock violations. */
	SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW,
	&vfs_badlock_backtrace, 0, "Print backtrace at lock violations");
	#endif

	static void
	vfs_badlock(const char msg, const char str, struct vnode *vp)
	{

	#ifdef KDB
	if (vfs_badlock_backtrace)
	kdb_backtrace();
	#endif
	if (vfs_badlock_vnode)
	vn_printf(vp, "vnode ");
	if (vfs_badlock_print)
	printf("%s: %p %s\n", str, (void *)vp, msg);
	if (vfs_badlock_ddb)
	kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
	}

	void
	assert_vi_locked(struct vnode vp, const char str)
	{

	if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp)))
	vfs_badlock("interlock is not locked but should be", str, vp);
	}

	void
	assert_vi_unlocked(struct vnode vp, const char str)
	{

	if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp)))
	vfs_badlock("interlock is locked but should not be", str, vp);
	}

	void
	assert_vop_locked(struct vnode vp, const char str)
	{
	int locked;

	if (!IGNORE_LOCK(vp)) {
	locked = VOP_ISLOCKED(vp);
	if (locked == 0 \|\| locked == LK_EXCLOTHER)
	vfs_badlock("is not locked but should be", str, vp);
	}
	}

	void
	assert_vop_unlocked(struct vnode vp, const char str)
	{

	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
	vfs_badlock("is locked but should not be", str, vp);
	}

	void
	assert_vop_elocked(struct vnode vp, const char str)
	{

	if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
	vfs_badlock("is not exclusive locked but should be", str, vp);
	}
	#endif /* DEBUG_VFS_LOCKS */

	void
	vop_rename_fail(struct vop_rename_args *ap)
	{

	if (ap->a_tvp != NULL)
	vput(ap->a_tvp);
	if (ap->a_tdvp == ap->a_tvp)
	vrele(ap->a_tdvp);
	else
	vput(ap->a_tdvp);
	vrele(ap->a_fdvp);
	vrele(ap->a_fvp);
	}

	void
	vop_rename_pre(void *ap)
	{
	struct vop_rename_args *a = ap;

	#ifdef DEBUG_VFS_LOCKS
	if (a->a_tvp)
	ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME");
	ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME");
	ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME");
	ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME");

	/* Check the source (from). */
	if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock &&
	(a->a_tvp == NULL \|\| a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock))
	ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked");
	if (a->a_tvp == NULL \|\| a->a_tvp->v_vnlock != a->a_fvp->v_vnlock)
	ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked");

	/* Check the target. */
	if (a->a_tvp)
	ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked");
	ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked");
	#endif
	/*
	* It may be tempting to add vn_seqc_write_begin/end calls here and
	* in vop_rename_post but that's not going to work out since some
	* filesystems relookup vnodes mid-rename. This is probably a bug.
	*
	* For now filesystems are expected to do the relevant calls after they
	* decide what vnodes to operate on.
	*/
	if (a->a_tdvp != a->a_fdvp)
	vhold(a->a_fdvp);
	if (a->a_tvp != a->a_fvp)
	vhold(a->a_fvp);
	vhold(a->a_tdvp);
	if (a->a_tvp)
	vhold(a->a_tvp);
	}

	#ifdef DEBUG_VFS_LOCKS
	void
	-vop_fplookup_vexec_pre(void *ap __unused)
	+vop_fplookup_vexec_debugpre(void *ap __unused)
	{

	VFS_SMR_ASSERT_ENTERED();
	}

	void
	-vop_fplookup_vexec_post(void *ap __unused, int rc __unused)
	+vop_fplookup_vexec_debugpost(void *ap __unused, int rc __unused)
	{

	VFS_SMR_ASSERT_ENTERED();
	}

	void
	-vop_strategy_pre(void *ap)
	+vop_strategy_debugpre(void *ap)
	{
	struct vop_strategy_args *a;
	struct buf *bp;

	a = ap;
	bp = a->a_bp;

	/*
	* Cluster ops lock their component buffers but not the IO container.
	*/
	if ((bp->b_flags & B_CLUSTER) != 0)
	return;

	if (!KERNEL_PANICKED() && !BUF_ISLOCKED(bp)) {
	if (vfs_badlock_print)
	printf(
	"VOP_STRATEGY: bp is not locked but should be\n");
	if (vfs_badlock_ddb)
	kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
	}
	}

	void
	-vop_lock_pre(void *ap)
	+vop_lock_debugpre(void *ap)
	{
	struct vop_lock1_args *a = ap;

	if ((a->a_flags & LK_INTERLOCK) == 0)
	ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
	else
	ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK");
	}

	void
	-vop_lock_post(void *ap, int rc)
	+vop_lock_debugpost(void *ap, int rc)
	{
	struct vop_lock1_args *a = ap;

	ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
	if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0)
	ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK");
	}

	void
	-vop_unlock_pre(void *ap)
	+vop_unlock_debugpre(void *ap)
	{
	struct vop_unlock_args *a = ap;

	ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK");
	}

	void
	-vop_need_inactive_pre(void *ap)
	+vop_need_inactive_debugpre(void *ap)
	{
	struct vop_need_inactive_args *a = ap;

	ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE");
	}

	void
	-vop_need_inactive_post(void *ap, int rc)
	+vop_need_inactive_debugpost(void *ap, int rc)
	{
	struct vop_need_inactive_args *a = ap;

	ASSERT_VI_LOCKED(a->a_vp, "VOP_NEED_INACTIVE");
	}
	#endif

	void
	vop_create_pre(void *ap)
	{
	struct vop_create_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_begin(dvp);
	}

	void
	vop_create_post(void *ap, int rc)
	{
	struct vop_create_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_end(dvp);
	if (!rc)
	VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
	}

	void
	vop_whiteout_pre(void *ap)
	{
	struct vop_whiteout_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_begin(dvp);
	}

	void
	vop_whiteout_post(void *ap, int rc)
	{
	struct vop_whiteout_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_end(dvp);
	}

	void
	vop_deleteextattr_pre(void *ap)
	{
	struct vop_deleteextattr_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_begin(vp);
	}

	void
	vop_deleteextattr_post(void *ap, int rc)
	{
	struct vop_deleteextattr_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_end(vp);
	if (!rc)
	VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
	}

	void
	vop_link_pre(void *ap)
	{
	struct vop_link_args *a;
	struct vnode vp, tdvp;

	a = ap;
	vp = a->a_vp;
	tdvp = a->a_tdvp;
	vn_seqc_write_begin(vp);
	vn_seqc_write_begin(tdvp);
	}

	void
	vop_link_post(void *ap, int rc)
	{
	struct vop_link_args *a;
	struct vnode vp, tdvp;

	a = ap;
	vp = a->a_vp;
	tdvp = a->a_tdvp;
	vn_seqc_write_end(vp);
	vn_seqc_write_end(tdvp);
	if (!rc) {
	VFS_KNOTE_LOCKED(vp, NOTE_LINK);
	VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE);
	}
	}

	void
	vop_mkdir_pre(void *ap)
	{
	struct vop_mkdir_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_begin(dvp);
	}

	void
	vop_mkdir_post(void *ap, int rc)
	{
	struct vop_mkdir_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_end(dvp);
	if (!rc)
	VFS_KNOTE_LOCKED(dvp, NOTE_WRITE \| NOTE_LINK);
	}

	void
	vop_mknod_pre(void *ap)
	{
	struct vop_mknod_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_begin(dvp);
	}

	void
	vop_mknod_post(void *ap, int rc)
	{
	struct vop_mknod_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_end(dvp);
	if (!rc)
	VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
	}

	void
	vop_reclaim_post(void *ap, int rc)
	{
	struct vop_reclaim_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	ASSERT_VOP_IN_SEQC(vp);
	if (!rc)
	VFS_KNOTE_LOCKED(vp, NOTE_REVOKE);
	}

	void
	vop_remove_pre(void *ap)
	{
	struct vop_remove_args *a;
	struct vnode dvp, vp;

	a = ap;
	dvp = a->a_dvp;
	vp = a->a_vp;
	vn_seqc_write_begin(dvp);
	vn_seqc_write_begin(vp);
	}

	void
	vop_remove_post(void *ap, int rc)
	{
	struct vop_remove_args *a;
	struct vnode dvp, vp;

	a = ap;
	dvp = a->a_dvp;
	vp = a->a_vp;
	vn_seqc_write_end(dvp);
	vn_seqc_write_end(vp);
	if (!rc) {
	VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
	VFS_KNOTE_LOCKED(vp, NOTE_DELETE);
	}
	}

	void
	vop_rename_post(void *ap, int rc)
	{
	struct vop_rename_args *a = ap;
	long hint;

	if (!rc) {
	hint = NOTE_WRITE;
	if (a->a_fdvp == a->a_tdvp) {
	if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR)
	hint \|= NOTE_LINK;
	VFS_KNOTE_UNLOCKED(a->a_fdvp, hint);
	VFS_KNOTE_UNLOCKED(a->a_tdvp, hint);
	} else {
	hint \|= NOTE_EXTEND;
	if (a->a_fvp->v_type == VDIR)
	hint \|= NOTE_LINK;
	VFS_KNOTE_UNLOCKED(a->a_fdvp, hint);

	if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL &&
	a->a_tvp->v_type == VDIR)
	hint &= ~NOTE_LINK;
	VFS_KNOTE_UNLOCKED(a->a_tdvp, hint);
	}

	VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME);
	if (a->a_tvp)
	VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE);
	}
	if (a->a_tdvp != a->a_fdvp)
	vdrop(a->a_fdvp);
	if (a->a_tvp != a->a_fvp)
	vdrop(a->a_fvp);
	vdrop(a->a_tdvp);
	if (a->a_tvp)
	vdrop(a->a_tvp);
	}

	void
	vop_rmdir_pre(void *ap)
	{
	struct vop_rmdir_args *a;
	struct vnode dvp, vp;

	a = ap;
	dvp = a->a_dvp;
	vp = a->a_vp;
	vn_seqc_write_begin(dvp);
	vn_seqc_write_begin(vp);
	}

	void
	vop_rmdir_post(void *ap, int rc)
	{
	struct vop_rmdir_args *a;
	struct vnode dvp, vp;

	a = ap;
	dvp = a->a_dvp;
	vp = a->a_vp;
	vn_seqc_write_end(dvp);
	vn_seqc_write_end(vp);
	if (!rc) {
	VFS_KNOTE_LOCKED(dvp, NOTE_WRITE \| NOTE_LINK);
	VFS_KNOTE_LOCKED(vp, NOTE_DELETE);
	}
	}

	void
	vop_setattr_pre(void *ap)
	{
	struct vop_setattr_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_begin(vp);
	}

	void
	vop_setattr_post(void *ap, int rc)
	{
	struct vop_setattr_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_end(vp);
	if (!rc)
	VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB);
	}

	void
	vop_setacl_pre(void *ap)
	{
	struct vop_setacl_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_begin(vp);
	}

	void
	vop_setacl_post(void *ap, int rc __unused)
	{
	struct vop_setacl_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_end(vp);
	}

	void
	vop_setextattr_pre(void *ap)
	{
	struct vop_setextattr_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_begin(vp);
	}

	void
	vop_setextattr_post(void *ap, int rc)
	{
	struct vop_setextattr_args *a;
	struct vnode *vp;

	a = ap;
	vp = a->a_vp;
	vn_seqc_write_end(vp);
	if (!rc)
	VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB);
	}

	void
	vop_symlink_pre(void *ap)
	{
	struct vop_symlink_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_begin(dvp);
	}

	void
	vop_symlink_post(void *ap, int rc)
	{
	struct vop_symlink_args *a;
	struct vnode *dvp;

	a = ap;
	dvp = a->a_dvp;
	vn_seqc_write_end(dvp);
	if (!rc)
	VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
	}

	void
	vop_open_post(void *ap, int rc)
	{
	struct vop_open_args *a = ap;

	if (!rc)
	VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN);
	}

	void
	vop_close_post(void *ap, int rc)
	{
	struct vop_close_args *a = ap;

	if (!rc && (a->a_cred != NOCRED \|\| /* filter out revokes */
	!VN_IS_DOOMED(a->a_vp))) {
	VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ?
	NOTE_CLOSE_WRITE : NOTE_CLOSE);
	}
	}

	void
	vop_read_post(void *ap, int rc)
	{
	struct vop_read_args *a = ap;

	if (!rc)
	VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
	}

	void
	vop_readdir_post(void *ap, int rc)
	{
	struct vop_readdir_args *a = ap;

	if (!rc)
	VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
	}

	static struct knlist fs_knlist;

	static void
	vfs_event_init(void *arg)
	{
	knlist_init_mtx(&fs_knlist, NULL);
	}
	/* XXX - correct order? */
	SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL);

	void
	vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
	{

	KNOTE_UNLOCKED(&fs_knlist, event);
	}

	static int filt_fsattach(struct knote *kn);
	static void filt_fsdetach(struct knote *kn);
	static int filt_fsevent(struct knote *kn, long hint);

	struct filterops fs_filtops = {
	.f_isfd = 0,
	.f_attach = filt_fsattach,
	.f_detach = filt_fsdetach,
	.f_event = filt_fsevent
	};

	static int
	filt_fsattach(struct knote *kn)
	{

	kn->kn_flags \|= EV_CLEAR;
	knlist_add(&fs_knlist, kn, 0);
	return (0);
	}

	static void
	filt_fsdetach(struct knote *kn)
	{

	knlist_remove(&fs_knlist, kn, 0);
	}

	static int
	filt_fsevent(struct knote *kn, long hint)
	{

	kn->kn_fflags \|= hint;
	return (kn->kn_fflags != 0);
	}

	static int
	sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
	{
	struct vfsidctl vc;
	int error;
	struct mount *mp;

	error = SYSCTL_IN(req, &vc, sizeof(vc));
	if (error)
	return (error);
	if (vc.vc_vers != VFS_CTL_VERS1)
	return (EINVAL);
	mp = vfs_getvfs(&vc.vc_fsid);
	if (mp == NULL)
	return (ENOENT);
	/* ensure that a specific sysctl goes to the right filesystem. */
	if (strcmp(vc.vc_fstypename, "*") != 0 &&
	strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) {
	vfs_rel(mp);
	return (EINVAL);
	}
	VCTLTOREQ(&vc, req);
	error = VFS_SYSCTL(mp, vc.vc_op, req);
	vfs_rel(mp);
	return (error);
	}

	SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| CTLFLAG_WR,
	NULL, 0, sysctl_vfs_ctl, "",
	"Sysctl by fsid");

	/*
	* Function to initialize a va_filerev field sensibly.
	* XXX: Wouldn't a random number make a lot more sense ??
	*/
	u_quad_t
	init_va_filerev(void)
	{
	struct bintime bt;

	getbinuptime(&bt);
	return (((u_quad_t)bt.sec << 32LL) \| (bt.frac >> 32LL));
	}

	static int filt_vfsread(struct knote *kn, long hint);
	static int filt_vfswrite(struct knote *kn, long hint);
	static int filt_vfsvnode(struct knote *kn, long hint);
	static void filt_vfsdetach(struct knote *kn);
	static struct filterops vfsread_filtops = {
	.f_isfd = 1,
	.f_detach = filt_vfsdetach,
	.f_event = filt_vfsread
	};
	static struct filterops vfswrite_filtops = {
	.f_isfd = 1,
	.f_detach = filt_vfsdetach,
	.f_event = filt_vfswrite
	};
	static struct filterops vfsvnode_filtops = {
	.f_isfd = 1,
	.f_detach = filt_vfsdetach,
	.f_event = filt_vfsvnode
	};

	static void
	vfs_knllock(void *arg)
	{
	struct vnode *vp = arg;

	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	}

	static void
	vfs_knlunlock(void *arg)
	{
	struct vnode *vp = arg;

	VOP_UNLOCK(vp);
	}

	static void
	vfs_knl_assert_locked(void *arg)
	{
	#ifdef DEBUG_VFS_LOCKS
	struct vnode *vp = arg;

	ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked");
	#endif
	}

	static void
	vfs_knl_assert_unlocked(void *arg)
	{
	#ifdef DEBUG_VFS_LOCKS
	struct vnode *vp = arg;

	ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked");
	#endif
	}

	int
	vfs_kqfilter(struct vop_kqfilter_args *ap)
	{
	struct vnode *vp = ap->a_vp;
	struct knote *kn = ap->a_kn;
	struct knlist *knl;

	switch (kn->kn_filter) {
	case EVFILT_READ:
	kn->kn_fop = &vfsread_filtops;
	break;
	case EVFILT_WRITE:
	kn->kn_fop = &vfswrite_filtops;
	break;
	case EVFILT_VNODE:
	kn->kn_fop = &vfsvnode_filtops;
	break;
	default:
	return (EINVAL);
	}

	kn->kn_hook = (caddr_t)vp;

	v_addpollinfo(vp);
	if (vp->v_pollinfo == NULL)
	return (ENOMEM);
	knl = &vp->v_pollinfo->vpi_selinfo.si_note;
	vhold(vp);
	knlist_add(knl, kn, 0);

	return (0);
	}

	/*
	* Detach knote from vnode
	*/
	static void
	filt_vfsdetach(struct knote *kn)
	{
	struct vnode vp = (struct vnode )kn->kn_hook;

	KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo"));
	knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0);
	vdrop(vp);
	}

	/ARGSUSED/
	static int
	filt_vfsread(struct knote *kn, long hint)
	{
	struct vnode vp = (struct vnode )kn->kn_hook;
	struct vattr va;
	int res;

	/*
	* filesystem is gone, so set the EOF flag and schedule
	* the knote for deletion.
	*/
	if (hint == NOTE_REVOKE \|\| (hint == 0 && vp->v_type == VBAD)) {
	VI_LOCK(vp);
	kn->kn_flags \|= (EV_EOF \| EV_ONESHOT);
	VI_UNLOCK(vp);
	return (1);
	}

	if (VOP_GETATTR(vp, &va, curthread->td_ucred))
	return (0);

	VI_LOCK(vp);
	kn->kn_data = va.va_size - kn->kn_fp->f_offset;
	res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 \|\| kn->kn_data != 0;
	VI_UNLOCK(vp);
	return (res);
	}

	/ARGSUSED/
	static int
	filt_vfswrite(struct knote *kn, long hint)
	{
	struct vnode vp = (struct vnode )kn->kn_hook;

	VI_LOCK(vp);

	/*
	* filesystem is gone, so set the EOF flag and schedule
	* the knote for deletion.
	*/
	if (hint == NOTE_REVOKE \|\| (hint == 0 && vp->v_type == VBAD))
	kn->kn_flags \|= (EV_EOF \| EV_ONESHOT);

	kn->kn_data = 0;
	VI_UNLOCK(vp);
	return (1);
	}

	static int
	filt_vfsvnode(struct knote *kn, long hint)
	{
	struct vnode vp = (struct vnode )kn->kn_hook;
	int res;

	VI_LOCK(vp);
	if (kn->kn_sfflags & hint)
	kn->kn_fflags \|= hint;
	if (hint == NOTE_REVOKE \|\| (hint == 0 && vp->v_type == VBAD)) {
	kn->kn_flags \|= EV_EOF;
	VI_UNLOCK(vp);
	return (1);
	}
	res = (kn->kn_fflags != 0);
	VI_UNLOCK(vp);
	return (res);
	}

	/*
	* Returns whether the directory is empty or not.
	* If it is empty, the return value is 0; otherwise
	* the return value is an error value (which may
	* be ENOTEMPTY).
	*/
	int
	vfs_emptydir(struct vnode *vp)
	{
	struct uio uio;
	struct iovec iov;
	struct dirent dirent, dp, *endp;
	int error, eof;

	error = 0;
	eof = 0;

	ASSERT_VOP_LOCKED(vp, "vfs_emptydir");

	dirent = malloc(sizeof(struct dirent), M_TEMP, M_WAITOK);
	iov.iov_base = dirent;
	iov.iov_len = sizeof(struct dirent);

	uio.uio_iov = &iov;
	uio.uio_iovcnt = 1;
	uio.uio_offset = 0;
	uio.uio_resid = sizeof(struct dirent);
	uio.uio_segflg = UIO_SYSSPACE;
	uio.uio_rw = UIO_READ;
	uio.uio_td = curthread;

	while (eof == 0 && error == 0) {
	error = VOP_READDIR(vp, &uio, curthread->td_ucred, &eof,
	NULL, NULL);
	if (error != 0)
	break;
	endp = (void )((uint8_t )dirent +
	sizeof(struct dirent) - uio.uio_resid);
	for (dp = dirent; dp < endp;
	dp = (void )((uint8_t )dp + GENERIC_DIRSIZ(dp))) {
	if (dp->d_type == DT_WHT)
	continue;
	if (dp->d_namlen == 0)
	continue;
	if (dp->d_type != DT_DIR &&
	dp->d_type != DT_UNKNOWN) {
	error = ENOTEMPTY;
	break;
	}
	if (dp->d_namlen > 2) {
	error = ENOTEMPTY;
	break;
	}
	if (dp->d_namlen == 1 &&
	dp->d_name[0] != '.') {
	error = ENOTEMPTY;
	break;
	}
	if (dp->d_namlen == 2 &&
	dp->d_name[1] != '.') {
	error = ENOTEMPTY;
	break;
	}
	uio.uio_resid = sizeof(struct dirent);
	}
	}
	free(dirent, M_TEMP);
	return (error);
	}

	int
	vfs_read_dirent(struct vop_readdir_args ap, struct dirent dp, off_t off)
	{
	int error;

	if (dp->d_reclen > ap->a_uio->uio_resid)
	return (ENAMETOOLONG);
	error = uiomove(dp, dp->d_reclen, ap->a_uio);
	if (error) {
	if (ap->a_ncookies != NULL) {
	if (ap->a_cookies != NULL)
	free(ap->a_cookies, M_TEMP);
	ap->a_cookies = NULL;
	*ap->a_ncookies = 0;
	}
	return (error);
	}
	if (ap->a_ncookies == NULL)
	return (0);

	KASSERT(ap->a_cookies,
	("NULL ap->a_cookies value with non-NULL ap->a_ncookies!"));

	ap->a_cookies = realloc(ap->a_cookies,
	(ap->a_ncookies + 1) sizeof(u_long), M_TEMP, M_WAITOK \| M_ZERO);
	(ap->a_cookies)[ap->a_ncookies] = off;
	*ap->a_ncookies += 1;
	return (0);
	}

	/*
	* The purpose of this routine is to remove granularity from accmode_t,
	* reducing it into standard unix access bits - VEXEC, VREAD, VWRITE,
	* VADMIN and VAPPEND.
	*
	* If it returns 0, the caller is supposed to continue with the usual
	* access checks using 'accmode' as modified by this routine. If it
	* returns nonzero value, the caller is supposed to return that value
	* as errno.
	*
	* Note that after this routine runs, accmode may be zero.
	*/
	int
	vfs_unixify_accmode(accmode_t *accmode)
	{
	/*
	* There is no way to specify explicit "deny" rule using
	* file mode or POSIX.1e ACLs.
	*/
	if (*accmode & VEXPLICIT_DENY) {
	*accmode = 0;
	return (0);
	}

	/*
	* None of these can be translated into usual access bits.
	* Also, the common case for NFSv4 ACLs is to not contain
	* either of these bits. Caller should check for VWRITE
	* on the containing directory instead.
	*/
	if (*accmode & (VDELETE_CHILD \| VDELETE))
	return (EPERM);

	if (*accmode & VADMIN_PERMS) {
	*accmode &= ~VADMIN_PERMS;
	*accmode \|= VADMIN;
	}

	/*
	* There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL
	* or VSYNCHRONIZE using file mode or POSIX.1e ACL.
	*/
	*accmode &= ~(VSTAT_PERMS \| VSYNCHRONIZE);

	return (0);
	}

	/*
	* Clear out a doomed vnode (if any) and replace it with a new one as long
	* as the fs is not being unmounted. Return the root vnode to the caller.
	*/
	static int __noinline
	vfs_cache_root_fallback(struct mount mp, int flags, struct vnode *vpp)
	{
	struct vnode *vp;
	int error;

	restart:
	if (mp->mnt_rootvnode != NULL) {
	MNT_ILOCK(mp);
	vp = mp->mnt_rootvnode;
	if (vp != NULL) {
	if (!VN_IS_DOOMED(vp)) {
	vrefact(vp);
	MNT_IUNLOCK(mp);
	error = vn_lock(vp, flags);
	if (error == 0) {
	*vpp = vp;
	return (0);
	}
	vrele(vp);
	goto restart;
	}
	/*
	* Clear the old one.
	*/
	mp->mnt_rootvnode = NULL;
	}
	MNT_IUNLOCK(mp);
	if (vp != NULL) {
	vfs_op_barrier_wait(mp);
	vrele(vp);
	}
	}
	error = VFS_CACHEDROOT(mp, flags, vpp);
	if (error != 0)
	return (error);
	if (mp->mnt_vfs_ops == 0) {
	MNT_ILOCK(mp);
	if (mp->mnt_vfs_ops != 0) {
	MNT_IUNLOCK(mp);
	return (0);
	}
	if (mp->mnt_rootvnode == NULL) {
	vrefact(*vpp);
	mp->mnt_rootvnode = *vpp;
	} else {
	if (mp->mnt_rootvnode != *vpp) {
	if (!VN_IS_DOOMED(mp->mnt_rootvnode)) {
	panic("%s: mismatch between vnode returned "
	" by VFS_CACHEDROOT and the one cached "
	" (%p != %p)",
	__func__, *vpp, mp->mnt_rootvnode);
	}
	}
	}
	MNT_IUNLOCK(mp);
	}
	return (0);
	}

	int
	vfs_cache_root(struct mount mp, int flags, struct vnode *vpp)
	{
	struct vnode *vp;
	int error;

	if (!vfs_op_thread_enter(mp))
	return (vfs_cache_root_fallback(mp, flags, vpp));
	vp = atomic_load_ptr(&mp->mnt_rootvnode);
	if (vp == NULL \|\| VN_IS_DOOMED(vp)) {
	vfs_op_thread_exit(mp);
	return (vfs_cache_root_fallback(mp, flags, vpp));
	}
	vrefact(vp);
	vfs_op_thread_exit(mp);
	error = vn_lock(vp, flags);
	if (error != 0) {
	vrele(vp);
	return (vfs_cache_root_fallback(mp, flags, vpp));
	}
	*vpp = vp;
	return (0);
	}

	struct vnode *
	vfs_cache_root_clear(struct mount *mp)
	{
	struct vnode *vp;

	/*
	* ops > 0 guarantees there is nobody who can see this vnode
	*/
	MPASS(mp->mnt_vfs_ops > 0);
	vp = mp->mnt_rootvnode;
	if (vp != NULL)
	vn_seqc_write_begin(vp);
	mp->mnt_rootvnode = NULL;
	return (vp);
	}

	void
	vfs_cache_root_set(struct mount mp, struct vnode vp)
	{

	MPASS(mp->mnt_vfs_ops > 0);
	vrefact(vp);
	mp->mnt_rootvnode = vp;
	}

	/*
	* These are helper functions for filesystems to traverse all
	* their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h.
	*
	* This interface replaces MNT_VNODE_FOREACH.
	*/

	struct vnode *
	__mnt_vnode_next_all(struct vnode *mvp, struct mount mp)
	{
	struct vnode *vp;

	if (should_yield())
	kern_yield(PRI_USER);
	MNT_ILOCK(mp);
	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
	for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL;
	vp = TAILQ_NEXT(vp, v_nmntvnodes)) {
	/* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */
	if (vp->v_type == VMARKER \|\| VN_IS_DOOMED(vp))
	continue;
	VI_LOCK(vp);
	if (VN_IS_DOOMED(vp)) {
	VI_UNLOCK(vp);
	continue;
	}
	break;
	}
	if (vp == NULL) {
	__mnt_vnode_markerfree_all(mvp, mp);
	/* MNT_IUNLOCK(mp); -- done in above function */
	mtx_assert(MNT_MTX(mp), MA_NOTOWNED);
	return (NULL);
	}
	TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
	TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
	MNT_IUNLOCK(mp);
	return (vp);
	}

	struct vnode *
	__mnt_vnode_first_all(struct vnode *mvp, struct mount mp)
	{
	struct vnode *vp;

	*mvp = vn_alloc_marker(mp);
	MNT_ILOCK(mp);
	MNT_REF(mp);

	TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
	/* Allow a racy peek at VIRF_DOOMED to save a lock acquisition. */
	if (vp->v_type == VMARKER \|\| VN_IS_DOOMED(vp))
	continue;
	VI_LOCK(vp);
	if (VN_IS_DOOMED(vp)) {
	VI_UNLOCK(vp);
	continue;
	}
	break;
	}
	if (vp == NULL) {
	MNT_REL(mp);
	MNT_IUNLOCK(mp);
	vn_free_marker(*mvp);
	*mvp = NULL;
	return (NULL);
	}
	TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
	MNT_IUNLOCK(mp);
	return (vp);
	}

	void
	__mnt_vnode_markerfree_all(struct vnode *mvp, struct mount mp)
	{

	if (*mvp == NULL) {
	MNT_IUNLOCK(mp);
	return;
	}

	mtx_assert(MNT_MTX(mp), MA_OWNED);

	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
	TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
	MNT_REL(mp);
	MNT_IUNLOCK(mp);
	vn_free_marker(*mvp);
	*mvp = NULL;
	}

	/*
	* These are helper functions for filesystems to traverse their
	* lazy vnodes. See MNT_VNODE_FOREACH_LAZY() in sys/mount.h
	*/
	static void
	mnt_vnode_markerfree_lazy(struct vnode *mvp, struct mount mp)
	{

	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));

	MNT_ILOCK(mp);
	MNT_REL(mp);
	MNT_IUNLOCK(mp);
	vn_free_marker(*mvp);
	*mvp = NULL;
	}

	/*
	* Relock the mp mount vnode list lock with the vp vnode interlock in the
	* conventional lock order during mnt_vnode_next_lazy iteration.
	*
	* On entry, the mount vnode list lock is held and the vnode interlock is not.
	* The list lock is dropped and reacquired. On success, both locks are held.
	* On failure, the mount vnode list lock is held but the vnode interlock is
	* not, and the procedure may have yielded.
	*/
	static bool
	mnt_vnode_next_lazy_relock(struct vnode mvp, struct mount mp,
	struct vnode *vp)
	{

	VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER &&
	TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp,
	("%s: bad marker", __func__));
	VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp,
	("%s: inappropriate vnode", __func__));
	ASSERT_VI_UNLOCKED(vp, __func__);
	mtx_assert(&mp->mnt_listmtx, MA_OWNED);

	TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist);
	TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist);

	/*
	* Note we may be racing against vdrop which transitioned the hold
	* count to 0 and now waits for the ->mnt_listmtx lock. This is fine,
	* if we are the only user after we get the interlock we will just
	* vdrop.
	*/
	vhold(vp);
	mtx_unlock(&mp->mnt_listmtx);
	VI_LOCK(vp);
	if (VN_IS_DOOMED(vp)) {
	VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp);
	goto out_lost;
	}
	VNPASS(vp->v_mflag & VMP_LAZYLIST, vp);
	/*
	* There is nothing to do if we are the last user.
	*/
	if (!refcount_release_if_not_last(&vp->v_holdcnt))
	goto out_lost;
	mtx_lock(&mp->mnt_listmtx);
	return (true);
	out_lost:
	vdropl(vp);
	maybe_yield();
	mtx_lock(&mp->mnt_listmtx);
	return (false);
	}

	static struct vnode *
	mnt_vnode_next_lazy(struct vnode *mvp, struct mount mp, mnt_lazy_cb_t *cb,
	void *cbarg)
	{
	struct vnode *vp;

	mtx_assert(&mp->mnt_listmtx, MA_OWNED);
	KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
	restart:
	vp = TAILQ_NEXT(*mvp, v_lazylist);
	while (vp != NULL) {
	if (vp->v_type == VMARKER) {
	vp = TAILQ_NEXT(vp, v_lazylist);
	continue;
	}
	/*
	* See if we want to process the vnode. Note we may encounter a
	* long string of vnodes we don't care about and hog the list
	* as a result. Check for it and requeue the marker.
	*/
	VNPASS(!VN_IS_DOOMED(vp), vp);
	if (!cb(vp, cbarg)) {
	if (!should_yield()) {
	vp = TAILQ_NEXT(vp, v_lazylist);
	continue;
	}
	TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp,
	v_lazylist);
	TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp,
	v_lazylist);
	mtx_unlock(&mp->mnt_listmtx);
	kern_yield(PRI_USER);
	mtx_lock(&mp->mnt_listmtx);
	goto restart;
	}
	/*
	* Try-lock because this is the wrong lock order.
	*/
	if (!VI_TRYLOCK(vp) &&
	!mnt_vnode_next_lazy_relock(*mvp, mp, vp))
	goto restart;
	KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp));
	KASSERT(vp->v_mount == mp \|\| vp->v_mount == NULL,
	("alien vnode on the lazy list %p %p", vp, mp));
	VNPASS(vp->v_mount == mp, vp);
	VNPASS(!VN_IS_DOOMED(vp), vp);
	break;
	}
	TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist);

	/* Check if we are done */
	if (vp == NULL) {
	mtx_unlock(&mp->mnt_listmtx);
	mnt_vnode_markerfree_lazy(mvp, mp);
	return (NULL);
	}
	TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist);
	mtx_unlock(&mp->mnt_listmtx);
	ASSERT_VI_LOCKED(vp, "lazy iter");
	return (vp);
	}

	struct vnode *
	__mnt_vnode_next_lazy(struct vnode *mvp, struct mount mp, mnt_lazy_cb_t *cb,
	void *cbarg)
	{

	if (should_yield())
	kern_yield(PRI_USER);
	mtx_lock(&mp->mnt_listmtx);
	return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg));
	}

	struct vnode *
	__mnt_vnode_first_lazy(struct vnode *mvp, struct mount mp, mnt_lazy_cb_t *cb,
	void *cbarg)
	{
	struct vnode *vp;

	if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist))
	return (NULL);

	*mvp = vn_alloc_marker(mp);
	MNT_ILOCK(mp);
	MNT_REF(mp);
	MNT_IUNLOCK(mp);

	mtx_lock(&mp->mnt_listmtx);
	vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist);
	if (vp == NULL) {
	mtx_unlock(&mp->mnt_listmtx);
	mnt_vnode_markerfree_lazy(mvp, mp);
	return (NULL);
	}
	TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist);
	return (mnt_vnode_next_lazy(mvp, mp, cb, cbarg));
	}

	void
	__mnt_vnode_markerfree_lazy(struct vnode *mvp, struct mount mp)
	{

	if (*mvp == NULL)
	return;

	mtx_lock(&mp->mnt_listmtx);
	TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist);
	mtx_unlock(&mp->mnt_listmtx);
	mnt_vnode_markerfree_lazy(mvp, mp);
	}

	int
	vn_dir_check_exec(struct vnode vp, struct componentname cnp)
	{

	if ((cnp->cn_flags & NOEXECCHECK) != 0) {
	cnp->cn_flags &= ~NOEXECCHECK;
	return (0);
	}

	return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, cnp->cn_thread));
	}

	void
	vn_seqc_write_begin_locked(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	VNPASS(vp->v_holdcnt > 0, vp);
	VNPASS(vp->v_seqc_users >= 0, vp);
	vp->v_seqc_users++;
	if (vp->v_seqc_users == 1)
	seqc_sleepable_write_begin(&vp->v_seqc);
	}

	void
	vn_seqc_write_begin(struct vnode *vp)
	{

	VI_LOCK(vp);
	vn_seqc_write_begin_locked(vp);
	VI_UNLOCK(vp);
	}

	void
	vn_seqc_write_end_locked(struct vnode *vp)
	{

	ASSERT_VI_LOCKED(vp, __func__);
	VNPASS(vp->v_seqc_users > 0, vp);
	vp->v_seqc_users--;
	if (vp->v_seqc_users == 0)
	seqc_sleepable_write_end(&vp->v_seqc);
	}

	void
	vn_seqc_write_end(struct vnode *vp)
	{

	VI_LOCK(vp);
	vn_seqc_write_end_locked(vp);
	VI_UNLOCK(vp);
	}
	diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
	index 2caf09f3412c..9e9c53d3327c 100644
	--- a/sys/kern/vfs_syscalls.c
	+++ b/sys/kern/vfs_syscalls.c
	@@ -1,4892 +1,4915 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	* (c) UNIX System Laboratories, Inc.
	* All or some portions of this file are derived from material licensed
	* to the University of California by American Telephone and Telegraph
	* Co. or Unix System Laboratories, Inc. and are reproduced herein with
	* the permission of UNIX System Laboratories, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)vfs_syscalls.c 8.13 (Berkeley) 4/15/94
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_capsicum.h"
	#include "opt_ktrace.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bio.h>
	#include <sys/buf.h>
	#include <sys/capsicum.h>
	#include <sys/disk.h>
	#include <sys/sysent.h>
	#include <sys/malloc.h>
	#include <sys/mount.h>
	#include <sys/mutex.h>
	#include <sys/sysproto.h>
	#include <sys/namei.h>
	#include <sys/filedesc.h>
	#include <sys/kernel.h>
	#include <sys/fcntl.h>
	#include <sys/file.h>
	#include <sys/filio.h>
	#include <sys/limits.h>
	#include <sys/linker.h>
	#include <sys/rwlock.h>
	#include <sys/sdt.h>
	#include <sys/stat.h>
	#include <sys/sx.h>
	#include <sys/unistd.h>
	#include <sys/vnode.h>
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/dirent.h>
	#include <sys/jail.h>
	#include <sys/syscallsubr.h>
	#include <sys/sysctl.h>
	#ifdef KTRACE
	#include <sys/ktrace.h>
	#endif

	#include <machine/stdarg.h>

	#include <security/audit/audit.h>
	#include <security/mac/mac_framework.h>

	#include <vm/vm.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/uma.h>

	#include <ufs/ufs/quota.h>

	MALLOC_DEFINE(M_FADVISE, "fadvise", "posix_fadvise(2) information");

	SDT_PROVIDER_DEFINE(vfs);
	SDT_PROBE_DEFINE2(vfs, , stat, mode, "char *", "int");
	SDT_PROBE_DEFINE2(vfs, , stat, reg, "char *", "int");

	static int kern_chflagsat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, u_long flags, int atflag);
	static int setfflags(struct thread td, struct vnode , u_long);
	static int getutimes(const struct timeval , enum uio_seg, struct timespec );
	static int getutimens(const struct timespec *, enum uio_seg,
	struct timespec , int );
	static int setutimes(struct thread td, struct vnode ,
	const struct timespec *, int, int);
	static int vn_access(struct vnode vp, int user_flags, struct ucred cred,
	struct thread *td);
	static int kern_fhlinkat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, fhandle_t *fhp);
	static int kern_getfhat(struct thread *td, int flags, int fd,
	const char path, enum uio_seg pathseg, fhandle_t fhp);
	static int kern_readlink_vp(struct vnode vp, char buf, enum uio_seg bufseg,
	size_t count, struct thread *td);
	static int kern_linkat_vp(struct thread td, struct vnode vp, int fd,
	const char *path, enum uio_seg segflag);

	int
	kern_sync(struct thread *td)
	{
	struct mount mp, nmp;
	int save;

	mtx_lock(&mountlist_mtx);
	for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
	if (vfs_busy(mp, MBF_NOWAIT \| MBF_MNTLSTLOCK)) {
	nmp = TAILQ_NEXT(mp, mnt_list);
	continue;
	}
	if ((mp->mnt_flag & MNT_RDONLY) == 0 &&
	vn_start_write(NULL, &mp, V_NOWAIT) == 0) {
	save = curthread_pflags_set(TDP_SYNCIO);
	vfs_periodic(mp, MNT_NOWAIT);
	VFS_SYNC(mp, MNT_NOWAIT);
	curthread_pflags_restore(save);
	vn_finished_write(mp);
	}
	mtx_lock(&mountlist_mtx);
	nmp = TAILQ_NEXT(mp, mnt_list);
	vfs_unbusy(mp);
	}
	mtx_unlock(&mountlist_mtx);
	return (0);
	}

	/*
	* Sync each mounted filesystem.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct sync_args {
	int dummy;
	};
	#endif
	/* ARGSUSED */
	int
	sys_sync(struct thread td, struct sync_args uap)
	{

	return (kern_sync(td));
	}

	/*
	* Change filesystem quotas.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct quotactl_args {
	char *path;
	int cmd;
	int uid;
	caddr_t arg;
	};
	#endif
	int
	sys_quotactl(struct thread td, struct quotactl_args uap)
	{
	struct mount *mp;
	struct nameidata nd;
	int error;

	AUDIT_ARG_CMD(uap->cmd);
	AUDIT_ARG_UID(uap->uid);
	if (!prison_allow(td->td_ucred, PR_ALLOW_QUOTAS))
	return (EPERM);
	NDINIT(&nd, LOOKUP, FOLLOW \| LOCKLEAF \| AUDITVNODE1, UIO_USERSPACE,
	uap->path, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	mp = nd.ni_vp->v_mount;
	vfs_ref(mp);
	vput(nd.ni_vp);
	error = vfs_busy(mp, 0);
	if (error != 0) {
	vfs_rel(mp);
	return (error);
	}
	error = VFS_QUOTACTL(mp, uap->cmd, uap->uid, uap->arg);

	/*
	* Since quota on operation typically needs to open quota
	* file, the Q_QUOTAON handler needs to unbusy the mount point
	* before calling into namei. Otherwise, unmount might be
	* started between two vfs_busy() invocations (first is our,
	* second is from mount point cross-walk code in lookup()),
	* causing deadlock.
	*
	* Require that Q_QUOTAON handles the vfs_busy() reference on
	* its own, always returning with ubusied mount point.
	*/
	if ((uap->cmd >> SUBCMDSHIFT) != Q_QUOTAON &&
	(uap->cmd >> SUBCMDSHIFT) != Q_QUOTAOFF)
	vfs_unbusy(mp);
	vfs_rel(mp);
	return (error);
	}

	/*
	* Used by statfs conversion routines to scale the block size up if
	* necessary so that all of the block counts are <= 'max_size'. Note
	* that 'max_size' should be a bitmask, i.e. 2^n - 1 for some non-zero
	* value of 'n'.
	*/
	void
	statfs_scale_blocks(struct statfs *sf, long max_size)
	{
	uint64_t count;
	int shift;

	KASSERT(powerof2(max_size + 1), ("%s: invalid max_size", __func__));

	/*
	* Attempt to scale the block counts to give a more accurate
	* overview to userland of the ratio of free space to used
	* space. To do this, find the largest block count and compute
	* a divisor that lets it fit into a signed integer <= max_size.
	*/
	if (sf->f_bavail < 0)
	count = -sf->f_bavail;
	else
	count = sf->f_bavail;
	count = MAX(sf->f_blocks, MAX(sf->f_bfree, count));
	if (count <= max_size)
	return;

	count >>= flsl(max_size);
	shift = 0;
	while (count > 0) {
	shift++;
	count >>=1;
	}

	sf->f_bsize <<= shift;
	sf->f_blocks >>= shift;
	sf->f_bfree >>= shift;
	sf->f_bavail >>= shift;
	}

	static int
	kern_do_statfs(struct thread td, struct mount mp, struct statfs *buf)
	{
	int error;

	if (mp == NULL)
	return (EBADF);
	error = vfs_busy(mp, 0);
	vfs_rel(mp);
	if (error != 0)
	return (error);
	#ifdef MAC
	error = mac_mount_check_stat(td->td_ucred, mp);
	if (error != 0)
	goto out;
	#endif
	error = VFS_STATFS(mp, buf);
	if (error != 0)
	goto out;
	if (priv_check_cred_vfs_generation(td->td_ucred)) {
	buf->f_fsid.val[0] = buf->f_fsid.val[1] = 0;
	prison_enforce_statfs(td->td_ucred, mp, buf);
	}
	out:
	vfs_unbusy(mp);
	return (error);
	}

	/*
	* Get filesystem statistics.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct statfs_args {
	char *path;
	struct statfs *buf;
	};
	#endif
	int
	sys_statfs(struct thread td, struct statfs_args uap)
	{
	struct statfs *sfp;
	int error;

	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_statfs(td, uap->path, UIO_USERSPACE, sfp);
	if (error == 0)
	error = copyout(sfp, uap->buf, sizeof(struct statfs));
	free(sfp, M_STATFS);
	return (error);
	}

	int
	kern_statfs(struct thread td, const char path, enum uio_seg pathseg,
	struct statfs *buf)
	{
	struct mount *mp;
	struct nameidata nd;
	int error;

	NDINIT(&nd, LOOKUP, FOLLOW \| LOCKSHARED \| LOCKLEAF \| AUDITVNODE1,
	pathseg, path, td);
	error = namei(&nd);
	if (error != 0)
	return (error);
	mp = nd.ni_vp->v_mount;
	vfs_ref(mp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_vp);
	return (kern_do_statfs(td, mp, buf));
	}

	/*
	* Get filesystem statistics.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fstatfs_args {
	int fd;
	struct statfs *buf;
	};
	#endif
	int
	sys_fstatfs(struct thread td, struct fstatfs_args uap)
	{
	struct statfs *sfp;
	int error;

	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_fstatfs(td, uap->fd, sfp);
	if (error == 0)
	error = copyout(sfp, uap->buf, sizeof(struct statfs));
	free(sfp, M_STATFS);
	return (error);
	}

	int
	kern_fstatfs(struct thread td, int fd, struct statfs buf)
	{
	struct file *fp;
	struct mount *mp;
	struct vnode *vp;
	int error;

	AUDIT_ARG_FD(fd);
	error = getvnode(td, fd, &cap_fstatfs_rights, &fp);
	if (error != 0)
	return (error);
	vp = fp->f_vnode;
	vn_lock(vp, LK_SHARED \| LK_RETRY);
	#ifdef AUDIT
	AUDIT_ARG_VNODE1(vp);
	#endif
	mp = vp->v_mount;
	if (mp != NULL)
	vfs_ref(mp);
	VOP_UNLOCK(vp);
	fdrop(fp, td);
	return (kern_do_statfs(td, mp, buf));
	}

	/*
	* Get statistics on all filesystems.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct getfsstat_args {
	struct statfs *buf;
	long bufsize;
	int mode;
	};
	#endif
	int
	sys_getfsstat(struct thread td, struct getfsstat_args uap)
	{
	size_t count;
	int error;

	if (uap->bufsize < 0 \|\| uap->bufsize > SIZE_MAX)
	return (EINVAL);
	error = kern_getfsstat(td, &uap->buf, uap->bufsize, &count,
	UIO_USERSPACE, uap->mode);
	if (error == 0)
	td->td_retval[0] = count;
	return (error);
	}

	/*
	* If (bufsize > 0 && bufseg == UIO_SYSSPACE)
	* The caller is responsible for freeing memory which will be allocated
	* in '*buf'.
	*/
	int
	kern_getfsstat(struct thread td, struct statfs *buf, size_t bufsize,
	size_t *countp, enum uio_seg bufseg, int mode)
	{
	struct mount mp, nmp;
	struct statfs sfsp, sp, sptmp, tofree;
	size_t count, maxcount;
	int error;

	switch (mode) {
	case MNT_WAIT:
	case MNT_NOWAIT:
	break;
	default:
	if (bufseg == UIO_SYSSPACE)
	*buf = NULL;
	return (EINVAL);
	}
	restart:
	maxcount = bufsize / sizeof(struct statfs);
	if (bufsize == 0) {
	sfsp = NULL;
	tofree = NULL;
	} else if (bufseg == UIO_USERSPACE) {
	sfsp = *buf;
	tofree = NULL;
	} else /* if (bufseg == UIO_SYSSPACE) */ {
	count = 0;
	mtx_lock(&mountlist_mtx);
	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
	count++;
	}
	mtx_unlock(&mountlist_mtx);
	if (maxcount > count)
	maxcount = count;
	tofree = sfsp = buf = malloc(maxcount sizeof(struct statfs),
	M_STATFS, M_WAITOK);
	}

	count = 0;

	/*
	* If there is no target buffer they only want the count.
	*
	* This could be TAILQ_FOREACH but it is open-coded to match the original
	* code below.
	*/
	if (sfsp == NULL) {
	mtx_lock(&mountlist_mtx);
	for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
	if (prison_canseemount(td->td_ucred, mp) != 0) {
	nmp = TAILQ_NEXT(mp, mnt_list);
	continue;
	}
	#ifdef MAC
	if (mac_mount_check_stat(td->td_ucred, mp) != 0) {
	nmp = TAILQ_NEXT(mp, mnt_list);
	continue;
	}
	#endif
	count++;
	nmp = TAILQ_NEXT(mp, mnt_list);
	}
	mtx_unlock(&mountlist_mtx);
	*countp = count;
	return (0);
	}

	/*
	* They want the entire thing.
	*
	* Short-circuit the corner case of no room for anything, avoids
	* relocking below.
	*/
	if (maxcount < 1) {
	goto out;
	}

	mtx_lock(&mountlist_mtx);
	for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
	if (prison_canseemount(td->td_ucred, mp) != 0) {
	nmp = TAILQ_NEXT(mp, mnt_list);
	continue;
	}
	#ifdef MAC
	if (mac_mount_check_stat(td->td_ucred, mp) != 0) {
	nmp = TAILQ_NEXT(mp, mnt_list);
	continue;
	}
	#endif
	if (mode == MNT_WAIT) {
	if (vfs_busy(mp, MBF_MNTLSTLOCK) != 0) {
	/*
	* If vfs_busy() failed, and MBF_NOWAIT
	* wasn't passed, then the mp is gone.
	* Furthermore, because of MBF_MNTLSTLOCK,
	* the mountlist_mtx was dropped. We have
	* no other choice than to start over.
	*/
	mtx_unlock(&mountlist_mtx);
	free(tofree, M_STATFS);
	goto restart;
	}
	} else {
	if (vfs_busy(mp, MBF_NOWAIT \| MBF_MNTLSTLOCK) != 0) {
	nmp = TAILQ_NEXT(mp, mnt_list);
	continue;
	}
	}
	sp = &mp->mnt_stat;
	/*
	* If MNT_NOWAIT is specified, do not refresh
	* the fsstat cache.
	*/
	if (mode != MNT_NOWAIT) {
	error = VFS_STATFS(mp, sp);
	if (error != 0) {
	mtx_lock(&mountlist_mtx);
	nmp = TAILQ_NEXT(mp, mnt_list);
	vfs_unbusy(mp);
	continue;
	}
	}
	if (priv_check_cred_vfs_generation(td->td_ucred)) {
	sptmp = malloc(sizeof(struct statfs), M_STATFS,
	M_WAITOK);
	sptmp = sp;
	sptmp->f_fsid.val[0] = sptmp->f_fsid.val[1] = 0;
	prison_enforce_statfs(td->td_ucred, mp, sptmp);
	sp = sptmp;
	} else
	sptmp = NULL;
	if (bufseg == UIO_SYSSPACE) {
	bcopy(sp, sfsp, sizeof(*sp));
	free(sptmp, M_STATFS);
	} else /* if (bufseg == UIO_USERSPACE) */ {
	error = copyout(sp, sfsp, sizeof(*sp));
	free(sptmp, M_STATFS);
	if (error != 0) {
	vfs_unbusy(mp);
	return (error);
	}
	}
	sfsp++;
	count++;

	if (count == maxcount) {
	vfs_unbusy(mp);
	goto out;
	}

	mtx_lock(&mountlist_mtx);
	nmp = TAILQ_NEXT(mp, mnt_list);
	vfs_unbusy(mp);
	}
	mtx_unlock(&mountlist_mtx);
	out:
	*countp = count;
	return (0);
	}

	#ifdef COMPAT_FREEBSD4
	/*
	* Get old format filesystem statistics.
	*/
	static void freebsd4_cvtstatfs(struct statfs , struct ostatfs );

	#ifndef _SYS_SYSPROTO_H_
	struct freebsd4_statfs_args {
	char *path;
	struct ostatfs *buf;
	};
	#endif
	int
	freebsd4_statfs(struct thread td, struct freebsd4_statfs_args uap)
	{
	struct ostatfs osb;
	struct statfs *sfp;
	int error;

	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_statfs(td, uap->path, UIO_USERSPACE, sfp);
	if (error == 0) {
	freebsd4_cvtstatfs(sfp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	}
	free(sfp, M_STATFS);
	return (error);
	}

	/*
	* Get filesystem statistics.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct freebsd4_fstatfs_args {
	int fd;
	struct ostatfs *buf;
	};
	#endif
	int
	freebsd4_fstatfs(struct thread td, struct freebsd4_fstatfs_args uap)
	{
	struct ostatfs osb;
	struct statfs *sfp;
	int error;

	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_fstatfs(td, uap->fd, sfp);
	if (error == 0) {
	freebsd4_cvtstatfs(sfp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	}
	free(sfp, M_STATFS);
	return (error);
	}

	/*
	* Get statistics on all filesystems.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct freebsd4_getfsstat_args {
	struct ostatfs *buf;
	long bufsize;
	int mode;
	};
	#endif
	int
	freebsd4_getfsstat(struct thread td, struct freebsd4_getfsstat_args uap)
	{
	struct statfs buf, sp;
	struct ostatfs osb;
	size_t count, size;
	int error;

	if (uap->bufsize < 0)
	return (EINVAL);
	count = uap->bufsize / sizeof(struct ostatfs);
	if (count > SIZE_MAX / sizeof(struct statfs))
	return (EINVAL);
	size = count * sizeof(struct statfs);
	error = kern_getfsstat(td, &buf, size, &count, UIO_SYSSPACE,
	uap->mode);
	if (error == 0)
	td->td_retval[0] = count;
	if (size != 0) {
	sp = buf;
	while (count != 0 && error == 0) {
	freebsd4_cvtstatfs(sp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	sp++;
	uap->buf++;
	count--;
	}
	free(buf, M_STATFS);
	}
	return (error);
	}

	/*
	* Implement fstatfs() for (NFS) file handles.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct freebsd4_fhstatfs_args {
	struct fhandle *u_fhp;
	struct ostatfs *buf;
	};
	#endif
	int
	freebsd4_fhstatfs(struct thread td, struct freebsd4_fhstatfs_args uap)
	{
	struct ostatfs osb;
	struct statfs *sfp;
	fhandle_t fh;
	int error;

	error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
	if (error != 0)
	return (error);
	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_fhstatfs(td, fh, sfp);
	if (error == 0) {
	freebsd4_cvtstatfs(sfp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	}
	free(sfp, M_STATFS);
	return (error);
	}

	/*
	* Convert a new format statfs structure to an old format statfs structure.
	*/
	static void
	freebsd4_cvtstatfs(struct statfs nsp, struct ostatfs osp)
	{

	statfs_scale_blocks(nsp, LONG_MAX);
	bzero(osp, sizeof(*osp));
	osp->f_bsize = nsp->f_bsize;
	osp->f_iosize = MIN(nsp->f_iosize, LONG_MAX);
	osp->f_blocks = nsp->f_blocks;
	osp->f_bfree = nsp->f_bfree;
	osp->f_bavail = nsp->f_bavail;
	osp->f_files = MIN(nsp->f_files, LONG_MAX);
	osp->f_ffree = MIN(nsp->f_ffree, LONG_MAX);
	osp->f_owner = nsp->f_owner;
	osp->f_type = nsp->f_type;
	osp->f_flags = nsp->f_flags;
	osp->f_syncwrites = MIN(nsp->f_syncwrites, LONG_MAX);
	osp->f_asyncwrites = MIN(nsp->f_asyncwrites, LONG_MAX);
	osp->f_syncreads = MIN(nsp->f_syncreads, LONG_MAX);
	osp->f_asyncreads = MIN(nsp->f_asyncreads, LONG_MAX);
	strlcpy(osp->f_fstypename, nsp->f_fstypename,
	MIN(MFSNAMELEN, OMFSNAMELEN));
	strlcpy(osp->f_mntonname, nsp->f_mntonname,
	MIN(MNAMELEN, OMNAMELEN));
	strlcpy(osp->f_mntfromname, nsp->f_mntfromname,
	MIN(MNAMELEN, OMNAMELEN));
	osp->f_fsid = nsp->f_fsid;
	}
	#endif /* COMPAT_FREEBSD4 */

	#if defined(COMPAT_FREEBSD11)
	/*
	* Get old format filesystem statistics.
	*/
	static void freebsd11_cvtstatfs(struct statfs , struct freebsd11_statfs );

	int
	freebsd11_statfs(struct thread td, struct freebsd11_statfs_args uap)
	{
	struct freebsd11_statfs osb;
	struct statfs *sfp;
	int error;

	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_statfs(td, uap->path, UIO_USERSPACE, sfp);
	if (error == 0) {
	freebsd11_cvtstatfs(sfp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	}
	free(sfp, M_STATFS);
	return (error);
	}

	/*
	* Get filesystem statistics.
	*/
	int
	freebsd11_fstatfs(struct thread td, struct freebsd11_fstatfs_args uap)
	{
	struct freebsd11_statfs osb;
	struct statfs *sfp;
	int error;

	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_fstatfs(td, uap->fd, sfp);
	if (error == 0) {
	freebsd11_cvtstatfs(sfp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	}
	free(sfp, M_STATFS);
	return (error);
	}

	/*
	* Get statistics on all filesystems.
	*/
	int
	freebsd11_getfsstat(struct thread td, struct freebsd11_getfsstat_args uap)
	{
	struct freebsd11_statfs osb;
	struct statfs buf, sp;
	size_t count, size;
	int error;

	count = uap->bufsize / sizeof(struct ostatfs);
	size = count * sizeof(struct statfs);
	error = kern_getfsstat(td, &buf, size, &count, UIO_SYSSPACE,
	uap->mode);
	if (error == 0)
	td->td_retval[0] = count;
	if (size > 0) {
	sp = buf;
	while (count > 0 && error == 0) {
	freebsd11_cvtstatfs(sp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	sp++;
	uap->buf++;
	count--;
	}
	free(buf, M_STATFS);
	}
	return (error);
	}

	/*
	* Implement fstatfs() for (NFS) file handles.
	*/
	int
	freebsd11_fhstatfs(struct thread td, struct freebsd11_fhstatfs_args uap)
	{
	struct freebsd11_statfs osb;
	struct statfs *sfp;
	fhandle_t fh;
	int error;

	error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
	if (error)
	return (error);
	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_fhstatfs(td, fh, sfp);
	if (error == 0) {
	freebsd11_cvtstatfs(sfp, &osb);
	error = copyout(&osb, uap->buf, sizeof(osb));
	}
	free(sfp, M_STATFS);
	return (error);
	}

	/*
	* Convert a new format statfs structure to an old format statfs structure.
	*/
	static void
	freebsd11_cvtstatfs(struct statfs nsp, struct freebsd11_statfs osp)
	{

	bzero(osp, sizeof(*osp));
	osp->f_version = FREEBSD11_STATFS_VERSION;
	osp->f_type = nsp->f_type;
	osp->f_flags = nsp->f_flags;
	osp->f_bsize = nsp->f_bsize;
	osp->f_iosize = nsp->f_iosize;
	osp->f_blocks = nsp->f_blocks;
	osp->f_bfree = nsp->f_bfree;
	osp->f_bavail = nsp->f_bavail;
	osp->f_files = nsp->f_files;
	osp->f_ffree = nsp->f_ffree;
	osp->f_syncwrites = nsp->f_syncwrites;
	osp->f_asyncwrites = nsp->f_asyncwrites;
	osp->f_syncreads = nsp->f_syncreads;
	osp->f_asyncreads = nsp->f_asyncreads;
	osp->f_namemax = nsp->f_namemax;
	osp->f_owner = nsp->f_owner;
	osp->f_fsid = nsp->f_fsid;
	strlcpy(osp->f_fstypename, nsp->f_fstypename,
	MIN(MFSNAMELEN, sizeof(osp->f_fstypename)));
	strlcpy(osp->f_mntonname, nsp->f_mntonname,
	MIN(MNAMELEN, sizeof(osp->f_mntonname)));
	strlcpy(osp->f_mntfromname, nsp->f_mntfromname,
	MIN(MNAMELEN, sizeof(osp->f_mntfromname)));
	}
	#endif /* COMPAT_FREEBSD11 */

	/*
	* Change current working directory to a given file descriptor.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fchdir_args {
	int fd;
	};
	#endif
	int
	sys_fchdir(struct thread td, struct fchdir_args uap)
	{
	struct vnode vp, tdp;
	struct mount *mp;
	struct file *fp;
	int error;

	AUDIT_ARG_FD(uap->fd);
	error = getvnode(td, uap->fd, &cap_fchdir_rights,
	&fp);
	if (error != 0)
	return (error);
	vp = fp->f_vnode;
	vrefact(vp);
	fdrop(fp, td);
	vn_lock(vp, LK_SHARED \| LK_RETRY);
	AUDIT_ARG_VNODE1(vp);
	error = change_dir(vp, td);
	while (!error && (mp = vp->v_mountedhere) != NULL) {
	if (vfs_busy(mp, 0))
	continue;
	error = VFS_ROOT(mp, LK_SHARED, &tdp);
	vfs_unbusy(mp);
	if (error != 0)
	break;
	vput(vp);
	vp = tdp;
	}
	if (error != 0) {
	vput(vp);
	return (error);
	}
	VOP_UNLOCK(vp);
	pwd_chdir(td, vp);
	return (0);
	}

	/*
	* Change current working directory (``.'').
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct chdir_args {
	char *path;
	};
	#endif
	int
	sys_chdir(struct thread td, struct chdir_args uap)
	{

	return (kern_chdir(td, uap->path, UIO_USERSPACE));
	}

	int
	kern_chdir(struct thread td, const char path, enum uio_seg pathseg)
	{
	struct nameidata nd;
	int error;

	NDINIT(&nd, LOOKUP, FOLLOW \| LOCKSHARED \| LOCKLEAF \| AUDITVNODE1,
	pathseg, path, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	if ((error = change_dir(nd.ni_vp, td)) != 0) {
	vput(nd.ni_vp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	return (error);
	}
	VOP_UNLOCK(nd.ni_vp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	pwd_chdir(td, nd.ni_vp);
	return (0);
	}

	/*
	* Change notion of root (``/'') directory.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct chroot_args {
	char *path;
	};
	#endif
	int
	sys_chroot(struct thread td, struct chroot_args uap)
	{
	struct nameidata nd;
	int error;

	error = priv_check(td, PRIV_VFS_CHROOT);
	if (error != 0)
	return (error);
	NDINIT(&nd, LOOKUP, FOLLOW \| LOCKSHARED \| LOCKLEAF \| AUDITVNODE1,
	UIO_USERSPACE, uap->path, td);
	error = namei(&nd);
	if (error != 0)
	goto error;
	error = change_dir(nd.ni_vp, td);
	if (error != 0)
	goto e_vunlock;
	#ifdef MAC
	error = mac_vnode_check_chroot(td->td_ucred, nd.ni_vp);
	if (error != 0)
	goto e_vunlock;
	#endif
	VOP_UNLOCK(nd.ni_vp);
	error = pwd_chroot(td, nd.ni_vp);
	vrele(nd.ni_vp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	return (error);
	e_vunlock:
	vput(nd.ni_vp);
	error:
	NDFREE(&nd, NDF_ONLY_PNBUF);
	return (error);
	}

	/*
	* Common routine for chroot and chdir. Callers must provide a locked vnode
	* instance.
	*/
	int
	change_dir(struct vnode vp, struct thread td)
	{
	#ifdef MAC
	int error;
	#endif

	ASSERT_VOP_LOCKED(vp, "change_dir(): vp not locked");
	if (vp->v_type != VDIR)
	return (ENOTDIR);
	#ifdef MAC
	error = mac_vnode_check_chdir(td->td_ucred, vp);
	if (error != 0)
	return (error);
	#endif
	return (VOP_ACCESS(vp, VEXEC, td->td_ucred, td));
	}

	static __inline void
	flags_to_rights(int flags, cap_rights_t *rightsp)
	{

	if (flags & O_EXEC) {
	cap_rights_set_one(rightsp, CAP_FEXECVE);
	} else {
	switch ((flags & O_ACCMODE)) {
	case O_RDONLY:
	cap_rights_set_one(rightsp, CAP_READ);
	break;
	case O_RDWR:
	cap_rights_set_one(rightsp, CAP_READ);
	/* FALLTHROUGH */
	case O_WRONLY:
	cap_rights_set_one(rightsp, CAP_WRITE);
	if (!(flags & (O_APPEND \| O_TRUNC)))
	cap_rights_set_one(rightsp, CAP_SEEK);
	break;
	}
	}

	if (flags & O_CREAT)
	cap_rights_set_one(rightsp, CAP_CREATE);

	if (flags & O_TRUNC)
	cap_rights_set_one(rightsp, CAP_FTRUNCATE);

	if (flags & (O_SYNC \| O_FSYNC))
	cap_rights_set_one(rightsp, CAP_FSYNC);

	if (flags & (O_EXLOCK \| O_SHLOCK))
	cap_rights_set_one(rightsp, CAP_FLOCK);
	}

	/*
	* Check permissions, allocate an open file structure, and call the device
	* open routine if any.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct open_args {
	char *path;
	int flags;
	int mode;
	};
	#endif
	int
	sys_open(struct thread td, struct open_args uap)
	{

	return (kern_openat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->flags, uap->mode));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct openat_args {
	int fd;
	char *path;
	int flag;
	int mode;
	};
	#endif
	int
	sys_openat(struct thread td, struct openat_args uap)
	{

	AUDIT_ARG_FD(uap->fd);
	return (kern_openat(td, uap->fd, uap->path, UIO_USERSPACE, uap->flag,
	uap->mode));
	}

	int
	kern_openat(struct thread td, int fd, const char path, enum uio_seg pathseg,
	int flags, int mode)
	{
	struct proc *p = td->td_proc;
	struct filedesc *fdp = p->p_fd;
	struct file *fp;
	struct vnode *vp;
	struct nameidata nd;
	cap_rights_t rights;
	int cmode, error, indx;

	indx = -1;

	AUDIT_ARG_FFLAGS(flags);
	AUDIT_ARG_MODE(mode);
	cap_rights_init_one(&rights, CAP_LOOKUP);
	flags_to_rights(flags, &rights);
	/*
	* Only one of the O_EXEC, O_RDONLY, O_WRONLY and O_RDWR flags
	* may be specified.
	*/
	if (flags & O_EXEC) {
	if (flags & O_ACCMODE)
	return (EINVAL);
	} else if ((flags & O_ACCMODE) == O_ACCMODE) {
	return (EINVAL);
	} else {
	flags = FFLAGS(flags);
	}

	/*
	* Allocate a file structure. The descriptor to reference it
	* is allocated and set by finstall() below.
	*/
	error = falloc_noinstall(td, &fp);
	if (error != 0)
	return (error);
	/*
	* An extra reference on `fp' has been held for us by
	* falloc_noinstall().
	*/
	/* Set the flags early so the finit in devfs can pick them up. */
	fp->f_flag = flags & FMASK;
	cmode = ((mode & ~fdp->fd_cmask) & ALLPERMS) & ~S_ISTXT;
	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW \| AUDITVNODE1, pathseg, path, fd,
	&rights, td);
	td->td_dupfd = -1; /* XXX check for fdopen */
	error = vn_open(&nd, &flags, cmode, fp);
	if (error != 0) {
	/*
	* If the vn_open replaced the method vector, something
	* wonderous happened deep below and we just pass it up
	* pretending we know what we do.
	*/
	if (error == ENXIO && fp->f_ops != &badfileops)
	goto success;

	/*
	* Handle special fdopen() case. bleh.
	*
	* Don't do this for relative (capability) lookups; we don't
	* understand exactly what would happen, and we don't think
	* that it ever should.
	*/
	if ((nd.ni_lcf & NI_LCF_STRICTRELATIVE) == 0 &&
	(error == ENODEV \|\| error == ENXIO) &&
	td->td_dupfd >= 0) {
	error = dupfdopen(td, fdp, td->td_dupfd, flags, error,
	&indx);
	if (error == 0)
	goto success;
	}

	goto bad;
	}
	td->td_dupfd = 0;
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vp = nd.ni_vp;

	/*
	* Store the vnode, for any f_type. Typically, the vnode use
	* count is decremented by direct call to vn_closefile() for
	* files that switched type in the cdevsw fdopen() method.
	*/
	fp->f_vnode = vp;
	/*
	* If the file wasn't claimed by devfs bind it to the normal
	* vnode operations here.
	*/
	if (fp->f_ops == &badfileops) {
	KASSERT(vp->v_type != VFIFO, ("Unexpected fifo."));
	fp->f_seqcount[UIO_READ] = 1;
	fp->f_seqcount[UIO_WRITE] = 1;
	finit(fp, (flags & FMASK) \| (fp->f_flag & FHASLOCK),
	DTYPE_VNODE, vp, &vnops);
	}

	VOP_UNLOCK(vp);
	if (flags & O_TRUNC) {
	error = fo_truncate(fp, 0, td->td_ucred, td);
	if (error != 0)
	goto bad;
	}
	success:
	/*
	* If we haven't already installed the FD (for dupfdopen), do so now.
	*/
	if (indx == -1) {
	struct filecaps *fcaps;

	#ifdef CAPABILITIES
	if ((nd.ni_lcf & NI_LCF_STRICTRELATIVE) != 0)
	fcaps = &nd.ni_filecaps;
	else
	#endif
	fcaps = NULL;
	error = finstall(td, fp, &indx, flags, fcaps);
	/* On success finstall() consumes fcaps. */
	if (error != 0) {
	filecaps_free(&nd.ni_filecaps);
	goto bad;
	}
	} else {
	filecaps_free(&nd.ni_filecaps);
	}

	/*
	* Release our private reference, leaving the one associated with
	* the descriptor table intact.
	*/
	fdrop(fp, td);
	td->td_retval[0] = indx;
	return (0);
	bad:
	KASSERT(indx == -1, ("indx=%d, should be -1", indx));
	fdrop(fp, td);
	return (error);
	}

	#ifdef COMPAT_43
	/*
	* Create a file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct ocreat_args {
	char *path;
	int mode;
	};
	#endif
	int
	ocreat(struct thread td, struct ocreat_args uap)
	{

	return (kern_openat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	O_WRONLY \| O_CREAT \| O_TRUNC, uap->mode));
	}
	#endif /* COMPAT_43 */

	/*
	* Create a special file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct mknodat_args {
	int fd;
	char *path;
	mode_t mode;
	dev_t dev;
	};
	#endif
	int
	sys_mknodat(struct thread td, struct mknodat_args uap)
	{

	return (kern_mknodat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode,
	uap->dev));
	}

	#if defined(COMPAT_FREEBSD11)
	int
	freebsd11_mknod(struct thread *td,
	struct freebsd11_mknod_args *uap)
	{

	return (kern_mknodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->mode, uap->dev));
	}

	int
	freebsd11_mknodat(struct thread *td,
	struct freebsd11_mknodat_args *uap)
	{

	return (kern_mknodat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode,
	uap->dev));
	}
	#endif /* COMPAT_FREEBSD11 */

	int
	kern_mknodat(struct thread td, int fd, const char path, enum uio_seg pathseg,
	int mode, dev_t dev)
	{
	struct vnode *vp;
	struct mount *mp;
	struct vattr vattr;
	struct nameidata nd;
	int error, whiteout = 0;

	AUDIT_ARG_MODE(mode);
	AUDIT_ARG_DEV(dev);
	switch (mode & S_IFMT) {
	case S_IFCHR:
	case S_IFBLK:
	error = priv_check(td, PRIV_VFS_MKNOD_DEV);
	if (error == 0 && dev == VNOVAL)
	error = EINVAL;
	break;
	case S_IFWHT:
	error = priv_check(td, PRIV_VFS_MKNOD_WHT);
	break;
	case S_IFIFO:
	if (dev == 0)
	return (kern_mkfifoat(td, fd, path, pathseg, mode));
	/* FALLTHROUGH */
	default:
	error = EINVAL;
	break;
	}
	if (error != 0)
	return (error);
	restart:
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT \| SAVENAME \| AUDITVNODE1 \|
	NOCACHE, pathseg, path, fd, &cap_mknodat_rights,
	td);
	if ((error = namei(&nd)) != 0)
	return (error);
	vp = nd.ni_vp;
	if (vp != NULL) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (vp == nd.ni_dvp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	vrele(vp);
	return (EEXIST);
	} else {
	VATTR_NULL(&vattr);
	vattr.va_mode = (mode & ALLPERMS) &
	~td->td_proc->p_fd->fd_cmask;
	vattr.va_rdev = dev;
	whiteout = 0;

	switch (mode & S_IFMT) {
	case S_IFCHR:
	vattr.va_type = VCHR;
	break;
	case S_IFBLK:
	vattr.va_type = VBLK;
	break;
	case S_IFWHT:
	whiteout = 1;
	break;
	default:
	panic("kern_mknod: invalid mode");
	}
	}
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if ((error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH)) != 0)
	return (error);
	goto restart;
	}
	#ifdef MAC
	if (error == 0 && !whiteout)
	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp,
	&nd.ni_cnd, &vattr);
	#endif
	if (error == 0) {
	if (whiteout)
	error = VOP_WHITEOUT(nd.ni_dvp, &nd.ni_cnd, CREATE);
	else {
	error = VOP_MKNOD(nd.ni_dvp, &nd.ni_vp,
	&nd.ni_cnd, &vattr);
	if (error == 0)
	vput(nd.ni_vp);
	}
	}
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Create a named pipe.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct mkfifo_args {
	char *path;
	int mode;
	};
	#endif
	int
	sys_mkfifo(struct thread td, struct mkfifo_args uap)
	{

	return (kern_mkfifoat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->mode));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct mkfifoat_args {
	int fd;
	char *path;
	mode_t mode;
	};
	#endif
	int
	sys_mkfifoat(struct thread td, struct mkfifoat_args uap)
	{

	return (kern_mkfifoat(td, uap->fd, uap->path, UIO_USERSPACE,
	uap->mode));
	}

	int
	kern_mkfifoat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, int mode)
	{
	struct mount *mp;
	struct vattr vattr;
	struct nameidata nd;
	int error;

	AUDIT_ARG_MODE(mode);
	restart:
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT \| SAVENAME \| AUDITVNODE1 \|
	NOCACHE, pathseg, path, fd, &cap_mkfifoat_rights,
	td);
	if ((error = namei(&nd)) != 0)
	return (error);
	if (nd.ni_vp != NULL) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (nd.ni_vp == nd.ni_dvp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	vrele(nd.ni_vp);
	return (EEXIST);
	}
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if ((error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH)) != 0)
	return (error);
	goto restart;
	}
	VATTR_NULL(&vattr);
	vattr.va_type = VFIFO;
	vattr.va_mode = (mode & ALLPERMS) & ~td->td_proc->p_fd->fd_cmask;
	#ifdef MAC
	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
	&vattr);
	if (error != 0)
	goto out;
	#endif
	error = VOP_MKNOD(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
	if (error == 0)
	vput(nd.ni_vp);
	#ifdef MAC
	out:
	#endif
	vput(nd.ni_dvp);
	vn_finished_write(mp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	return (error);
	}

	/*
	* Make a hard file link.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct link_args {
	char *path;
	char *link;
	};
	#endif
	int
	sys_link(struct thread td, struct link_args uap)
	{

	return (kern_linkat(td, AT_FDCWD, AT_FDCWD, uap->path, uap->link,
	UIO_USERSPACE, FOLLOW));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct linkat_args {
	int fd1;
	char *path1;
	int fd2;
	char *path2;
	int flag;
	};
	#endif
	int
	sys_linkat(struct thread td, struct linkat_args uap)
	{
	int flag;

	flag = uap->flag;
	if ((flag & ~(AT_SYMLINK_FOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);

	return (kern_linkat(td, uap->fd1, uap->fd2, uap->path1, uap->path2,
	UIO_USERSPACE, ((flag & AT_SYMLINK_FOLLOW) != 0 ? FOLLOW :
	NOFOLLOW) \| ((flag & AT_BENEATH) != 0 ? BENEATH : 0)));
	}

	int hardlink_check_uid = 0;
	SYSCTL_INT(_security_bsd, OID_AUTO, hardlink_check_uid, CTLFLAG_RW,
	&hardlink_check_uid, 0,
	"Unprivileged processes cannot create hard links to files owned by other "
	"users");
	static int hardlink_check_gid = 0;
	SYSCTL_INT(_security_bsd, OID_AUTO, hardlink_check_gid, CTLFLAG_RW,
	&hardlink_check_gid, 0,
	"Unprivileged processes cannot create hard links to files owned by other "
	"groups");

	static int
	can_hardlink(struct vnode vp, struct ucred cred)
	{
	struct vattr va;
	int error;

	if (!hardlink_check_uid && !hardlink_check_gid)
	return (0);

	error = VOP_GETATTR(vp, &va, cred);
	if (error != 0)
	return (error);

	if (hardlink_check_uid && cred->cr_uid != va.va_uid) {
	error = priv_check_cred(cred, PRIV_VFS_LINK);
	if (error != 0)
	return (error);
	}

	if (hardlink_check_gid && !groupmember(va.va_gid, cred)) {
	error = priv_check_cred(cred, PRIV_VFS_LINK);
	if (error != 0)
	return (error);
	}

	return (0);
	}

	int
	kern_linkat(struct thread td, int fd1, int fd2, const char path1,
	const char *path2, enum uio_seg segflag, int follow)
	{
	struct nameidata nd;
	int error;

	do {
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, LOOKUP, follow \| AUDITVNODE1, segflag,
	path1, fd1, &cap_linkat_source_rights, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = kern_linkat_vp(td, nd.ni_vp, fd2, path2, segflag);
	} while (error == EAGAIN);
	return (error);
	}

	static int
	kern_linkat_vp(struct thread td, struct vnode vp, int fd, const char *path,
	enum uio_seg segflag)
	{
	struct nameidata nd;
	struct mount *mp;
	int error;

	if (vp->v_type == VDIR) {
	vrele(vp);
	return (EPERM); /* POSIX */
	}
	NDINIT_ATRIGHTS(&nd, CREATE,
	LOCKPARENT \| SAVENAME \| AUDITVNODE2 \| NOCACHE, segflag, path, fd,
	&cap_linkat_target_rights, td);
	if ((error = namei(&nd)) == 0) {
	if (nd.ni_vp != NULL) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (nd.ni_dvp == nd.ni_vp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	vrele(nd.ni_vp);
	vrele(vp);
	return (EEXIST);
	} else if (nd.ni_dvp->v_mount != vp->v_mount) {
	/*
	* Cross-device link. No need to recheck
	* vp->v_type, since it cannot change, except
	* to VBAD.
	*/
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	vrele(vp);
	return (EXDEV);
	} else if ((error = vn_lock(vp, LK_EXCLUSIVE)) == 0) {
	error = can_hardlink(vp, td->td_ucred);
	#ifdef MAC
	if (error == 0)
	error = mac_vnode_check_link(td->td_ucred,
	nd.ni_dvp, vp, &nd.ni_cnd);
	#endif
	if (error != 0) {
	vput(vp);
	vput(nd.ni_dvp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	return (error);
	}
	error = vn_start_write(vp, &mp, V_NOWAIT);
	if (error != 0) {
	vput(vp);
	vput(nd.ni_dvp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = vn_start_write(NULL, &mp,
	V_XSLEEP \| PCATCH);
	if (error != 0)
	return (error);
	return (EAGAIN);
	}
	error = VOP_LINK(nd.ni_dvp, vp, &nd.ni_cnd);
	VOP_UNLOCK(vp);
	vput(nd.ni_dvp);
	vn_finished_write(mp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	} else {
	vput(nd.ni_dvp);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vrele(vp);
	return (EAGAIN);
	}
	}
	vrele(vp);
	return (error);
	}

	/*
	* Make a symbolic link.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct symlink_args {
	char *path;
	char *link;
	};
	#endif
	int
	sys_symlink(struct thread td, struct symlink_args uap)
	{

	return (kern_symlinkat(td, uap->path, AT_FDCWD, uap->link,
	UIO_USERSPACE));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct symlinkat_args {
	char *path;
	int fd;
	char *path2;
	};
	#endif
	int
	sys_symlinkat(struct thread td, struct symlinkat_args uap)
	{

	return (kern_symlinkat(td, uap->path1, uap->fd, uap->path2,
	UIO_USERSPACE));
	}

	int
	kern_symlinkat(struct thread td, const char path1, int fd, const char *path2,
	enum uio_seg segflg)
	{
	struct mount *mp;
	struct vattr vattr;
	const char *syspath;
	char *tmppath;
	struct nameidata nd;
	int error;

	if (segflg == UIO_SYSSPACE) {
	syspath = path1;
	} else {
	tmppath = uma_zalloc(namei_zone, M_WAITOK);
	if ((error = copyinstr(path1, tmppath, MAXPATHLEN, NULL)) != 0)
	goto out;
	syspath = tmppath;
	}
	AUDIT_ARG_TEXT(syspath);
	restart:
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT \| SAVENAME \| AUDITVNODE1 \|
	NOCACHE, segflg, path2, fd, &cap_symlinkat_rights,
	td);
	if ((error = namei(&nd)) != 0)
	goto out;
	if (nd.ni_vp) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (nd.ni_vp == nd.ni_dvp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	vrele(nd.ni_vp);
	error = EEXIST;
	goto out;
	}
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if ((error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH)) != 0)
	goto out;
	goto restart;
	}
	VATTR_NULL(&vattr);
	vattr.va_mode = ACCESSPERMS &~ td->td_proc->p_fd->fd_cmask;
	#ifdef MAC
	vattr.va_type = VLNK;
	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
	&vattr);
	if (error != 0)
	goto out2;
	#endif
	error = VOP_SYMLINK(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr, syspath);
	if (error == 0)
	vput(nd.ni_vp);
	#ifdef MAC
	out2:
	#endif
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	vn_finished_write(mp);
	out:
	if (segflg != UIO_SYSSPACE)
	uma_zfree(namei_zone, tmppath);
	return (error);
	}

	/*
	* Delete a whiteout from the filesystem.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct undelete_args {
	char *path;
	};
	#endif
	int
	sys_undelete(struct thread td, struct undelete_args uap)
	{
	struct mount *mp;
	struct nameidata nd;
	int error;

	restart:
	bwillwrite();
	NDINIT(&nd, DELETE, LOCKPARENT \| DOWHITEOUT \| AUDITVNODE1,
	UIO_USERSPACE, uap->path, td);
	error = namei(&nd);
	if (error != 0)
	return (error);

	if (nd.ni_vp != NULLVP \|\| !(nd.ni_cnd.cn_flags & ISWHITEOUT)) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (nd.ni_vp == nd.ni_dvp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	if (nd.ni_vp)
	vrele(nd.ni_vp);
	return (EEXIST);
	}
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if ((error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH)) != 0)
	return (error);
	goto restart;
	}
	error = VOP_WHITEOUT(nd.ni_dvp, &nd.ni_cnd, DELETE);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Delete a name from the filesystem.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct unlink_args {
	char *path;
	};
	#endif
	int
	sys_unlink(struct thread td, struct unlink_args uap)
	{

	return (kern_funlinkat(td, AT_FDCWD, uap->path, FD_NONE, UIO_USERSPACE,
	0, 0));
	}

	static int
	kern_funlinkat_ex(struct thread td, int dfd, const char path, int fd,
	int flag, enum uio_seg pathseg, ino_t oldinum)
	{

	if ((flag & ~AT_REMOVEDIR) != 0)
	return (EINVAL);

	if ((flag & AT_REMOVEDIR) != 0)
	return (kern_frmdirat(td, dfd, path, fd, UIO_USERSPACE, 0));

	return (kern_funlinkat(td, dfd, path, fd, UIO_USERSPACE, 0, 0));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct unlinkat_args {
	int fd;
	char *path;
	int flag;
	};
	#endif
	int
	sys_unlinkat(struct thread td, struct unlinkat_args uap)
	{

	return (kern_funlinkat_ex(td, uap->fd, uap->path, FD_NONE, uap->flag,
	UIO_USERSPACE, 0));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct funlinkat_args {
	int dfd;
	const char *path;
	int fd;
	int flag;
	};
	#endif
	int
	sys_funlinkat(struct thread td, struct funlinkat_args uap)
	{

	return (kern_funlinkat_ex(td, uap->dfd, uap->path, uap->fd, uap->flag,
	UIO_USERSPACE, 0));
	}

	int
	kern_funlinkat(struct thread td, int dfd, const char path, int fd,
	enum uio_seg pathseg, int flag, ino_t oldinum)
	{
	struct mount *mp;
	struct file *fp;
	struct vnode *vp;
	struct nameidata nd;
	struct stat sb;
	int error;

	fp = NULL;
	if (fd != FD_NONE) {
	error = getvnode(td, fd, &cap_no_rights, &fp);
	if (error != 0)
	return (error);
	}

	restart:
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT \| LOCKLEAF \| AUDITVNODE1 \|
	((flag & AT_BENEATH) != 0 ? BENEATH : 0),
	pathseg, path, dfd, &cap_unlinkat_rights, td);
	if ((error = namei(&nd)) != 0) {
	if (error == EINVAL)
	error = EPERM;
	goto fdout;
	}
	vp = nd.ni_vp;
	if (vp->v_type == VDIR && oldinum == 0) {
	error = EPERM; /* POSIX */
	} else if (oldinum != 0 &&
	((error = vn_stat(vp, &sb, td->td_ucred, NOCRED, td)) == 0) &&
	sb.st_ino != oldinum) {
	error = EIDRM; /* Identifier removed */
	} else if (fp != NULL && fp->f_vnode != vp) {
	if (VN_IS_DOOMED(fp->f_vnode))
	error = EBADF;
	else
	error = EDEADLK;
	} else {
	/*
	* The root of a mounted filesystem cannot be deleted.
	*
	* XXX: can this only be a VDIR case?
	*/
	if (vp->v_vflag & VV_ROOT)
	error = EBUSY;
	}
	if (error == 0) {
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if (vp == nd.ni_dvp)
	vrele(vp);
	else
	vput(vp);
	if ((error = vn_start_write(NULL, &mp,
	V_XSLEEP \| PCATCH)) != 0) {
	goto fdout;
	}
	goto restart;
	}
	#ifdef MAC
	error = mac_vnode_check_unlink(td->td_ucred, nd.ni_dvp, vp,
	&nd.ni_cnd);
	if (error != 0)
	goto out;
	#endif
	vfs_notify_upper(vp, VFS_NOTIFY_UPPER_UNLINK);
	error = VOP_REMOVE(nd.ni_dvp, vp, &nd.ni_cnd);
	#ifdef MAC
	out:
	#endif
	vn_finished_write(mp);
	}
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if (vp == nd.ni_dvp)
	vrele(vp);
	else
	vput(vp);
	fdout:
	if (fp != NULL)
	fdrop(fp, td);
	return (error);
	}

	/*
	* Reposition read/write file offset.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct lseek_args {
	int fd;
	int pad;
	off_t offset;
	int whence;
	};
	#endif
	int
	sys_lseek(struct thread td, struct lseek_args uap)
	{

	return (kern_lseek(td, uap->fd, uap->offset, uap->whence));
	}

	int
	kern_lseek(struct thread *td, int fd, off_t offset, int whence)
	{
	struct file *fp;
	int error;

	AUDIT_ARG_FD(fd);
	error = fget(td, fd, &cap_seek_rights, &fp);
	if (error != 0)
	return (error);
	error = (fp->f_ops->fo_flags & DFLAG_SEEKABLE) != 0 ?
	fo_seek(fp, offset, whence, td) : ESPIPE;
	fdrop(fp, td);
	return (error);
	}

	#if defined(COMPAT_43)
	/*
	* Reposition read/write file offset.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct olseek_args {
	int fd;
	long offset;
	int whence;
	};
	#endif
	int
	olseek(struct thread td, struct olseek_args uap)
	{

	return (kern_lseek(td, uap->fd, uap->offset, uap->whence));
	}
	#endif /* COMPAT_43 */

	#if defined(COMPAT_FREEBSD6)
	/* Version with the 'pad' argument */
	int
	freebsd6_lseek(struct thread td, struct freebsd6_lseek_args uap)
	{

	return (kern_lseek(td, uap->fd, uap->offset, uap->whence));
	}
	#endif

	/*
	* Check access permissions using passed credentials.
	*/
	static int
	vn_access(struct vnode vp, int user_flags, struct ucred cred,
	struct thread *td)
	{
	accmode_t accmode;
	int error;

	/* Flags == 0 means only check for existence. */
	if (user_flags == 0)
	return (0);

	accmode = 0;
	if (user_flags & R_OK)
	accmode \|= VREAD;
	if (user_flags & W_OK)
	accmode \|= VWRITE;
	if (user_flags & X_OK)
	accmode \|= VEXEC;
	#ifdef MAC
	error = mac_vnode_check_access(cred, vp, accmode);
	if (error != 0)
	return (error);
	#endif
	if ((accmode & VWRITE) == 0 \|\| (error = vn_writechk(vp)) == 0)
	error = VOP_ACCESS(vp, accmode, cred, td);
	return (error);
	}

	/*
	* Check access permissions using "real" credentials.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct access_args {
	char *path;
	int amode;
	};
	#endif
	int
	sys_access(struct thread td, struct access_args uap)
	{

	return (kern_accessat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	0, uap->amode));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct faccessat_args {
	int dirfd;
	char *path;
	int amode;
	int flag;
	}
	#endif
	int
	sys_faccessat(struct thread td, struct faccessat_args uap)
	{

	return (kern_accessat(td, uap->fd, uap->path, UIO_USERSPACE, uap->flag,
	uap->amode));
	}

	int
	kern_accessat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, int flag, int amode)
	{
	struct ucred cred, usecred;
	struct vnode *vp;
	struct nameidata nd;
	int error;

	if ((flag & ~(AT_EACCESS \| AT_BENEATH)) != 0)
	return (EINVAL);
	if (amode != F_OK && (amode & ~(R_OK \| W_OK \| X_OK)) != 0)
	return (EINVAL);

	/*
	* Create and modify a temporary credential instead of one that
	* is potentially shared (if we need one).
	*/
	cred = td->td_ucred;
	if ((flag & AT_EACCESS) == 0 &&
	((cred->cr_uid != cred->cr_ruid \|\|
	cred->cr_rgid != cred->cr_groups[0]))) {
	usecred = crdup(cred);
	usecred->cr_uid = cred->cr_ruid;
	usecred->cr_groups[0] = cred->cr_rgid;
	td->td_ucred = usecred;
	} else
	usecred = cred;
	AUDIT_ARG_VALUE(amode);
	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW \| LOCKSHARED \| LOCKLEAF \|
	AUDITVNODE1 \| ((flag & AT_BENEATH) != 0 ? BENEATH : 0),
	pathseg, path, fd, &cap_fstat_rights, td);
	if ((error = namei(&nd)) != 0)
	goto out;
	vp = nd.ni_vp;

	error = vn_access(vp, amode, usecred, td);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(vp);
	out:
	if (usecred != cred) {
	td->td_ucred = cred;
	crfree(usecred);
	}
	return (error);
	}

	/*
	* Check access permissions using "effective" credentials.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct eaccess_args {
	char *path;
	int amode;
	};
	#endif
	int
	sys_eaccess(struct thread td, struct eaccess_args uap)
	{

	return (kern_accessat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	AT_EACCESS, uap->amode));
	}

	#if defined(COMPAT_43)
	/*
	* Get file status; this version follows links.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct ostat_args {
	char *path;
	struct ostat *ub;
	};
	#endif
	int
	ostat(struct thread td, struct ostat_args uap)
	{
	struct stat sb;
	struct ostat osb;
	int error;

	error = kern_statat(td, 0, AT_FDCWD, uap->path, UIO_USERSPACE,
	&sb, NULL);
	if (error != 0)
	return (error);
	cvtstat(&sb, &osb);
	return (copyout(&osb, uap->ub, sizeof (osb)));
	}

	/*
	* Get file status; this version does not follow links.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct olstat_args {
	char *path;
	struct ostat *ub;
	};
	#endif
	int
	olstat(struct thread td, struct olstat_args uap)
	{
	struct stat sb;
	struct ostat osb;
	int error;

	error = kern_statat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->path,
	UIO_USERSPACE, &sb, NULL);
	if (error != 0)
	return (error);
	cvtstat(&sb, &osb);
	return (copyout(&osb, uap->ub, sizeof (osb)));
	}

	/*
	* Convert from an old to a new stat structure.
	* XXX: many values are blindly truncated.
	*/
	void
	cvtstat(struct stat st, struct ostat ost)
	{

	bzero(ost, sizeof(*ost));
	ost->st_dev = st->st_dev;
	ost->st_ino = st->st_ino;
	ost->st_mode = st->st_mode;
	ost->st_nlink = st->st_nlink;
	ost->st_uid = st->st_uid;
	ost->st_gid = st->st_gid;
	ost->st_rdev = st->st_rdev;
	ost->st_size = MIN(st->st_size, INT32_MAX);
	ost->st_atim = st->st_atim;
	ost->st_mtim = st->st_mtim;
	ost->st_ctim = st->st_ctim;
	ost->st_blksize = st->st_blksize;
	ost->st_blocks = st->st_blocks;
	ost->st_flags = st->st_flags;
	ost->st_gen = st->st_gen;
	}
	#endif /* COMPAT_43 */

	#if defined(COMPAT_43) \|\| defined(COMPAT_FREEBSD11)
	int ino64_trunc_error;
	SYSCTL_INT(_vfs, OID_AUTO, ino64_trunc_error, CTLFLAG_RW,
	&ino64_trunc_error, 0,
	"Error on truncation of device, file or inode number, or link count");

	int
	freebsd11_cvtstat(struct stat st, struct freebsd11_stat ost)
	{

	ost->st_dev = st->st_dev;
	if (ost->st_dev != st->st_dev) {
	switch (ino64_trunc_error) {
	default:
	/*
	* Since dev_t is almost raw, don't clamp to the
	* maximum for case 2, but ignore the error.
	*/
	break;
	case 1:
	return (EOVERFLOW);
	}
	}
	ost->st_ino = st->st_ino;
	if (ost->st_ino != st->st_ino) {
	switch (ino64_trunc_error) {
	default:
	case 0:
	break;
	case 1:
	return (EOVERFLOW);
	case 2:
	ost->st_ino = UINT32_MAX;
	break;
	}
	}
	ost->st_mode = st->st_mode;
	ost->st_nlink = st->st_nlink;
	if (ost->st_nlink != st->st_nlink) {
	switch (ino64_trunc_error) {
	default:
	case 0:
	break;
	case 1:
	return (EOVERFLOW);
	case 2:
	ost->st_nlink = UINT16_MAX;
	break;
	}
	}
	ost->st_uid = st->st_uid;
	ost->st_gid = st->st_gid;
	ost->st_rdev = st->st_rdev;
	if (ost->st_rdev != st->st_rdev) {
	switch (ino64_trunc_error) {
	default:
	break;
	case 1:
	return (EOVERFLOW);
	}
	}
	ost->st_atim = st->st_atim;
	ost->st_mtim = st->st_mtim;
	ost->st_ctim = st->st_ctim;
	ost->st_size = st->st_size;
	ost->st_blocks = st->st_blocks;
	ost->st_blksize = st->st_blksize;
	ost->st_flags = st->st_flags;
	ost->st_gen = st->st_gen;
	ost->st_lspare = 0;
	ost->st_birthtim = st->st_birthtim;
	bzero((char *)&ost->st_birthtim + sizeof(ost->st_birthtim),
	sizeof(*ost) - offsetof(struct freebsd11_stat,
	st_birthtim) - sizeof(ost->st_birthtim));
	return (0);
	}

	int
	freebsd11_stat(struct thread td, struct freebsd11_stat_args uap)
	{
	struct stat sb;
	struct freebsd11_stat osb;
	int error;

	error = kern_statat(td, 0, AT_FDCWD, uap->path, UIO_USERSPACE,
	&sb, NULL);
	if (error != 0)
	return (error);
	error = freebsd11_cvtstat(&sb, &osb);
	if (error == 0)
	error = copyout(&osb, uap->ub, sizeof(osb));
	return (error);
	}

	int
	freebsd11_lstat(struct thread td, struct freebsd11_lstat_args uap)
	{
	struct stat sb;
	struct freebsd11_stat osb;
	int error;

	error = kern_statat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->path,
	UIO_USERSPACE, &sb, NULL);
	if (error != 0)
	return (error);
	error = freebsd11_cvtstat(&sb, &osb);
	if (error == 0)
	error = copyout(&osb, uap->ub, sizeof(osb));
	return (error);
	}

	int
	freebsd11_fhstat(struct thread td, struct freebsd11_fhstat_args uap)
	{
	struct fhandle fh;
	struct stat sb;
	struct freebsd11_stat osb;
	int error;

	error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
	if (error != 0)
	return (error);
	error = kern_fhstat(td, fh, &sb);
	if (error != 0)
	return (error);
	error = freebsd11_cvtstat(&sb, &osb);
	if (error == 0)
	error = copyout(&osb, uap->sb, sizeof(osb));
	return (error);
	}

	int
	freebsd11_fstatat(struct thread td, struct freebsd11_fstatat_args uap)
	{
	struct stat sb;
	struct freebsd11_stat osb;
	int error;

	error = kern_statat(td, uap->flag, uap->fd, uap->path,
	UIO_USERSPACE, &sb, NULL);
	if (error != 0)
	return (error);
	error = freebsd11_cvtstat(&sb, &osb);
	if (error == 0)
	error = copyout(&osb, uap->buf, sizeof(osb));
	return (error);
	}
	#endif /* COMPAT_FREEBSD11 */

	/*
	* Get file status
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fstatat_args {
	int fd;
	char *path;
	struct stat *buf;
	int flag;
	}
	#endif
	int
	sys_fstatat(struct thread td, struct fstatat_args uap)
	{
	struct stat sb;
	int error;

	error = kern_statat(td, uap->flag, uap->fd, uap->path,
	UIO_USERSPACE, &sb, NULL);
	if (error == 0)
	error = copyout(&sb, uap->buf, sizeof (sb));
	return (error);
	}

	int
	kern_statat(struct thread td, int flag, int fd, const char path,
	enum uio_seg pathseg, struct stat *sbp,
	void (hook)(struct vnode vp, struct stat *sbp))
	{
	struct nameidata nd;
	int error;

	if ((flag & ~(AT_SYMLINK_NOFOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);

	NDINIT_ATRIGHTS(&nd, LOOKUP, ((flag & AT_SYMLINK_NOFOLLOW) != 0 ?
	NOFOLLOW : FOLLOW) \| ((flag & AT_BENEATH) != 0 ? BENEATH : 0) \|
	LOCKSHARED \| LOCKLEAF \| AUDITVNODE1, pathseg, path, fd,
	&cap_fstat_rights, td);

	if ((error = namei(&nd)) != 0)
	return (error);
	error = vn_stat(nd.ni_vp, sbp, td->td_ucred, NOCRED, td);
	if (error == 0) {
	SDT_PROBE2(vfs, , stat, mode, path, sbp->st_mode);
	if (S_ISREG(sbp->st_mode))
	SDT_PROBE2(vfs, , stat, reg, path, pathseg);
	if (__predict_false(hook != NULL))
	hook(nd.ni_vp, sbp);
	}
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_vp);
	#ifdef __STAT_TIME_T_EXT
	sbp->st_atim_ext = 0;
	sbp->st_mtim_ext = 0;
	sbp->st_ctim_ext = 0;
	sbp->st_btim_ext = 0;
	#endif
	#ifdef KTRACE
	if (KTRPOINT(td, KTR_STRUCT))
	ktrstat_error(sbp, error);
	#endif
	return (error);
	}

	#if defined(COMPAT_FREEBSD11)
	/*
	* Implementation of the NetBSD [l]stat() functions.
	*/
	void
	freebsd11_cvtnstat(struct stat sb, struct nstat nsb)
	{

	bzero(nsb, sizeof(*nsb));
	nsb->st_dev = sb->st_dev;
	nsb->st_ino = sb->st_ino;
	nsb->st_mode = sb->st_mode;
	nsb->st_nlink = sb->st_nlink;
	nsb->st_uid = sb->st_uid;
	nsb->st_gid = sb->st_gid;
	nsb->st_rdev = sb->st_rdev;
	nsb->st_atim = sb->st_atim;
	nsb->st_mtim = sb->st_mtim;
	nsb->st_ctim = sb->st_ctim;
	nsb->st_size = sb->st_size;
	nsb->st_blocks = sb->st_blocks;
	nsb->st_blksize = sb->st_blksize;
	nsb->st_flags = sb->st_flags;
	nsb->st_gen = sb->st_gen;
	nsb->st_birthtim = sb->st_birthtim;
	}

	#ifndef _SYS_SYSPROTO_H_
	struct freebsd11_nstat_args {
	char *path;
	struct nstat *ub;
	};
	#endif
	int
	freebsd11_nstat(struct thread td, struct freebsd11_nstat_args uap)
	{
	struct stat sb;
	struct nstat nsb;
	int error;

	error = kern_statat(td, 0, AT_FDCWD, uap->path, UIO_USERSPACE,
	&sb, NULL);
	if (error != 0)
	return (error);
	freebsd11_cvtnstat(&sb, &nsb);
	return (copyout(&nsb, uap->ub, sizeof (nsb)));
	}

	/*
	* NetBSD lstat. Get file status; this version does not follow links.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct freebsd11_nlstat_args {
	char *path;
	struct nstat *ub;
	};
	#endif
	int
	freebsd11_nlstat(struct thread td, struct freebsd11_nlstat_args uap)
	{
	struct stat sb;
	struct nstat nsb;
	int error;

	error = kern_statat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->path,
	UIO_USERSPACE, &sb, NULL);
	if (error != 0)
	return (error);
	freebsd11_cvtnstat(&sb, &nsb);
	return (copyout(&nsb, uap->ub, sizeof (nsb)));
	}
	#endif /* COMPAT_FREEBSD11 */

	/*
	* Get configurable pathname variables.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct pathconf_args {
	char *path;
	int name;
	};
	#endif
	int
	sys_pathconf(struct thread td, struct pathconf_args uap)
	{
	long value;
	int error;

	error = kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name, FOLLOW,
	&value);
	if (error == 0)
	td->td_retval[0] = value;
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct lpathconf_args {
	char *path;
	int name;
	};
	#endif
	int
	sys_lpathconf(struct thread td, struct lpathconf_args uap)
	{
	long value;
	int error;

	error = kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name,
	NOFOLLOW, &value);
	if (error == 0)
	td->td_retval[0] = value;
	return (error);
	}

	int
	kern_pathconf(struct thread td, const char path, enum uio_seg pathseg,
	int name, u_long flags, long *valuep)
	{
	struct nameidata nd;
	int error;

	NDINIT(&nd, LOOKUP, LOCKSHARED \| LOCKLEAF \| AUDITVNODE1 \| flags,
	pathseg, path, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);

	error = VOP_PATHCONF(nd.ni_vp, name, valuep);
	vput(nd.ni_vp);
	return (error);
	}

	/*
	* Return target name of a symbolic link.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct readlink_args {
	char *path;
	char *buf;
	size_t count;
	};
	#endif
	int
	sys_readlink(struct thread td, struct readlink_args uap)
	{

	return (kern_readlinkat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->buf, UIO_USERSPACE, uap->count));
	}
	#ifndef _SYS_SYSPROTO_H_
	struct readlinkat_args {
	int fd;
	char *path;
	char *buf;
	size_t bufsize;
	};
	#endif
	int
	sys_readlinkat(struct thread td, struct readlinkat_args uap)
	{

	return (kern_readlinkat(td, uap->fd, uap->path, UIO_USERSPACE,
	uap->buf, UIO_USERSPACE, uap->bufsize));
	}

	int
	kern_readlinkat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, char *buf, enum uio_seg bufseg, size_t count)
	{
	struct vnode *vp;
	struct nameidata nd;
	int error;

	if (count > IOSIZE_MAX)
	return (EINVAL);

	NDINIT_AT(&nd, LOOKUP, NOFOLLOW \| LOCKSHARED \| LOCKLEAF \| AUDITVNODE1,
	pathseg, path, fd, td);

	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vp = nd.ni_vp;

	error = kern_readlink_vp(vp, buf, bufseg, count, td);
	vput(vp);

	return (error);
	}

	/*
	* Helper function to readlink from a vnode
	*/
	static int
	kern_readlink_vp(struct vnode vp, char buf, enum uio_seg bufseg, size_t count,
	struct thread *td)
	{
	struct iovec aiov;
	struct uio auio;
	int error;

	ASSERT_VOP_LOCKED(vp, "kern_readlink_vp(): vp not locked");
	#ifdef MAC
	error = mac_vnode_check_readlink(td->td_ucred, vp);
	if (error != 0)
	return (error);
	#endif
	if (vp->v_type != VLNK && (vp->v_vflag & VV_READLINK) == 0)
	return (EINVAL);

	aiov.iov_base = buf;
	aiov.iov_len = count;
	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_offset = 0;
	auio.uio_rw = UIO_READ;
	auio.uio_segflg = bufseg;
	auio.uio_td = td;
	auio.uio_resid = count;
	error = VOP_READLINK(vp, &auio, td->td_ucred);
	td->td_retval[0] = count - auio.uio_resid;
	return (error);
	}

	/*
	* Common implementation code for chflags() and fchflags().
	*/
	static int
	setfflags(struct thread td, struct vnode vp, u_long flags)
	{
	struct mount *mp;
	struct vattr vattr;
	int error;

	/* We can't support the value matching VNOVAL. */
	if (flags == VNOVAL)
	return (EOPNOTSUPP);

	/*
	* Prevent non-root users from setting flags on devices. When
	* a device is reused, users can retain ownership of the device
	* if they are allowed to set flags and programs assume that
	* chown can't fail when done as root.
	*/
	if (vp->v_type == VCHR \|\| vp->v_type == VBLK) {
	error = priv_check(td, PRIV_VFS_CHFLAGS_DEV);
	if (error != 0)
	return (error);
	}

	if ((error = vn_start_write(vp, &mp, V_WAIT \| PCATCH)) != 0)
	return (error);
	VATTR_NULL(&vattr);
	vattr.va_flags = flags;
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	#ifdef MAC
	error = mac_vnode_check_setflags(td->td_ucred, vp, vattr.va_flags);
	if (error == 0)
	#endif
	error = VOP_SETATTR(vp, &vattr, td->td_ucred);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Change flags of a file given a path name.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct chflags_args {
	const char *path;
	u_long flags;
	};
	#endif
	int
	sys_chflags(struct thread td, struct chflags_args uap)
	{

	return (kern_chflagsat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->flags, 0));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct chflagsat_args {
	int fd;
	const char *path;
	u_long flags;
	int atflag;
	}
	#endif
	int
	sys_chflagsat(struct thread td, struct chflagsat_args uap)
	{

	if ((uap->atflag & ~(AT_SYMLINK_NOFOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);

	return (kern_chflagsat(td, uap->fd, uap->path, UIO_USERSPACE,
	uap->flags, uap->atflag));
	}

	/*
	* Same as chflags() but doesn't follow symlinks.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct lchflags_args {
	const char *path;
	u_long flags;
	};
	#endif
	int
	sys_lchflags(struct thread td, struct lchflags_args uap)
	{

	return (kern_chflagsat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->flags, AT_SYMLINK_NOFOLLOW));
	}

	static int
	kern_chflagsat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, u_long flags, int atflag)
	{
	struct nameidata nd;
	int error, follow;

	AUDIT_ARG_FFLAGS(flags);
	follow = (atflag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW : FOLLOW;
	follow \|= (atflag & AT_BENEATH) != 0 ? BENEATH : 0;
	NDINIT_ATRIGHTS(&nd, LOOKUP, follow \| AUDITVNODE1, pathseg, path, fd,
	&cap_fchflags_rights, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = setfflags(td, nd.ni_vp, flags);
	vrele(nd.ni_vp);
	return (error);
	}

	/*
	* Change flags of a file given a file descriptor.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fchflags_args {
	int fd;
	u_long flags;
	};
	#endif
	int
	sys_fchflags(struct thread td, struct fchflags_args uap)
	{
	struct file *fp;
	int error;

	AUDIT_ARG_FD(uap->fd);
	AUDIT_ARG_FFLAGS(uap->flags);
	error = getvnode(td, uap->fd, &cap_fchflags_rights,
	&fp);
	if (error != 0)
	return (error);
	#ifdef AUDIT
	vn_lock(fp->f_vnode, LK_SHARED \| LK_RETRY);
	AUDIT_ARG_VNODE1(fp->f_vnode);
	VOP_UNLOCK(fp->f_vnode);
	#endif
	error = setfflags(td, fp->f_vnode, uap->flags);
	fdrop(fp, td);
	return (error);
	}

	/*
	* Common implementation code for chmod(), lchmod() and fchmod().
	*/
	int
	setfmode(struct thread td, struct ucred cred, struct vnode *vp, int mode)
	{
	struct mount *mp;
	struct vattr vattr;
	int error;

	if ((error = vn_start_write(vp, &mp, V_WAIT \| PCATCH)) != 0)
	return (error);
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	VATTR_NULL(&vattr);
	vattr.va_mode = mode & ALLPERMS;
	#ifdef MAC
	error = mac_vnode_check_setmode(cred, vp, vattr.va_mode);
	if (error == 0)
	#endif
	error = VOP_SETATTR(vp, &vattr, cred);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Change mode of a file given path name.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct chmod_args {
	char *path;
	int mode;
	};
	#endif
	int
	sys_chmod(struct thread td, struct chmod_args uap)
	{

	return (kern_fchmodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->mode, 0));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct fchmodat_args {
	int dirfd;
	char *path;
	mode_t mode;
	int flag;
	}
	#endif
	int
	sys_fchmodat(struct thread td, struct fchmodat_args uap)
	{

	if ((uap->flag & ~(AT_SYMLINK_NOFOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);

	return (kern_fchmodat(td, uap->fd, uap->path, UIO_USERSPACE,
	uap->mode, uap->flag));
	}

	/*
	* Change mode of a file given path name (don't follow links.)
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct lchmod_args {
	char *path;
	int mode;
	};
	#endif
	int
	sys_lchmod(struct thread td, struct lchmod_args uap)
	{

	return (kern_fchmodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->mode, AT_SYMLINK_NOFOLLOW));
	}

	int
	kern_fchmodat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, mode_t mode, int flag)
	{
	struct nameidata nd;
	int error, follow;

	AUDIT_ARG_MODE(mode);
	follow = (flag & AT_SYMLINK_NOFOLLOW) != 0 ? NOFOLLOW : FOLLOW;
	follow \|= (flag & AT_BENEATH) != 0 ? BENEATH : 0;
	NDINIT_ATRIGHTS(&nd, LOOKUP, follow \| AUDITVNODE1, pathseg, path, fd,
	&cap_fchmod_rights, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = setfmode(td, td->td_ucred, nd.ni_vp, mode);
	vrele(nd.ni_vp);
	return (error);
	}

	/*
	* Change mode of a file given a file descriptor.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fchmod_args {
	int fd;
	int mode;
	};
	#endif
	int
	sys_fchmod(struct thread td, struct fchmod_args uap)
	{
	struct file *fp;
	int error;

	AUDIT_ARG_FD(uap->fd);
	AUDIT_ARG_MODE(uap->mode);

	error = fget(td, uap->fd, &cap_fchmod_rights, &fp);
	if (error != 0)
	return (error);
	error = fo_chmod(fp, uap->mode, td->td_ucred, td);
	fdrop(fp, td);
	return (error);
	}

	/*
	* Common implementation for chown(), lchown(), and fchown()
	*/
	int
	setfown(struct thread td, struct ucred cred, struct vnode *vp, uid_t uid,
	gid_t gid)
	{
	struct mount *mp;
	struct vattr vattr;
	int error;

	if ((error = vn_start_write(vp, &mp, V_WAIT \| PCATCH)) != 0)
	return (error);
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	VATTR_NULL(&vattr);
	vattr.va_uid = uid;
	vattr.va_gid = gid;
	#ifdef MAC
	error = mac_vnode_check_setowner(cred, vp, vattr.va_uid,
	vattr.va_gid);
	if (error == 0)
	#endif
	error = VOP_SETATTR(vp, &vattr, cred);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Set ownership given a path name.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct chown_args {
	char *path;
	int uid;
	int gid;
	};
	#endif
	int
	sys_chown(struct thread td, struct chown_args uap)
	{

	return (kern_fchownat(td, AT_FDCWD, uap->path, UIO_USERSPACE, uap->uid,
	uap->gid, 0));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct fchownat_args {
	int fd;
	const char * path;
	uid_t uid;
	gid_t gid;
	int flag;
	};
	#endif
	int
	sys_fchownat(struct thread td, struct fchownat_args uap)
	{

	if ((uap->flag & ~(AT_SYMLINK_NOFOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);

	return (kern_fchownat(td, uap->fd, uap->path, UIO_USERSPACE, uap->uid,
	uap->gid, uap->flag));
	}

	int
	kern_fchownat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, int uid, int gid, int flag)
	{
	struct nameidata nd;
	int error, follow;

	AUDIT_ARG_OWNER(uid, gid);
	follow = (flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW : FOLLOW;
	follow \|= (flag & AT_BENEATH) != 0 ? BENEATH : 0;
	NDINIT_ATRIGHTS(&nd, LOOKUP, follow \| AUDITVNODE1, pathseg, path, fd,
	&cap_fchown_rights, td);

	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = setfown(td, td->td_ucred, nd.ni_vp, uid, gid);
	vrele(nd.ni_vp);
	return (error);
	}

	/*
	* Set ownership given a path name, do not cross symlinks.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct lchown_args {
	char *path;
	int uid;
	int gid;
	};
	#endif
	int
	sys_lchown(struct thread td, struct lchown_args uap)
	{

	return (kern_fchownat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->uid, uap->gid, AT_SYMLINK_NOFOLLOW));
	}

	/*
	* Set ownership given a file descriptor.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fchown_args {
	int fd;
	int uid;
	int gid;
	};
	#endif
	int
	sys_fchown(struct thread td, struct fchown_args uap)
	{
	struct file *fp;
	int error;

	AUDIT_ARG_FD(uap->fd);
	AUDIT_ARG_OWNER(uap->uid, uap->gid);
	error = fget(td, uap->fd, &cap_fchown_rights, &fp);
	if (error != 0)
	return (error);
	error = fo_chown(fp, uap->uid, uap->gid, td->td_ucred, td);
	fdrop(fp, td);
	return (error);
	}

	/*
	* Common implementation code for utimes(), lutimes(), and futimes().
	*/
	static int
	getutimes(const struct timeval *usrtvp, enum uio_seg tvpseg,
	struct timespec *tsp)
	{
	struct timeval tv[2];
	const struct timeval *tvp;
	int error;

	if (usrtvp == NULL) {
	vfs_timestamp(&tsp[0]);
	tsp[1] = tsp[0];
	} else {
	if (tvpseg == UIO_SYSSPACE) {
	tvp = usrtvp;
	} else {
	if ((error = copyin(usrtvp, tv, sizeof(tv))) != 0)
	return (error);
	tvp = tv;
	}

	if (tvp[0].tv_usec < 0 \|\| tvp[0].tv_usec >= 1000000 \|\|
	tvp[1].tv_usec < 0 \|\| tvp[1].tv_usec >= 1000000)
	return (EINVAL);
	TIMEVAL_TO_TIMESPEC(&tvp[0], &tsp[0]);
	TIMEVAL_TO_TIMESPEC(&tvp[1], &tsp[1]);
	}
	return (0);
	}

	/*
	* Common implementation code for futimens(), utimensat().
	*/
	#define UTIMENS_NULL 0x1
	#define UTIMENS_EXIT 0x2
	static int
	getutimens(const struct timespec *usrtsp, enum uio_seg tspseg,
	struct timespec tsp, int retflags)
	{
	struct timespec tsnow;
	int error;

	vfs_timestamp(&tsnow);
	*retflags = 0;
	if (usrtsp == NULL) {
	tsp[0] = tsnow;
	tsp[1] = tsnow;
	*retflags \|= UTIMENS_NULL;
	return (0);
	}
	if (tspseg == UIO_SYSSPACE) {
	tsp[0] = usrtsp[0];
	tsp[1] = usrtsp[1];
	} else if ((error = copyin(usrtsp, tsp, sizeof(tsp) 2)) != 0)
	return (error);
	if (tsp[0].tv_nsec == UTIME_OMIT && tsp[1].tv_nsec == UTIME_OMIT)
	*retflags \|= UTIMENS_EXIT;
	if (tsp[0].tv_nsec == UTIME_NOW && tsp[1].tv_nsec == UTIME_NOW)
	*retflags \|= UTIMENS_NULL;
	if (tsp[0].tv_nsec == UTIME_OMIT)
	tsp[0].tv_sec = VNOVAL;
	else if (tsp[0].tv_nsec == UTIME_NOW)
	tsp[0] = tsnow;
	else if (tsp[0].tv_nsec < 0 \|\| tsp[0].tv_nsec >= 1000000000L)
	return (EINVAL);
	if (tsp[1].tv_nsec == UTIME_OMIT)
	tsp[1].tv_sec = VNOVAL;
	else if (tsp[1].tv_nsec == UTIME_NOW)
	tsp[1] = tsnow;
	else if (tsp[1].tv_nsec < 0 \|\| tsp[1].tv_nsec >= 1000000000L)
	return (EINVAL);

	return (0);
	}

	/*
	* Common implementation code for utimes(), lutimes(), futimes(), futimens(),
	* and utimensat().
	*/
	static int
	setutimes(struct thread td, struct vnode vp, const struct timespec *ts,
	int numtimes, int nullflag)
	{
	struct mount *mp;
	struct vattr vattr;
	int error, setbirthtime;

	if ((error = vn_start_write(vp, &mp, V_WAIT \| PCATCH)) != 0)
	return (error);
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	setbirthtime = 0;
	if (numtimes < 3 && !VOP_GETATTR(vp, &vattr, td->td_ucred) &&
	timespeccmp(&ts[1], &vattr.va_birthtime, < ))
	setbirthtime = 1;
	VATTR_NULL(&vattr);
	vattr.va_atime = ts[0];
	vattr.va_mtime = ts[1];
	if (setbirthtime)
	vattr.va_birthtime = ts[1];
	if (numtimes > 2)
	vattr.va_birthtime = ts[2];
	if (nullflag)
	vattr.va_vaflags \|= VA_UTIMES_NULL;
	#ifdef MAC
	error = mac_vnode_check_setutimes(td->td_ucred, vp, vattr.va_atime,
	vattr.va_mtime);
	#endif
	if (error == 0)
	error = VOP_SETATTR(vp, &vattr, td->td_ucred);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Set the access and modification times of a file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct utimes_args {
	char *path;
	struct timeval *tptr;
	};
	#endif
	int
	sys_utimes(struct thread td, struct utimes_args uap)
	{

	return (kern_utimesat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->tptr, UIO_USERSPACE));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct futimesat_args {
	int fd;
	const char * path;
	const struct timeval * times;
	};
	#endif
	int
	sys_futimesat(struct thread td, struct futimesat_args uap)
	{

	return (kern_utimesat(td, uap->fd, uap->path, UIO_USERSPACE,
	uap->times, UIO_USERSPACE));
	}

	int
	kern_utimesat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, struct timeval *tptr, enum uio_seg tptrseg)
	{
	struct nameidata nd;
	struct timespec ts[2];
	int error;

	if ((error = getutimes(tptr, tptrseg, ts)) != 0)
	return (error);
	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW \| AUDITVNODE1, pathseg, path, fd,
	&cap_futimes_rights, td);

	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = setutimes(td, nd.ni_vp, ts, 2, tptr == NULL);
	vrele(nd.ni_vp);
	return (error);
	}

	/*
	* Set the access and modification times of a file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct lutimes_args {
	char *path;
	struct timeval *tptr;
	};
	#endif
	int
	sys_lutimes(struct thread td, struct lutimes_args uap)
	{

	return (kern_lutimes(td, uap->path, UIO_USERSPACE, uap->tptr,
	UIO_USERSPACE));
	}

	int
	kern_lutimes(struct thread td, const char path, enum uio_seg pathseg,
	struct timeval *tptr, enum uio_seg tptrseg)
	{
	struct timespec ts[2];
	struct nameidata nd;
	int error;

	if ((error = getutimes(tptr, tptrseg, ts)) != 0)
	return (error);
	NDINIT(&nd, LOOKUP, NOFOLLOW \| AUDITVNODE1, pathseg, path, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	error = setutimes(td, nd.ni_vp, ts, 2, tptr == NULL);
	vrele(nd.ni_vp);
	return (error);
	}

	/*
	* Set the access and modification times of a file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct futimes_args {
	int fd;
	struct timeval *tptr;
	};
	#endif
	int
	sys_futimes(struct thread td, struct futimes_args uap)
	{

	return (kern_futimes(td, uap->fd, uap->tptr, UIO_USERSPACE));
	}

	int
	kern_futimes(struct thread td, int fd, struct timeval tptr,
	enum uio_seg tptrseg)
	{
	struct timespec ts[2];
	struct file *fp;
	int error;

	AUDIT_ARG_FD(fd);
	error = getutimes(tptr, tptrseg, ts);
	if (error != 0)
	return (error);
	error = getvnode(td, fd, &cap_futimes_rights, &fp);
	if (error != 0)
	return (error);
	#ifdef AUDIT
	vn_lock(fp->f_vnode, LK_SHARED \| LK_RETRY);
	AUDIT_ARG_VNODE1(fp->f_vnode);
	VOP_UNLOCK(fp->f_vnode);
	#endif
	error = setutimes(td, fp->f_vnode, ts, 2, tptr == NULL);
	fdrop(fp, td);
	return (error);
	}

	int
	sys_futimens(struct thread td, struct futimens_args uap)
	{

	return (kern_futimens(td, uap->fd, uap->times, UIO_USERSPACE));
	}

	int
	kern_futimens(struct thread td, int fd, struct timespec tptr,
	enum uio_seg tptrseg)
	{
	struct timespec ts[2];
	struct file *fp;
	int error, flags;

	AUDIT_ARG_FD(fd);
	error = getutimens(tptr, tptrseg, ts, &flags);
	if (error != 0)
	return (error);
	if (flags & UTIMENS_EXIT)
	return (0);
	error = getvnode(td, fd, &cap_futimes_rights, &fp);
	if (error != 0)
	return (error);
	#ifdef AUDIT
	vn_lock(fp->f_vnode, LK_SHARED \| LK_RETRY);
	AUDIT_ARG_VNODE1(fp->f_vnode);
	VOP_UNLOCK(fp->f_vnode);
	#endif
	error = setutimes(td, fp->f_vnode, ts, 2, flags & UTIMENS_NULL);
	fdrop(fp, td);
	return (error);
	}

	int
	sys_utimensat(struct thread td, struct utimensat_args uap)
	{

	return (kern_utimensat(td, uap->fd, uap->path, UIO_USERSPACE,
	uap->times, UIO_USERSPACE, uap->flag));
	}

	int
	kern_utimensat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, struct timespec *tptr, enum uio_seg tptrseg,
	int flag)
	{
	struct nameidata nd;
	struct timespec ts[2];
	int error, flags;

	if ((flag & ~(AT_SYMLINK_NOFOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);

	if ((error = getutimens(tptr, tptrseg, ts, &flags)) != 0)
	return (error);
	NDINIT_ATRIGHTS(&nd, LOOKUP, ((flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW :
	FOLLOW) \| ((flag & AT_BENEATH) != 0 ? BENEATH : 0) \| AUDITVNODE1,
	pathseg, path, fd, &cap_futimes_rights, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	/*
	* We are allowed to call namei() regardless of 2xUTIME_OMIT.
	* POSIX states:
	* "If both tv_nsec fields are UTIME_OMIT... EACCESS may be detected."
	* "Search permission is denied by a component of the path prefix."
	*/
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if ((flags & UTIMENS_EXIT) == 0)
	error = setutimes(td, nd.ni_vp, ts, 2, flags & UTIMENS_NULL);
	vrele(nd.ni_vp);
	return (error);
	}

	/*
	* Truncate a file given its path name.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct truncate_args {
	char *path;
	int pad;
	off_t length;
	};
	#endif
	int
	sys_truncate(struct thread td, struct truncate_args uap)
	{

	return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
	}

	int
	kern_truncate(struct thread td, const char path, enum uio_seg pathseg,
	off_t length)
	{
	struct mount *mp;
	struct vnode *vp;
	void *rl_cookie;
	struct vattr vattr;
	struct nameidata nd;
	int error;

	if (length < 0)
	return(EINVAL);
	NDINIT(&nd, LOOKUP, FOLLOW \| AUDITVNODE1, pathseg, path, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	vp = nd.ni_vp;
	rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
	if ((error = vn_start_write(vp, &mp, V_WAIT \| PCATCH)) != 0) {
	vn_rangelock_unlock(vp, rl_cookie);
	vrele(vp);
	return (error);
	}
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vn_lock(vp, LK_EXCLUSIVE \| LK_RETRY);
	if (vp->v_type == VDIR)
	error = EISDIR;
	#ifdef MAC
	else if ((error = mac_vnode_check_write(td->td_ucred, NOCRED, vp))) {
	}
	#endif
	else if ((error = vn_writechk(vp)) == 0 &&
	(error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td)) == 0) {
	VATTR_NULL(&vattr);
	vattr.va_size = length;
	error = VOP_SETATTR(vp, &vattr, td->td_ucred);
	}
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	vn_rangelock_unlock(vp, rl_cookie);
	vrele(vp);
	return (error);
	}

	#if defined(COMPAT_43)
	/*
	* Truncate a file given its path name.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct otruncate_args {
	char *path;
	long length;
	};
	#endif
	int
	otruncate(struct thread td, struct otruncate_args uap)
	{

	return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
	}
	#endif /* COMPAT_43 */

	#if defined(COMPAT_FREEBSD6)
	/* Versions with the pad argument */
	int
	freebsd6_truncate(struct thread td, struct freebsd6_truncate_args uap)
	{

	return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
	}

	int
	freebsd6_ftruncate(struct thread td, struct freebsd6_ftruncate_args uap)
	{

	return (kern_ftruncate(td, uap->fd, uap->length));
	}
	#endif

	int
	kern_fsync(struct thread *td, int fd, bool fullsync)
	{
	struct vnode *vp;
	struct mount *mp;
	struct file *fp;
	int error, lock_flags;

	AUDIT_ARG_FD(fd);
	error = getvnode(td, fd, &cap_fsync_rights, &fp);
	if (error != 0)
	return (error);
	vp = fp->f_vnode;
	#if 0
	if (!fullsync)
	/* XXXKIB: compete outstanding aio writes */;
	#endif
	error = vn_start_write(vp, &mp, V_WAIT \| PCATCH);
	if (error != 0)
	goto drop;
	if (MNT_SHARED_WRITES(mp) \|\|
	((mp == NULL) && MNT_SHARED_WRITES(vp->v_mount))) {
	lock_flags = LK_SHARED;
	} else {
	lock_flags = LK_EXCLUSIVE;
	}
	vn_lock(vp, lock_flags \| LK_RETRY);
	AUDIT_ARG_VNODE1(vp);
	if (vp->v_object != NULL) {
	VM_OBJECT_WLOCK(vp->v_object);
	vm_object_page_clean(vp->v_object, 0, 0, 0);
	VM_OBJECT_WUNLOCK(vp->v_object);
	}
	error = fullsync ? VOP_FSYNC(vp, MNT_WAIT, td) : VOP_FDATASYNC(vp, td);
	VOP_UNLOCK(vp);
	vn_finished_write(mp);
	drop:
	fdrop(fp, td);
	return (error);
	}

	/*
	* Sync an open file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fsync_args {
	int fd;
	};
	#endif
	int
	sys_fsync(struct thread td, struct fsync_args uap)
	{

	return (kern_fsync(td, uap->fd, true));
	}

	int
	sys_fdatasync(struct thread td, struct fdatasync_args uap)
	{

	return (kern_fsync(td, uap->fd, false));
	}

	/*
	* Rename files. Source and destination must either both be directories, or
	* both not be directories. If target is a directory, it must be empty.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct rename_args {
	char *from;
	char *to;
	};
	#endif
	int
	sys_rename(struct thread td, struct rename_args uap)
	{

	return (kern_renameat(td, AT_FDCWD, uap->from, AT_FDCWD,
	uap->to, UIO_USERSPACE));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct renameat_args {
	int oldfd;
	char *old;
	int newfd;
	char *new;
	};
	#endif
	int
	sys_renameat(struct thread td, struct renameat_args uap)
	{

	return (kern_renameat(td, uap->oldfd, uap->old, uap->newfd, uap->new,
	UIO_USERSPACE));
	}

	+#ifdef MAC
	+static int
	+kern_renameat_mac(struct thread td, int oldfd, const char old, int newfd,
	+ const char new, enum uio_seg pathseg, struct nameidata fromnd)
	+{
	+ int error;
	+
	+ NDINIT_ATRIGHTS(fromnd, DELETE, LOCKPARENT \| LOCKLEAF \| SAVESTART \|
	+ AUDITVNODE1, pathseg, old, oldfd, &cap_renameat_source_rights, td);
	+ if ((error = namei(fromnd)) != 0)
	+ return (error);
	+ error = mac_vnode_check_rename_from(td->td_ucred, fromnd->ni_dvp,
	+ fromnd->ni_vp, &fromnd->ni_cnd);
	+ VOP_UNLOCK(fromnd->ni_dvp);
	+ if (fromnd->ni_dvp != fromnd->ni_vp)
	+ VOP_UNLOCK(fromnd->ni_vp);
	+ if (error != 0) {
	+ NDFREE(fromnd, NDF_ONLY_PNBUF);
	+ vrele(fromnd->ni_dvp);
	+ vrele(fromnd->ni_vp);
	+ if (fromnd->ni_startdir)
	+ vrele(fromnd->ni_startdir);
	+ }
	+ return (error);
	+}
	+#endif
	+
	int
	kern_renameat(struct thread td, int oldfd, const char old, int newfd,
	const char *new, enum uio_seg pathseg)
	{
	struct mount *mp = NULL;
	struct vnode tvp, fvp, *tdvp;
	struct nameidata fromnd, tond;
	int error;

	again:
	bwillwrite();
	#ifdef MAC
	- NDINIT_ATRIGHTS(&fromnd, DELETE, LOCKPARENT \| LOCKLEAF \| SAVESTART \|
	- AUDITVNODE1, pathseg, old, oldfd,
	- &cap_renameat_source_rights, td);
	-#else
	- NDINIT_ATRIGHTS(&fromnd, DELETE, WANTPARENT \| SAVESTART \| AUDITVNODE1,
	- pathseg, old, oldfd,
	- &cap_renameat_source_rights, td);
	+ if (mac_vnode_check_rename_from_enabled()) {
	+ error = kern_renameat_mac(td, oldfd, old, newfd, new, pathseg,
	+ &fromnd);
	+ if (error != 0)
	+ return (error);
	+ } else {
	#endif
	-
	+ NDINIT_ATRIGHTS(&fromnd, DELETE, WANTPARENT \| SAVESTART \| AUDITVNODE1,
	+ pathseg, old, oldfd, &cap_renameat_source_rights, td);
	if ((error = namei(&fromnd)) != 0)
	return (error);
	#ifdef MAC
	- error = mac_vnode_check_rename_from(td->td_ucred, fromnd.ni_dvp,
	- fromnd.ni_vp, &fromnd.ni_cnd);
	- VOP_UNLOCK(fromnd.ni_dvp);
	- if (fromnd.ni_dvp != fromnd.ni_vp)
	- VOP_UNLOCK(fromnd.ni_vp);
	+ }
	#endif
	fvp = fromnd.ni_vp;
	NDINIT_ATRIGHTS(&tond, RENAME, LOCKPARENT \| LOCKLEAF \| NOCACHE \|
	SAVESTART \| AUDITVNODE2, pathseg, new, newfd,
	&cap_renameat_target_rights, td);
	if (fromnd.ni_vp->v_type == VDIR)
	tond.ni_cnd.cn_flags \|= WILLBEDIR;
	if ((error = namei(&tond)) != 0) {
	/* Translate error code for rename("dir1", "dir2/."). */
	if (error == EISDIR && fvp->v_type == VDIR)
	error = EINVAL;
	NDFREE(&fromnd, NDF_ONLY_PNBUF);
	vrele(fromnd.ni_dvp);
	vrele(fvp);
	goto out1;
	}
	tdvp = tond.ni_dvp;
	tvp = tond.ni_vp;
	error = vn_start_write(fvp, &mp, V_NOWAIT);
	if (error != 0) {
	NDFREE(&fromnd, NDF_ONLY_PNBUF);
	NDFREE(&tond, NDF_ONLY_PNBUF);
	if (tvp != NULL)
	vput(tvp);
	if (tdvp == tvp)
	vrele(tdvp);
	else
	vput(tdvp);
	vrele(fromnd.ni_dvp);
	vrele(fvp);
	vrele(tond.ni_startdir);
	if (fromnd.ni_startdir != NULL)
	vrele(fromnd.ni_startdir);
	error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH);
	if (error != 0)
	return (error);
	goto again;
	}
	if (tvp != NULL) {
	if (fvp->v_type == VDIR && tvp->v_type != VDIR) {
	error = ENOTDIR;
	goto out;
	} else if (fvp->v_type != VDIR && tvp->v_type == VDIR) {
	error = EISDIR;
	goto out;
	}
	#ifdef CAPABILITIES
	if (newfd != AT_FDCWD && (tond.ni_resflags & NIRES_ABS) == 0) {
	/*
	* If the target already exists we require CAP_UNLINKAT
	* from 'newfd', when newfd was used for the lookup.
	*/
	error = cap_check(&tond.ni_filecaps.fc_rights,
	&cap_unlinkat_rights);
	if (error != 0)
	goto out;
	}
	#endif
	}
	if (fvp == tdvp) {
	error = EINVAL;
	goto out;
	}
	/*
	* If the source is the same as the destination (that is, if they
	* are links to the same vnode), then there is nothing to do.
	*/
	if (fvp == tvp)
	error = -1;
	#ifdef MAC
	else
	error = mac_vnode_check_rename_to(td->td_ucred, tdvp,
	tond.ni_vp, fromnd.ni_dvp == tdvp, &tond.ni_cnd);
	#endif
	out:
	if (error == 0) {
	error = VOP_RENAME(fromnd.ni_dvp, fromnd.ni_vp, &fromnd.ni_cnd,
	tond.ni_dvp, tond.ni_vp, &tond.ni_cnd);
	NDFREE(&fromnd, NDF_ONLY_PNBUF);
	NDFREE(&tond, NDF_ONLY_PNBUF);
	} else {
	NDFREE(&fromnd, NDF_ONLY_PNBUF);
	NDFREE(&tond, NDF_ONLY_PNBUF);
	if (tvp != NULL)
	vput(tvp);
	if (tdvp == tvp)
	vrele(tdvp);
	else
	vput(tdvp);
	vrele(fromnd.ni_dvp);
	vrele(fvp);
	}
	vrele(tond.ni_startdir);
	vn_finished_write(mp);
	out1:
	if (fromnd.ni_startdir)
	vrele(fromnd.ni_startdir);
	if (error == -1)
	return (0);
	return (error);
	}

	/*
	* Make a directory file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct mkdir_args {
	char *path;
	int mode;
	};
	#endif
	int
	sys_mkdir(struct thread td, struct mkdir_args uap)
	{

	return (kern_mkdirat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
	uap->mode));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct mkdirat_args {
	int fd;
	char *path;
	mode_t mode;
	};
	#endif
	int
	sys_mkdirat(struct thread td, struct mkdirat_args uap)
	{

	return (kern_mkdirat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode));
	}

	int
	kern_mkdirat(struct thread td, int fd, const char path, enum uio_seg segflg,
	int mode)
	{
	struct mount *mp;
	struct vnode *vp;
	struct vattr vattr;
	struct nameidata nd;
	int error;

	AUDIT_ARG_MODE(mode);
	restart:
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT \| SAVENAME \| AUDITVNODE1 \|
	NOCACHE, segflg, path, fd, &cap_mkdirat_rights,
	td);
	nd.ni_cnd.cn_flags \|= WILLBEDIR;
	if ((error = namei(&nd)) != 0)
	return (error);
	vp = nd.ni_vp;
	if (vp != NULL) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	/*
	* XXX namei called with LOCKPARENT but not LOCKLEAF has
	* the strange behaviour of leaving the vnode unlocked
	* if the target is the same vnode as the parent.
	*/
	if (vp == nd.ni_dvp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	vrele(vp);
	return (EEXIST);
	}
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if ((error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH)) != 0)
	return (error);
	goto restart;
	}
	VATTR_NULL(&vattr);
	vattr.va_type = VDIR;
	vattr.va_mode = (mode & ACCESSPERMS) &~ td->td_proc->p_fd->fd_cmask;
	#ifdef MAC
	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
	&vattr);
	if (error != 0)
	goto out;
	#endif
	error = VOP_MKDIR(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
	#ifdef MAC
	out:
	#endif
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(nd.ni_dvp);
	if (error == 0)
	vput(nd.ni_vp);
	vn_finished_write(mp);
	return (error);
	}

	/*
	* Remove a directory file.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct rmdir_args {
	char *path;
	};
	#endif
	int
	sys_rmdir(struct thread td, struct rmdir_args uap)
	{

	return (kern_frmdirat(td, AT_FDCWD, uap->path, FD_NONE, UIO_USERSPACE,
	0));
	}

	int
	kern_frmdirat(struct thread td, int dfd, const char path, int fd,
	enum uio_seg pathseg, int flag)
	{
	struct mount *mp;
	struct vnode *vp;
	struct file *fp;
	struct nameidata nd;
	cap_rights_t rights;
	int error;

	fp = NULL;
	if (fd != FD_NONE) {
	error = getvnode(td, fd, cap_rights_init_one(&rights, CAP_LOOKUP),
	&fp);
	if (error != 0)
	return (error);
	}

	restart:
	bwillwrite();
	NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT \| LOCKLEAF \| AUDITVNODE1 \|
	((flag & AT_BENEATH) != 0 ? BENEATH : 0),
	pathseg, path, dfd, &cap_unlinkat_rights, td);
	if ((error = namei(&nd)) != 0)
	goto fdout;
	vp = nd.ni_vp;
	if (vp->v_type != VDIR) {
	error = ENOTDIR;
	goto out;
	}
	/*
	* No rmdir "." please.
	*/
	if (nd.ni_dvp == vp) {
	error = EINVAL;
	goto out;
	}
	/*
	* The root of a mounted filesystem cannot be deleted.
	*/
	if (vp->v_vflag & VV_ROOT) {
	error = EBUSY;
	goto out;
	}

	if (fp != NULL && fp->f_vnode != vp) {
	if (VN_IS_DOOMED(fp->f_vnode))
	error = EBADF;
	else
	error = EDEADLK;
	goto out;
	}

	#ifdef MAC
	error = mac_vnode_check_unlink(td->td_ucred, nd.ni_dvp, vp,
	&nd.ni_cnd);
	if (error != 0)
	goto out;
	#endif
	if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(vp);
	if (nd.ni_dvp == vp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	if ((error = vn_start_write(NULL, &mp, V_XSLEEP \| PCATCH)) != 0)
	goto fdout;
	goto restart;
	}
	vfs_notify_upper(vp, VFS_NOTIFY_UPPER_UNLINK);
	error = VOP_RMDIR(nd.ni_dvp, nd.ni_vp, &nd.ni_cnd);
	vn_finished_write(mp);
	out:
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vput(vp);
	if (nd.ni_dvp == vp)
	vrele(nd.ni_dvp);
	else
	vput(nd.ni_dvp);
	fdout:
	if (fp != NULL)
	fdrop(fp, td);
	return (error);
	}

	#if defined(COMPAT_43) \|\| defined(COMPAT_FREEBSD11)
	int
	freebsd11_kern_getdirentries(struct thread td, int fd, char ubuf, u_int count,
	long basep, void (func)(struct freebsd11_dirent *))
	{
	struct freebsd11_dirent dstdp;
	struct dirent dp, edp;
	char *dirbuf;
	off_t base;
	ssize_t resid, ucount;
	int error;

	/* XXX arbitrary sanity limit on `count'. */
	count = min(count, 64 * 1024);

	dirbuf = malloc(count, M_TEMP, M_WAITOK);

	error = kern_getdirentries(td, fd, dirbuf, count, &base, &resid,
	UIO_SYSSPACE);
	if (error != 0)
	goto done;
	if (basep != NULL)
	*basep = base;

	ucount = 0;
	for (dp = (struct dirent *)dirbuf,
	edp = (struct dirent *)&dirbuf[count - resid];
	ucount < count && dp < edp; ) {
	if (dp->d_reclen == 0)
	break;
	MPASS(dp->d_reclen >= _GENERIC_DIRLEN(0));
	if (dp->d_namlen >= sizeof(dstdp.d_name))
	continue;
	dstdp.d_type = dp->d_type;
	dstdp.d_namlen = dp->d_namlen;
	dstdp.d_fileno = dp->d_fileno; /* truncate */
	if (dstdp.d_fileno != dp->d_fileno) {
	switch (ino64_trunc_error) {
	default:
	case 0:
	break;
	case 1:
	error = EOVERFLOW;
	goto done;
	case 2:
	dstdp.d_fileno = UINT32_MAX;
	break;
	}
	}
	dstdp.d_reclen = sizeof(dstdp) - sizeof(dstdp.d_name) +
	((dp->d_namlen + 1 + 3) &~ 3);
	bcopy(dp->d_name, dstdp.d_name, dstdp.d_namlen);
	bzero(dstdp.d_name + dstdp.d_namlen,
	dstdp.d_reclen - offsetof(struct freebsd11_dirent, d_name) -
	dstdp.d_namlen);
	MPASS(dstdp.d_reclen <= dp->d_reclen);
	MPASS(ucount + dstdp.d_reclen <= count);
	if (func != NULL)
	func(&dstdp);
	error = copyout(&dstdp, ubuf + ucount, dstdp.d_reclen);
	if (error != 0)
	break;
	dp = (struct dirent )((char )dp + dp->d_reclen);
	ucount += dstdp.d_reclen;
	}

	done:
	free(dirbuf, M_TEMP);
	if (error == 0)
	td->td_retval[0] = ucount;
	return (error);
	}
	#endif /* COMPAT */

	#ifdef COMPAT_43
	static void
	ogetdirentries_cvt(struct freebsd11_dirent *dp)
	{
	#if (BYTE_ORDER == LITTLE_ENDIAN)
	/*
	* The expected low byte of dp->d_namlen is our dp->d_type.
	* The high MBZ byte of dp->d_namlen is our dp->d_namlen.
	*/
	dp->d_type = dp->d_namlen;
	dp->d_namlen = 0;
	#else
	/*
	* The dp->d_type is the high byte of the expected dp->d_namlen,
	* so must be zero'ed.
	*/
	dp->d_type = 0;
	#endif
	}

	/*
	* Read a block of directory entries in a filesystem independent format.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct ogetdirentries_args {
	int fd;
	char *buf;
	u_int count;
	long *basep;
	};
	#endif
	int
	ogetdirentries(struct thread td, struct ogetdirentries_args uap)
	{
	long loff;
	int error;

	error = kern_ogetdirentries(td, uap, &loff);
	if (error == 0)
	error = copyout(&loff, uap->basep, sizeof(long));
	return (error);
	}

	int
	kern_ogetdirentries(struct thread td, struct ogetdirentries_args uap,
	long *ploff)
	{
	long base;
	int error;

	/* XXX arbitrary sanity limit on `count'. */
	if (uap->count > 64 * 1024)
	return (EINVAL);

	error = freebsd11_kern_getdirentries(td, uap->fd, uap->buf, uap->count,
	&base, ogetdirentries_cvt);

	if (error == 0 && uap->basep != NULL)
	error = copyout(&base, uap->basep, sizeof(long));

	return (error);
	}
	#endif /* COMPAT_43 */

	#if defined(COMPAT_FREEBSD11)
	#ifndef _SYS_SYSPROTO_H_
	struct freebsd11_getdirentries_args {
	int fd;
	char *buf;
	u_int count;
	long *basep;
	};
	#endif
	int
	freebsd11_getdirentries(struct thread *td,
	struct freebsd11_getdirentries_args *uap)
	{
	long base;
	int error;

	error = freebsd11_kern_getdirentries(td, uap->fd, uap->buf, uap->count,
	&base, NULL);

	if (error == 0 && uap->basep != NULL)
	error = copyout(&base, uap->basep, sizeof(long));
	return (error);
	}

	int
	freebsd11_getdents(struct thread td, struct freebsd11_getdents_args uap)
	{
	struct freebsd11_getdirentries_args ap;

	ap.fd = uap->fd;
	ap.buf = uap->buf;
	ap.count = uap->count;
	ap.basep = NULL;
	return (freebsd11_getdirentries(td, &ap));
	}
	#endif /* COMPAT_FREEBSD11 */

	/*
	* Read a block of directory entries in a filesystem independent format.
	*/
	int
	sys_getdirentries(struct thread td, struct getdirentries_args uap)
	{
	off_t base;
	int error;

	error = kern_getdirentries(td, uap->fd, uap->buf, uap->count, &base,
	NULL, UIO_USERSPACE);
	if (error != 0)
	return (error);
	if (uap->basep != NULL)
	error = copyout(&base, uap->basep, sizeof(off_t));
	return (error);
	}

	int
	kern_getdirentries(struct thread td, int fd, char buf, size_t count,
	off_t basep, ssize_t residp, enum uio_seg bufseg)
	{
	struct vnode *vp;
	struct file *fp;
	struct uio auio;
	struct iovec aiov;
	off_t loff;
	int error, eofflag;
	off_t foffset;

	AUDIT_ARG_FD(fd);
	if (count > IOSIZE_MAX)
	return (EINVAL);
	auio.uio_resid = count;
	error = getvnode(td, fd, &cap_read_rights, &fp);
	if (error != 0)
	return (error);
	if ((fp->f_flag & FREAD) == 0) {
	fdrop(fp, td);
	return (EBADF);
	}
	vp = fp->f_vnode;
	foffset = foffset_lock(fp, 0);
	unionread:
	if (vp->v_type != VDIR) {
	error = EINVAL;
	goto fail;
	}
	aiov.iov_base = buf;
	aiov.iov_len = count;
	auio.uio_iov = &aiov;
	auio.uio_iovcnt = 1;
	auio.uio_rw = UIO_READ;
	auio.uio_segflg = bufseg;
	auio.uio_td = td;
	vn_lock(vp, LK_SHARED \| LK_RETRY);
	AUDIT_ARG_VNODE1(vp);
	loff = auio.uio_offset = foffset;
	#ifdef MAC
	error = mac_vnode_check_readdir(td->td_ucred, vp);
	if (error == 0)
	#endif
	error = VOP_READDIR(vp, &auio, fp->f_cred, &eofflag, NULL,
	NULL);
	foffset = auio.uio_offset;
	if (error != 0) {
	VOP_UNLOCK(vp);
	goto fail;
	}
	if (count == auio.uio_resid &&
	(vp->v_vflag & VV_ROOT) &&
	(vp->v_mount->mnt_flag & MNT_UNION)) {
	struct vnode *tvp = vp;

	vp = vp->v_mount->mnt_vnodecovered;
	VREF(vp);
	fp->f_vnode = vp;
	fp->f_data = vp;
	foffset = 0;
	vput(tvp);
	goto unionread;
	}
	VOP_UNLOCK(vp);
	*basep = loff;
	if (residp != NULL)
	*residp = auio.uio_resid;
	td->td_retval[0] = count - auio.uio_resid;
	fail:
	foffset_unlock(fp, foffset, 0);
	fdrop(fp, td);
	return (error);
	}

	/*
	* Set the mode mask for creation of filesystem nodes.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct umask_args {
	int newmask;
	};
	#endif
	int
	sys_umask(struct thread td, struct umask_args uap)
	{
	struct filedesc *fdp;

	fdp = td->td_proc->p_fd;
	FILEDESC_XLOCK(fdp);
	td->td_retval[0] = fdp->fd_cmask;
	fdp->fd_cmask = uap->newmask & ALLPERMS;
	FILEDESC_XUNLOCK(fdp);
	return (0);
	}

	/*
	* Void all references to file by ripping underlying filesystem away from
	* vnode.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct revoke_args {
	char *path;
	};
	#endif
	int
	sys_revoke(struct thread td, struct revoke_args uap)
	{
	struct vnode *vp;
	struct vattr vattr;
	struct nameidata nd;
	int error;

	NDINIT(&nd, LOOKUP, FOLLOW \| LOCKLEAF \| AUDITVNODE1, UIO_USERSPACE,
	uap->path, td);
	if ((error = namei(&nd)) != 0)
	return (error);
	vp = nd.ni_vp;
	NDFREE(&nd, NDF_ONLY_PNBUF);
	if (vp->v_type != VCHR \|\| vp->v_rdev == NULL) {
	error = EINVAL;
	goto out;
	}
	#ifdef MAC
	error = mac_vnode_check_revoke(td->td_ucred, vp);
	if (error != 0)
	goto out;
	#endif
	error = VOP_GETATTR(vp, &vattr, td->td_ucred);
	if (error != 0)
	goto out;
	if (td->td_ucred->cr_uid != vattr.va_uid) {
	error = priv_check(td, PRIV_VFS_ADMIN);
	if (error != 0)
	goto out;
	}
	if (vp->v_usecount > 1 \|\| vcount(vp) > 1)
	VOP_REVOKE(vp, REVOKEALL);
	out:
	vput(vp);
	return (error);
	}

	/*
	* Convert a user file descriptor to a kernel file entry and check that, if it
	* is a capability, the correct rights are present. A reference on the file
	* entry is held upon returning.
	*/
	int
	getvnode(struct thread td, int fd, cap_rights_t rightsp, struct file **fpp)
	{
	struct file *fp;
	int error;

	error = fget_unlocked(td->td_proc->p_fd, fd, rightsp, &fp);
	if (error != 0)
	return (error);

	/*
	* The file could be not of the vnode type, or it may be not
	* yet fully initialized, in which case the f_vnode pointer
	* may be set, but f_ops is still badfileops. E.g.,
	* devfs_open() transiently create such situation to
	* facilitate csw d_fdopen().
	*
	* Dupfdopen() handling in kern_openat() installs the
	* half-baked file into the process descriptor table, allowing
	* other thread to dereference it. Guard against the race by
	* checking f_ops.
	*/
	if (fp->f_vnode == NULL \|\| fp->f_ops == &badfileops) {
	fdrop(fp, td);
	return (EINVAL);
	}
	*fpp = fp;
	return (0);
	}

	/*
	* Get an (NFS) file handle.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct lgetfh_args {
	char *fname;
	fhandle_t *fhp;
	};
	#endif
	int
	sys_lgetfh(struct thread td, struct lgetfh_args uap)
	{

	return (kern_getfhat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->fname,
	UIO_USERSPACE, uap->fhp));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct getfh_args {
	char *fname;
	fhandle_t *fhp;
	};
	#endif
	int
	sys_getfh(struct thread td, struct getfh_args uap)
	{

	return (kern_getfhat(td, 0, AT_FDCWD, uap->fname, UIO_USERSPACE,
	uap->fhp));
	}

	/*
	* syscall for the rpc.lockd to use to translate an open descriptor into
	* a NFS file handle.
	*
	* warning: do not remove the priv_check() call or this becomes one giant
	* security hole.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct getfhat_args {
	int fd;
	char *path;
	fhandle_t *fhp;
	int flags;
	};
	#endif
	int
	sys_getfhat(struct thread td, struct getfhat_args uap)
	{

	if ((uap->flags & ~(AT_SYMLINK_NOFOLLOW \| AT_BENEATH)) != 0)
	return (EINVAL);
	return (kern_getfhat(td, uap->flags, uap->fd, uap->path, UIO_USERSPACE,
	uap->fhp));
	}

	static int
	kern_getfhat(struct thread td, int flags, int fd, const char path,
	enum uio_seg pathseg, fhandle_t *fhp)
	{
	struct nameidata nd;
	fhandle_t fh;
	struct vnode *vp;
	int error;

	error = priv_check(td, PRIV_VFS_GETFH);
	if (error != 0)
	return (error);
	NDINIT_AT(&nd, LOOKUP, ((flags & AT_SYMLINK_NOFOLLOW) != 0 ? NOFOLLOW :
	FOLLOW) \| ((flags & AT_BENEATH) != 0 ? BENEATH : 0) \| LOCKLEAF \|
	AUDITVNODE1, pathseg, path, fd, td);
	error = namei(&nd);
	if (error != 0)
	return (error);
	NDFREE(&nd, NDF_ONLY_PNBUF);
	vp = nd.ni_vp;
	bzero(&fh, sizeof(fh));
	fh.fh_fsid = vp->v_mount->mnt_stat.f_fsid;
	error = VOP_VPTOFH(vp, &fh.fh_fid);
	vput(vp);
	if (error == 0)
	error = copyout(&fh, fhp, sizeof (fh));
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct fhlink_args {
	fhandle_t *fhp;
	const char *to;
	};
	#endif
	int
	sys_fhlink(struct thread td, struct fhlink_args uap)
	{

	return (kern_fhlinkat(td, AT_FDCWD, uap->to, UIO_USERSPACE, uap->fhp));
	}

	#ifndef _SYS_SYSPROTO_H_
	struct fhlinkat_args {
	fhandle_t *fhp;
	int tofd;
	const char *to;
	};
	#endif
	int
	sys_fhlinkat(struct thread td, struct fhlinkat_args uap)
	{

	return (kern_fhlinkat(td, uap->tofd, uap->to, UIO_USERSPACE, uap->fhp));
	}

	static int
	kern_fhlinkat(struct thread td, int fd, const char path,
	enum uio_seg pathseg, fhandle_t *fhp)
	{
	fhandle_t fh;
	struct mount *mp;
	struct vnode *vp;
	int error;

	error = priv_check(td, PRIV_VFS_GETFH);
	if (error != 0)
	return (error);
	error = copyin(fhp, &fh, sizeof(fh));
	if (error != 0)
	return (error);
	do {
	bwillwrite();
	if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
	return (ESTALE);
	error = VFS_FHTOVP(mp, &fh.fh_fid, LK_SHARED, &vp);
	vfs_unbusy(mp);
	if (error != 0)
	return (error);
	VOP_UNLOCK(vp);
	} while ((error = kern_linkat_vp(td, vp, fd, path, pathseg)) == EAGAIN);
	return (error);
	}

	#ifndef _SYS_SYSPROTO_H_
	struct fhreadlink_args {
	fhandle_t *fhp;
	char *buf;
	size_t bufsize;
	};
	#endif
	int
	sys_fhreadlink(struct thread td, struct fhreadlink_args uap)
	{
	fhandle_t fh;
	struct mount *mp;
	struct vnode *vp;
	int error;

	error = priv_check(td, PRIV_VFS_GETFH);
	if (error != 0)
	return (error);
	if (uap->bufsize > IOSIZE_MAX)
	return (EINVAL);
	error = copyin(uap->fhp, &fh, sizeof(fh));
	if (error != 0)
	return (error);
	if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
	return (ESTALE);
	error = VFS_FHTOVP(mp, &fh.fh_fid, LK_SHARED, &vp);
	vfs_unbusy(mp);
	if (error != 0)
	return (error);
	error = kern_readlink_vp(vp, uap->buf, UIO_USERSPACE, uap->bufsize, td);
	vput(vp);
	return (error);
	}

	/*
	* syscall for the rpc.lockd to use to translate a NFS file handle into an
	* open descriptor.
	*
	* warning: do not remove the priv_check() call or this becomes one giant
	* security hole.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fhopen_args {
	const struct fhandle *u_fhp;
	int flags;
	};
	#endif
	int
	sys_fhopen(struct thread td, struct fhopen_args uap)
	{
	struct mount *mp;
	struct vnode *vp;
	struct fhandle fhp;
	struct file *fp;
	int fmode, error;
	int indx;

	error = priv_check(td, PRIV_VFS_FHOPEN);
	if (error != 0)
	return (error);
	indx = -1;
	fmode = FFLAGS(uap->flags);
	/* why not allow a non-read/write open for our lockd? */
	if (((fmode & (FREAD \| FWRITE)) == 0) \|\| (fmode & O_CREAT))
	return (EINVAL);
	error = copyin(uap->u_fhp, &fhp, sizeof(fhp));
	if (error != 0)
	return(error);
	/* find the mount point */
	mp = vfs_busyfs(&fhp.fh_fsid);
	if (mp == NULL)
	return (ESTALE);
	/* now give me my vnode, it gets returned to me locked */
	error = VFS_FHTOVP(mp, &fhp.fh_fid, LK_EXCLUSIVE, &vp);
	vfs_unbusy(mp);
	if (error != 0)
	return (error);

	error = falloc_noinstall(td, &fp);
	if (error != 0) {
	vput(vp);
	return (error);
	}
	/*
	* An extra reference on `fp' has been held for us by
	* falloc_noinstall().
	*/

	#ifdef INVARIANTS
	td->td_dupfd = -1;
	#endif
	error = vn_open_vnode(vp, fmode, td->td_ucred, td, fp);
	if (error != 0) {
	KASSERT(fp->f_ops == &badfileops,
	("VOP_OPEN in fhopen() set f_ops"));
	KASSERT(td->td_dupfd < 0,
	("fhopen() encountered fdopen()"));

	vput(vp);
	goto bad;
	}
	#ifdef INVARIANTS
	td->td_dupfd = 0;
	#endif
	fp->f_vnode = vp;
	fp->f_seqcount[UIO_READ] = 1;
	fp->f_seqcount[UIO_WRITE] = 1;
	finit(fp, (fmode & FMASK) \| (fp->f_flag & FHASLOCK), DTYPE_VNODE, vp,
	&vnops);
	VOP_UNLOCK(vp);
	if ((fmode & O_TRUNC) != 0) {
	error = fo_truncate(fp, 0, td->td_ucred, td);
	if (error != 0)
	goto bad;
	}

	error = finstall(td, fp, &indx, fmode, NULL);
	bad:
	fdrop(fp, td);
	td->td_retval[0] = indx;
	return (error);
	}

	/*
	* Stat an (NFS) file handle.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fhstat_args {
	struct fhandle *u_fhp;
	struct stat *sb;
	};
	#endif
	int
	sys_fhstat(struct thread td, struct fhstat_args uap)
	{
	struct stat sb;
	struct fhandle fh;
	int error;

	error = copyin(uap->u_fhp, &fh, sizeof(fh));
	if (error != 0)
	return (error);
	error = kern_fhstat(td, fh, &sb);
	if (error == 0)
	error = copyout(&sb, uap->sb, sizeof(sb));
	return (error);
	}

	int
	kern_fhstat(struct thread td, struct fhandle fh, struct stat sb)
	{
	struct mount *mp;
	struct vnode *vp;
	int error;

	error = priv_check(td, PRIV_VFS_FHSTAT);
	if (error != 0)
	return (error);
	if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
	return (ESTALE);
	error = VFS_FHTOVP(mp, &fh.fh_fid, LK_EXCLUSIVE, &vp);
	vfs_unbusy(mp);
	if (error != 0)
	return (error);
	error = vn_stat(vp, sb, td->td_ucred, NOCRED, td);
	vput(vp);
	return (error);
	}

	/*
	* Implement fstatfs() for (NFS) file handles.
	*/
	#ifndef _SYS_SYSPROTO_H_
	struct fhstatfs_args {
	struct fhandle *u_fhp;
	struct statfs *buf;
	};
	#endif
	int
	sys_fhstatfs(struct thread td, struct fhstatfs_args uap)
	{
	struct statfs *sfp;
	fhandle_t fh;
	int error;

	error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
	if (error != 0)
	return (error);
	sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
	error = kern_fhstatfs(td, fh, sfp);
	if (error == 0)
	error = copyout(sfp, uap->buf, sizeof(*sfp));
	free(sfp, M_STATFS);
	return (error);
	}

	int
	kern_fhstatfs(struct thread td, fhandle_t fh, struct statfs buf)
	{
	struct mount *mp;
	struct vnode *vp;
	int error;

	error = priv_check(td, PRIV_VFS_FHSTATFS);
	if (error != 0)
	return (error);
	if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
	return (ESTALE);
	error = VFS_FHTOVP(mp, &fh.fh_fid, LK_EXCLUSIVE, &vp);
	if (error != 0) {
	vfs_unbusy(mp);
	return (error);
	}
	vput(vp);
	error = prison_canseemount(td->td_ucred, mp);
	if (error != 0)
	goto out;
	#ifdef MAC
	error = mac_mount_check_stat(td->td_ucred, mp);
	if (error != 0)
	goto out;
	#endif
	error = VFS_STATFS(mp, buf);
	out:
	vfs_unbusy(mp);
	return (error);
	}

	/*
	* Unlike madvise(2), we do not make a best effort to remember every
	* possible caching hint. Instead, we remember the last setting with
	* the exception that we will allow POSIX_FADV_NORMAL to adjust the
	* region of any current setting.
	*/
	int
	kern_posix_fadvise(struct thread *td, int fd, off_t offset, off_t len,
	int advice)
	{
	struct fadvise_info fa, new;
	struct file *fp;
	struct vnode *vp;
	off_t end;
	int error;

	if (offset < 0 \|\| len < 0 \|\| offset > OFF_MAX - len)
	return (EINVAL);
	AUDIT_ARG_VALUE(advice);
	switch (advice) {
	case POSIX_FADV_SEQUENTIAL:
	case POSIX_FADV_RANDOM:
	case POSIX_FADV_NOREUSE:
	new = malloc(sizeof(*fa), M_FADVISE, M_WAITOK);
	break;
	case POSIX_FADV_NORMAL:
	case POSIX_FADV_WILLNEED:
	case POSIX_FADV_DONTNEED:
	new = NULL;
	break;
	default:
	return (EINVAL);
	}
	/* XXX: CAP_POSIX_FADVISE? */
	AUDIT_ARG_FD(fd);
	error = fget(td, fd, &cap_no_rights, &fp);
	if (error != 0)
	goto out;
	AUDIT_ARG_FILE(td->td_proc, fp);
	if ((fp->f_ops->fo_flags & DFLAG_SEEKABLE) == 0) {
	error = ESPIPE;
	goto out;
	}
	if (fp->f_type != DTYPE_VNODE) {
	error = ENODEV;
	goto out;
	}
	vp = fp->f_vnode;
	if (vp->v_type != VREG) {
	error = ENODEV;
	goto out;
	}
	if (len == 0)
	end = OFF_MAX;
	else
	end = offset + len - 1;
	switch (advice) {
	case POSIX_FADV_SEQUENTIAL:
	case POSIX_FADV_RANDOM:
	case POSIX_FADV_NOREUSE:
	/*
	* Try to merge any existing non-standard region with
	* this new region if possible, otherwise create a new
	* non-standard region for this request.
	*/
	mtx_pool_lock(mtxpool_sleep, fp);
	fa = fp->f_advice;
	if (fa != NULL && fa->fa_advice == advice &&
	((fa->fa_start <= end && fa->fa_end >= offset) \|\|
	(end != OFF_MAX && fa->fa_start == end + 1) \|\|
	(fa->fa_end != OFF_MAX && fa->fa_end + 1 == offset))) {
	if (offset < fa->fa_start)
	fa->fa_start = offset;
	if (end > fa->fa_end)
	fa->fa_end = end;
	} else {
	new->fa_advice = advice;
	new->fa_start = offset;
	new->fa_end = end;
	fp->f_advice = new;
	new = fa;
	}
	mtx_pool_unlock(mtxpool_sleep, fp);
	break;
	case POSIX_FADV_NORMAL:
	/*
	* If a the "normal" region overlaps with an existing
	* non-standard region, trim or remove the
	* non-standard region.
	*/
	mtx_pool_lock(mtxpool_sleep, fp);
	fa = fp->f_advice;
	if (fa != NULL) {
	if (offset <= fa->fa_start && end >= fa->fa_end) {
	new = fa;
	fp->f_advice = NULL;
	} else if (offset <= fa->fa_start &&
	end >= fa->fa_start)
	fa->fa_start = end + 1;
	else if (offset <= fa->fa_end && end >= fa->fa_end)
	fa->fa_end = offset - 1;
	else if (offset >= fa->fa_start && end <= fa->fa_end) {
	/*
	* If the "normal" region is a middle
	* portion of the existing
	* non-standard region, just remove
	* the whole thing rather than picking
	* one side or the other to
	* preserve.
	*/
	new = fa;
	fp->f_advice = NULL;
	}
	}
	mtx_pool_unlock(mtxpool_sleep, fp);
	break;
	case POSIX_FADV_WILLNEED:
	case POSIX_FADV_DONTNEED:
	error = VOP_ADVISE(vp, offset, end, advice);
	break;
	}
	out:
	if (fp != NULL)
	fdrop(fp, td);
	free(new, M_FADVISE);
	return (error);
	}

	int
	sys_posix_fadvise(struct thread td, struct posix_fadvise_args uap)
	{
	int error;

	error = kern_posix_fadvise(td, uap->fd, uap->offset, uap->len,
	uap->advice);
	return (kern_posix_error(td, error));
	}

	int
	kern_copy_file_range(struct thread td, int infd, off_t inoffp, int outfd,
	off_t *outoffp, size_t len, unsigned int flags)
	{
	struct file infp, outfp;
	struct vnode invp, outvp;
	int error;
	size_t retlen;
	void rl_rcookie, rl_wcookie;
	off_t savinoff, savoutoff;

	infp = outfp = NULL;
	rl_rcookie = rl_wcookie = NULL;
	savinoff = -1;
	error = 0;
	retlen = 0;

	if (flags != 0) {
	error = EINVAL;
	goto out;
	}
	if (len > SSIZE_MAX)
	/*
	* Although the len argument is size_t, the return argument
	* is ssize_t (which is signed). Therefore a size that won't
	* fit in ssize_t can't be returned.
	*/
	len = SSIZE_MAX;

	/* Get the file structures for the file descriptors. */
	error = fget_read(td, infd, &cap_read_rights, &infp);
	if (error != 0)
	goto out;
	if (infp->f_ops == &badfileops) {
	error = EBADF;
	goto out;
	}
	if (infp->f_vnode == NULL) {
	error = EINVAL;
	goto out;
	}
	error = fget_write(td, outfd, &cap_write_rights, &outfp);
	if (error != 0)
	goto out;
	if (outfp->f_ops == &badfileops) {
	error = EBADF;
	goto out;
	}
	if (outfp->f_vnode == NULL) {
	error = EINVAL;
	goto out;
	}

	/* Set the offset pointers to the correct place. */
	if (inoffp == NULL)
	inoffp = &infp->f_offset;
	if (outoffp == NULL)
	outoffp = &outfp->f_offset;
	savinoff = *inoffp;
	savoutoff = *outoffp;

	invp = infp->f_vnode;
	outvp = outfp->f_vnode;
	/* Sanity check the f_flag bits. */
	if ((outfp->f_flag & (FWRITE \| FAPPEND)) != FWRITE \|\|
	(infp->f_flag & FREAD) == 0) {
	error = EBADF;
	goto out;
	}

	/* If len == 0, just return 0. */
	if (len == 0)
	goto out;

	/*
	* If infp and outfp refer to the same file, the byte ranges cannot
	* overlap.
	*/
	if (invp == outvp && ((savinoff <= savoutoff && savinoff + len >
	savoutoff) \|\| (savinoff > savoutoff && savoutoff + len >
	savinoff))) {
	error = EINVAL;
	goto out;
	}

	/* Range lock the byte ranges for both invp and outvp. */
	for (;;) {
	rl_wcookie = vn_rangelock_wlock(outvp, outoffp, outoffp +
	len);
	rl_rcookie = vn_rangelock_tryrlock(invp, inoffp, inoffp +
	len);
	if (rl_rcookie != NULL)
	break;
	vn_rangelock_unlock(outvp, rl_wcookie);
	rl_rcookie = vn_rangelock_rlock(invp, inoffp, inoffp + len);
	vn_rangelock_unlock(invp, rl_rcookie);
	}

	retlen = len;
	error = vn_copy_file_range(invp, inoffp, outvp, outoffp, &retlen,
	flags, infp->f_cred, outfp->f_cred, td);
	out:
	if (rl_rcookie != NULL)
	vn_rangelock_unlock(invp, rl_rcookie);
	if (rl_wcookie != NULL)
	vn_rangelock_unlock(outvp, rl_wcookie);
	if (savinoff != -1 && (error == EINTR \|\| error == ERESTART)) {
	*inoffp = savinoff;
	*outoffp = savoutoff;
	}
	if (outfp != NULL)
	fdrop(outfp, td);
	if (infp != NULL)
	fdrop(infp, td);
	td->td_retval[0] = retlen;
	return (error);
	}

	int
	sys_copy_file_range(struct thread td, struct copy_file_range_args uap)
	{
	off_t inoff, outoff, inoffp, outoffp;
	int error;

	inoffp = outoffp = NULL;
	if (uap->inoffp != NULL) {
	error = copyin(uap->inoffp, &inoff, sizeof(off_t));
	if (error != 0)
	return (error);
	inoffp = &inoff;
	}
	if (uap->outoffp != NULL) {
	error = copyin(uap->outoffp, &outoff, sizeof(off_t));
	if (error != 0)
	return (error);
	outoffp = &outoff;
	}
	error = kern_copy_file_range(td, uap->infd, inoffp, uap->outfd,
	outoffp, uap->len, uap->flags);
	if (error == 0 && uap->inoffp != NULL)
	error = copyout(inoffp, uap->inoffp, sizeof(off_t));
	if (error == 0 && uap->outoffp != NULL)
	error = copyout(outoffp, uap->outoffp, sizeof(off_t));
	return (error);
	}
	diff --git a/sys/kern/vnode_if.src b/sys/kern/vnode_if.src
	index 5c0649fdadaf..e5a7b389fb30 100644
	--- a/sys/kern/vnode_if.src
	+++ b/sys/kern/vnode_if.src
	@@ -1,789 +1,789 @@
	#-
	# Copyright (c) 1992, 1993
	# The Regents of the University of California. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	# 3. Neither the name of the University nor the names of its contributors
	# may be used to endorse or promote products derived from this software
	# without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	# SUCH DAMAGE.
	#
	# @(#)vnode_if.src 8.12 (Berkeley) 5/14/95
	# $FreeBSD$
	#

	#
	# Above each of the vop descriptors in lines starting with %%
	# is a specification of the locking protocol used by each vop call.
	# The first column is the name of the variable, the remaining three
	# columns are in, out and error respectively. The "in" column defines
	# the lock state on input, the "out" column defines the state on successful
	# return, and the "error" column defines the locking state on error exit.
	#
	# The locking value can take the following values:
	# L: locked; not converted to type of lock.
	# E: locked with exclusive lock for this process.
	# U: unlocked.
	# -: not applicable. vnode does not yet (or no longer) exists.
	# =: the same on input and output, may be either L or U.
	#
	# The paramater named "vpp" is assumed to be always used with double
	# indirection (**vpp) and that name is hard-coded in vnode_if.awk !
	#
	# Lines starting with %! specify a pre or post-condition function
	# to call before/after the vop call.
	#
	# If other such parameters are introduced, they have to be added to
	# the AWK script at the head of the definition of "add_debug_code()".
	#

	vop_islocked {
	IN struct vnode *vp;
	};


	%% lookup dvp L L L
	%% lookup vpp - L -

	# XXX - the lookup locking protocol defies simple description and depends
	# on the flags and operation fields in the (cnp) structure. Note
	# especially that *vpp may equal dvp and both may be locked.

	vop_lookup {
	IN struct vnode *dvp;
	INOUT struct vnode **vpp;
	IN struct componentname *cnp;
	};


	%% cachedlookup dvp L L L
	%% cachedlookup vpp - L -

	# This must be an exact copy of lookup. See kern/vfs_cache.c for details.

	vop_cachedlookup {
	IN struct vnode *dvp;
	INOUT struct vnode **vpp;
	IN struct componentname *cnp;
	};


	%% create dvp E E E
	%% create vpp - L -
	%! create pre vop_create_pre
	%! create post vop_create_post

	vop_create {
	IN struct vnode *dvp;
	OUT struct vnode **vpp;
	IN struct componentname *cnp;
	IN struct vattr *vap;
	};


	%% whiteout dvp E E E
	%! whiteout pre vop_whiteout_pre
	%! whiteout post vop_whiteout_post

	vop_whiteout {
	IN struct vnode *dvp;
	IN struct componentname *cnp;
	IN int flags;
	};


	%% mknod dvp E E E
	%% mknod vpp - L -
	%! mknod pre vop_mknod_pre
	%! mknod post vop_mknod_post

	vop_mknod {
	IN struct vnode *dvp;
	OUT struct vnode **vpp;
	IN struct componentname *cnp;
	IN struct vattr *vap;
	};


	%% open vp L L L
	%! open post vop_open_post

	vop_open {
	IN struct vnode *vp;
	IN int mode;
	IN struct ucred *cred;
	IN struct thread *td;
	IN struct file *fp;
	};


	%% close vp L L L
	%! close post vop_close_post

	vop_close {
	IN struct vnode *vp;
	IN int fflag;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% fplookup_vexec vp - - -
	-%! fplookup_vexec pre vop_fplookup_vexec_pre
	-%! fplookup_vexec post vop_fplookup_vexec_post
	+%! fplookup_vexec debugpre vop_fplookup_vexec_debugpre
	+%! fplookup_vexec debugpost vop_fplookup_vexec_debugpost

	vop_fplookup_vexec {
	IN struct vnode *vp;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% access vp L L L

	vop_access {
	IN struct vnode *vp;
	IN accmode_t accmode;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% accessx vp L L L

	vop_accessx {
	IN struct vnode *vp;
	IN accmode_t accmode;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% getattr vp L L L

	vop_getattr {
	IN struct vnode *vp;
	OUT struct vattr *vap;
	IN struct ucred *cred;
	};


	%% setattr vp E E E
	%! setattr pre vop_setattr_pre
	%! setattr post vop_setattr_post

	vop_setattr {
	IN struct vnode *vp;
	IN struct vattr *vap;
	IN struct ucred *cred;
	};


	%% mmapped vp L L L

	vop_mmapped {
	IN struct vnode *vp;
	};


	%% read vp L L L
	%! read post vop_read_post

	vop_read {
	IN struct vnode *vp;
	INOUT struct uio *uio;
	IN int ioflag;
	IN struct ucred *cred;
	};


	%% write vp L L L
	%! write pre VOP_WRITE_PRE
	%! write post VOP_WRITE_POST

	vop_write {
	IN struct vnode *vp;
	INOUT struct uio *uio;
	IN int ioflag;
	IN struct ucred *cred;
	};


	%% ioctl vp U U U

	vop_ioctl {
	IN struct vnode *vp;
	IN u_long command;
	IN void *data;
	IN int fflag;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% poll vp U U U

	vop_poll {
	IN struct vnode *vp;
	IN int events;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% kqfilter vp U U U

	vop_kqfilter {
	IN struct vnode *vp;
	IN struct knote *kn;
	};


	%% revoke vp L L L

	vop_revoke {
	IN struct vnode *vp;
	IN int flags;
	};


	%% fsync vp L L L

	vop_fsync {
	IN struct vnode *vp;
	IN int waitfor;
	IN struct thread *td;
	};


	%% remove dvp E E E
	%% remove vp E E E
	%! remove pre vop_remove_pre
	%! remove post vop_remove_post

	vop_remove {
	IN struct vnode *dvp;
	IN struct vnode *vp;
	IN struct componentname *cnp;
	};


	%% link tdvp E E E
	%% link vp E E E
	%! link pre vop_link_pre
	%! link post vop_link_post

	vop_link {
	IN struct vnode *tdvp;
	IN struct vnode *vp;
	IN struct componentname *cnp;
	};


	%! rename pre vop_rename_pre
	%! rename post vop_rename_post

	vop_rename {
	IN WILLRELE struct vnode *fdvp;
	IN WILLRELE struct vnode *fvp;
	IN struct componentname *fcnp;
	IN WILLRELE struct vnode *tdvp;
	IN WILLRELE struct vnode *tvp;
	IN struct componentname *tcnp;
	};


	%% mkdir dvp E E E
	%% mkdir vpp - E -
	%! mkdir pre vop_mkdir_pre
	%! mkdir post vop_mkdir_post

	vop_mkdir {
	IN struct vnode *dvp;
	OUT struct vnode **vpp;
	IN struct componentname *cnp;
	IN struct vattr *vap;
	};


	%% rmdir dvp E E E
	%% rmdir vp E E E
	%! rmdir pre vop_rmdir_pre
	%! rmdir post vop_rmdir_post

	vop_rmdir {
	IN struct vnode *dvp;
	IN struct vnode *vp;
	IN struct componentname *cnp;
	};


	%% symlink dvp E E E
	%% symlink vpp - E -
	%! symlink pre vop_symlink_pre
	%! symlink post vop_symlink_post

	vop_symlink {
	IN struct vnode *dvp;
	OUT struct vnode **vpp;
	IN struct componentname *cnp;
	IN struct vattr *vap;
	IN const char *target;
	};


	%% readdir vp L L L
	%! readdir post vop_readdir_post

	vop_readdir {
	IN struct vnode *vp;
	INOUT struct uio *uio;
	IN struct ucred *cred;
	INOUT int *eofflag;
	OUT int *ncookies;
	INOUT u_long **cookies;
	};


	%% readlink vp L L L

	vop_readlink {
	IN struct vnode *vp;
	INOUT struct uio *uio;
	IN struct ucred *cred;
	};


	%% inactive vp E E E

	vop_inactive {
	IN struct vnode *vp;
	IN struct thread *td;
	};

	-%! need_inactive pre vop_need_inactive_pre
	-%! need_inactive post vop_need_inactive_post
	+%! need_inactive debugpre vop_need_inactive_debugpre
	+%! need_inactive debugpost vop_need_inactive_debugpost

	vop_need_inactive {
	IN struct vnode *vp;
	};

	%% reclaim vp E E E
	%! reclaim post vop_reclaim_post

	vop_reclaim {
	IN struct vnode *vp;
	IN struct thread *td;
	};


	-%! lock1 pre vop_lock_pre
	-%! lock1 post vop_lock_post
	+%! lock1 debugpre vop_lock_debugpre
	+%! lock1 debugpost vop_lock_debugpost

	vop_lock1 {
	IN struct vnode *vp;
	IN int flags;
	IN const char *file;
	IN int line;
	};


	-%! unlock pre vop_unlock_pre
	+%! unlock debugpre vop_unlock_debugpre

	vop_unlock {
	IN struct vnode *vp;
	};


	%% bmap vp L L L

	vop_bmap {
	IN struct vnode *vp;
	IN daddr_t bn;
	OUT struct bufobj **bop;
	IN daddr_t *bnp;
	OUT int *runp;
	OUT int *runb;
	};


	%% strategy vp L L L
	-%! strategy pre vop_strategy_pre
	+%! strategy debugpre vop_strategy_debugpre

	vop_strategy {
	IN struct vnode *vp;
	IN struct buf *bp;
	};


	%% getwritemount vp = = =

	vop_getwritemount {
	IN struct vnode *vp;
	OUT struct mount **mpp;
	};


	%% print vp - - -

	vop_print {
	IN struct vnode *vp;
	};


	%% pathconf vp L L L

	vop_pathconf {
	IN struct vnode *vp;
	IN int name;
	OUT long *retval;
	};


	%% advlock vp U U U

	vop_advlock {
	IN struct vnode *vp;
	IN void *id;
	IN int op;
	IN struct flock *fl;
	IN int flags;
	};


	%% advlockasync vp U U U

	vop_advlockasync {
	IN struct vnode *vp;
	IN void *id;
	IN int op;
	IN struct flock *fl;
	IN int flags;
	IN struct task *task;
	INOUT void **cookiep;
	};


	%% advlockpurge vp E E E

	vop_advlockpurge {
	IN struct vnode *vp;
	};


	%% reallocblks vp E E E

	vop_reallocblks {
	IN struct vnode *vp;
	IN struct cluster_save *buflist;
	};


	%% getpages vp L L L

	vop_getpages {
	IN struct vnode *vp;
	IN vm_page_t *m;
	IN int count;
	IN int *rbehind;
	IN int *rahead;
	};


	%% getpages_async vp L L L

	vop_getpages_async {
	IN struct vnode *vp;
	IN vm_page_t *m;
	IN int count;
	IN int *rbehind;
	IN int *rahead;
	IN vop_getpages_iodone_t *iodone;
	IN void *arg;
	};


	%% putpages vp L L L

	vop_putpages {
	IN struct vnode *vp;
	IN vm_page_t *m;
	IN int count;
	IN int sync;
	IN int *rtvals;
	};


	%% getacl vp L L L

	vop_getacl {
	IN struct vnode *vp;
	IN acl_type_t type;
	OUT struct acl *aclp;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% setacl vp E E E
	%! setacl pre vop_setacl_pre
	%! setacl post vop_setacl_post

	vop_setacl {
	IN struct vnode *vp;
	IN acl_type_t type;
	IN struct acl *aclp;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% aclcheck vp = = =

	vop_aclcheck {
	IN struct vnode *vp;
	IN acl_type_t type;
	IN struct acl *aclp;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% closeextattr vp L L L

	vop_closeextattr {
	IN struct vnode *vp;
	IN int commit;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% getextattr vp L L L

	vop_getextattr {
	IN struct vnode *vp;
	IN int attrnamespace;
	IN const char *name;
	INOUT struct uio *uio;
	OUT size_t *size;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% listextattr vp L L L

	vop_listextattr {
	IN struct vnode *vp;
	IN int attrnamespace;
	INOUT struct uio *uio;
	OUT size_t *size;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% openextattr vp L L L

	vop_openextattr {
	IN struct vnode *vp;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% deleteextattr vp E E E
	%! deleteextattr pre vop_deleteextattr_pre
	%! deleteextattr post vop_deleteextattr_post

	vop_deleteextattr {
	IN struct vnode *vp;
	IN int attrnamespace;
	IN const char *name;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% setextattr vp E E E
	%! setextattr pre vop_setextattr_pre
	%! setextattr post vop_setextattr_post

	vop_setextattr {
	IN struct vnode *vp;
	IN int attrnamespace;
	IN const char *name;
	INOUT struct uio *uio;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% setlabel vp E E E

	vop_setlabel {
	IN struct vnode *vp;
	IN struct label *label;
	IN struct ucred *cred;
	IN struct thread *td;
	};


	%% vptofh vp = = =

	vop_vptofh {
	IN struct vnode *vp;
	IN struct fid *fhp;
	};


	%% vptocnp vp L L L
	%% vptocnp vpp - U -

	vop_vptocnp {
	IN struct vnode *vp;
	OUT struct vnode **vpp;
	IN struct ucred *cred;
	INOUT char *buf;
	INOUT size_t *buflen;
	};


	%% allocate vp E E E

	vop_allocate {
	IN struct vnode *vp;
	INOUT off_t *offset;
	INOUT off_t *len;
	};


	%% advise vp U U U

	vop_advise {
	IN struct vnode *vp;
	IN off_t start;
	IN off_t end;
	IN int advice;
	};


	%% unp_bind vp E E E

	vop_unp_bind {
	IN struct vnode *vp;
	IN struct unpcb *unpcb;
	};


	%% unp_connect vp L L L

	vop_unp_connect {
	IN struct vnode *vp;
	OUT struct unpcb **unpcb;
	};


	%% unp_detach vp = = =

	vop_unp_detach {
	IN struct vnode *vp;
	};


	%% is_text vp L L L

	vop_is_text {
	IN struct vnode *vp;
	};


	%% set_text vp = = =

	vop_set_text {
	IN struct vnode *vp;
	};


	%% vop_unset_text vp L L L

	vop_unset_text {
	IN struct vnode *vp;
	};


	%% add_writecount vp L L L

	vop_add_writecount {
	IN struct vnode *vp;
	IN int inc;
	};


	%% fdatasync vp L L L

	vop_fdatasync {
	IN struct vnode *vp;
	IN struct thread *td;
	};


	%% copy_file_range invp U U U
	%% copy_file_range outvp U U U

	vop_copy_file_range {
	IN struct vnode *invp;
	INOUT off_t *inoffp;
	IN struct vnode *outvp;
	INOUT off_t *outoffp;
	INOUT size_t *lenp;
	IN unsigned int flags;
	IN struct ucred *incred;
	IN struct ucred *outcred;
	IN struct thread *fsizetd;
	};


	# The VOPs below are spares at the end of the table to allow new VOPs to be
	# added in stable branches without breaking the KBI. New VOPs in HEAD should
	# be added above these spares. When merging a new VOP to a stable branch,
	# the new VOP should replace one of the spares.

	vop_spare1 {
	IN struct vnode *vp;
	};

	vop_spare2 {
	IN struct vnode *vp;
	};

	vop_spare3 {
	IN struct vnode *vp;
	};

	vop_spare4 {
	IN struct vnode *vp;
	};

	vop_spare5 {
	IN struct vnode *vp;
	};
	diff --git a/sys/mips/conf/std.MALTA b/sys/mips/conf/std.MALTA
	index 26940db1b92f..7b951b926824 100644
	--- a/sys/mips/conf/std.MALTA
	+++ b/sys/mips/conf/std.MALTA
	@@ -1,57 +1,64 @@
	# MALTA_COMMON -- Common kernel config options for MALTA boards
	#
	# $FreeBSD$

	options YAMON

	options TICK_USE_YAMON_FREQ=defined
	#options TICK_USE_MALTA_RTC=defined

	include "../malta/std.malta"

	hints "MALTA.hints" #Default places to look for devices.

	makeoptions DEBUG=-g #Build kernel with gdb(1) debug symbols

	options DDB
	options KDB

	options SCHED_4BSD #4BSD scheduler
	options INET #InterNETworking
	options TCP_HHOOK # hhook(9) framework for TCP
	options NFSCL #Network Filesystem Client
	options NFS_ROOT #NFS usable as /, requires NFSCL
	options PSEUDOFS #Pseudo-filesystem framework
	options _KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions
	options CAPABILITY_MODE # Capsicum capability mode
	options CAPABILITIES # Capsicum capabilities

	options TMPFS #Efficient memory filesystem

	options FFS #Berkeley Fast Filesystem
	options SOFTUPDATES #Enable FFS soft updates support
	options UFS_ACL #Support for access control lists
	options UFS_DIRHASH #Improve performance on big directories
	options ROOTDEVNAME=\"ufs:ada0\"

	options GEOM_LABEL # Provides labelization
	options GEOM_PART_GPT # GUID Partition Tables.
	options GEOM_RAID # Soft RAID functionality.

	# Debugging for use in -current
	#options DEADLKRES #Enable the deadlock resolver
	options INVARIANTS #Enable calls of extra sanity checking
	options INVARIANT_SUPPORT #Extra sanity checks of internal structures, required by INVARIANTS
	#options WITNESS #Enable checks to detect deadlocks and cycles
	#options WITNESS_SKIPSPIN #Don't run witness on spinlocks for speed

	# Kernel dump features.
	options ZSTDIO # zstd-compressed kernel and user dumps

	device loop
	device ether
	device le
	device miibus
	device bpf
	device md
	device uart
	+
	+# VirtIO support
	+device virtio # Generic VirtIO bus (required)
	+device virtio_pci # VirtIO PCI Interface
	+device vtnet # VirtIO Ethernet device
	+device virtio_blk # VirtIO Block device
	+device virtio_random # VirtIO Entropy device
	diff --git a/sys/mips/mips/trap.c b/sys/mips/mips/trap.c
	index 26f7c1515818..6ada543577c9 100644
	--- a/sys/mips/mips/trap.c
	+++ b/sys/mips/mips/trap.c
	@@ -1,1704 +1,1704 @@
	/* $OpenBSD: trap.c,v 1.19 1998/09/30 12:40:41 pefo Exp $ */
	/* tracked to 1.23 */
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1988 University of Utah.
	* Copyright (c) 1992, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* the Systems Programming Group of the University of Utah Computer
	* Science Department and Ralph Campbell.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* from: Utah Hdr: trap.c 1.32 91/04/06
	*
	* from: @(#)trap.c 8.5 (Berkeley) 1/11/94
	* JNPR: trap.c,v 1.13.2.2 2007/08/29 10:03:49 girish
	*/
	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_ddb.h"
	#include "opt_ktrace.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/sysent.h>
	#include <sys/proc.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/signalvar.h>
	#include <sys/syscall.h>
	#include <sys/lock.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_page.h>
	#include <vm/vm_map.h>
	#include <vm/vm_param.h>
	#include <sys/vmmeter.h>
	#include <sys/ptrace.h>
	#include <sys/user.h>
	#include <sys/buf.h>
	#include <sys/vnode.h>
	#include <sys/sysctl.h>
	#include <sys/syslog.h>
	#include <sys/bus.h>
	#ifdef KTRACE
	#include <sys/ktrace.h>
	#endif
	#include <net/netisr.h>

	#include <machine/trap.h>
	#include <machine/cpu.h>
	#include <machine/cpuinfo.h>
	#include <machine/pte.h>
	#include <machine/pmap.h>
	#include <machine/md_var.h>
	#include <machine/mips_opcode.h>
	#include <machine/frame.h>
	#include <machine/regnum.h>
	#include <machine/tls.h>

	#ifdef DDB
	#include <machine/db_machdep.h>
	#include <ddb/db_sym.h>
	#include <ddb/ddb.h>
	#include <sys/kdb.h>
	#endif

	#ifdef KDTRACE_HOOKS
	#include <sys/dtrace_bsd.h>
	#endif

	#ifdef TRAP_DEBUG
	int trap_debug = 0;
	SYSCTL_INT(_machdep, OID_AUTO, trap_debug, CTLFLAG_RW,
	&trap_debug, 0, "Debug information on all traps");
	#endif

	#define lbu_macro(data, addr) \
	__asm __volatile ("lbu %0, 0x0(%1)" \
	: "=r" (data) /* outputs */ \
	: "r" (addr)); /* inputs */

	#define lb_macro(data, addr) \
	__asm __volatile ("lb %0, 0x0(%1)" \
	: "=r" (data) /* outputs */ \
	: "r" (addr)); /* inputs */

	#define lwl_macro(data, addr) \
	__asm __volatile ("lwl %0, 0x0(%1)" \
	: "+r" (data) /* outputs */ \
	: "r" (addr)); /* inputs */

	#define lwr_macro(data, addr) \
	__asm __volatile ("lwr %0, 0x0(%1)" \
	: "+r" (data) /* outputs */ \
	: "r" (addr)); /* inputs */

	#define ldl_macro(data, addr) \
	__asm __volatile ("ldl %0, 0x0(%1)" \
	: "+r" (data) /* outputs */ \
	: "r" (addr)); /* inputs */

	#define ldr_macro(data, addr) \
	__asm __volatile ("ldr %0, 0x0(%1)" \
	: "+r" (data) /* outputs */ \
	: "r" (addr)); /* inputs */

	#define sb_macro(data, addr) \
	__asm __volatile ("sb %0, 0x0(%1)" \
	: /* outputs */ \
	: "r" (data), "r" (addr)); /* inputs */

	#define swl_macro(data, addr) \
	__asm __volatile ("swl %0, 0x0(%1)" \
	: /* outputs */ \
	: "r" (data), "r" (addr)); /* inputs */

	#define swr_macro(data, addr) \
	__asm __volatile ("swr %0, 0x0(%1)" \
	: /* outputs */ \
	: "r" (data), "r" (addr)); /* inputs */

	#define sdl_macro(data, addr) \
	__asm __volatile ("sdl %0, 0x0(%1)" \
	: /* outputs */ \
	: "r" (data), "r" (addr)); /* inputs */

	#define sdr_macro(data, addr) \
	__asm __volatile ("sdr %0, 0x0(%1)" \
	: /* outputs */ \
	: "r" (data), "r" (addr)); /* inputs */

	static void log_illegal_instruction(const char , struct trapframe );
	static void log_bad_page_fault(char , struct trapframe , int);
	static void log_frame_dump(struct trapframe *frame);
	static void get_mapping_info(vm_offset_t, pd_entry_t , pt_entry_t );

	int (dtrace_invop_jump_addr)(struct trapframe );

	#ifdef TRAP_DEBUG
	static void trap_frame_dump(struct trapframe *frame);
	#endif

	void (*machExceptionTable[]) (void)= {
	/*
	* The kernel exception handlers.
	*/
	MipsKernIntr, /* external interrupt */
	MipsKernGenException, /* TLB modification */
	MipsTLBInvalidException,/* TLB miss (load or instr. fetch) */
	MipsTLBInvalidException,/* TLB miss (store) */
	MipsKernGenException, /* address error (load or I-fetch) */
	MipsKernGenException, /* address error (store) */
	MipsKernGenException, /* bus error (I-fetch) */
	MipsKernGenException, /* bus error (load or store) */
	MipsKernGenException, /* system call */
	MipsKernGenException, /* breakpoint */
	MipsKernGenException, /* reserved instruction */
	MipsKernGenException, /* coprocessor unusable */
	MipsKernGenException, /* arithmetic overflow */
	MipsKernGenException, /* trap exception */
	MipsKernGenException, /* virtual coherence exception inst */
	MipsKernGenException, /* floating point exception */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* watch exception */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* reserved */
	MipsKernGenException, /* virtual coherence exception data */
	/*
	* The user exception handlers.
	*/
	MipsUserIntr, /* 0 */
	MipsUserGenException, /* 1 */
	MipsTLBInvalidException,/* 2 */
	MipsTLBInvalidException,/* 3 */
	MipsUserGenException, /* 4 */
	MipsUserGenException, /* 5 */
	MipsUserGenException, /* 6 */
	MipsUserGenException, /* 7 */
	MipsUserGenException, /* 8 */
	MipsUserGenException, /* 9 */
	MipsUserGenException, /* 10 */
	MipsUserGenException, /* 11 */
	MipsUserGenException, /* 12 */
	MipsUserGenException, /* 13 */
	MipsUserGenException, /* 14 */
	MipsUserGenException, /* 15 */
	MipsUserGenException, /* 16 */
	MipsUserGenException, /* 17 */
	MipsUserGenException, /* 18 */
	MipsUserGenException, /* 19 */
	MipsUserGenException, /* 20 */
	MipsUserGenException, /* 21 */
	MipsUserGenException, /* 22 */
	MipsUserGenException, /* 23 */
	MipsUserGenException, /* 24 */
	MipsUserGenException, /* 25 */
	MipsUserGenException, /* 26 */
	MipsUserGenException, /* 27 */
	MipsUserGenException, /* 28 */
	MipsUserGenException, /* 29 */
	MipsUserGenException, /* 20 */
	MipsUserGenException, /* 31 */
	};

	char *trap_type[] = {
	"external interrupt",
	"TLB modification",
	"TLB miss (load or instr. fetch)",
	"TLB miss (store)",
	"address error (load or I-fetch)",
	"address error (store)",
	"bus error (I-fetch)",
	"bus error (load or store)",
	"system call",
	"breakpoint",
	"reserved instruction",
	"coprocessor unusable",
	"arithmetic overflow",
	"trap",
	"virtual coherency instruction",
	"floating point",
	"reserved 16",
	"reserved 17",
	"reserved 18",
	"reserved 19",
	"reserved 20",
	"reserved 21",
	"reserved 22",
	"watch",
	"reserved 24",
	"reserved 25",
	"reserved 26",
	"reserved 27",
	"reserved 28",
	"reserved 29",
	"reserved 30",
	"virtual coherency data",
	};

	#if !defined(SMP) && (defined(DDB) \|\| defined(DEBUG))
	struct trapdebug trapdebug[TRAPSIZE], *trp = trapdebug;
	#endif

	#define KERNLAND(x) ((vm_offset_t)(x) >= VM_MIN_KERNEL_ADDRESS && (vm_offset_t)(x) < VM_MAX_KERNEL_ADDRESS)
	#define DELAYBRANCH(x) ((x) & MIPS_CR_BR_DELAY)

	/*
	* MIPS load/store access type
	*/
	enum {
	MIPS_LHU_ACCESS = 1,
	MIPS_LH_ACCESS,
	MIPS_LWU_ACCESS,
	MIPS_LW_ACCESS,
	MIPS_LD_ACCESS,
	MIPS_SH_ACCESS,
	MIPS_SW_ACCESS,
	MIPS_SD_ACCESS
	};

	char *access_name[] = {
	"Load Halfword Unsigned",
	"Load Halfword",
	"Load Word Unsigned",
	"Load Word",
	"Load Doubleword",
	"Store Halfword",
	"Store Word",
	"Store Doubleword"
	};

	#ifdef CPU_CNMIPS
	#include <machine/octeon_cop2.h>
	#endif

	static int allow_unaligned_acc = 1;

	SYSCTL_INT(_vm, OID_AUTO, allow_unaligned_acc, CTLFLAG_RW,
	&allow_unaligned_acc, 0, "Allow unaligned accesses");

	/*
	* FP emulation is assumed to work on O32, but the code is outdated and crufty
	* enough that it's a more sensible default to have it disabled when using
	* other ABIs. At the very least, it needs a lot of help in using
	* type-semantic ABI-oblivious macros for everything it does.
	*/
	#if defined(__mips_o32)
	static int emulate_fp = 1;
	#else
	static int emulate_fp = 0;
	#endif
	SYSCTL_INT(_machdep, OID_AUTO, emulate_fp, CTLFLAG_RW,
	&emulate_fp, 0, "Emulate unimplemented FPU instructions");

	static int emulate_unaligned_access(struct trapframe *frame, int mode);

	extern void fswintrberr(void); /* XXX */

	int
	cpu_fetch_syscall_args(struct thread *td)
	{
	struct trapframe *locr0;
	struct sysentvec *se;
	struct syscall_args *sa;
	int error, nsaved;

	locr0 = td->td_frame;
	sa = &td->td_sa;

	bzero(sa->args, sizeof(sa->args));

	/* compute next PC after syscall instruction */
	td->td_pcb->pcb_tpc = sa->trapframe->pc; /* Remember if restart */
	if (DELAYBRANCH(sa->trapframe->cause)) /* Check BD bit */
	locr0->pc = MipsEmulateBranch(locr0, sa->trapframe->pc, 0, 0);
	else
	locr0->pc += sizeof(int);
	sa->code = locr0->v0;

	switch (sa->code) {
	case SYS___syscall:
	case SYS_syscall:
	/*
	* This is an indirect syscall, in which the code is the first argument.
	*/
	#if (!defined(__mips_n32) && !defined(__mips_n64)) \|\| defined(COMPAT_FREEBSD32)
	if (sa->code == SYS___syscall && SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
	/*
	* Like syscall, but code is a quad, so as to maintain alignment
	* for the rest of the arguments.
	*/
	if (_QUAD_LOWWORD == 0)
	sa->code = locr0->a0;
	else
	sa->code = locr0->a1;
	sa->args[0] = locr0->a2;
	sa->args[1] = locr0->a3;
	nsaved = 2;
	break;
	}
	#endif
	/*
	* This is either not a quad syscall, or is a quad syscall with a
	* new ABI in which quads fit in a single register.
	*/
	sa->code = locr0->a0;
	sa->args[0] = locr0->a1;
	sa->args[1] = locr0->a2;
	sa->args[2] = locr0->a3;
	nsaved = 3;
	#if defined(__mips_n32) \|\| defined(__mips_n64)
	#ifdef COMPAT_FREEBSD32
	if (!SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
	#endif
	/*
	* Non-o32 ABIs support more arguments in registers.
	*/
	sa->args[3] = locr0->a4;
	sa->args[4] = locr0->a5;
	sa->args[5] = locr0->a6;
	sa->args[6] = locr0->a7;
	nsaved += 4;
	#ifdef COMPAT_FREEBSD32
	}
	#endif
	#endif
	break;
	default:
	/*
	* A direct syscall, arguments are just parameters to the syscall.
	*/
	sa->args[0] = locr0->a0;
	sa->args[1] = locr0->a1;
	sa->args[2] = locr0->a2;
	sa->args[3] = locr0->a3;
	nsaved = 4;
	#if defined (__mips_n32) \|\| defined(__mips_n64)
	#ifdef COMPAT_FREEBSD32
	if (!SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
	#endif
	/*
	* Non-o32 ABIs support more arguments in registers.
	*/
	sa->args[4] = locr0->a4;
	sa->args[5] = locr0->a5;
	sa->args[6] = locr0->a6;
	sa->args[7] = locr0->a7;
	nsaved += 4;
	#ifdef COMPAT_FREEBSD32
	}
	#endif
	#endif
	break;
	}

	#ifdef TRAP_DEBUG
	if (trap_debug)
	printf("SYSCALL #%d pid:%u\n", sa->code, td->td_proc->p_pid);
	#endif

	se = td->td_proc->p_sysent;
	/*
	* XXX
	* Shouldn't this go before switching on the code?
	*/

	if (sa->code >= se->sv_size)
	sa->callp = &se->sv_table[0];
	else
	sa->callp = &se->sv_table[sa->code];

	sa->narg = sa->callp->sy_narg;

	if (sa->narg > nsaved) {
	#if defined(__mips_n32) \|\| defined(__mips_n64)
	/*
	* XXX
	* Is this right for new ABIs? I think the 4 there
	* should be 8, size there are 8 registers to skip,
	* not 4, but I'm not certain.
	*/
	#ifdef COMPAT_FREEBSD32
	if (!SV_PROC_FLAG(td->td_proc, SV_ILP32))
	#endif
	printf("SYSCALL #%u pid:%u, narg (%u) > nsaved (%u).\n",
	sa->code, td->td_proc->p_pid, sa->narg, nsaved);
	#endif
	#if (defined(__mips_n32) \|\| defined(__mips_n64)) && defined(COMPAT_FREEBSD32)
	if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) {
	unsigned i;
	int32_t arg;

	error = 0; /* XXX GCC is awful. */
	for (i = nsaved; i < sa->narg; i++) {
	error = copyin((caddr_t)(intptr_t)(locr0->sp +
	(4 + (i - nsaved)) * sizeof(int32_t)),
	(caddr_t)&arg, sizeof arg);
	if (error != 0)
	break;
	sa->args[i] = arg;
	}
	} else
	#endif
	error = copyin((caddr_t)(intptr_t)(locr0->sp +
	4 * sizeof(register_t)), (caddr_t)&sa->args[nsaved],
	(u_int)(sa->narg - nsaved) * sizeof(register_t));
	if (error != 0) {
	locr0->v0 = error;
	locr0->a3 = 1;
	}
	} else
	error = 0;

	if (error == 0) {
	td->td_retval[0] = 0;
	td->td_retval[1] = locr0->v1;
	}

	return (error);
	}

	#undef __FBSDID
	#define __FBSDID(x)
	#include "../../kern/subr_syscall.c"

	/*
	* Handle an exception.
	* Called from MipsKernGenException() or MipsUserGenException()
	* when a processor trap occurs.
	* In the case of a kernel trap, we return the pc where to resume if
	* p->p_addr->u_pcb.pcb_onfault is set, otherwise, return old pc.
	*/
	register_t
	trap(struct trapframe *trapframe)
	{
	int type, usermode;
	int i = 0;
	unsigned ucode = 0;
	struct thread *td = curthread;
	struct proc *p = curproc;
	vm_prot_t ftype;
	pmap_t pmap;
	int access_type;
	ksiginfo_t ksi;
	char *msg = NULL;
	intptr_t addr = 0;
	register_t pc;
	int cop, error;
	register_t *frame_regs;

	trapdebug_enter(trapframe, 0);
	#ifdef KDB
	if (kdb_active) {
	kdb_reenter();
	return (0);
	}
	#endif
	type = (trapframe->cause & MIPS_CR_EXC_CODE) >> MIPS_CR_EXC_CODE_SHIFT;
	if (TRAPF_USERMODE(trapframe)) {
	type \|= T_USER;
	usermode = 1;
	} else {
	usermode = 0;
	}

	/*
	* Enable hardware interrupts if they were on before the trap. If it
	* was off disable all so we don't accidently enable it when doing a
	* return to userland.
	*/
	if (trapframe->sr & MIPS_SR_INT_IE) {
	set_intr_mask(trapframe->sr & MIPS_SR_INT_MASK);
	intr_enable();
	} else {
	intr_disable();
	}

	#ifdef TRAP_DEBUG
	if (trap_debug) {
	static vm_offset_t last_badvaddr = 0;
	static vm_offset_t this_badvaddr = 0;
	static int count = 0;
	u_int32_t pid;

	printf("trap type %x (%s - ", type,
	trap_type[type & (~T_USER)]);

	if (type & T_USER)
	printf("user mode)\n");
	else
	printf("kernel mode)\n");

	#ifdef SMP
	printf("cpuid = %d\n", PCPU_GET(cpuid));
	#endif
	pid = mips_rd_entryhi() & TLBHI_ASID_MASK;
	printf("badaddr = %#jx, pc = %#jx, ra = %#jx, sp = %#jx, sr = %jx, pid = %d, ASID = %u\n",
	(intmax_t)trapframe->badvaddr, (intmax_t)trapframe->pc, (intmax_t)trapframe->ra,
	(intmax_t)trapframe->sp, (intmax_t)trapframe->sr,
	(curproc ? curproc->p_pid : -1), pid);

	switch (type & ~T_USER) {
	case T_TLB_MOD:
	case T_TLB_LD_MISS:
	case T_TLB_ST_MISS:
	case T_ADDR_ERR_LD:
	case T_ADDR_ERR_ST:
	this_badvaddr = trapframe->badvaddr;
	break;
	case T_SYSCALL:
	this_badvaddr = trapframe->ra;
	break;
	default:
	this_badvaddr = trapframe->pc;
	break;
	}
	if ((last_badvaddr == this_badvaddr) &&
	((type & ~T_USER) != T_SYSCALL) &&
	((type & ~T_USER) != T_COP_UNUSABLE)) {
	if (++count == 3) {
	trap_frame_dump(trapframe);
	panic("too many faults at %p\n", (void *)last_badvaddr);
	}
	} else {
	last_badvaddr = this_badvaddr;
	count = 0;
	}
	}
	#endif

	#ifdef KDTRACE_HOOKS
	/*
	* A trap can occur while DTrace executes a probe. Before
	* executing the probe, DTrace blocks re-scheduling and sets
	* a flag in its per-cpu flags to indicate that it doesn't
	* want to fault. On returning from the probe, the no-fault
	* flag is cleared and finally re-scheduling is enabled.
	*
	* If the DTrace kernel module has registered a trap handler,
	* call it and if it returns non-zero, assume that it has
	* handled the trap and modified the trap frame so that this
	* function can return normally.
	*/
	/*
	* XXXDTRACE: add pid probe handler here (if ever)
	*/
	if (!usermode) {
	if (dtrace_trap_func != NULL &&
	(*dtrace_trap_func)(trapframe, type) != 0)
	return (trapframe->pc);
	}
	#endif

	switch (type) {
	case T_MCHECK:
	#ifdef DDB
	kdb_trap(type, 0, trapframe);
	#endif
	panic("MCHECK\n");
	break;
	case T_TLB_MOD:
	/* check for kernel address */
	if (KERNLAND(trapframe->badvaddr)) {
	if (pmap_emulate_modified(kernel_pmap,
	trapframe->badvaddr) != 0) {
	ftype = VM_PROT_WRITE;
	goto kernel_fault;
	}
	return (trapframe->pc);
	}
	/* FALLTHROUGH */

	case T_TLB_MOD + T_USER:
	pmap = &p->p_vmspace->vm_pmap;
	if (pmap_emulate_modified(pmap, trapframe->badvaddr) != 0) {
	ftype = VM_PROT_WRITE;
	goto dofault;
	}
	if (!usermode)
	return (trapframe->pc);
	goto out;

	case T_TLB_LD_MISS:
	case T_TLB_ST_MISS:
	ftype = (type == T_TLB_ST_MISS) ? VM_PROT_WRITE : VM_PROT_READ;
	/* check for kernel address */
	if (KERNLAND(trapframe->badvaddr)) {
	vm_offset_t va;
	int rv;

	kernel_fault:
	va = (vm_offset_t)trapframe->badvaddr;
	rv = vm_fault_trap(kernel_map, va, ftype,
	VM_FAULT_NORMAL, NULL, NULL);
	if (rv == KERN_SUCCESS)
	return (trapframe->pc);
	if (td->td_pcb->pcb_onfault != NULL) {
	pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault;
	td->td_pcb->pcb_onfault = NULL;
	return (pc);
	}
	goto err;
	}

	/*
	* It is an error for the kernel to access user space except
	* through the copyin/copyout routines.
	*/
	if (td->td_pcb->pcb_onfault == NULL)
	goto err;

	goto dofault;

	case T_TLB_LD_MISS + T_USER:
	ftype = VM_PROT_READ;
	goto dofault;

	case T_TLB_ST_MISS + T_USER:
	ftype = VM_PROT_WRITE;
	dofault:
	{
	vm_offset_t va;
	struct vmspace *vm;
	vm_map_t map;
	int rv = 0;

	vm = p->p_vmspace;
	map = &vm->vm_map;
	va = (vm_offset_t)trapframe->badvaddr;
	if (KERNLAND(trapframe->badvaddr)) {
	/*
	* Don't allow user-mode faults in kernel
	* address space.
	*/
	goto nogo;
	}

	rv = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL,
	&i, &ucode);
	/*
	* XXXDTRACE: add dtrace_doubletrap_func here?
	*/
	#ifdef VMFAULT_TRACE
	printf("vm_fault(%p (pmap %p), %p (%p), %x, %d) -> %x at pc %p\n",
	map, &vm->vm_pmap, (void )va, (void )(intptr_t)trapframe->badvaddr,
	ftype, VM_FAULT_NORMAL, rv, (void *)(intptr_t)trapframe->pc);
	#endif

	if (rv == KERN_SUCCESS) {
	if (!usermode) {
	return (trapframe->pc);
	}
	goto out;
	}
	nogo:
	if (!usermode) {
	if (td->td_pcb->pcb_onfault != NULL) {
	pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault;
	td->td_pcb->pcb_onfault = NULL;
	return (pc);
	}
	goto err;
	}
	addr = trapframe->badvaddr;

	msg = "BAD_PAGE_FAULT";
	log_bad_page_fault(msg, trapframe, type);

	break;
	}

	case T_ADDR_ERR_LD + T_USER: /* misaligned or kseg access */
	case T_ADDR_ERR_ST + T_USER: /* misaligned or kseg access */
	if (trapframe->badvaddr < 0 \|\|
	trapframe->badvaddr >= VM_MAXUSER_ADDRESS) {
	msg = "ADDRESS_SPACE_ERR";
	} else if (allow_unaligned_acc) {
	int mode;

	if (type == (T_ADDR_ERR_LD + T_USER))
	mode = VM_PROT_READ;
	else
	mode = VM_PROT_WRITE;

	access_type = emulate_unaligned_access(trapframe, mode);
	if (access_type != 0)
	goto out;
	msg = "ALIGNMENT_FIX_ERR";
	} else {
	msg = "ADDRESS_ERR";
	}

	/* FALL THROUGH */

	case T_BUS_ERR_IFETCH + T_USER: /* BERR asserted to cpu */
	case T_BUS_ERR_LD_ST + T_USER: /* BERR asserted to cpu */
	ucode = 0; /* XXX should be VM_PROT_something */
	i = SIGBUS;
	addr = trapframe->pc;
	if (!msg)
	msg = "BUS_ERR";
	log_bad_page_fault(msg, trapframe, type);
	break;

	case T_SYSCALL + T_USER:
	{
	td->td_sa.trapframe = trapframe;
	syscallenter(td);

	#if !defined(SMP) && (defined(DDB) \|\| defined(DEBUG))
	if (trp == trapdebug)
	trapdebug[TRAPSIZE - 1].code = td->td_sa.code;
	else
	trp[-1].code = td->td_sa.code;
	#endif
	trapdebug_enter(td->td_frame, -td->td_sa.code);

	/*
	* The sync'ing of I & D caches for SYS_ptrace() is
	* done by procfs_domem() through procfs_rwmem()
	* instead of being done here under a special check
	* for SYS_ptrace().
	*/
	syscallret(td);
	return (trapframe->pc);
	}

	#if defined(KDTRACE_HOOKS) \|\| defined(DDB)
	case T_BREAK:
	#ifdef KDTRACE_HOOKS
	if (!usermode && dtrace_invop_jump_addr != NULL &&
	dtrace_invop_jump_addr(trapframe) == 0)
	return (trapframe->pc);
	#endif
	#ifdef DDB
	kdb_trap(type, 0, trapframe);
	return (trapframe->pc);
	#endif
	#endif

	case T_BREAK + T_USER:
	{
	intptr_t va;
	uint32_t instr;

	i = SIGTRAP;
	ucode = TRAP_BRKPT;
	- addr = trapframe->pc;

	/* compute address of break instruction */
	va = trapframe->pc;
	if (DELAYBRANCH(trapframe->cause))
	va += sizeof(int);
	+ addr = va;

	if (td->td_md.md_ss_addr != va)
	break;

	/* read break instruction */
	instr = fuword32((caddr_t)va);

	if (instr != MIPS_BREAK_SSTEP)
	break;

	CTR3(KTR_PTRACE,
	"trap: tid %d, single step at %#lx: %#08x",
	td->td_tid, va, instr);
	PROC_LOCK(p);
	_PHOLD(p);
	error = ptrace_clear_single_step(td);
	_PRELE(p);
	PROC_UNLOCK(p);
	if (error == 0)
	ucode = TRAP_TRACE;
	break;
	}

	case T_IWATCH + T_USER:
	case T_DWATCH + T_USER:
	{
	intptr_t va;

	/* compute address of trapped instruction */
	va = trapframe->pc;
	if (DELAYBRANCH(trapframe->cause))
	va += sizeof(int);
	printf("watch exception @ %p\n", (void *)va);
	i = SIGTRAP;
	ucode = TRAP_BRKPT;
	addr = va;
	break;
	}

	case T_TRAP + T_USER:
	{
	intptr_t va;
	struct trapframe *locr0 = td->td_frame;

	/* compute address of trap instruction */
	va = trapframe->pc;
	if (DELAYBRANCH(trapframe->cause))
	va += sizeof(int);

	if (DELAYBRANCH(trapframe->cause)) { /* Check BD bit */
	locr0->pc = MipsEmulateBranch(locr0, trapframe->pc, 0,
	0);
	} else {
	locr0->pc += sizeof(int);
	}
	addr = va;
	i = SIGEMT; /* Stuff it with something for now */
	break;
	}

	case T_RES_INST + T_USER:
	{
	InstFmt inst;
	inst = (InstFmt )(intptr_t)trapframe->pc;
	switch (inst.RType.op) {
	case OP_SPECIAL3:
	switch (inst.RType.func) {
	case OP_RDHWR:
	/* Register 29 used for TLS */
	if (inst.RType.rd == 29) {
	frame_regs = &(trapframe->zero);
	frame_regs[inst.RType.rt] = (register_t)(intptr_t)td->td_md.md_tls;
	frame_regs[inst.RType.rt] += td->td_proc->p_md.md_tls_tcb_offset;
	trapframe->pc += sizeof(int);
	goto out;
	}
	break;
	}
	break;
	}

	log_illegal_instruction("RES_INST", trapframe);
	i = SIGILL;
	addr = trapframe->pc;
	}
	break;
	case T_C2E:
	case T_C2E + T_USER:
	goto err;
	break;
	case T_COP_UNUSABLE:
	#ifdef CPU_CNMIPS
	cop = (trapframe->cause & MIPS_CR_COP_ERR) >> MIPS_CR_COP_ERR_SHIFT;
	/* Handle only COP2 exception */
	if (cop != 2)
	goto err;

	addr = trapframe->pc;
	/* save userland cop2 context if it has been touched */
	if ((td->td_md.md_flags & MDTD_COP2USED) &&
	(td->td_md.md_cop2owner == COP2_OWNER_USERLAND)) {
	if (td->td_md.md_ucop2)
	octeon_cop2_save(td->td_md.md_ucop2);
	else
	panic("COP2 was used in user mode but md_ucop2 is NULL");
	}

	if (td->td_md.md_cop2 == NULL) {
	td->td_md.md_cop2 = octeon_cop2_alloc_ctx();
	if (td->td_md.md_cop2 == NULL)
	panic("Failed to allocate COP2 context");
	memset(td->td_md.md_cop2, 0, sizeof(*td->td_md.md_cop2));
	}

	octeon_cop2_restore(td->td_md.md_cop2);

	/* Make userland re-request its context */
	td->td_frame->sr &= ~MIPS_SR_COP_2_BIT;
	td->td_md.md_flags \|= MDTD_COP2USED;
	td->td_md.md_cop2owner = COP2_OWNER_KERNEL;
	/* Enable COP2, it will be disabled in cpu_switch */
	mips_wr_status(mips_rd_status() \| MIPS_SR_COP_2_BIT);
	return (trapframe->pc);
	#else
	goto err;
	break;
	#endif

	case T_COP_UNUSABLE + T_USER:
	cop = (trapframe->cause & MIPS_CR_COP_ERR) >> MIPS_CR_COP_ERR_SHIFT;
	if (cop == 1) {
	/* FP (COP1) instruction */
	if (cpuinfo.fpu_id == 0) {
	log_illegal_instruction("COP1_UNUSABLE",
	trapframe);
	i = SIGILL;
	break;
	}
	addr = trapframe->pc;
	MipsSwitchFPState(PCPU_GET(fpcurthread), td->td_frame);
	PCPU_SET(fpcurthread, td);
	#if defined(__mips_n32) \|\| defined(__mips_n64)
	td->td_frame->sr \|= MIPS_SR_COP_1_BIT \| MIPS_SR_FR;
	#else
	td->td_frame->sr \|= MIPS_SR_COP_1_BIT;
	#endif
	td->td_md.md_flags \|= MDTD_FPUSED;
	goto out;
	}
	#ifdef CPU_CNMIPS
	else if (cop == 2) {
	addr = trapframe->pc;
	if ((td->td_md.md_flags & MDTD_COP2USED) &&
	(td->td_md.md_cop2owner == COP2_OWNER_KERNEL)) {
	if (td->td_md.md_cop2)
	octeon_cop2_save(td->td_md.md_cop2);
	else
	panic("COP2 was used in kernel mode but md_cop2 is NULL");
	}

	if (td->td_md.md_ucop2 == NULL) {
	td->td_md.md_ucop2 = octeon_cop2_alloc_ctx();
	if (td->td_md.md_ucop2 == NULL)
	panic("Failed to allocate userland COP2 context");
	memset(td->td_md.md_ucop2, 0, sizeof(*td->td_md.md_ucop2));
	}

	octeon_cop2_restore(td->td_md.md_ucop2);

	td->td_frame->sr \|= MIPS_SR_COP_2_BIT;
	td->td_md.md_flags \|= MDTD_COP2USED;
	td->td_md.md_cop2owner = COP2_OWNER_USERLAND;
	goto out;
	}
	#endif
	else {
	log_illegal_instruction("COPn_UNUSABLE", trapframe);
	i = SIGILL; /* only FPU instructions allowed */
	break;
	}

	case T_FPE:
	#if !defined(SMP) && (defined(DDB) \|\| defined(DEBUG))
	trapDump("fpintr");
	#else
	printf("FPU Trap: PC %#jx CR %x SR %x\n",
	(intmax_t)trapframe->pc, (unsigned)trapframe->cause, (unsigned)trapframe->sr);
	goto err;
	#endif

	case T_FPE + T_USER:
	if (!emulate_fp) {
	i = SIGFPE;
	addr = trapframe->pc;
	break;
	}
	MipsFPTrap(trapframe->sr, trapframe->cause, trapframe->pc);
	goto out;

	case T_OVFLOW + T_USER:
	i = SIGFPE;
	addr = trapframe->pc;
	break;

	case T_ADDR_ERR_LD: /* misaligned access */
	case T_ADDR_ERR_ST: /* misaligned access */
	#ifdef TRAP_DEBUG
	if (trap_debug) {
	printf("+++ ADDR_ERR: type = %d, badvaddr = %#jx\n", type,
	(intmax_t)trapframe->badvaddr);
	}
	#endif
	/* Only allow emulation on a user address */
	if (allow_unaligned_acc &&
	((vm_offset_t)trapframe->badvaddr < VM_MAXUSER_ADDRESS)) {
	int mode;

	if (type == T_ADDR_ERR_LD)
	mode = VM_PROT_READ;
	else
	mode = VM_PROT_WRITE;

	access_type = emulate_unaligned_access(trapframe, mode);
	if (access_type != 0)
	return (trapframe->pc);
	}
	/* FALLTHROUGH */

	case T_BUS_ERR_LD_ST: /* BERR asserted to cpu */
	if (td->td_pcb->pcb_onfault != NULL) {
	pc = (register_t)(intptr_t)td->td_pcb->pcb_onfault;
	td->td_pcb->pcb_onfault = NULL;
	return (pc);
	}

	/* FALLTHROUGH */

	default:
	err:

	#if !defined(SMP) && defined(DEBUG)
	trapDump("trap");
	#endif
	#ifdef SMP
	printf("cpu:%d-", PCPU_GET(cpuid));
	#endif
	printf("Trap cause = %d (%s - ", type,
	trap_type[type & (~T_USER)]);

	if (type & T_USER)
	printf("user mode)\n");
	else
	printf("kernel mode)\n");

	#ifdef TRAP_DEBUG
	if (trap_debug)
	printf("badvaddr = %#jx, pc = %#jx, ra = %#jx, sr = %#jxx\n",
	(intmax_t)trapframe->badvaddr, (intmax_t)trapframe->pc, (intmax_t)trapframe->ra,
	(intmax_t)trapframe->sr);
	#endif

	#ifdef KDB
	if (debugger_on_trap) {
	kdb_why = KDB_WHY_TRAP;
	kdb_trap(type, 0, trapframe);
	kdb_why = KDB_WHY_UNSET;
	}
	#endif
	panic("trap");
	}
	td->td_frame->pc = trapframe->pc;
	td->td_frame->cause = trapframe->cause;
	td->td_frame->badvaddr = trapframe->badvaddr;
	ksiginfo_init_trap(&ksi);
	ksi.ksi_signo = i;
	ksi.ksi_code = ucode;
	ksi.ksi_addr = (void *)addr;
	- ksi.ksi_trapno = type;
	+ ksi.ksi_trapno = type & ~T_USER;
	trapsignal(td, &ksi);
	out:

	/*
	* Note: we should only get here if returning to user mode.
	*/
	userret(td, trapframe);
	return (trapframe->pc);
	}

	#if !defined(SMP) && (defined(DDB) \|\| defined(DEBUG))
	void
	trapDump(char *msg)
	{
	register_t s;
	int i;

	s = intr_disable();
	printf("trapDump(%s)\n", msg);
	for (i = 0; i < TRAPSIZE; i++) {
	if (trp == trapdebug) {
	trp = &trapdebug[TRAPSIZE - 1];
	} else {
	trp--;
	}

	if (trp->cause == 0)
	break;

	printf("%s: ADR %jx PC %jx CR %jx SR %jx\n",
	trap_type[(trp->cause & MIPS_CR_EXC_CODE) >>
	MIPS_CR_EXC_CODE_SHIFT],
	(intmax_t)trp->vadr, (intmax_t)trp->pc,
	(intmax_t)trp->cause, (intmax_t)trp->status);

	printf(" RA %jx SP %jx code %d\n", (intmax_t)trp->ra,
	(intmax_t)trp->sp, (int)trp->code);
	}
	intr_restore(s);
	}
	#endif


	/*
	* Return the resulting PC as if the branch was executed.
	*/
	uintptr_t
	MipsEmulateBranch(struct trapframe *framePtr, uintptr_t instPC, int fpcCSR,
	uintptr_t instptr)
	{
	InstFmt inst;
	register_t regsPtr = (register_t ) framePtr;
	uintptr_t retAddr = 0;
	int condition;

	#define GetBranchDest(InstPtr, inst) \
	(InstPtr + 4 + ((short)inst.IType.imm << 2))


	if (instptr) {
	if (instptr < MIPS_KSEG0_START)
	inst.word = fuword32((void *)instptr);
	else
	inst = (InstFmt ) instptr;
	} else {
	if ((vm_offset_t)instPC < MIPS_KSEG0_START)
	inst.word = fuword32((void *)instPC);
	else
	inst = (InstFmt ) instPC;
	}

	switch ((int)inst.JType.op) {
	case OP_SPECIAL:
	switch ((int)inst.RType.func) {
	case OP_JR:
	case OP_JALR:
	retAddr = regsPtr[inst.RType.rs];
	break;

	default:
	retAddr = instPC + 4;
	break;
	}
	break;

	case OP_BCOND:
	switch ((int)inst.IType.rt) {
	case OP_BLTZ:
	case OP_BLTZL:
	case OP_BLTZAL:
	case OP_BLTZALL:
	if ((int)(regsPtr[inst.RType.rs]) < 0)
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	case OP_BGEZ:
	case OP_BGEZL:
	case OP_BGEZAL:
	case OP_BGEZALL:
	if ((int)(regsPtr[inst.RType.rs]) >= 0)
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	case OP_TGEI:
	case OP_TGEIU:
	case OP_TLTI:
	case OP_TLTIU:
	case OP_TEQI:
	case OP_TNEI:
	retAddr = instPC + 4; /* Like syscall... */
	break;

	default:
	panic("MipsEmulateBranch: Bad branch cond");
	}
	break;

	case OP_J:
	case OP_JAL:
	retAddr = (inst.JType.target << 2) \|
	((unsigned)(instPC + 4) & 0xF0000000);
	break;

	case OP_BEQ:
	case OP_BEQL:
	if (regsPtr[inst.RType.rs] == regsPtr[inst.RType.rt])
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	case OP_BNE:
	case OP_BNEL:
	if (regsPtr[inst.RType.rs] != regsPtr[inst.RType.rt])
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	case OP_BLEZ:
	case OP_BLEZL:
	if ((int)(regsPtr[inst.RType.rs]) <= 0)
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	case OP_BGTZ:
	case OP_BGTZL:
	if ((int)(regsPtr[inst.RType.rs]) > 0)
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	case OP_COP1:
	switch (inst.RType.rs) {
	case OP_BCx:
	case OP_BCy:
	if ((inst.RType.rt & COPz_BC_TF_MASK) == COPz_BC_TRUE)
	condition = fpcCSR & MIPS_FPU_COND_BIT;
	else
	condition = !(fpcCSR & MIPS_FPU_COND_BIT);
	if (condition)
	retAddr = GetBranchDest(instPC, inst);
	else
	retAddr = instPC + 8;
	break;

	default:
	retAddr = instPC + 4;
	}
	break;

	default:
	retAddr = instPC + 4;
	}
	return (retAddr);
	}

	static void
	log_frame_dump(struct trapframe *frame)
	{
	log(LOG_ERR, "Trapframe Register Dump:\n");
	log(LOG_ERR, "\tzero: %#jx\tat: %#jx\tv0: %#jx\tv1: %#jx\n",
	(intmax_t)0, (intmax_t)frame->ast, (intmax_t)frame->v0, (intmax_t)frame->v1);

	log(LOG_ERR, "\ta0: %#jx\ta1: %#jx\ta2: %#jx\ta3: %#jx\n",
	(intmax_t)frame->a0, (intmax_t)frame->a1, (intmax_t)frame->a2, (intmax_t)frame->a3);

	#if defined(__mips_n32) \|\| defined(__mips_n64)
	log(LOG_ERR, "\ta4: %#jx\ta5: %#jx\ta6: %#jx\ta6: %#jx\n",
	(intmax_t)frame->a4, (intmax_t)frame->a5, (intmax_t)frame->a6, (intmax_t)frame->a7);

	log(LOG_ERR, "\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
	(intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);
	#else
	log(LOG_ERR, "\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
	(intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);

	log(LOG_ERR, "\tt4: %#jx\tt5: %#jx\tt6: %#jx\tt7: %#jx\n",
	(intmax_t)frame->t4, (intmax_t)frame->t5, (intmax_t)frame->t6, (intmax_t)frame->t7);
	#endif
	log(LOG_ERR, "\tt8: %#jx\tt9: %#jx\ts0: %#jx\ts1: %#jx\n",
	(intmax_t)frame->t8, (intmax_t)frame->t9, (intmax_t)frame->s0, (intmax_t)frame->s1);

	log(LOG_ERR, "\ts2: %#jx\ts3: %#jx\ts4: %#jx\ts5: %#jx\n",
	(intmax_t)frame->s2, (intmax_t)frame->s3, (intmax_t)frame->s4, (intmax_t)frame->s5);

	log(LOG_ERR, "\ts6: %#jx\ts7: %#jx\tk0: %#jx\tk1: %#jx\n",
	(intmax_t)frame->s6, (intmax_t)frame->s7, (intmax_t)frame->k0, (intmax_t)frame->k1);

	log(LOG_ERR, "\tgp: %#jx\tsp: %#jx\ts8: %#jx\tra: %#jx\n",
	(intmax_t)frame->gp, (intmax_t)frame->sp, (intmax_t)frame->s8, (intmax_t)frame->ra);

	log(LOG_ERR, "\tsr: %#jx\tmullo: %#jx\tmulhi: %#jx\tbadvaddr: %#jx\n",
	(intmax_t)frame->sr, (intmax_t)frame->mullo, (intmax_t)frame->mulhi, (intmax_t)frame->badvaddr);

	log(LOG_ERR, "\tcause: %#jx\tpc: %#jx\n",
	(intmax_t)frame->cause, (intmax_t)frame->pc);
	}

	#ifdef TRAP_DEBUG
	static void
	trap_frame_dump(struct trapframe *frame)
	{
	printf("Trapframe Register Dump:\n");
	printf("\tzero: %#jx\tat: %#jx\tv0: %#jx\tv1: %#jx\n",
	(intmax_t)0, (intmax_t)frame->ast, (intmax_t)frame->v0, (intmax_t)frame->v1);

	printf("\ta0: %#jx\ta1: %#jx\ta2: %#jx\ta3: %#jx\n",
	(intmax_t)frame->a0, (intmax_t)frame->a1, (intmax_t)frame->a2, (intmax_t)frame->a3);
	#if defined(__mips_n32) \|\| defined(__mips_n64)
	printf("\ta4: %#jx\ta5: %#jx\ta6: %#jx\ta7: %#jx\n",
	(intmax_t)frame->a4, (intmax_t)frame->a5, (intmax_t)frame->a6, (intmax_t)frame->a7);

	printf("\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
	(intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);
	#else
	printf("\tt0: %#jx\tt1: %#jx\tt2: %#jx\tt3: %#jx\n",
	(intmax_t)frame->t0, (intmax_t)frame->t1, (intmax_t)frame->t2, (intmax_t)frame->t3);

	printf("\tt4: %#jx\tt5: %#jx\tt6: %#jx\tt7: %#jx\n",
	(intmax_t)frame->t4, (intmax_t)frame->t5, (intmax_t)frame->t6, (intmax_t)frame->t7);
	#endif
	printf("\tt8: %#jx\tt9: %#jx\ts0: %#jx\ts1: %#jx\n",
	(intmax_t)frame->t8, (intmax_t)frame->t9, (intmax_t)frame->s0, (intmax_t)frame->s1);

	printf("\ts2: %#jx\ts3: %#jx\ts4: %#jx\ts5: %#jx\n",
	(intmax_t)frame->s2, (intmax_t)frame->s3, (intmax_t)frame->s4, (intmax_t)frame->s5);

	printf("\ts6: %#jx\ts7: %#jx\tk0: %#jx\tk1: %#jx\n",
	(intmax_t)frame->s6, (intmax_t)frame->s7, (intmax_t)frame->k0, (intmax_t)frame->k1);

	printf("\tgp: %#jx\tsp: %#jx\ts8: %#jx\tra: %#jx\n",
	(intmax_t)frame->gp, (intmax_t)frame->sp, (intmax_t)frame->s8, (intmax_t)frame->ra);

	printf("\tsr: %#jx\tmullo: %#jx\tmulhi: %#jx\tbadvaddr: %#jx\n",
	(intmax_t)frame->sr, (intmax_t)frame->mullo, (intmax_t)frame->mulhi, (intmax_t)frame->badvaddr);

	printf("\tcause: %#jx\tpc: %#jx\n",
	(intmax_t)frame->cause, (intmax_t)frame->pc);
	}

	#endif


	static void
	get_mapping_info(vm_offset_t va, pd_entry_t pdepp, pt_entry_t ptepp)
	{
	pt_entry_t *ptep;
	pd_entry_t *pdep;
	struct proc *p = curproc;

	pdep = (&(p->p_vmspace->vm_pmap.pm_segtab[(va >> SEGSHIFT) & (NPDEPG - 1)]));
	if (*pdep)
	ptep = pmap_pte(&p->p_vmspace->vm_pmap, va);
	else
	ptep = (pt_entry_t *)0;

	*pdepp = pdep;
	*ptepp = ptep;
	}

	static void
	log_illegal_instruction(const char msg, struct trapframe frame)
	{
	pt_entry_t *ptep;
	pd_entry_t *pdep;
	unsigned int *addr, instr[4];
	struct thread *td;
	struct proc *p;
	register_t pc;

	td = curthread;
	p = td->td_proc;

	#ifdef SMP
	printf("cpuid = %d\n", PCPU_GET(cpuid));
	#endif
	pc = frame->pc + (DELAYBRANCH(frame->cause) ? 4 : 0);
	log(LOG_ERR, "%s: pid %d tid %ld (%s), uid %d: pc %#jx ra %#jx\n",
	msg, p->p_pid, (long)td->td_tid, p->p_comm,
	p->p_ucred ? p->p_ucred->cr_uid : -1,
	(intmax_t)pc,
	(intmax_t)frame->ra);

	/* log registers in trap frame */
	log_frame_dump(frame);

	get_mapping_info((vm_offset_t)pc, &pdep, &ptep);

	/*
	* Dump a few words around faulting instruction, if the addres is
	* valid.
	*/
	addr = (unsigned int *)(intptr_t)pc;
	if ((pc & 3) == 0 && copyin(addr, instr, sizeof(instr)) == 0) {
	/* dump page table entry for faulting instruction */
	log(LOG_ERR, "Page table info for pc address %#jx: pde = %p, pte = %#jx\n",
	(intmax_t)pc, (void )(intptr_t)pdep, (uintmax_t)(ptep ? *ptep : 0));

	log(LOG_ERR, "Dumping 4 words starting at pc address %p: \n",
	addr);
	log(LOG_ERR, "%08x %08x %08x %08x\n",
	instr[0], instr[1], instr[2], instr[3]);
	} else {
	log(LOG_ERR, "pc address %#jx is inaccessible, pde = %p, pte = %#jx\n",
	(intmax_t)pc, (void )(intptr_t)pdep, (uintmax_t)(ptep ? *ptep : 0));
	}
	}

	static void
	log_bad_page_fault(char msg, struct trapframe frame, int trap_type)
	{
	pt_entry_t *ptep;
	pd_entry_t *pdep;
	unsigned int *addr, instr[4];
	struct thread *td;
	struct proc *p;
	char *read_or_write;
	register_t pc;

	trap_type &= ~T_USER;

	td = curthread;
	p = td->td_proc;

	#ifdef SMP
	printf("cpuid = %d\n", PCPU_GET(cpuid));
	#endif
	switch (trap_type) {
	case T_TLB_MOD:
	case T_TLB_ST_MISS:
	case T_ADDR_ERR_ST:
	read_or_write = "write";
	break;
	case T_TLB_LD_MISS:
	case T_ADDR_ERR_LD:
	case T_BUS_ERR_IFETCH:
	read_or_write = "read";
	break;
	default:
	read_or_write = "unknown";
	}

	pc = frame->pc + (DELAYBRANCH(frame->cause) ? 4 : 0);
	log(LOG_ERR, "%s: pid %d tid %ld (%s), uid %d: pc %#jx got a %s fault "
	"(type %#x) at %#jx\n",
	msg, p->p_pid, (long)td->td_tid, p->p_comm,
	p->p_ucred ? p->p_ucred->cr_uid : -1,
	(intmax_t)pc,
	read_or_write,
	trap_type,
	(intmax_t)frame->badvaddr);

	/* log registers in trap frame */
	log_frame_dump(frame);

	get_mapping_info((vm_offset_t)pc, &pdep, &ptep);

	/*
	* Dump a few words around faulting instruction, if the addres is
	* valid.
	*/
	addr = (unsigned int *)(intptr_t)pc;
	if ((pc & 3) == 0 && pc != frame->badvaddr &&
	trap_type != T_BUS_ERR_IFETCH &&
	copyin((caddr_t)(intptr_t)pc, instr, sizeof(instr)) == 0) {
	/* dump page table entry for faulting instruction */
	log(LOG_ERR, "Page table info for pc address %#jx: pde = %p, pte = %#jx\n",
	(intmax_t)pc, (void )(intptr_t)pdep, (uintmax_t)(ptep ? *ptep : 0));

	log(LOG_ERR, "Dumping 4 words starting at pc address %p: \n",
	addr);
	log(LOG_ERR, "%08x %08x %08x %08x\n",
	instr[0], instr[1], instr[2], instr[3]);
	} else {
	log(LOG_ERR, "pc address %#jx is inaccessible, pde = %p, pte = %#jx\n",
	(intmax_t)pc, (void )(intptr_t)pdep, (uintmax_t)(ptep ? *ptep : 0));
	}

	get_mapping_info((vm_offset_t)frame->badvaddr, &pdep, &ptep);
	log(LOG_ERR, "Page table info for bad address %#jx: pde = %p, pte = %#jx\n",
	(intmax_t)frame->badvaddr, (void )(intptr_t)pdep, (uintmax_t)(ptep ? *ptep : 0));
	}


	/*
	* Unaligned load/store emulation
	*/
	static int
	mips_unaligned_load_store(struct trapframe *frame, int mode, register_t addr, register_t pc)
	{
	register_t reg = (register_t ) frame;
	u_int32_t inst = ((u_int32_t )(intptr_t)pc);
	register_t value_msb = 0, value = 0;
	unsigned size;

	/*
	* ADDR_ERR faults have higher priority than TLB
	* Miss faults. Therefore, it is necessary to
	* verify that the faulting address is a valid
	* virtual address within the process' address space
	* before trying to emulate the unaligned access.
	*/
	switch (MIPS_INST_OPCODE(inst)) {
	case OP_LHU: case OP_LH:
	case OP_SH:
	size = 2;
	break;
	case OP_LWU: case OP_LW:
	case OP_SW:
	size = 4;
	break;
	case OP_LD:
	case OP_SD:
	size = 8;
	break;
	default:
	printf("%s: unhandled opcode in address error: %#x\n", __func__, MIPS_INST_OPCODE(inst));
	return (0);
	}

	if (!useracc((void )rounddown2((vm_offset_t)addr, size), size 2, mode))
	return (0);

	/*
	* XXX
	* Handle LL/SC LLD/SCD.
	*/
	switch (MIPS_INST_OPCODE(inst)) {
	case OP_LHU:
	KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
	lbu_macro(value_msb, addr);
	addr += 1;
	lbu_macro(value, addr);
	value \|= value_msb << 8;
	reg[MIPS_INST_RT(inst)] = value;
	return (MIPS_LHU_ACCESS);

	case OP_LH:
	KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
	lb_macro(value_msb, addr);
	addr += 1;
	lbu_macro(value, addr);
	value \|= value_msb << 8;
	reg[MIPS_INST_RT(inst)] = value;
	return (MIPS_LH_ACCESS);

	case OP_LWU:
	KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
	lwl_macro(value, addr);
	addr += 3;
	lwr_macro(value, addr);
	value &= 0xffffffff;
	reg[MIPS_INST_RT(inst)] = value;
	return (MIPS_LWU_ACCESS);

	case OP_LW:
	KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
	lwl_macro(value, addr);
	addr += 3;
	lwr_macro(value, addr);
	reg[MIPS_INST_RT(inst)] = value;
	return (MIPS_LW_ACCESS);

	#if defined(__mips_n32) \|\| defined(__mips_n64)
	case OP_LD:
	KASSERT(mode == VM_PROT_READ, ("access mode must be read for load instruction."));
	ldl_macro(value, addr);
	addr += 7;
	ldr_macro(value, addr);
	reg[MIPS_INST_RT(inst)] = value;
	return (MIPS_LD_ACCESS);
	#endif

	case OP_SH:
	KASSERT(mode == VM_PROT_WRITE, ("access mode must be write for store instruction."));
	value = reg[MIPS_INST_RT(inst)];
	value_msb = value >> 8;
	sb_macro(value_msb, addr);
	addr += 1;
	sb_macro(value, addr);
	return (MIPS_SH_ACCESS);

	case OP_SW:
	KASSERT(mode == VM_PROT_WRITE, ("access mode must be write for store instruction."));
	value = reg[MIPS_INST_RT(inst)];
	swl_macro(value, addr);
	addr += 3;
	swr_macro(value, addr);
	return (MIPS_SW_ACCESS);

	#if defined(__mips_n32) \|\| defined(__mips_n64)
	case OP_SD:
	KASSERT(mode == VM_PROT_WRITE, ("access mode must be write for store instruction."));
	value = reg[MIPS_INST_RT(inst)];
	sdl_macro(value, addr);
	addr += 7;
	sdr_macro(value, addr);
	return (MIPS_SD_ACCESS);
	#endif
	}
	panic("%s: should not be reached.", __func__);
	}


	/*
	* XXX TODO: SMP?
	*/
	static struct timeval unaligned_lasterr;
	static int unaligned_curerr;

	static int unaligned_pps_log_limit = 4;

	SYSCTL_INT(_machdep, OID_AUTO, unaligned_log_pps_limit, CTLFLAG_RWTUN,
	&unaligned_pps_log_limit, 0,
	"limit number of userland unaligned log messages per second");

	static int
	emulate_unaligned_access(struct trapframe *frame, int mode)
	{
	register_t pc;
	int access_type = 0;
	struct thread *td = curthread;
	struct proc *p = curproc;

	pc = frame->pc + (DELAYBRANCH(frame->cause) ? 4 : 0);

	/*
	* Fall through if it's instruction fetch exception
	*/
	if (!((pc & 3) \|\| (pc == frame->badvaddr))) {

	/*
	* Handle unaligned load and store
	*/

	/*
	* Return access type if the instruction was emulated.
	* Otherwise restore pc and fall through.
	*/
	access_type = mips_unaligned_load_store(frame,
	mode, frame->badvaddr, pc);

	if (access_type) {
	if (DELAYBRANCH(frame->cause))
	frame->pc = MipsEmulateBranch(frame, frame->pc,
	0, 0);
	else
	frame->pc += 4;

	if (ppsratecheck(&unaligned_lasterr,
	&unaligned_curerr, unaligned_pps_log_limit)) {
	/* XXX TODO: keep global/tid/pid counters? */
	log(LOG_INFO,
	"Unaligned %s: pid=%ld (%s), tid=%ld, "
	"pc=%#jx, badvaddr=%#jx\n",
	access_name[access_type - 1],
	(long) p->p_pid,
	p->p_comm,
	(long) td->td_tid,
	(intmax_t)pc,
	(intmax_t)frame->badvaddr);
	}
	}
	}
	return access_type;
	}
	diff --git a/sys/modules/Makefile b/sys/modules/Makefile
	index dff871fa7338..dc4e04736670 100644
	--- a/sys/modules/Makefile
	+++ b/sys/modules/Makefile
	@@ -1,823 +1,823 @@
	# $FreeBSD$

	SYSDIR?=${SRCTOP}/sys
	.include "${SYSDIR}/conf/kern.opts.mk"

	SUBDIR_PARALLEL=

	# Modules that include binary-only blobs of microcode should be selectable by
	# MK_SOURCELESS_UCODE option (see below).

	.include "${SYSDIR}/conf/config.mk"

	.if defined(MODULES_OVERRIDE) && !defined(ALL_MODULES)
	SUBDIR=${MODULES_OVERRIDE}
	.else
	SUBDIR= \
	${_3dfx} \
	${_3dfx_linux} \
	${_aac} \
	${_aacraid} \
	accf_data \
	accf_dns \
	accf_http \
	acl_nfs4 \
	acl_posix1e \
	${_acpi} \
	ae \
	${_aesni} \
	age \
	${_agp} \
	ahci \
	aic7xxx \
	alc \
	ale \
	alq \
	${_amd_ecc_inject} \
	${_amdgpio} \
	${_amdsbwd} \
	${_amdsmn} \
	${_amdtemp} \
	amr \
	${_an} \
	${_aout} \
	${_apm} \
	${_arcmsr} \
	${_allwinner} \
	${_armv8crypto} \
	${_asmc} \
	ata \
	ath \
	ath_dfs \
	ath_hal \
	ath_hal_ar5210 \
	ath_hal_ar5211 \
	ath_hal_ar5212 \
	ath_hal_ar5416 \
	ath_hal_ar9300 \
	ath_main \
	ath_rate \
	ath_pci \
	${_autofs} \
	${_bce} \
	${_bcm283x_clkman} \
	${_bcm283x_pwm} \
	bfe \
	bge \
	bhnd \
	${_bxe} \
	${_bios} \
	${_blake2} \
	bnxt \
	bridgestp \
	bwi \
	bwn \
	${_bytgpio} \
	${_chvgpio} \
	cam \
	${_cardbus} \
	${_carp} \
	cas \
	${_cbb} \
	cc \
	${_ccp} \
	cd9660 \
	cd9660_iconv \
	${_ce} \
	${_cfi} \
	${_chromebook_platform} \
	${_ciss} \
	cloudabi \
	${_cloudabi32} \
	${_cloudabi64} \
	${_cmx} \
	${_coretemp} \
	${_cp} \
	${_cpsw} \
	${_cpuctl} \
	${_cpufreq} \
	${_crypto} \
	${_cryptodev} \
	ctl \
	${_cxgb} \
	${_cxgbe} \
	dc \
	dcons \
	dcons_crom \
	${_dpms} \
	dummynet \
	${_efirt} \
	${_em} \
	${_ena} \
	esp \
	${_et} \
	evdev \
	${_exca} \
	ext2fs \
	fdc \
	fdescfs \
	${_ffec} \
	filemon \
	firewire \
	firmware \
	fusefs \
	${_fxp} \
	gem \
	geom \
	${_glxiic} \
	${_glxsb} \
	gpio \
	hifn \
	hme \
	${_hpt27xx} \
	${_hptiop} \
	${_hptmv} \
	${_hptnr} \
	${_hptrr} \
	hwpmc \
	${_hwpmc_mips24k} \
	${_hwpmc_mips74k} \
	${_hyperv} \
	i2c \
	${_iavf} \
	${_ibcore} \
	${_ichwd} \
	${_ice} \
	${_ice_ddp} \
	${_ida} \
	if_bridge \
	if_disc \
	if_edsc \
	${_if_enc} \
	if_epair \
	${_if_gif} \
	${_if_gre} \
	${_if_me} \
	if_lagg \
	${_if_ndis} \
	${_if_stf} \
	if_tuntap \
	if_vlan \
	if_vxlan \
	iflib \
	${_iir} \
	imgact_binmisc \
	${_intelspi} \
	${_io} \
	${_ioat} \
	${_ipoib} \
	${_ipdivert} \
	${_ipfilter} \
	${_ipfw} \
	ipfw_nat \
	${_ipfw_nat64} \
	${_ipfw_nptv6} \
	${_ipfw_pmod} \
	${_ipmi} \
	ip6_mroute_mod \
	ip_mroute_mod \
	${_ips} \
	${_ipsec} \
	${_ipw} \
	${_ipwfw} \
	${_isci} \
	${_iser} \
	isp \
	${_ispfw} \
	${_itwd} \
	${_iwi} \
	${_iwifw} \
	${_iwm} \
	${_iwmfw} \
	${_iwn} \
	${_iwnfw} \
	${_ix} \
	${_ixv} \
	${_ixl} \
	jme \
	kbdmux \
	kgssapi \
	kgssapi_krb5 \
	khelp \
	krpc \
	ksyms \
	${_ktls_ocf} \
	le \
	lge \
	libalias \
	libiconv \
	libmchain \
	lindebugfs \
	linuxkpi \
	${_lio} \
	lpt \
	mac_biba \
	mac_bsdextended \
	mac_ifoff \
	mac_lomac \
	mac_mls \
	mac_none \
	mac_ntpd \
	mac_partition \
	mac_portacl \
	mac_seeotheruids \
	mac_stub \
	mac_test \
	malo \
	md \
	mdio \
	mem \
	mfi \
	mii \
	mlx \
	mlxfw \
	${_mlx4} \
	${_mlx4ib} \
	${_mlx4en} \
	${_mlx5} \
	${_mlx5en} \
	${_mlx5ib} \
	${_mly} \
	mmc \
	mmcsd \
	${_mpr} \
	${_mps} \
	mpt \
	mqueue \
	mrsas \
	msdosfs \
	msdosfs_iconv \
	msk \
	${_mthca} \
	mvs \
	mwl \
	${_mwlfw} \
	mxge \
	my \
	${_nctgpio} \
	${_ndis} \
	${_netgraph} \
	${_nfe} \
	nfscl \
	nfscommon \
	nfsd \
	nfslockd \
	nfssvc \
	nge \
	nmdm \
	nullfs \
	${_ntb} \
	${_nvd} \
	${_nvdimm} \
	${_nvme} \
	${_nvram} \
	oce \
	${_ocs_fc} \
	otus \
	${_otusfw} \
	ow \
	${_padlock} \
	${_padlock_rng} \
	${_pccard} \
	${_pchtherm} \
	${_pcfclock} \
	${_pf} \
	${_pflog} \
	${_pfsync} \
	plip \
	${_pms} \
	ppbus \
	ppc \
	ppi \
	pps \
	procfs \
	proto \
	pseudofs \
	${_pst} \
	pty \
	puc \
	pwm \
	${_qlxge} \
	${_qlxgb} \
	${_qlxgbe} \
	${_qlnx} \
	ral \
	${_ralfw} \
	${_random_fortuna} \
	${_random_other} \
	rc4 \
	${_rdma} \
	${_rdrand_rng} \
	re \
	rl \
	${_rockchip} \
	rtwn \
	rtwn_pci \
	rtwn_usb \
	${_rtwnfw} \
	${_s3} \
	${_safe} \
	safexcel \
	${_sbni} \
	scc \
	${_sctp} \
	sdhci \
	${_sdhci_acpi} \
	sdhci_pci \
	sdio \
	sem \
	send \
	${_sfxge} \
	sge \
	${_sgx} \
	${_sgx_linux} \
	siftr \
	siis \
	sis \
	sk \
	${_smartpqi} \
	smbfs \
	snp \
	sound \
	${_speaker} \
	spi \
	${_splash} \
	${_sppp} \
	ste \
	stge \
	${_superio} \
	${_sym} \
	${_syscons} \
	sysvipc \
	tcp \
	${_ti} \
	tmpfs \
	${_toecore} \
	${_tpm} \
	${_twa} \
	twe \
	tws \
	uart \
	udf \
	udf_iconv \
	ufs \
	uinput \
	unionfs \
	usb \
	${_vesa} \
	${_virtio} \
	vge \
	${_viawd} \
	videomode \
	vkbd \
	${_vmd} \
	${_vmm} \
	${_vmware} \
	vr \
	vte \
	${_wbwd} \
	${_wi} \
	wlan \
	wlan_acl \
	wlan_amrr \
	wlan_ccmp \
	wlan_rssadapt \
	wlan_tkip \
	wlan_wep \
	wlan_xauth \
	${_wpi} \
	${_wpifw} \
	${_x86bios} \
	xdr \
	xl \
	xz \
	zlib

	.if ${MK_AUTOFS} != "no" \|\| defined(ALL_MODULES)
	_autofs= autofs
	.endif

	.if ${MK_CDDL} != "no" \|\| defined(ALL_MODULES)
	.if (${MACHINE_CPUARCH} != "arm" \|\| ${MACHINE_ARCH:Marmv[67]*} != "") && \
	${MACHINE_CPUARCH} != "mips"
	.if ${KERN_OPTS:MKDTRACE_HOOKS}
	SUBDIR+= dtrace
	.endif
	.endif
	SUBDIR+= opensolaris
	.endif

	.if ${MK_CRYPT} != "no" \|\| defined(ALL_MODULES)
	.if exists(${SRCTOP}/sys/opencrypto)
	_crypto= crypto
	_cryptodev= cryptodev
	_random_fortuna=random_fortuna
	_random_other= random_other
	_ktls_ocf= ktls_ocf
	.endif
	.endif

	.if ${MK_CUSE} != "no" \|\| defined(ALL_MODULES)
	SUBDIR+= cuse
	.endif

	.if ${MK_EFI} != "no"
	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64"
	_efirt= efirt
	.endif
	.endif

	.if (${MK_INET_SUPPORT} != "no" \|\| ${MK_INET6_SUPPORT} != "no") \|\| \
	defined(ALL_MODULES)
	_carp= carp
	_toecore= toecore
	_if_enc= if_enc
	_if_gif= if_gif
	_if_gre= if_gre
	_ipfw_pmod= ipfw_pmod
	.if ${KERN_OPTS:MIPSEC_SUPPORT} && !${KERN_OPTS:MIPSEC}
	_ipsec= ipsec
	.endif
	.if ${KERN_OPTS:MSCTP_SUPPORT} \|\| ${KERN_OPTS:MSCTP}
	_sctp= sctp
	.endif
	.endif

	.if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") \|\| \
	defined(ALL_MODULES)
	_if_stf= if_stf
	.endif

	.if ${MK_INET_SUPPORT} != "no" \|\| defined(ALL_MODULES)
	_if_me= if_me
	_ipdivert= ipdivert
	_ipfw= ipfw
	.if ${MK_INET6_SUPPORT} != "no" \|\| defined(ALL_MODULES)
	_ipfw_nat64= ipfw_nat64
	.endif
	.endif

	.if ${MK_INET6_SUPPORT} != "no" \|\| defined(ALL_MODULES)
	_ipfw_nptv6= ipfw_nptv6
	.endif

	.if ${MK_IPFILTER} != "no" \|\| defined(ALL_MODULES)
	_ipfilter= ipfilter
	.endif

	.if ${MK_ISCSI} != "no" \|\| defined(ALL_MODULES)
	SUBDIR+= cfiscsi
	SUBDIR+= iscsi
	SUBDIR+= iscsi_initiator
	.endif

	.if !empty(OPT_FDT)
	SUBDIR+= fdt
	.endif

	# Linuxulator
	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64" \|\| \
	${MACHINE_CPUARCH} == "i386"
	SUBDIR+= linprocfs
	SUBDIR+= linsysfs
	.endif
	.if ${MACHINE_CPUARCH} == "amd64" \|\| ${MACHINE_CPUARCH} == "i386"
	SUBDIR+= linux
	.endif
	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64"
	SUBDIR+= linux64
	SUBDIR+= linux_common
	.endif

	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64" \|\| \
	${MACHINE_CPUARCH} == "i386"
	_ena= ena
	.if ${MK_OFED} != "no" \|\| defined(ALL_MODULES)
	_ibcore= ibcore
	_ipoib= ipoib
	_iser= iser
	.endif
	_mlx4= mlx4
	_mlx5= mlx5
	.if (${MK_INET_SUPPORT} != "no" && ${MK_INET6_SUPPORT} != "no") \|\| \
	defined(ALL_MODULES)
	_mlx4en= mlx4en
	_mlx5en= mlx5en
	.endif
	.if ${MK_OFED} != "no" \|\| defined(ALL_MODULES)
	_mthca= mthca
	_mlx4ib= mlx4ib
	_mlx5ib= mlx5ib
	.endif
	.endif

	.if ${MK_NETGRAPH} != "no" \|\| defined(ALL_MODULES)
	_netgraph= netgraph
	.endif

	.if (${MK_PF} != "no" && (${MK_INET_SUPPORT} != "no" \|\| \
	${MK_INET6_SUPPORT} != "no")) \|\| defined(ALL_MODULES)
	_pf= pf
	_pflog= pflog
	.if ${MK_INET_SUPPORT} != "no"
	_pfsync= pfsync
	.endif
	.endif

	.if ${MK_SOURCELESS_UCODE} != "no"
	_bce= bce
	_fxp= fxp
	_ispfw= ispfw
	_ti= ti

	.if ${MACHINE_CPUARCH} != "mips"
	_mwlfw= mwlfw
	_otusfw= otusfw
	_ralfw= ralfw
	_rtwnfw= rtwnfw
	.endif
	.endif

	.if ${MK_SOURCELESS_UCODE} != "no" && ${MACHINE_CPUARCH} != "arm" && \
	${MACHINE_CPUARCH} != "mips" && \
	${MACHINE_ARCH} != "powerpc" && ${MACHINE_ARCH} != "powerpcspe" && \
	${MACHINE_CPUARCH} != "riscv"
	_cxgbe= cxgbe
	.endif

	.if ${MACHINE_ARCH} == "amd64" \|\| ${MACHINE_ARCH} == "arm64"
	_ice= ice
	.if ${MK_SOURCELESS_UCODE} != "no"
	_ice_ddp= ice_ddp
	.endif
	.endif

	# These rely on 64bit atomics
	.if ${MACHINE_ARCH} != "powerpc" && ${MACHINE_ARCH} != "powerpcspe" && \
	${MACHINE_CPUARCH} != "mips"
	_mps= mps
	_mpr= mpr
	.endif

	.if ${MK_TESTS} != "no" \|\| defined(ALL_MODULES)
	SUBDIR+= tests
	.endif

	.if ${MK_ZFS} != "no" \|\| defined(ALL_MODULES)
	SUBDIR+= zfs
	.endif

	.if (${MACHINE_CPUARCH} == "mips" && ${MACHINE_ARCH:Mmips64} == "")
	_hwpmc_mips24k= hwpmc_mips24k
	_hwpmc_mips74k= hwpmc_mips74k
	.endif

	.if ${MACHINE_CPUARCH} != "aarch64" && ${MACHINE_CPUARCH} != "arm" && \
	${MACHINE_CPUARCH} != "mips" && ${MACHINE_CPUARCH} != "powerpc" && \
	${MACHINE_CPUARCH} != "riscv"
	_syscons= syscons
	.endif

	.if ${MACHINE_CPUARCH} != "mips"
	# no BUS_SPACE_UNSPECIFIED
	# No barrier instruction support (specific to this driver)
	_sym= sym
	# intr_disable() is a macro, causes problems
	.if ${MK_SOURCELESS_UCODE} != "no"
	_cxgb= cxgb
	.endif
	.endif

	.if ${MACHINE_CPUARCH} == "aarch64"
	_allwinner= allwinner
	_armv8crypto= armv8crypto
	_em= em
	_rockchip= rockchip
	.endif

	.if ${MACHINE_CPUARCH} == "i386" \|\| ${MACHINE_CPUARCH} == "amd64"
	_agp= agp
	_an= an
	_aout= aout
	_bios= bios
	.if ${MK_SOURCELESS_UCODE} != "no"
	_bxe= bxe
	.endif
	_cardbus= cardbus
	_cbb= cbb
	_cpuctl= cpuctl
	_cpufreq= cpufreq
	_dpms= dpms
	_em= em
	_et= et
	_exca= exca
	_if_ndis= if_ndis
	_io= io
	_itwd= itwd
	_ix= ix
	_ixv= ixv
	.if ${MK_SOURCELESS_UCODE} != "no"
	_lio= lio
	.endif
	_nctgpio= nctgpio
	_ndis= ndis
	_ntb= ntb
	_ocs_fc= ocs_fc
	_pccard= pccard
	.if ${MK_OFED} != "no" \|\| defined(ALL_MODULES)
	_rdma= rdma
	.endif
	_safe= safe
	_speaker= speaker
	_splash= splash
	_sppp= sppp
	_vmware= vmware
	_wbwd= wbwd
	_wi= wi

	_aac= aac
	_aacraid= aacraid
	_acpi= acpi
	.if ${MK_CRYPT} != "no" \|\| defined(ALL_MODULES)
	_aesni= aesni
	.endif
	_amd_ecc_inject=amd_ecc_inject
	_amdsbwd= amdsbwd
	_amdsmn= amdsmn
	_amdtemp= amdtemp
	_arcmsr= arcmsr
	_asmc= asmc
	.if ${MK_CRYPT} != "no" \|\| defined(ALL_MODULES)
	_blake2= blake2
	.endif
	_bytgpio= bytgpio
	_chvgpio= chvgpio
	_ciss= ciss
	_chromebook_platform= chromebook_platform
	_cmx= cmx
	_coretemp= coretemp
	.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED)
	_hpt27xx= hpt27xx
	.endif
	_hptiop= hptiop
	.if ${MK_SOURCELESS_HOST} != "no" && empty(KCSAN_ENABLED)
	_hptmv= hptmv
	_hptnr= hptnr
	_hptrr= hptrr
	.endif
	_hyperv= hyperv
	_ichwd= ichwd
	_ida= ida
	_iir= iir
	_intelspi= intelspi
	_ipmi= ipmi
	_ips= ips
	_isci= isci
	_ipw= ipw
	_iwi= iwi
	_iwm= iwm
	_iwn= iwn
	.if ${MK_SOURCELESS_UCODE} != "no"
	_ipwfw= ipwfw
	_iwifw= iwifw
	_iwmfw= iwmfw
	_iwnfw= iwnfw
	.endif
	_mly= mly
	_nfe= nfe
	_nvd= nvd
	_nvme= nvme
	_nvram= nvram
	.if ${MK_CRYPT} != "no" \|\| defined(ALL_MODULES)
	_padlock= padlock
	_padlock_rng= padlock_rng
	_rdrand_rng= rdrand_rng
	.endif
	_pchtherm = pchtherm
	_s3= s3
	_sdhci_acpi= sdhci_acpi
	_superio= superio
	_tpm= tpm
	_twa= twa
	_vesa= vesa
	_viawd= viawd
	_virtio= virtio
	_wpi= wpi
	.if ${MK_SOURCELESS_UCODE} != "no"
	_wpifw= wpifw
	.endif
	_x86bios= x86bios
	.endif

	.if ${MACHINE_CPUARCH} == "amd64"
	_amdgpio= amdgpio
	_ccp= ccp
	_iavf= iavf
	_ioat= ioat
	_ixl= ixl
	_nvdimm= nvdimm
	_pms= pms
	_qlxge= qlxge
	_qlxgb= qlxgb
	_vmd= vmd
	.if ${MK_SOURCELESS_UCODE} != "no"
	_qlxgbe= qlxgbe
	_qlnx= qlnx
	.endif
	_sfxge= sfxge
	_sgx= sgx
	_sgx_linux= sgx_linux
	_smartpqi= smartpqi

	.if ${MK_BHYVE} != "no" \|\| defined(ALL_MODULES)
	.if ${KERN_OPTS:MSMP}
	_vmm= vmm
	.endif
	.endif
	.endif

	.if ${MACHINE_CPUARCH} == "i386"
	# XXX some of these can move to the general case when de-i386'ed
	# XXX some of these can move now, but are untested on other architectures.
	_3dfx= 3dfx
	_3dfx_linux= 3dfx_linux
	_apm= apm
	.if ${MK_SOURCELESS_HOST} != "no"
	_ce= ce
	.endif
	.if ${MK_SOURCELESS_UCODE} != "no"
	_cp= cp
	.endif
	_glxiic= glxiic
	_glxsb= glxsb
	_pcfclock= pcfclock
	_pst= pst
	_sbni= sbni
	.endif

	-.if ${MACHINE_CPUARCH} == "arm"
	+.if ${MACHINE_ARCH} == "armv7"
	_cfi= cfi
	_cpsw= cpsw
	.endif

	.if ${MACHINE_CPUARCH} == "powerpc"
	_aacraid= aacraid
	_agp= agp
	_an= an
	_cardbus= cardbus
	_cbb= cbb
	_cfi= cfi
	_cpufreq= cpufreq
	_exca= exca
	_ffec= ffec
	_nvd= nvd
	_nvme= nvme
	_pccard= pccard
	_wi= wi
	_virtio= virtio
	.endif

	.if ${MACHINE_ARCH} == "powerpc64"
	_ipmi= ipmi
	_ixl= ixl
	_nvram= opal_nvram
	.endif
	.if ${MACHINE_ARCH} == "powerpc64" \|\| ${MACHINE_ARCH} == "powerpc"
	# Don't build powermac_nvram for powerpcspe, it's never supported.
	_nvram+= powermac_nvram
	.endif

	.if (${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64" \|\| \
	${MACHINE_ARCH:Marmv[67]*} != "" \|\| ${MACHINE_CPUARCH} == "i386")
	_cloudabi32= cloudabi32
	.endif
	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64"
	_cloudabi64= cloudabi64
	.endif

	.endif

	.if ${MACHINE_ARCH:Marmv[67]*} != "" \|\| ${MACHINE_CPUARCH} == "aarch64"
	_bcm283x_clkman= bcm283x_clkman
	_bcm283x_pwm= bcm283x_pwm
	.endif

	SUBDIR+=${MODULES_EXTRA}

	.for reject in ${WITHOUT_MODULES}
	SUBDIR:= ${SUBDIR:N${reject}}
	.endfor

	# Calling kldxref(8) for each module is expensive.
	.if !defined(NO_XREF)
	.MAKEFLAGS+= -DNO_XREF
	afterinstall: .PHONY
	@if type kldxref >/dev/null 2>&1; then \
	${ECHO} ${KLDXREF_CMD} ${DESTDIR}${KMODDIR}; \
	${KLDXREF_CMD} ${DESTDIR}${KMODDIR}; \
	fi
	.endif

	SUBDIR:= ${SUBDIR:u:O}

	.include <bsd.subdir.mk>
	diff --git a/sys/modules/esp/Makefile b/sys/modules/esp/Makefile
	index 1880d385dfb8..8540b4434ae3 100644
	--- a/sys/modules/esp/Makefile
	+++ b/sys/modules/esp/Makefile
	@@ -1,9 +1,9 @@
	# $FreeBSD$

	.PATH: ${SRCTOP}/sys/dev/esp

	KMOD= esp
	-SRCS= device_if.h esp_pci.c ${esp_sbus} bus_if.h ncr53c9x.c ${ofw_bus_if}
	+SRCS= device_if.h esp_pci.c bus_if.h ncr53c9x.c
	SRCS+= opt_cam.h pci_if.h

	.include <bsd.kmod.mk>
	diff --git a/sys/modules/rc/Makefile b/sys/modules/rc/Makefile
	deleted file mode 100644
	index a30f91d516e3..000000000000
	--- a/sys/modules/rc/Makefile
	+++ /dev/null
	@@ -1,8 +0,0 @@
	-# $FreeBSD$
	-
	-.PATH: ${SRCTOP}/sys/dev/rc
	-
	-KMOD= rc
	-SRCS= rc.c device_if.h bus_if.h isa_if.h
	-
	-.include <bsd.kmod.mk>
	diff --git a/sys/modules/rp/Makefile b/sys/modules/rp/Makefile
	deleted file mode 100644
	index 530fc12898ce..000000000000
	--- a/sys/modules/rp/Makefile
	+++ /dev/null
	@@ -1,8 +0,0 @@
	-# $FreeBSD$
	-
	-.PATH: ${SRCTOP}/sys/dev/rp
	-
	-KMOD= rp
	-SRCS= rp.c rp_pci.c device_if.h bus_if.h pci_if.h
	-
	-.include <bsd.kmod.mk>
	diff --git a/sys/modules/uart/Makefile b/sys/modules/uart/Makefile
	index b6bba88da9ca..5832f4c1696d 100644
	--- a/sys/modules/uart/Makefile
	+++ b/sys/modules/uart/Makefile
	@@ -1,43 +1,43 @@
	# $FreeBSD$

	.PATH: ${SRCTOP}/sys/dev/uart

	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "amd64" \|\| \
	${MACHINE_CPUARCH} == "i386"
	uart_bus_acpi=uart_bus_acpi.c
	uart_cpu_acpi=uart_cpu_acpi.c
	.endif

	.if ${MACHINE_CPUARCH} == "aarch64" \|\| ${MACHINE_CPUARCH} == "arm" \|\| \
	${MACHINE_CPUARCH} == "powerpc" \|\| ${MACHINE_CPUARCH} == "riscv"
	ofw_bus_if= ofw_bus_if.h
	.endif

	.if ${MACHINE} == "i386" \|\| ${MACHINE} == "amd64"
	_uart_cpu=uart_cpu_x86.c
	.else
	_uart_cpu=uart_cpu_${MACHINE}.c
	.endif
	.if exists(${.CURDIR:H:H}/dev/uart/${_uart_cpu})
	uart_cpu_machine= ${_uart_cpu}
	.endif

	.if ${MACHINE} == "arm64"
	uart_dev_mvebu=uart_dev_mvebu.c
	uart_dev_mu=uart_dev_mu.c
	.endif

	KMOD= uart
	-SRCS= ${uart_bus_acpi} ${uart_bus_ebus} uart_bus_isa.c uart_bus_pccard.c \
	+SRCS= ${uart_bus_acpi} uart_bus_isa.c uart_bus_pccard.c \
	uart_bus_pci.c uart_bus_puc.c uart_bus_scc.c \
	uart_core.c ${uart_cpu_acpi} ${uart_cpu_machine} uart_dbg.c \
	${uart_dev_mvebu} uart_dev_ns8250.c ${uart_dev_mu} \
	uart_dev_quicc.c uart_dev_sab82532.c uart_dev_z8530.c \
	uart_if.c uart_if.h uart_subr.c uart_tty.c

	SRCS+= acpi_if.h bus_if.h card_if.h device_if.h isa_if.h ${ofw_bus_if} \
	pci_if.h \
	power_if.h pccarddevs.h serdev_if.h
	SRCS+= opt_acpi.h opt_platform.h opt_uart.h

	.include <bsd.kmod.mk>
	diff --git a/sys/net/iflib.c b/sys/net/iflib.c
	index f87b31251ffe..9a37f30b43f6 100644
	--- a/sys/net/iflib.c
	+++ b/sys/net/iflib.c
	@@ -1,6896 +1,6898 @@
	/*-
	* Copyright (c) 2014-2018, Matthew Macy <mmacy@mattmacy.io>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions are met:
	*
	* 1. Redistributions of source code must retain the above copyright notice,
	* this list of conditions and the following disclaimer.
	*
	* 2. Neither the name of Matthew Macy nor the names of its
	* contributors may be used to endorse or promote products derived from
	* this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_acpi.h"
	#include "opt_sched.h"

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/bus.h>
	#include <sys/eventhandler.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/module.h>
	#include <sys/kobj.h>
	#include <sys/rman.h>
	#include <sys/sbuf.h>
	#include <sys/smp.h>
	#include <sys/socket.h>
	#include <sys/sockio.h>
	#include <sys/sysctl.h>
	#include <sys/syslog.h>
	#include <sys/taskqueue.h>
	#include <sys/limits.h>

	#include <net/if.h>
	#include <net/if_var.h>
	#include <net/if_types.h>
	#include <net/if_media.h>
	#include <net/bpf.h>
	#include <net/ethernet.h>
	#include <net/mp_ring.h>
	#include <net/debugnet.h>
	#include <net/pfil.h>
	#include <net/vnet.h>

	#include <netinet/in.h>
	#include <netinet/in_pcb.h>
	#include <netinet/tcp_lro.h>
	#include <netinet/in_systm.h>
	#include <netinet/if_ether.h>
	#include <netinet/ip.h>
	#include <netinet/ip6.h>
	#include <netinet/tcp.h>
	#include <netinet/ip_var.h>
	#include <netinet6/ip6_var.h>

	#include <machine/bus.h>
	#include <machine/in_cksum.h>

	#include <vm/vm.h>
	#include <vm/pmap.h>

	#include <dev/led/led.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	#include <dev/pci/pci_private.h>

	#include <net/iflib.h>
	#include <net/iflib_private.h>

	#include "ifdi_if.h"

	#ifdef PCI_IOV
	#include <dev/pci/pci_iov.h>
	#endif

	#include <sys/bitstring.h>
	/*
	* enable accounting of every mbuf as it comes in to and goes out of
	* iflib's software descriptor references
	*/
	#define MEMORY_LOGGING 0
	/*
	* Enable mbuf vectors for compressing long mbuf chains
	*/

	/*
	* NB:
	* - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
	* we prefetch needs to be determined by the time spent in m_free vis a vis
	* the cost of a prefetch. This will of course vary based on the workload:
	* - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
	* is quite expensive, thus suggesting very little prefetch.
	* - small packet forwarding which is just returning a single mbuf to
	* UMA will typically be very fast vis a vis the cost of a memory
	* access.
	*/


	/*
	* File organization:
	* - private structures
	* - iflib private utility functions
	* - ifnet functions
	* - vlan registry and other exported functions
	* - iflib public core functions
	*
	*
	*/
	MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library");

	#define IFLIB_RXEOF_MORE (1U << 0)
	#define IFLIB_RXEOF_EMPTY (2U << 0)

	struct iflib_txq;
	typedef struct iflib_txq *iflib_txq_t;
	struct iflib_rxq;
	typedef struct iflib_rxq *iflib_rxq_t;
	struct iflib_fl;
	typedef struct iflib_fl *iflib_fl_t;

	struct iflib_ctx;

	static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid);
	static void iflib_timer(void *arg);

	typedef struct iflib_filter_info {
	driver_filter_t *ifi_filter;
	void *ifi_filter_arg;
	struct grouptask *ifi_task;
	void *ifi_ctx;
	} *iflib_filter_info_t;

	struct iflib_ctx {
	KOBJ_FIELDS;
	/*
	* Pointer to hardware driver's softc
	*/
	void *ifc_softc;
	device_t ifc_dev;
	if_t ifc_ifp;

	cpuset_t ifc_cpus;
	if_shared_ctx_t ifc_sctx;
	struct if_softc_ctx ifc_softc_ctx;

	struct sx ifc_ctx_sx;
	struct mtx ifc_state_mtx;

	iflib_txq_t ifc_txqs;
	iflib_rxq_t ifc_rxqs;
	uint32_t ifc_if_flags;
	uint32_t ifc_flags;
	uint32_t ifc_max_fl_buf_size;
	uint32_t ifc_rx_mbuf_sz;

	int ifc_link_state;
	int ifc_watchdog_events;
	struct cdev *ifc_led_dev;
	struct resource *ifc_msix_mem;

	struct if_irq ifc_legacy_irq;
	struct grouptask ifc_admin_task;
	struct grouptask ifc_vflr_task;
	struct iflib_filter_info ifc_filter_info;
	struct ifmedia ifc_media;
	struct ifmedia *ifc_mediap;

	struct sysctl_oid *ifc_sysctl_node;
	uint16_t ifc_sysctl_ntxqs;
	uint16_t ifc_sysctl_nrxqs;
	uint16_t ifc_sysctl_qs_eq_override;
	uint16_t ifc_sysctl_rx_budget;
	uint16_t ifc_sysctl_tx_abdicate;
	uint16_t ifc_sysctl_core_offset;
	#define CORE_OFFSET_UNSPECIFIED 0xffff
	uint8_t ifc_sysctl_separate_txrx;

	qidx_t ifc_sysctl_ntxds[8];
	qidx_t ifc_sysctl_nrxds[8];
	struct if_txrx ifc_txrx;
	#define isc_txd_encap ifc_txrx.ift_txd_encap
	#define isc_txd_flush ifc_txrx.ift_txd_flush
	#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update
	#define isc_rxd_available ifc_txrx.ift_rxd_available
	#define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
	#define isc_rxd_refill ifc_txrx.ift_rxd_refill
	#define isc_rxd_flush ifc_txrx.ift_rxd_flush
	#define isc_legacy_intr ifc_txrx.ift_legacy_intr
	eventhandler_tag ifc_vlan_attach_event;
	eventhandler_tag ifc_vlan_detach_event;
	struct ether_addr ifc_mac;
	};

	void *
	iflib_get_softc(if_ctx_t ctx)
	{

	return (ctx->ifc_softc);
	}

	device_t
	iflib_get_dev(if_ctx_t ctx)
	{

	return (ctx->ifc_dev);
	}

	if_t
	iflib_get_ifp(if_ctx_t ctx)
	{

	return (ctx->ifc_ifp);
	}

	struct ifmedia *
	iflib_get_media(if_ctx_t ctx)
	{

	return (ctx->ifc_mediap);
	}

	uint32_t
	iflib_get_flags(if_ctx_t ctx)
	{
	return (ctx->ifc_flags);
	}

	void
	iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN])
	{

	bcopy(mac, ctx->ifc_mac.octet, ETHER_ADDR_LEN);
	}

	if_softc_ctx_t
	iflib_get_softc_ctx(if_ctx_t ctx)
	{

	return (&ctx->ifc_softc_ctx);
	}

	if_shared_ctx_t
	iflib_get_sctx(if_ctx_t ctx)
	{

	return (ctx->ifc_sctx);
	}

	#define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2)
	#define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*))
	#define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1)))

	#define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
	#define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)

	typedef struct iflib_sw_rx_desc_array {
	bus_dmamap_t ifsd_map; / bus_dma maps for packet */
	struct mbuf *ifsd_m; / pkthdr mbufs */
	caddr_t ifsd_cl; / direct cluster pointer for rx */
	bus_addr_t ifsd_ba; / bus addr of cluster for rx */
	} iflib_rxsd_array_t;

	typedef struct iflib_sw_tx_desc_array {
	bus_dmamap_t ifsd_map; / bus_dma maps for packet */
	bus_dmamap_t ifsd_tso_map; / bus_dma maps for TSO packet */
	struct mbuf *ifsd_m; / pkthdr mbufs */
	} if_txsd_vec_t;

	/* magic number that should be high enough for any hardware */
	#define IFLIB_MAX_TX_SEGS 128
	#define IFLIB_RX_COPY_THRESH 128
	#define IFLIB_MAX_RX_REFRESH 32
	/* The minimum descriptors per second before we start coalescing */
	#define IFLIB_MIN_DESC_SEC 16384
	#define IFLIB_DEFAULT_TX_UPDATE_FREQ 16
	#define IFLIB_QUEUE_IDLE 0
	#define IFLIB_QUEUE_HUNG 1
	#define IFLIB_QUEUE_WORKING 2
	/* maximum number of txqs that can share an rx interrupt */
	#define IFLIB_MAX_TX_SHARED_INTR 4

	/* this should really scale with ring size - this is a fairly arbitrary value */
	#define TX_BATCH_SIZE 32

	#define IFLIB_RESTART_BUDGET 8

	#define CSUM_OFFLOAD (CSUM_IP_TSO\|CSUM_IP6_TSO\|CSUM_IP\| \
	CSUM_IP_UDP\|CSUM_IP_TCP\|CSUM_IP_SCTP\| \
	CSUM_IP6_UDP\|CSUM_IP6_TCP\|CSUM_IP6_SCTP)

	struct iflib_txq {
	qidx_t ift_in_use;
	qidx_t ift_cidx;
	qidx_t ift_cidx_processed;
	qidx_t ift_pidx;
	uint8_t ift_gen;
	uint8_t ift_br_offset;
	uint16_t ift_npending;
	uint16_t ift_db_pending;
	uint16_t ift_rs_pending;
	/* implicit pad */
	uint8_t ift_txd_size[8];
	uint64_t ift_processed;
	uint64_t ift_cleaned;
	uint64_t ift_cleaned_prev;
	#if MEMORY_LOGGING
	uint64_t ift_enqueued;
	uint64_t ift_dequeued;
	#endif
	uint64_t ift_no_tx_dma_setup;
	uint64_t ift_no_desc_avail;
	uint64_t ift_mbuf_defrag_failed;
	uint64_t ift_mbuf_defrag;
	uint64_t ift_map_failed;
	uint64_t ift_txd_encap_efbig;
	uint64_t ift_pullups;
	uint64_t ift_last_timer_tick;

	struct mtx ift_mtx;
	struct mtx ift_db_mtx;

	/* constant values */
	if_ctx_t ift_ctx;
	struct ifmp_ring *ift_br;
	struct grouptask ift_task;
	qidx_t ift_size;
	uint16_t ift_id;
	struct callout ift_timer;

	if_txsd_vec_t ift_sds;
	uint8_t ift_qstatus;
	uint8_t ift_closed;
	uint8_t ift_update_freq;
	struct iflib_filter_info ift_filter_info;
	bus_dma_tag_t ift_buf_tag;
	bus_dma_tag_t ift_tso_buf_tag;
	iflib_dma_info_t ift_ifdi;
	#define MTX_NAME_LEN 32
	char ift_mtx_name[MTX_NAME_LEN];
	bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE);
	#ifdef IFLIB_DIAGNOSTICS
	uint64_t ift_cpu_exec_count[256];
	#endif
	} __aligned(CACHE_LINE_SIZE);

	struct iflib_fl {
	qidx_t ifl_cidx;
	qidx_t ifl_pidx;
	qidx_t ifl_credits;
	uint8_t ifl_gen;
	uint8_t ifl_rxd_size;
	#if MEMORY_LOGGING
	uint64_t ifl_m_enqueued;
	uint64_t ifl_m_dequeued;
	uint64_t ifl_cl_enqueued;
	uint64_t ifl_cl_dequeued;
	#endif
	/* implicit pad */
	bitstr_t *ifl_rx_bitmap;
	qidx_t ifl_fragidx;
	/* constant */
	qidx_t ifl_size;
	uint16_t ifl_buf_size;
	uint16_t ifl_cltype;
	uma_zone_t ifl_zone;
	iflib_rxsd_array_t ifl_sds;
	iflib_rxq_t ifl_rxq;
	uint8_t ifl_id;
	bus_dma_tag_t ifl_buf_tag;
	iflib_dma_info_t ifl_ifdi;
	uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
	qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH];
	} __aligned(CACHE_LINE_SIZE);

	static inline qidx_t
	get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen)
	{
	qidx_t used;

	if (pidx > cidx)
	used = pidx - cidx;
	else if (pidx < cidx)
	used = size - cidx + pidx;
	else if (gen == 0 && pidx == cidx)
	used = 0;
	else if (gen == 1 && pidx == cidx)
	used = size;
	else
	panic("bad state");

	return (used);
	}

	#define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))

	#define IDXDIFF(head, tail, wrap) \
	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))

	struct iflib_rxq {
	if_ctx_t ifr_ctx;
	iflib_fl_t ifr_fl;
	uint64_t ifr_rx_irq;
	struct pfil_head *pfil;
	/*
	* If there is a separate completion queue (IFLIB_HAS_RXCQ), this is
	* the command queue consumer index. Otherwise it's unused.
	*/
	qidx_t ifr_cq_cidx;
	uint16_t ifr_id;
	uint8_t ifr_nfl;
	uint8_t ifr_ntxqirq;
	uint8_t ifr_txqid[IFLIB_MAX_TX_SHARED_INTR];
	uint8_t ifr_fl_offset;
	struct lro_ctrl ifr_lc;
	struct grouptask ifr_task;
	struct callout ifr_watchdog;
	struct iflib_filter_info ifr_filter_info;
	iflib_dma_info_t ifr_ifdi;

	/* dynamically allocate if any drivers need a value substantially larger than this */
	struct if_rxd_frag ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE);
	#ifdef IFLIB_DIAGNOSTICS
	uint64_t ifr_cpu_exec_count[256];
	#endif
	} __aligned(CACHE_LINE_SIZE);

	typedef struct if_rxsd {
	caddr_t *ifsd_cl;
	iflib_fl_t ifsd_fl;
	} *if_rxsd_t;

	/* multiple of word size */
	#ifdef __LP64__
	#define PKT_INFO_SIZE 6
	#define RXD_INFO_SIZE 5
	#define PKT_TYPE uint64_t
	#else
	#define PKT_INFO_SIZE 11
	#define RXD_INFO_SIZE 8
	#define PKT_TYPE uint32_t
	#endif
	#define PKT_LOOP_BOUND ((PKT_INFO_SIZE/3)*3)
	#define RXD_LOOP_BOUND ((RXD_INFO_SIZE/4)*4)

	typedef struct if_pkt_info_pad {
	PKT_TYPE pkt_val[PKT_INFO_SIZE];
	} *if_pkt_info_pad_t;
	typedef struct if_rxd_info_pad {
	PKT_TYPE rxd_val[RXD_INFO_SIZE];
	} *if_rxd_info_pad_t;

	CTASSERT(sizeof(struct if_pkt_info_pad) == sizeof(struct if_pkt_info));
	CTASSERT(sizeof(struct if_rxd_info_pad) == sizeof(struct if_rxd_info));


	static inline void
	pkt_info_zero(if_pkt_info_t pi)
	{
	if_pkt_info_pad_t pi_pad;

	pi_pad = (if_pkt_info_pad_t)pi;
	pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0;
	pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0;
	#ifndef __LP64__
	pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0;
	pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0;
	#endif
	}

	static device_method_t iflib_pseudo_methods[] = {
	DEVMETHOD(device_attach, noop_attach),
	DEVMETHOD(device_detach, iflib_pseudo_detach),
	DEVMETHOD_END
	};

	driver_t iflib_pseudodriver = {
	"iflib_pseudo", iflib_pseudo_methods, sizeof(struct iflib_ctx),
	};

	static inline void
	rxd_info_zero(if_rxd_info_t ri)
	{
	if_rxd_info_pad_t ri_pad;
	int i;

	ri_pad = (if_rxd_info_pad_t)ri;
	for (i = 0; i < RXD_LOOP_BOUND; i += 4) {
	ri_pad->rxd_val[i] = 0;
	ri_pad->rxd_val[i+1] = 0;
	ri_pad->rxd_val[i+2] = 0;
	ri_pad->rxd_val[i+3] = 0;
	}
	#ifdef __LP64__
	ri_pad->rxd_val[RXD_INFO_SIZE-1] = 0;
	#endif
	}

	/*
	* Only allow a single packet to take up most 1/nth of the tx ring
	*/
	#define MAX_SINGLE_PACKET_FRACTION 12
	#define IF_BAD_DMA (bus_addr_t)-1

	#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))

	#define CTX_LOCK_INIT(_sc) sx_init(&(_sc)->ifc_ctx_sx, "iflib ctx lock")
	#define CTX_LOCK(ctx) sx_xlock(&(ctx)->ifc_ctx_sx)
	#define CTX_UNLOCK(ctx) sx_xunlock(&(ctx)->ifc_ctx_sx)
	#define CTX_LOCK_DESTROY(ctx) sx_destroy(&(ctx)->ifc_ctx_sx)

	#define STATE_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_state_mtx, _name, "iflib state lock", MTX_DEF)
	#define STATE_LOCK(ctx) mtx_lock(&(ctx)->ifc_state_mtx)
	#define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx)
	#define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx)

	#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
	#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)

	void
	iflib_set_detach(if_ctx_t ctx)
	{
	STATE_LOCK(ctx);
	ctx->ifc_flags \|= IFC_IN_DETACH;
	STATE_UNLOCK(ctx);
	}

	/* Our boot-time initialization hook */
	static int iflib_module_event_handler(module_t, int, void *);

	static moduledata_t iflib_moduledata = {
	"iflib",
	iflib_module_event_handler,
	NULL
	};

	DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY);
	MODULE_VERSION(iflib, 1);

	MODULE_DEPEND(iflib, pci, 1, 1, 1);
	MODULE_DEPEND(iflib, ether, 1, 1, 1);

	TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
	TASKQGROUP_DEFINE(if_config_tqg, 1, 1);

	#ifndef IFLIB_DEBUG_COUNTERS
	#ifdef INVARIANTS
	#define IFLIB_DEBUG_COUNTERS 1
	#else
	#define IFLIB_DEBUG_COUNTERS 0
	#endif /* !INVARIANTS */
	#endif

	static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"iflib driver parameters");

	/*
	* XXX need to ensure that this can't accidentally cause the head to be moved backwards
	*/
	static int iflib_min_tx_latency = 0;
	SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
	&iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput");
	static int iflib_no_tx_batch = 0;
	SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW,
	&iflib_no_tx_batch, 0, "minimize transmit latency at the possible expense of throughput");


	#if IFLIB_DEBUG_COUNTERS

	static int iflib_tx_seen;
	static int iflib_tx_sent;
	static int iflib_tx_encap;
	static int iflib_rx_allocs;
	static int iflib_fl_refills;
	static int iflib_fl_refills_large;
	static int iflib_tx_frees;

	SYSCTL_INT(_net_iflib, OID_AUTO, tx_seen, CTLFLAG_RD,
	&iflib_tx_seen, 0, "# TX mbufs seen");
	SYSCTL_INT(_net_iflib, OID_AUTO, tx_sent, CTLFLAG_RD,
	&iflib_tx_sent, 0, "# TX mbufs sent");
	SYSCTL_INT(_net_iflib, OID_AUTO, tx_encap, CTLFLAG_RD,
	&iflib_tx_encap, 0, "# TX mbufs encapped");
	SYSCTL_INT(_net_iflib, OID_AUTO, tx_frees, CTLFLAG_RD,
	&iflib_tx_frees, 0, "# TX frees");
	SYSCTL_INT(_net_iflib, OID_AUTO, rx_allocs, CTLFLAG_RD,
	&iflib_rx_allocs, 0, "# RX allocations");
	SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD,
	&iflib_fl_refills, 0, "# refills");
	SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD,
	&iflib_fl_refills_large, 0, "# large refills");


	static int iflib_txq_drain_flushing;
	static int iflib_txq_drain_oactive;
	static int iflib_txq_drain_notready;

	SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_flushing, CTLFLAG_RD,
	&iflib_txq_drain_flushing, 0, "# drain flushes");
	SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD,
	&iflib_txq_drain_oactive, 0, "# drain oactives");
	SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD,
	&iflib_txq_drain_notready, 0, "# drain notready");


	static int iflib_encap_load_mbuf_fail;
	static int iflib_encap_pad_mbuf_fail;
	static int iflib_encap_txq_avail_fail;
	static int iflib_encap_txd_encap_fail;

	SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
	&iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
	SYSCTL_INT(_net_iflib, OID_AUTO, encap_pad_mbuf_fail, CTLFLAG_RD,
	&iflib_encap_pad_mbuf_fail, 0, "# runt frame pad failures");
	SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
	&iflib_encap_txq_avail_fail, 0, "# txq avail failures");
	SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
	&iflib_encap_txd_encap_fail, 0, "# driver encap failures");

	static int iflib_task_fn_rxs;
	static int iflib_rx_intr_enables;
	static int iflib_fast_intrs;
	static int iflib_rx_unavail;
	static int iflib_rx_ctx_inactive;
	static int iflib_rx_if_input;
	static int iflib_rxd_flush;

	static int iflib_verbose_debug;

	SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD,
	&iflib_task_fn_rxs, 0, "# task_fn_rx calls");
	SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD,
	&iflib_rx_intr_enables, 0, "# RX intr enables");
	SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD,
	&iflib_fast_intrs, 0, "# fast_intr calls");
	SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD,
	&iflib_rx_unavail, 0, "# times rxeof called with no available data");
	SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD,
	&iflib_rx_ctx_inactive, 0, "# times rxeof called with inactive context");
	SYSCTL_INT(_net_iflib, OID_AUTO, rx_if_input, CTLFLAG_RD,
	&iflib_rx_if_input, 0, "# times rxeof called if_input");
	SYSCTL_INT(_net_iflib, OID_AUTO, rxd_flush, CTLFLAG_RD,
	&iflib_rxd_flush, 0, "# times rxd_flush called");
	SYSCTL_INT(_net_iflib, OID_AUTO, verbose_debug, CTLFLAG_RW,
	&iflib_verbose_debug, 0, "enable verbose debugging");

	#define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
	static void
	iflib_debug_reset(void)
	{
	iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs =
	iflib_fl_refills = iflib_fl_refills_large = iflib_tx_frees =
	iflib_txq_drain_flushing = iflib_txq_drain_oactive =
	iflib_txq_drain_notready =
	iflib_encap_load_mbuf_fail = iflib_encap_pad_mbuf_fail =
	iflib_encap_txq_avail_fail = iflib_encap_txd_encap_fail =
	iflib_task_fn_rxs = iflib_rx_intr_enables = iflib_fast_intrs =
	iflib_rx_unavail =
	iflib_rx_ctx_inactive = iflib_rx_if_input =
	iflib_rxd_flush = 0;
	}

	#else
	#define DBG_COUNTER_INC(name)
	static void iflib_debug_reset(void) {}
	#endif

	#define IFLIB_DEBUG 0

	static void iflib_tx_structures_free(if_ctx_t ctx);
	static void iflib_rx_structures_free(if_ctx_t ctx);
	static int iflib_queues_alloc(if_ctx_t ctx);
	static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
	static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget);
	static int iflib_qset_structures_setup(if_ctx_t ctx);
	static int iflib_msix_init(if_ctx_t ctx);
	static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void filterarg, int rid, const char *str);
	static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
	static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
	#ifdef ALTQ
	static void iflib_altq_if_start(if_t ifp);
	static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m);
	#endif
	static int iflib_register(if_ctx_t);
	static void iflib_deregister(if_ctx_t);
	static void iflib_unregister_vlan_handlers(if_ctx_t ctx);
	static uint16_t iflib_get_mbuf_size_for(unsigned int size);
	static void iflib_init_locked(if_ctx_t ctx);
	static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
	static void iflib_add_device_sysctl_post(if_ctx_t ctx);
	static void iflib_ifmp_purge(iflib_txq_t txq);
	static void _iflib_pre_assert(if_softc_ctx_t scctx);
	static void iflib_if_init_locked(if_ctx_t ctx);
	static void iflib_free_intr_mem(if_ctx_t ctx);
	#ifndef __NO_STRICT_ALIGNMENT
	static struct mbuf * iflib_fixup_rx(struct mbuf *m);
	#endif

	static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets =
	SLIST_HEAD_INITIALIZER(cpu_offsets);
	struct cpu_offset {
	SLIST_ENTRY(cpu_offset) entries;
	cpuset_t set;
	unsigned int refcount;
	uint16_t offset;
	};
	static struct mtx cpu_offset_mtx;
	MTX_SYSINIT(iflib_cpu_offset, &cpu_offset_mtx, "iflib_cpu_offset lock",
	MTX_DEF);

	DEBUGNET_DEFINE(iflib);

	static int
	iflib_num_rx_descs(if_ctx_t ctx)
	{
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	uint16_t first_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0;

	return scctx->isc_nrxd[first_rxq];
	}

	static int
	iflib_num_tx_descs(if_ctx_t ctx)
	{
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	uint16_t first_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0;

	return scctx->isc_ntxd[first_txq];
	}

	#ifdef DEV_NETMAP
	#include <sys/selinfo.h>
	#include <net/netmap.h>
	#include <dev/netmap/netmap_kern.h>

	MODULE_DEPEND(iflib, netmap, 1, 1, 1);

	static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init);

	/*
	* device-specific sysctl variables:
	*
	* iflib_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
	* During regular operations the CRC is stripped, but on some
	* hardware reception of frames not multiple of 64 is slower,
	* so using crcstrip=0 helps in benchmarks.
	*
	* iflib_rx_miss, iflib_rx_miss_bufs:
	* count packets that might be missed due to lost interrupts.
	*/
	SYSCTL_DECL(_dev_netmap);
	/*
	* The xl driver by default strips CRCs and we do not override it.
	*/

	int iflib_crcstrip = 1;
	SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_crcstrip,
	CTLFLAG_RW, &iflib_crcstrip, 1, "strip CRC on RX frames");

	int iflib_rx_miss, iflib_rx_miss_bufs;
	SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss,
	CTLFLAG_RW, &iflib_rx_miss, 0, "potentially missed RX intr");
	SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss_bufs,
	CTLFLAG_RW, &iflib_rx_miss_bufs, 0, "potentially missed RX intr bufs");

	/*
	* Register/unregister. We are already under netmap lock.
	* Only called on the first register or the last unregister.
	*/
	static int
	iflib_netmap_register(struct netmap_adapter *na, int onoff)
	{
	if_t ifp = na->ifp;
	if_ctx_t ctx = ifp->if_softc;
	int status;

	CTX_LOCK(ctx);
	IFDI_INTR_DISABLE(ctx);

	/* Tell the stack that the interface is no longer active */
	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING \| IFF_DRV_OACTIVE);

	if (!CTX_IS_VF(ctx))
	IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip);

	iflib_stop(ctx);

	/*
	* Enable (or disable) netmap flags, and intercept (or restore)
	* ifp->if_transmit. This is done once the device has been stopped
	* to prevent race conditions.
	*/
	if (onoff) {
	nm_set_native_flags(na);
	} else {
	nm_clear_native_flags(na);
	}

	iflib_init_locked(ctx);
	IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ?
	status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1;
	if (status)
	nm_clear_native_flags(na);
	CTX_UNLOCK(ctx);
	return (status);
	}

	static int
	netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init)
	{
	struct netmap_adapter *na = kring->na;
	u_int const lim = kring->nkr_num_slots - 1;
	u_int head = kring->rhead;
	struct netmap_ring *ring = kring->ring;
	bus_dmamap_t *map;
	struct if_rxd_update iru;
	if_ctx_t ctx = rxq->ifr_ctx;
	iflib_fl_t fl = &rxq->ifr_fl[0];
	uint32_t refill_pidx, nic_i;
	#if IFLIB_DEBUG_COUNTERS
	int rf_count = 0;
	#endif

	if (nm_i == head && __predict_true(!init))
	return 0;
	iru_init(&iru, rxq, 0 /* flid */);
	map = fl->ifl_sds.ifsd_map;
	refill_pidx = netmap_idx_k2n(kring, nm_i);
	/*
	* IMPORTANT: we must leave one free slot in the ring,
	* so move head back by one unit
	*/
	head = nm_prev(head, lim);
	nic_i = UINT_MAX;
	DBG_COUNTER_INC(fl_refills);
	while (nm_i != head) {
	#if IFLIB_DEBUG_COUNTERS
	if (++rf_count == 9)
	DBG_COUNTER_INC(fl_refills_large);
	#endif
	for (int tmp_pidx = 0; tmp_pidx < IFLIB_MAX_RX_REFRESH && nm_i != head; tmp_pidx++) {
	struct netmap_slot *slot = &ring->slot[nm_i];
	void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]);
	uint32_t nic_i_dma = refill_pidx;
	nic_i = netmap_idx_k2n(kring, nm_i);

	MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH);

	if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
	return netmap_ring_reinit(kring);

	if (__predict_false(init)) {
	netmap_load_map(na, fl->ifl_buf_tag,
	map[nic_i], addr);
	} else if (slot->flags & NS_BUF_CHANGED) {
	/* buffer has changed, reload map */
	netmap_reload_map(na, fl->ifl_buf_tag,
	map[nic_i], addr);
	}
	slot->flags &= ~NS_BUF_CHANGED;

	nm_i = nm_next(nm_i, lim);
	fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim);
	if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1)
	continue;

	iru.iru_pidx = refill_pidx;
	iru.iru_count = tmp_pidx+1;
	ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
	refill_pidx = nic_i;
	for (int n = 0; n < iru.iru_count; n++) {
	bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i_dma],
	BUS_DMASYNC_PREREAD);
	/* XXX - change this to not use the netmap func*/
	nic_i_dma = nm_next(nic_i_dma, lim);
	}
	}
	}
	kring->nr_hwcur = head;

	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	if (__predict_true(nic_i != UINT_MAX)) {
	ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
	DBG_COUNTER_INC(rxd_flush);
	}
	return (0);
	}

	/*
	* Reconcile kernel and user view of the transmit ring.
	*
	* All information is in the kring.
	* Userspace wants to send packets up to the one before kring->rhead,
	* kernel knows kring->nr_hwcur is the first unsent packet.
	*
	* Here we push packets out (as many as possible), and possibly
	* reclaim buffers from previously completed transmission.
	*
	* The caller (netmap) guarantees that there is only one instance
	* running at any time. Any interference with other driver
	* methods should be handled by the individual drivers.
	*/
	static int
	iflib_netmap_txsync(struct netmap_kring *kring, int flags)
	{
	struct netmap_adapter *na = kring->na;
	if_t ifp = na->ifp;
	struct netmap_ring *ring = kring->ring;
	u_int nm_i; /* index into the netmap kring */
	u_int nic_i; /* index into the NIC ring */
	u_int n;
	u_int const lim = kring->nkr_num_slots - 1;
	u_int const head = kring->rhead;
	struct if_pkt_info pi;

	/*
	* interrupts on every tx packet are expensive so request
	* them every half ring, or where NS_REPORT is set
	*/
	u_int report_frequency = kring->nkr_num_slots >> 1;
	/* device-specific */
	if_ctx_t ctx = ifp->if_softc;
	iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];

	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);

	/*
	* First part: process new packets to send.
	* nm_i is the current index in the netmap kring,
	* nic_i is the corresponding index in the NIC ring.
	*
	* If we have packets to send (nm_i != head)
	* iterate over the netmap ring, fetch length and update
	* the corresponding slot in the NIC ring. Some drivers also
	* need to update the buffer's physical address in the NIC slot
	* even NS_BUF_CHANGED is not set (PNMB computes the addresses).
	*
	* The netmap_reload_map() calls is especially expensive,
	* even when (as in this case) the tag is 0, so do only
	* when the buffer has actually changed.
	*
	* If possible do not set the report/intr bit on all slots,
	* but only a few times per ring or when NS_REPORT is set.
	*
	* Finally, on 10G and faster drivers, it might be useful
	* to prefetch the next slot and txr entry.
	*/

	nm_i = kring->nr_hwcur;
	if (nm_i != head) { /* we have new packets to send */
	pkt_info_zero(&pi);
	pi.ipi_segs = txq->ift_segs;
	pi.ipi_qsidx = kring->ring_id;
	nic_i = netmap_idx_k2n(kring, nm_i);

	__builtin_prefetch(&ring->slot[nm_i]);
	__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
	__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);

	for (n = 0; nm_i != head; n++) {
	struct netmap_slot *slot = &ring->slot[nm_i];
	u_int len = slot->len;
	uint64_t paddr;
	void *addr = PNMB(na, slot, &paddr);
	int flags = (slot->flags & NS_REPORT \|\|
	nic_i == 0 \|\| nic_i == report_frequency) ?
	IPI_TX_INTR : 0;

	/* device-specific */
	pi.ipi_len = len;
	pi.ipi_segs[0].ds_addr = paddr;
	pi.ipi_segs[0].ds_len = len;
	pi.ipi_nsegs = 1;
	pi.ipi_ndescs = 0;
	pi.ipi_pidx = nic_i;
	pi.ipi_flags = flags;

	/* Fill the slot in the NIC ring. */
	ctx->isc_txd_encap(ctx->ifc_softc, &pi);
	DBG_COUNTER_INC(tx_encap);

	/* prefetch for next round */
	__builtin_prefetch(&ring->slot[nm_i + 1]);
	__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
	__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);

	NM_CHECK_ADDR_LEN(na, addr, len);

	if (slot->flags & NS_BUF_CHANGED) {
	/* buffer has changed, reload map */
	netmap_reload_map(na, txq->ift_buf_tag,
	txq->ift_sds.ifsd_map[nic_i], addr);
	}
	/* make sure changes to the buffer are synced */
	bus_dmamap_sync(txq->ift_buf_tag,
	txq->ift_sds.ifsd_map[nic_i],
	BUS_DMASYNC_PREWRITE);

	slot->flags &= ~(NS_REPORT \| NS_BUF_CHANGED);
	nm_i = nm_next(nm_i, lim);
	nic_i = nm_next(nic_i, lim);
	}
	kring->nr_hwcur = nm_i;

	/* synchronize the NIC ring */
	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);

	/* (re)start the tx unit up to slot nic_i (excluded) */
	ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
	}

	/*
	* Second part: reclaim buffers for completed transmissions.
	*
	* If there are unclaimed buffers, attempt to reclaim them.
	* If none are reclaimed, and TX IRQs are not in use, do an initial
	* minimal delay, then trigger the tx handler which will spin in the
	* group task queue.
	*/
	if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) {
	if (iflib_tx_credits_update(ctx, txq)) {
	/* some tx completed, increment avail */
	nic_i = txq->ift_cidx_processed;
	kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
	}
	}
	if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ))
	if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) {
	callout_reset_on(&txq->ift_timer, hz < 2000 ? 1 : hz / 1000,
	iflib_timer, txq, txq->ift_timer.c_cpu);
	}
	return (0);
	}

	/*
	* Reconcile kernel and user view of the receive ring.
	* Same as for the txsync, this routine must be efficient.
	* The caller guarantees a single invocations, but races against
	* the rest of the driver should be handled here.
	*
	* On call, kring->rhead is the first packet that userspace wants
	* to keep, and kring->rcur is the wakeup point.
	* The kernel has previously reported packets up to kring->rtail.
	*
	* If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
	* of whether or not we received an interrupt.
	*/
	static int
	iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
	{
	struct netmap_adapter *na = kring->na;
	struct netmap_ring *ring = kring->ring;
	if_t ifp = na->ifp;
	iflib_fl_t fl;
	uint32_t nm_i; /* index into the netmap ring */
	uint32_t nic_i; /* index into the NIC ring */
	u_int i, n;
	u_int const lim = kring->nkr_num_slots - 1;
	u_int const head = kring->rhead;
	int force_update = (flags & NAF_FORCE_READ) \|\| kring->nr_kflags & NKR_PENDINTR;
	struct if_rxd_info ri;

	if_ctx_t ctx = ifp->if_softc;
	iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
	if (head > lim)
	return netmap_ring_reinit(kring);

	/*
	* XXX netmap_fl_refill() only ever (re)fills free list 0 so far.
	*/

	for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) {
	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	}

	/*
	* First part: import newly received packets.
	*
	* nm_i is the index of the next free slot in the netmap ring,
	* nic_i is the index of the next received packet in the NIC ring,
	* and they may differ in case if_init() has been called while
	* in netmap mode. For the receive ring we have
	*
	* nic_i = rxr->next_check;
	* nm_i = kring->nr_hwtail (previous)
	* and
	* nm_i == (nic_i + kring->nkr_hwofs) % ring_size
	*
	* rxr->next_check is set to 0 on a ring reinit
	*/
	if (netmap_no_pendintr \|\| force_update) {
	uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim);
	int crclen = iflib_crcstrip ? 0 : 4;
	int error, avail;

	for (i = 0; i < rxq->ifr_nfl; i++) {
	fl = &rxq->ifr_fl[i];
	nic_i = fl->ifl_cidx;
	nm_i = netmap_idx_n2k(kring, nic_i);
	avail = ctx->isc_rxd_available(ctx->ifc_softc,
	rxq->ifr_id, nic_i, USHRT_MAX);
	for (n = 0; avail > 0 && nm_i != hwtail_lim; n++, avail--) {
	rxd_info_zero(&ri);
	ri.iri_frags = rxq->ifr_frags;
	ri.iri_qsidx = kring->ring_id;
	ri.iri_ifp = ctx->ifc_ifp;
	ri.iri_cidx = nic_i;

	error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
	ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen;
	ring->slot[nm_i].flags = 0;
	bus_dmamap_sync(fl->ifl_buf_tag,
	fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD);
	nm_i = nm_next(nm_i, lim);
	nic_i = nm_next(nic_i, lim);
	}
	if (n) { /* update the state variables */
	if (netmap_no_pendintr && !force_update) {
	/* diagnostics */
	iflib_rx_miss ++;
	iflib_rx_miss_bufs += n;
	}
	fl->ifl_cidx = nic_i;
	kring->nr_hwtail = nm_i;
	}
	kring->nr_kflags &= ~NKR_PENDINTR;
	}
	}
	/*
	* Second part: skip past packets that userspace has released.
	* (kring->nr_hwcur to head excluded),
	* and make the buffers available for reception.
	* As usual nm_i is the index in the netmap ring,
	* nic_i is the index in the NIC ring, and
	* nm_i == (nic_i + kring->nkr_hwofs) % ring_size
	*/
	/* XXX not sure how this will work with multiple free lists */
	nm_i = kring->nr_hwcur;

	return (netmap_fl_refill(rxq, kring, nm_i, false));
	}

	static void
	iflib_netmap_intr(struct netmap_adapter *na, int onoff)
	{
	if_ctx_t ctx = na->ifp->if_softc;

	CTX_LOCK(ctx);
	if (onoff) {
	IFDI_INTR_ENABLE(ctx);
	} else {
	IFDI_INTR_DISABLE(ctx);
	}
	CTX_UNLOCK(ctx);
	}


	static int
	iflib_netmap_attach(if_ctx_t ctx)
	{
	struct netmap_adapter na;

	bzero(&na, sizeof(na));

	na.ifp = ctx->ifc_ifp;
	na.na_flags = NAF_BDG_MAYSLEEP;
	MPASS(ctx->ifc_softc_ctx.isc_ntxqsets);
	MPASS(ctx->ifc_softc_ctx.isc_nrxqsets);

	na.num_tx_desc = iflib_num_tx_descs(ctx);
	na.num_rx_desc = iflib_num_rx_descs(ctx);
	na.nm_txsync = iflib_netmap_txsync;
	na.nm_rxsync = iflib_netmap_rxsync;
	na.nm_register = iflib_netmap_register;
	na.nm_intr = iflib_netmap_intr;
	na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets;
	na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets;
	return (netmap_attach(&na));
	}

	static int
	iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
	{
	struct netmap_adapter *na = NA(ctx->ifc_ifp);
	struct netmap_slot *slot;

	slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
	if (slot == NULL)
	return (0);
	for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) {

	/*
	* In netmap mode, set the map for the packet buffer.
	* NOTE: Some drivers (not this one) also need to set
	* the physical buffer address in the NIC ring.
	* netmap_idx_n2k() maps a nic index, i, into the corresponding
	* netmap slot index, si
	*/
	int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i);
	netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i],
	NMB(na, slot + si));
	}
	return (1);
	}

	static int
	iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
	{
	struct netmap_adapter *na = NA(ctx->ifc_ifp);
	struct netmap_kring *kring;
	struct netmap_slot *slot;
	uint32_t nm_i;

	slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
	if (slot == NULL)
	return (0);
	kring = na->rx_rings[rxq->ifr_id];
	nm_i = netmap_idx_n2k(kring, 0);
	netmap_fl_refill(rxq, kring, nm_i, true);
	return (1);
	}

	static void
	iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on)
	{
	struct netmap_kring *kring;
	uint16_t txqid;

	txqid = txq->ift_id;
	kring = netmap_kring_on(NA(ctx->ifc_ifp), txqid, NR_TX);
	if (kring == NULL)
	return;

	if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) {
	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD);
	if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false))
	netmap_tx_irq(ctx->ifc_ifp, txqid);
	if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) {
	if (hz < 2000)
	*reset_on = 1;
	else
	*reset_on = hz / 1000;
	}
	}
	}

	#define iflib_netmap_detach(ifp) netmap_detach(ifp)

	#else
	#define iflib_netmap_txq_init(ctx, txq) (0)
	#define iflib_netmap_rxq_init(ctx, rxq) (0)
	#define iflib_netmap_detach(ifp)

	#define iflib_netmap_attach(ctx) (0)
	#define netmap_rx_irq(ifp, qid, budget) (0)
	#define netmap_tx_irq(ifp, qid) do {} while (0)
	#define iflib_netmap_timer_adjust(ctx, txq, reset_on)
	#endif

	#if defined(__i386__) \|\| defined(__amd64__)
	static __inline void
	prefetch(void *x)
	{
	__asm volatile("prefetcht0 %0" :: "m" ((unsigned long )x));
	}
	static __inline void
	prefetch2cachelines(void *x)
	{
	__asm volatile("prefetcht0 %0" :: "m" ((unsigned long )x));
	#if (CACHE_LINE_SIZE < 128)
	__asm volatile("prefetcht0 %0" :: "m" ((((unsigned long )x)+CACHE_LINE_SIZE/(sizeof(unsigned long)))));
	#endif
	}
	#else
	#define prefetch(x)
	#define prefetch2cachelines(x)
	#endif

	static void
	iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid)
	{
	iflib_fl_t fl;

	fl = &rxq->ifr_fl[flid];
	iru->iru_paddrs = fl->ifl_bus_addrs;
	iru->iru_idxs = fl->ifl_rxd_idxs;
	iru->iru_qsidx = rxq->ifr_id;
	iru->iru_buf_size = fl->ifl_buf_size;
	iru->iru_flidx = fl->ifl_id;
	}

	static void
	_iflib_dmamap_cb(void arg, bus_dma_segment_t segs, int nseg, int err)
	{
	if (err)
	return;
	(bus_addr_t ) arg = segs[0].ds_addr;
	}

	int
	iflib_dma_alloc_align(if_ctx_t ctx, int size, int align, iflib_dma_info_t dma, int mapflags)
	{
	int err;
	device_t dev = ctx->ifc_dev;

	err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
	align, 0, /* alignment, bounds */
	BUS_SPACE_MAXADDR, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	size, /* maxsize */
	1, /* nsegments */
	size, /* maxsegsize */
	BUS_DMA_ALLOCNOW, /* flags */
	NULL, /* lockfunc */
	NULL, /* lockarg */
	&dma->idi_tag);
	if (err) {
	device_printf(dev,
	"%s: bus_dma_tag_create failed: %d\n",
	__func__, err);
	goto fail_0;
	}

	err = bus_dmamem_alloc(dma->idi_tag, (void**) &dma->idi_vaddr,
	BUS_DMA_NOWAIT \| BUS_DMA_COHERENT \| BUS_DMA_ZERO, &dma->idi_map);
	if (err) {
	device_printf(dev,
	"%s: bus_dmamem_alloc(%ju) failed: %d\n",
	__func__, (uintmax_t)size, err);
	goto fail_1;
	}

	dma->idi_paddr = IF_BAD_DMA;
	err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr,
	size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags \| BUS_DMA_NOWAIT);
	if (err \|\| dma->idi_paddr == IF_BAD_DMA) {
	device_printf(dev,
	"%s: bus_dmamap_load failed: %d\n",
	__func__, err);
	goto fail_2;
	}

	dma->idi_size = size;
	return (0);

	fail_2:
	bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
	fail_1:
	bus_dma_tag_destroy(dma->idi_tag);
	fail_0:
	dma->idi_tag = NULL;

	return (err);
	}

	int
	iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags)
	{
	if_shared_ctx_t sctx = ctx->ifc_sctx;

	KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized"));

	return (iflib_dma_alloc_align(ctx, size, sctx->isc_q_align, dma, mapflags));
	}

	int
	iflib_dma_alloc_multi(if_ctx_t ctx, int sizes, iflib_dma_info_t dmalist, int mapflags, int count)
	{
	int i, err;
	iflib_dma_info_t *dmaiter;

	dmaiter = dmalist;
	for (i = 0; i < count; i++, dmaiter++) {
	if ((err = iflib_dma_alloc(ctx, sizes[i], *dmaiter, mapflags)) != 0)
	break;
	}
	if (err)
	iflib_dma_free_multi(dmalist, i);
	return (err);
	}

	void
	iflib_dma_free(iflib_dma_info_t dma)
	{
	if (dma->idi_tag == NULL)
	return;
	if (dma->idi_paddr != IF_BAD_DMA) {
	bus_dmamap_sync(dma->idi_tag, dma->idi_map,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(dma->idi_tag, dma->idi_map);
	dma->idi_paddr = IF_BAD_DMA;
	}
	if (dma->idi_vaddr != NULL) {
	bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
	dma->idi_vaddr = NULL;
	}
	bus_dma_tag_destroy(dma->idi_tag);
	dma->idi_tag = NULL;
	}

	void
	iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count)
	{
	int i;
	iflib_dma_info_t *dmaiter = dmalist;

	for (i = 0; i < count; i++, dmaiter++)
	iflib_dma_free(*dmaiter);
	}

	static int
	iflib_fast_intr(void *arg)
	{
	iflib_filter_info_t info = arg;
	struct grouptask *gtask = info->ifi_task;
	int result;

	DBG_COUNTER_INC(fast_intrs);
	if (info->ifi_filter != NULL) {
	result = info->ifi_filter(info->ifi_filter_arg);
	if ((result & FILTER_SCHEDULE_THREAD) == 0)
	return (result);
	}

	GROUPTASK_ENQUEUE(gtask);
	return (FILTER_HANDLED);
	}

	static int
	iflib_fast_intr_rxtx(void *arg)
	{
	iflib_filter_info_t info = arg;
	struct grouptask *gtask = info->ifi_task;
	if_ctx_t ctx;
	iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx;
	iflib_txq_t txq;
	void *sc;
	int i, cidx, result;
	qidx_t txqid;
	bool intr_enable, intr_legacy;

	DBG_COUNTER_INC(fast_intrs);
	if (info->ifi_filter != NULL) {
	result = info->ifi_filter(info->ifi_filter_arg);
	if ((result & FILTER_SCHEDULE_THREAD) == 0)
	return (result);
	}

	ctx = rxq->ifr_ctx;
	sc = ctx->ifc_softc;
	intr_enable = false;
	intr_legacy = !!(ctx->ifc_flags & IFC_LEGACY);
	MPASS(rxq->ifr_ntxqirq);
	for (i = 0; i < rxq->ifr_ntxqirq; i++) {
	txqid = rxq->ifr_txqid[i];
	txq = &ctx->ifc_txqs[txqid];
	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD);
	if (!ctx->isc_txd_credits_update(sc, txqid, false)) {
	if (intr_legacy)
	intr_enable = true;
	else
	IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid);
	continue;
	}
	GROUPTASK_ENQUEUE(&txq->ift_task);
	}
	if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ)
	cidx = rxq->ifr_cq_cidx;
	else
	cidx = rxq->ifr_fl[0].ifl_cidx;
	if (iflib_rxd_avail(ctx, rxq, cidx, 1))
	GROUPTASK_ENQUEUE(gtask);
	else {
	if (intr_legacy)
	intr_enable = true;
	else
	IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
	DBG_COUNTER_INC(rx_intr_enables);
	}
	if (intr_enable)
	IFDI_INTR_ENABLE(ctx);
	return (FILTER_HANDLED);
	}


	static int
	iflib_fast_intr_ctx(void *arg)
	{
	iflib_filter_info_t info = arg;
	struct grouptask *gtask = info->ifi_task;
	int result;

	DBG_COUNTER_INC(fast_intrs);
	if (info->ifi_filter != NULL) {
	result = info->ifi_filter(info->ifi_filter_arg);
	if ((result & FILTER_SCHEDULE_THREAD) == 0)
	return (result);
	}

	GROUPTASK_ENQUEUE(gtask);
	return (FILTER_HANDLED);
	}

	static int
	_iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
	driver_filter_t filter, driver_intr_t handler, void *arg,
	const char *name)
	{
	struct resource *res;
	void *tag = NULL;
	device_t dev = ctx->ifc_dev;
	int flags, i, rc;

	flags = RF_ACTIVE;
	if (ctx->ifc_flags & IFC_LEGACY)
	flags \|= RF_SHAREABLE;
	MPASS(rid < 512);
	i = rid;
	res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i, flags);
	if (res == NULL) {
	device_printf(dev,
	"failed to allocate IRQ for rid %d, name %s.\n", rid, name);
	return (ENOMEM);
	}
	irq->ii_res = res;
	KASSERT(filter == NULL \|\| handler == NULL, ("filter and handler can't both be non-NULL"));
	rc = bus_setup_intr(dev, res, INTR_MPSAFE \| INTR_TYPE_NET,
	filter, handler, arg, &tag);
	if (rc != 0) {
	device_printf(dev,
	"failed to setup interrupt for rid %d, name %s: %d\n",
	rid, name ? name : "unknown", rc);
	return (rc);
	} else if (name)
	bus_describe_intr(dev, res, tag, "%s", name);

	irq->ii_tag = tag;
	return (0);
	}

	/*********************************************************************
	*
	* Allocate DMA resources for TX buffers as well as memory for the TX
	* mbuf map. TX DMA maps (non-TSO/TSO) and TX mbuf map are kept in a
	* iflib_sw_tx_desc_array structure, storing all the information that
	* is needed to transmit a packet on the wire. This is called only
	* once at attach, setup is done every reset.
	*
	**********************************************************************/
	static int
	iflib_txsd_alloc(iflib_txq_t txq)
	{
	if_ctx_t ctx = txq->ift_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	device_t dev = ctx->ifc_dev;
	bus_size_t tsomaxsize;
	int err, nsegments, ntsosegments;
	bool tso;

	nsegments = scctx->isc_tx_nsegments;
	ntsosegments = scctx->isc_tx_tso_segments_max;
	tsomaxsize = scctx->isc_tx_tso_size_max;
	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_VLAN_MTU)
	tsomaxsize += sizeof(struct ether_vlan_header);
	MPASS(scctx->isc_ntxd[0] > 0);
	MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0);
	MPASS(nsegments > 0);
	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) {
	MPASS(ntsosegments > 0);
	MPASS(sctx->isc_tso_maxsize >= tsomaxsize);
	}

	/*
	* Set up DMA tags for TX buffers.
	*/
	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
	1, 0, /* alignment, bounds */
	BUS_SPACE_MAXADDR, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	sctx->isc_tx_maxsize, /* maxsize */
	nsegments, /* nsegments */
	sctx->isc_tx_maxsegsize, /* maxsegsize */
	0, /* flags */
	NULL, /* lockfunc */
	NULL, /* lockfuncarg */
	&txq->ift_buf_tag))) {
	device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err);
	device_printf(dev,"maxsize: %ju nsegments: %d maxsegsize: %ju\n",
	(uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize);
	goto fail;
	}
	tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0;
	if (tso && (err = bus_dma_tag_create(bus_get_dma_tag(dev),
	1, 0, /* alignment, bounds */
	BUS_SPACE_MAXADDR, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	tsomaxsize, /* maxsize */
	ntsosegments, /* nsegments */
	sctx->isc_tso_maxsegsize,/* maxsegsize */
	0, /* flags */
	NULL, /* lockfunc */
	NULL, /* lockfuncarg */
	&txq->ift_tso_buf_tag))) {
	device_printf(dev, "Unable to allocate TSO TX DMA tag: %d\n",
	err);
	goto fail;
	}

	/* Allocate memory for the TX mbuf map. */
	if (!(txq->ift_sds.ifsd_m =
	(struct mbuf *) malloc(sizeof(struct mbuf ) *
	scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev, "Unable to allocate TX mbuf map memory\n");
	err = ENOMEM;
	goto fail;
	}

	/*
	* Create the DMA maps for TX buffers.
	*/
	if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc(
	sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
	M_IFLIB, M_NOWAIT \| M_ZERO)) == NULL) {
	device_printf(dev,
	"Unable to allocate TX buffer DMA map memory\n");
	err = ENOMEM;
	goto fail;
	}
	if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc(
	sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
	M_IFLIB, M_NOWAIT \| M_ZERO)) == NULL) {
	device_printf(dev,
	"Unable to allocate TSO TX buffer map memory\n");
	err = ENOMEM;
	goto fail;
	}
	for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
	err = bus_dmamap_create(txq->ift_buf_tag, 0,
	&txq->ift_sds.ifsd_map[i]);
	if (err != 0) {
	device_printf(dev, "Unable to create TX DMA map\n");
	goto fail;
	}
	if (!tso)
	continue;
	err = bus_dmamap_create(txq->ift_tso_buf_tag, 0,
	&txq->ift_sds.ifsd_tso_map[i]);
	if (err != 0) {
	device_printf(dev, "Unable to create TSO TX DMA map\n");
	goto fail;
	}
	}
	return (0);
	fail:
	/* We free all, it handles case where we are in the middle */
	iflib_tx_structures_free(ctx);
	return (err);
	}

	static void
	iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
	{
	bus_dmamap_t map;

	if (txq->ift_sds.ifsd_map != NULL) {
	map = txq->ift_sds.ifsd_map[i];
	bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(txq->ift_buf_tag, map);
	bus_dmamap_destroy(txq->ift_buf_tag, map);
	txq->ift_sds.ifsd_map[i] = NULL;
	}

	if (txq->ift_sds.ifsd_tso_map != NULL) {
	map = txq->ift_sds.ifsd_tso_map[i];
	bus_dmamap_sync(txq->ift_tso_buf_tag, map,
	BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(txq->ift_tso_buf_tag, map);
	bus_dmamap_destroy(txq->ift_tso_buf_tag, map);
	txq->ift_sds.ifsd_tso_map[i] = NULL;
	}
	}

	static void
	iflib_txq_destroy(iflib_txq_t txq)
	{
	if_ctx_t ctx = txq->ift_ctx;

	for (int i = 0; i < txq->ift_size; i++)
	iflib_txsd_destroy(ctx, txq, i);

	if (txq->ift_br != NULL) {
	ifmp_ring_free(txq->ift_br);
	txq->ift_br = NULL;
	}

	mtx_destroy(&txq->ift_mtx);

	if (txq->ift_sds.ifsd_map != NULL) {
	free(txq->ift_sds.ifsd_map, M_IFLIB);
	txq->ift_sds.ifsd_map = NULL;
	}
	if (txq->ift_sds.ifsd_tso_map != NULL) {
	free(txq->ift_sds.ifsd_tso_map, M_IFLIB);
	txq->ift_sds.ifsd_tso_map = NULL;
	}
	if (txq->ift_sds.ifsd_m != NULL) {
	free(txq->ift_sds.ifsd_m, M_IFLIB);
	txq->ift_sds.ifsd_m = NULL;
	}
	if (txq->ift_buf_tag != NULL) {
	bus_dma_tag_destroy(txq->ift_buf_tag);
	txq->ift_buf_tag = NULL;
	}
	if (txq->ift_tso_buf_tag != NULL) {
	bus_dma_tag_destroy(txq->ift_tso_buf_tag);
	txq->ift_tso_buf_tag = NULL;
	}
	if (txq->ift_ifdi != NULL) {
	free(txq->ift_ifdi, M_IFLIB);
	}
	}

	static void
	iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
	{
	struct mbuf **mp;

	mp = &txq->ift_sds.ifsd_m[i];
	if (*mp == NULL)
	return;

	if (txq->ift_sds.ifsd_map != NULL) {
	bus_dmamap_sync(txq->ift_buf_tag,
	txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]);
	}
	if (txq->ift_sds.ifsd_tso_map != NULL) {
	bus_dmamap_sync(txq->ift_tso_buf_tag,
	txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(txq->ift_tso_buf_tag,
	txq->ift_sds.ifsd_tso_map[i]);
	}
	m_free(*mp);
	DBG_COUNTER_INC(tx_frees);
	*mp = NULL;
	}

	static int
	iflib_txq_setup(iflib_txq_t txq)
	{
	if_ctx_t ctx = txq->ift_ctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	iflib_dma_info_t di;
	int i;

	/* Set number of descriptors available */
	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	/* XXX make configurable */
	txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;

	/* Reset indices */
	txq->ift_cidx_processed = 0;
	txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
	txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];

	for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
	bzero((void *)di->idi_vaddr, di->idi_size);

	IFDI_TXQ_SETUP(ctx, txq->ift_id);
	for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
	bus_dmamap_sync(di->idi_tag, di->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	return (0);
	}

	/*********************************************************************
	*
	* Allocate DMA resources for RX buffers as well as memory for the RX
	* mbuf map, direct RX cluster pointer map and RX cluster bus address
	* map. RX DMA map, RX mbuf map, direct RX cluster pointer map and
	* RX cluster map are kept in a iflib_sw_rx_desc_array structure.
	* Since we use use one entry in iflib_sw_rx_desc_array per received
	* packet, the maximum number of entries we'll need is equal to the
	* number of hardware receive descriptors that we've allocated.
	*
	**********************************************************************/
	static int
	iflib_rxsd_alloc(iflib_rxq_t rxq)
	{
	if_ctx_t ctx = rxq->ifr_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	device_t dev = ctx->ifc_dev;
	iflib_fl_t fl;
	int err;

	MPASS(scctx->isc_nrxd[0] > 0);
	MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0);

	fl = rxq->ifr_fl;
	for (int i = 0; i < rxq->ifr_nfl; i++, fl++) {
	fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */
	/* Set up DMA tag for RX buffers. */
	err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
	1, 0, /* alignment, bounds */
	BUS_SPACE_MAXADDR, /* lowaddr */
	BUS_SPACE_MAXADDR, /* highaddr */
	NULL, NULL, /* filter, filterarg */
	sctx->isc_rx_maxsize, /* maxsize */
	sctx->isc_rx_nsegments, /* nsegments */
	sctx->isc_rx_maxsegsize, /* maxsegsize */
	0, /* flags */
	NULL, /* lockfunc */
	NULL, /* lockarg */
	&fl->ifl_buf_tag);
	if (err) {
	device_printf(dev,
	"Unable to allocate RX DMA tag: %d\n", err);
	goto fail;
	}

	/* Allocate memory for the RX mbuf map. */
	if (!(fl->ifl_sds.ifsd_m =
	(struct mbuf *) malloc(sizeof(struct mbuf ) *
	scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev,
	"Unable to allocate RX mbuf map memory\n");
	err = ENOMEM;
	goto fail;
	}

	/* Allocate memory for the direct RX cluster pointer map. */
	if (!(fl->ifl_sds.ifsd_cl =
	(caddr_t ) malloc(sizeof(caddr_t)
	scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev,
	"Unable to allocate RX cluster map memory\n");
	err = ENOMEM;
	goto fail;
	}

	/* Allocate memory for the RX cluster bus address map. */
	if (!(fl->ifl_sds.ifsd_ba =
	(bus_addr_t ) malloc(sizeof(bus_addr_t)
	scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev,
	"Unable to allocate RX bus address map memory\n");
	err = ENOMEM;
	goto fail;
	}

	/*
	* Create the DMA maps for RX buffers.
	*/
	if (!(fl->ifl_sds.ifsd_map =
	(bus_dmamap_t ) malloc(sizeof(bus_dmamap_t) scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev,
	"Unable to allocate RX buffer DMA map memory\n");
	err = ENOMEM;
	goto fail;
	}
	for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) {
	err = bus_dmamap_create(fl->ifl_buf_tag, 0,
	&fl->ifl_sds.ifsd_map[i]);
	if (err != 0) {
	device_printf(dev, "Unable to create RX buffer DMA map\n");
	goto fail;
	}
	}
	}
	return (0);

	fail:
	iflib_rx_structures_free(ctx);
	return (err);
	}


	/*
	* Internal service routines
	*/

	struct rxq_refill_cb_arg {
	int error;
	bus_dma_segment_t seg;
	int nseg;
	};

	static void
	_rxq_refill_cb(void arg, bus_dma_segment_t segs, int nseg, int error)
	{
	struct rxq_refill_cb_arg *cb_arg = arg;

	cb_arg->error = error;
	cb_arg->seg = segs[0];
	cb_arg->nseg = nseg;
	}

	/**
	* iflib_fl_refill - refill an rxq free-buffer list
	* @ctx: the iflib context
	* @fl: the free list to refill
	* @count: the number of new buffers to allocate
	*
	* (Re)populate an rxq free-buffer list with up to @count new packet buffers.
	* The caller must assure that @count does not exceed the queue's capacity.
	*/
	static uint8_t
	iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
	{
	struct if_rxd_update iru;
	struct rxq_refill_cb_arg cb_arg;
	struct mbuf *m;
	caddr_t cl, *sd_cl;
	struct mbuf **sd_m;
	bus_dmamap_t *sd_map;
	bus_addr_t bus_addr, *sd_ba;
	int err, frag_idx, i, idx, n, pidx;
	qidx_t credits;

	sd_m = fl->ifl_sds.ifsd_m;
	sd_map = fl->ifl_sds.ifsd_map;
	sd_cl = fl->ifl_sds.ifsd_cl;
	sd_ba = fl->ifl_sds.ifsd_ba;
	pidx = fl->ifl_pidx;
	idx = pidx;
	frag_idx = fl->ifl_fragidx;
	credits = fl->ifl_credits;

	i = 0;
	n = count;
	MPASS(n > 0);
	MPASS(credits + n <= fl->ifl_size);

	if (pidx < fl->ifl_cidx)
	MPASS(pidx + n <= fl->ifl_cidx);
	if (pidx == fl->ifl_cidx && (credits < fl->ifl_size))
	MPASS(fl->ifl_gen == 0);
	if (pidx > fl->ifl_cidx)
	MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);

	DBG_COUNTER_INC(fl_refills);
	if (n > 8)
	DBG_COUNTER_INC(fl_refills_large);
	iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
	while (n-- > 0) {
	/*
	* We allocate an uninitialized mbuf + cluster, mbuf is
	* initialized after rx.
	*
	* If the cluster is still set then we know a minimum sized
	* packet was received
	*/
	bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size,
	&frag_idx);
	if (frag_idx < 0)
	bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
	MPASS(frag_idx >= 0);
	if ((cl = sd_cl[frag_idx]) == NULL) {
	cl = uma_zalloc(fl->ifl_zone, M_NOWAIT);
	if (__predict_false(cl == NULL))
	break;

	cb_arg.error = 0;
	MPASS(sd_map != NULL);
	err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx],
	cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg,
	BUS_DMA_NOWAIT);
	if (__predict_false(err != 0 \|\| cb_arg.error)) {
	uma_zfree(fl->ifl_zone, cl);
	break;
	}

	sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr;
	sd_cl[frag_idx] = cl;
	#if MEMORY_LOGGING
	fl->ifl_cl_enqueued++;
	#endif
	} else {
	bus_addr = sd_ba[frag_idx];
	}
	bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx],
	BUS_DMASYNC_PREREAD);

	if (sd_m[frag_idx] == NULL) {
	m = m_gethdr(M_NOWAIT, MT_NOINIT);
	if (__predict_false(m == NULL))
	break;
	sd_m[frag_idx] = m;
	}
	bit_set(fl->ifl_rx_bitmap, frag_idx);
	#if MEMORY_LOGGING
	fl->ifl_m_enqueued++;
	#endif

	DBG_COUNTER_INC(rx_allocs);
	fl->ifl_rxd_idxs[i] = frag_idx;
	fl->ifl_bus_addrs[i] = bus_addr;
	credits++;
	i++;
	MPASS(credits <= fl->ifl_size);
	if (++idx == fl->ifl_size) {
	#ifdef INVARIANTS
	fl->ifl_gen = 1;
	#endif
	idx = 0;
	}
	if (n == 0 \|\| i == IFLIB_MAX_RX_REFRESH) {
	iru.iru_pidx = pidx;
	iru.iru_count = i;
	ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
	fl->ifl_pidx = idx;
	fl->ifl_credits = credits;
	pidx = idx;
	i = 0;
	}
	}

	if (n < count - 1) {
	if (i != 0) {
	iru.iru_pidx = pidx;
	iru.iru_count = i;
	ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
	fl->ifl_pidx = idx;
	fl->ifl_credits = credits;
	}
	DBG_COUNTER_INC(rxd_flush);
	if (fl->ifl_pidx == 0)
	pidx = fl->ifl_size - 1;
	else
	pidx = fl->ifl_pidx - 1;

	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id,
	fl->ifl_id, pidx);
	if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) {
	fl->ifl_fragidx = frag_idx + 1;
	if (fl->ifl_fragidx == fl->ifl_size)
	fl->ifl_fragidx = 0;
	} else {
	fl->ifl_fragidx = frag_idx;
	}
	}

	return (n == -1 ? 0 : IFLIB_RXEOF_EMPTY);
	}

	static inline uint8_t
	iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl)
	{
	/* we avoid allowing pidx to catch up with cidx as it confuses ixl */
	int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
	#ifdef INVARIANTS
	int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
	#endif

	MPASS(fl->ifl_credits <= fl->ifl_size);
	MPASS(reclaimable == delta);

	if (reclaimable > 0)
	return (iflib_fl_refill(ctx, fl, reclaimable));
	return (0);
	}

	uint8_t
	iflib_in_detach(if_ctx_t ctx)
	{
	bool in_detach;

	STATE_LOCK(ctx);
	in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH);
	STATE_UNLOCK(ctx);
	return (in_detach);
	}

	static void
	iflib_fl_bufs_free(iflib_fl_t fl)
	{
	iflib_dma_info_t idi = fl->ifl_ifdi;
	bus_dmamap_t sd_map;
	uint32_t i;

	for (i = 0; i < fl->ifl_size; i++) {
	struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
	caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];

	if (*sd_cl != NULL) {
	sd_map = fl->ifl_sds.ifsd_map[i];
	bus_dmamap_sync(fl->ifl_buf_tag, sd_map,
	BUS_DMASYNC_POSTREAD);
	bus_dmamap_unload(fl->ifl_buf_tag, sd_map);
	uma_zfree(fl->ifl_zone, *sd_cl);
	*sd_cl = NULL;
	if (*sd_m != NULL) {
	m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
	uma_zfree(zone_mbuf, *sd_m);
	*sd_m = NULL;
	}
	} else {
	MPASS(*sd_m == NULL);
	}
	#if MEMORY_LOGGING
	fl->ifl_m_dequeued++;
	fl->ifl_cl_dequeued++;
	#endif
	}
	#ifdef INVARIANTS
	for (i = 0; i < fl->ifl_size; i++) {
	MPASS(fl->ifl_sds.ifsd_cl[i] == NULL);
	MPASS(fl->ifl_sds.ifsd_m[i] == NULL);
	}
	#endif
	/*
	* Reset free list values
	*/
	fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0;
	bzero(idi->idi_vaddr, idi->idi_size);
	}

	/*********************************************************************
	*
	* Initialize a free list and its buffers.
	*
	**********************************************************************/
	static int
	iflib_fl_setup(iflib_fl_t fl)
	{
	iflib_rxq_t rxq = fl->ifl_rxq;
	if_ctx_t ctx = rxq->ifr_ctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	int qidx;

	bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1);
	/*
	** Free current RX buffer structs and their mbufs
	*/
	iflib_fl_bufs_free(fl);
	/* Now replenish the mbufs */
	MPASS(fl->ifl_credits == 0);
	qidx = rxq->ifr_fl_offset + fl->ifl_id;
	if (scctx->isc_rxd_buf_size[qidx] != 0)
	fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx];
	else
	fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz;
	/*
	* ifl_buf_size may be a driver-supplied value, so pull it up
	* to the selected mbuf size.
	*/
	fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size);
	if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
	ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
	fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
	fl->ifl_zone = m_getzone(fl->ifl_buf_size);


	/* avoid pre-allocating zillions of clusters to an idle card
	* potentially speeding up attach
	*/
	(void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
	MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
	if (min(128, fl->ifl_size) != fl->ifl_credits)
	return (ENOBUFS);
	/*
	* handle failure
	*/
	MPASS(rxq != NULL);
	MPASS(fl->ifl_ifdi != NULL);
	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	return (0);
	}

	/*********************************************************************
	*
	* Free receive ring data structures
	*
	**********************************************************************/
	static void
	iflib_rx_sds_free(iflib_rxq_t rxq)
	{
	iflib_fl_t fl;
	int i, j;

	if (rxq->ifr_fl != NULL) {
	for (i = 0; i < rxq->ifr_nfl; i++) {
	fl = &rxq->ifr_fl[i];
	if (fl->ifl_buf_tag != NULL) {
	if (fl->ifl_sds.ifsd_map != NULL) {
	for (j = 0; j < fl->ifl_size; j++) {
	bus_dmamap_sync(
	fl->ifl_buf_tag,
	fl->ifl_sds.ifsd_map[j],
	BUS_DMASYNC_POSTREAD);
	bus_dmamap_unload(
	fl->ifl_buf_tag,
	fl->ifl_sds.ifsd_map[j]);
	bus_dmamap_destroy(
	fl->ifl_buf_tag,
	fl->ifl_sds.ifsd_map[j]);
	}
	}
	bus_dma_tag_destroy(fl->ifl_buf_tag);
	fl->ifl_buf_tag = NULL;
	}
	free(fl->ifl_sds.ifsd_m, M_IFLIB);
	free(fl->ifl_sds.ifsd_cl, M_IFLIB);
	free(fl->ifl_sds.ifsd_ba, M_IFLIB);
	free(fl->ifl_sds.ifsd_map, M_IFLIB);
	fl->ifl_sds.ifsd_m = NULL;
	fl->ifl_sds.ifsd_cl = NULL;
	fl->ifl_sds.ifsd_ba = NULL;
	fl->ifl_sds.ifsd_map = NULL;
	}
	free(rxq->ifr_fl, M_IFLIB);
	rxq->ifr_fl = NULL;
	free(rxq->ifr_ifdi, M_IFLIB);
	rxq->ifr_ifdi = NULL;
	rxq->ifr_cq_cidx = 0;
	}
	}

	/*
	* Timer routine
	*/
	static void
	iflib_timer(void *arg)
	{
	iflib_txq_t txq = arg;
	if_ctx_t ctx = txq->ift_ctx;
	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
	uint64_t this_tick = ticks;
	uint32_t reset_on = hz / 2;

	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
	return;

	/*
	** Check on the state of the TX queue(s), this
	** can be done without the lock because its RO
	** and the HUNG state will be static if set.
	*/
	if (this_tick - txq->ift_last_timer_tick >= hz / 2) {
	txq->ift_last_timer_tick = this_tick;
	IFDI_TIMER(ctx, txq->ift_id);
	if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
	((txq->ift_cleaned_prev == txq->ift_cleaned) \|\|
	(sctx->isc_pause_frames == 0)))
	goto hung;

	if (txq->ift_qstatus != IFLIB_QUEUE_IDLE &&
	ifmp_ring_is_stalled(txq->ift_br)) {
	KASSERT(ctx->ifc_link_state == LINK_STATE_UP, ("queue can't be marked as hung if interface is down"));
	txq->ift_qstatus = IFLIB_QUEUE_HUNG;
	}
	txq->ift_cleaned_prev = txq->ift_cleaned;
	}
	#ifdef DEV_NETMAP
	if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP)
	iflib_netmap_timer_adjust(ctx, txq, &reset_on);
	#endif
	/* handle any laggards */
	if (txq->ift_db_pending)
	GROUPTASK_ENQUEUE(&txq->ift_task);

	sctx->isc_pause_frames = 0;
	if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)
	callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu);
	return;

	hung:
	device_printf(ctx->ifc_dev,
	"Watchdog timeout (TX: %d desc avail: %d pidx: %d) -- resetting\n",
	txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
	STATE_LOCK(ctx);
	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
	ctx->ifc_flags \|= (IFC_DO_WATCHDOG\|IFC_DO_RESET);
	iflib_admin_intr_deferred(ctx);
	STATE_UNLOCK(ctx);
	}

	static uint16_t
	iflib_get_mbuf_size_for(unsigned int size)
	{

	if (size <= MCLBYTES)
	return (MCLBYTES);
	else
	return (MJUMPAGESIZE);
	}

	static void
	iflib_calc_rx_mbuf_sz(if_ctx_t ctx)
	{
	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;

	/*
	* XXX don't set the max_frame_size to larger
	* than the hardware can handle
	*/
	ctx->ifc_rx_mbuf_sz =
	iflib_get_mbuf_size_for(sctx->isc_max_frame_size);
	}

	uint32_t
	iflib_get_rx_mbuf_sz(if_ctx_t ctx)
	{

	return (ctx->ifc_rx_mbuf_sz);
	}

	static void
	iflib_init_locked(if_ctx_t ctx)
	{
	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_t ifp = ctx->ifc_ifp;
	iflib_fl_t fl;
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	int i, j, tx_ip_csum_flags, tx_ip6_csum_flags;

	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
	IFDI_INTR_DISABLE(ctx);

	tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP \| CSUM_TCP \| CSUM_UDP \| CSUM_SCTP);
	tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP \| CSUM_IP6_UDP \| CSUM_IP6_SCTP);
	/* Set hardware offload abilities */
	if_clearhwassist(ifp);
	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
	if_sethwassistbits(ifp, tx_ip_csum_flags, 0);
	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
	if_sethwassistbits(ifp, tx_ip6_csum_flags, 0);
	if (if_getcapenable(ifp) & IFCAP_TSO4)
	if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
	if (if_getcapenable(ifp) & IFCAP_TSO6)
	if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);

	for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	CALLOUT_UNLOCK(txq);
	iflib_netmap_txq_init(ctx, txq);
	}

	/*
	* Calculate a suitable Rx mbuf size prior to calling IFDI_INIT, so
	* that drivers can use the value when setting up the hardware receive
	* buffers.
	*/
	iflib_calc_rx_mbuf_sz(ctx);

	#ifdef INVARIANTS
	i = if_getdrvflags(ifp);
	#endif
	IFDI_INIT(ctx);
	MPASS(if_getdrvflags(ifp) == i);
	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
	if (iflib_netmap_rxq_init(ctx, rxq) > 0) {
	/* This rxq is in netmap mode. Skip normal init. */
	continue;
	}
	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
	if (iflib_fl_setup(fl)) {
	device_printf(ctx->ifc_dev,
	"setting up free list %d failed - "
	"check cluster settings\n", j);
	goto done;
	}
	}
	}
	done:
	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
	IFDI_INTR_ENABLE(ctx);
	txq = ctx->ifc_txqs;
	for (i = 0; i < sctx->isc_ntxqsets; i++, txq++)
	callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq,
	txq->ift_timer.c_cpu);
	}

	static int
	iflib_media_change(if_t ifp)
	{
	if_ctx_t ctx = if_getsoftc(ifp);
	int err;

	CTX_LOCK(ctx);
	if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0)
	iflib_init_locked(ctx);
	CTX_UNLOCK(ctx);
	return (err);
	}

	static void
	iflib_media_status(if_t ifp, struct ifmediareq *ifmr)
	{
	if_ctx_t ctx = if_getsoftc(ifp);

	CTX_LOCK(ctx);
	IFDI_UPDATE_ADMIN_STATUS(ctx);
	IFDI_MEDIA_STATUS(ctx, ifmr);
	CTX_UNLOCK(ctx);
	}

	void
	iflib_stop(if_ctx_t ctx)
	{
	iflib_txq_t txq = ctx->ifc_txqs;
	iflib_rxq_t rxq = ctx->ifc_rxqs;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	iflib_dma_info_t di;
	iflib_fl_t fl;
	int i, j;

	/* Tell the stack that the interface is no longer active */
	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);

	IFDI_INTR_DISABLE(ctx);
	DELAY(1000);
	IFDI_STOP(ctx);
	DELAY(1000);

	iflib_debug_reset();
	/* Wait for current tx queue users to exit to disarm watchdog timer. */
	for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
	/* make sure all transmitters have completed before proceeding XXX */

	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	CALLOUT_UNLOCK(txq);

	/* clean any enqueued buffers */
	iflib_ifmp_purge(txq);
	/* Free any existing tx buffers. */
	for (j = 0; j < txq->ift_size; j++) {
	iflib_txsd_free(ctx, txq, j);
	}
	txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
	txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
	txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
	txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
	txq->ift_pullups = 0;
	ifmp_ring_reset_stats(txq->ift_br);
	for (j = 0, di = txq->ift_ifdi; j < sctx->isc_ntxqs; j++, di++)
	bzero((void *)di->idi_vaddr, di->idi_size);
	}
	for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) {
	/* make sure all transmitters have completed before proceeding XXX */

	rxq->ifr_cq_cidx = 0;
	for (j = 0, di = rxq->ifr_ifdi; j < sctx->isc_nrxqs; j++, di++)
	bzero((void *)di->idi_vaddr, di->idi_size);
	/* also resets the free lists pidx/cidx */
	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	iflib_fl_bufs_free(fl);
	}
	}

	static inline caddr_t
	calc_next_rxd(iflib_fl_t fl, int cidx)
	{
	qidx_t size;
	int nrxd;
	caddr_t start, end, cur, next;

	nrxd = fl->ifl_size;
	size = fl->ifl_rxd_size;
	start = fl->ifl_ifdi->idi_vaddr;

	if (__predict_false(size == 0))
	return (start);
	cur = start + size*cidx;
	end = start + size*nrxd;
	next = CACHE_PTR_NEXT(cur);
	return (next < end ? next : start);
	}

	static inline void
	prefetch_pkts(iflib_fl_t fl, int cidx)
	{
	int nextptr;
	int nrxd = fl->ifl_size;
	caddr_t next_rxd;


	nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd-1);
	prefetch(&fl->ifl_sds.ifsd_m[nextptr]);
	prefetch(&fl->ifl_sds.ifsd_cl[nextptr]);
	next_rxd = calc_next_rxd(fl, cidx);
	prefetch(next_rxd);
	prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]);
	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]);
	}

	static struct mbuf *
	rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, bool unload, if_rxsd_t sd,
	int *pf_rv, if_rxd_info_t ri)
	{
	bus_dmamap_t map;
	iflib_fl_t fl;
	caddr_t payload;
	struct mbuf *m;
	int flid, cidx, len, next;

	map = NULL;
	flid = irf->irf_flid;
	cidx = irf->irf_idx;
	fl = &rxq->ifr_fl[flid];
	sd->ifsd_fl = fl;
	m = fl->ifl_sds.ifsd_m[cidx];
	sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx];
	fl->ifl_credits--;
	#if MEMORY_LOGGING
	fl->ifl_m_dequeued++;
	#endif
	if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH)
	prefetch_pkts(fl, cidx);
	next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1);
	prefetch(&fl->ifl_sds.ifsd_map[next]);
	map = fl->ifl_sds.ifsd_map[cidx];

	bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD);

	if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL &&
	irf->irf_len != 0) {
	payload = *sd->ifsd_cl;
	payload += ri->iri_pad;
	len = ri->iri_len - ri->iri_pad;
	*pf_rv = pfil_run_hooks(rxq->pfil, payload, ri->iri_ifp,
	len \| PFIL_MEMPTR \| PFIL_IN, NULL);
	switch (*pf_rv) {
	case PFIL_DROPPED:
	case PFIL_CONSUMED:
	/*
	* The filter ate it. Everything is recycled.
	*/
	m = NULL;
	unload = 0;
	break;
	case PFIL_REALLOCED:
	/*
	* The filter copied it. Everything is recycled.
	*/
	m = pfil_mem2mbuf(payload);
	unload = 0;
	break;
	case PFIL_PASS:
	/*
	* Filter said it was OK, so receive like
	* normal
	*/
	fl->ifl_sds.ifsd_m[cidx] = NULL;
	break;
	default:
	MPASS(0);
	}
	} else {
	fl->ifl_sds.ifsd_m[cidx] = NULL;
	*pf_rv = PFIL_PASS;
	}

	if (unload && irf->irf_len != 0)
	bus_dmamap_unload(fl->ifl_buf_tag, map);
	fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1);
	if (__predict_false(fl->ifl_cidx == 0))
	fl->ifl_gen = 0;
	bit_clear(fl->ifl_rx_bitmap, cidx);
	return (m);
	}

	static struct mbuf *
	assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd, int *pf_rv)
	{
	struct mbuf m, mh, *mt;
	caddr_t cl;
	int *pf_rv_ptr, flags, i, padlen;
	bool consumed;

	i = 0;
	mh = NULL;
	consumed = false;
	*pf_rv = PFIL_PASS;
	pf_rv_ptr = pf_rv;
	do {
	m = rxd_frag_to_sd(rxq, &ri->iri_frags[i], !consumed, sd,
	pf_rv_ptr, ri);

	MPASS(*sd->ifsd_cl != NULL);

	/*
	* Exclude zero-length frags & frags from
	* packets the filter has consumed or dropped
	*/
	if (ri->iri_frags[i].irf_len == 0 \|\| consumed \|\|
	pf_rv == PFIL_CONSUMED \|\| pf_rv == PFIL_DROPPED) {
	if (mh == NULL) {
	/* everything saved here */
	consumed = true;
	pf_rv_ptr = NULL;
	continue;
	}
	/* XXX we can save the cluster here, but not the mbuf */
	m_init(m, M_NOWAIT, MT_DATA, 0);
	m_free(m);
	continue;
	}
	if (mh == NULL) {
	flags = M_PKTHDR\|M_EXT;
	mh = mt = m;
	padlen = ri->iri_pad;
	} else {
	flags = M_EXT;
	mt->m_next = m;
	mt = m;
	/* assuming padding is only on the first fragment */
	padlen = 0;
	}
	cl = *sd->ifsd_cl;
	*sd->ifsd_cl = NULL;

	/* Can these two be made one ? */
	m_init(m, M_NOWAIT, MT_DATA, flags);
	m_cljset(m, cl, sd->ifsd_fl->ifl_cltype);
	/*
	* These must follow m_init and m_cljset
	*/
	m->m_data += padlen;
	ri->iri_len -= padlen;
	m->m_len = ri->iri_frags[i].irf_len;
	} while (++i < ri->iri_nfrags);

	return (mh);
	}

	/*
	* Process one software descriptor
	*/
	static struct mbuf *
	iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
	{
	struct if_rxsd sd;
	struct mbuf *m;
	int pf_rv;

	/* should I merge this back in now that the two paths are basically duplicated? */
	if (ri->iri_nfrags == 1 &&
	ri->iri_frags[0].irf_len != 0 &&
	ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) {
	m = rxd_frag_to_sd(rxq, &ri->iri_frags[0], false, &sd,
	&pf_rv, ri);
	if (pf_rv != PFIL_PASS && pf_rv != PFIL_REALLOCED)
	return (m);
	if (pf_rv == PFIL_PASS) {
	m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR);
	#ifndef __NO_STRICT_ALIGNMENT
	if (!IP_ALIGNED(m))
	m->m_data += 2;
	#endif
	memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len);
	m->m_len = ri->iri_frags[0].irf_len;
	}
	} else {
	m = assemble_segments(rxq, ri, &sd, &pf_rv);
	if (m == NULL)
	return (NULL);
	if (pf_rv != PFIL_PASS && pf_rv != PFIL_REALLOCED)
	return (m);
	}
	m->m_pkthdr.len = ri->iri_len;
	m->m_pkthdr.rcvif = ri->iri_ifp;
	m->m_flags \|= ri->iri_flags;
	m->m_pkthdr.ether_vtag = ri->iri_vtag;
	m->m_pkthdr.flowid = ri->iri_flowid;
	M_HASHTYPE_SET(m, ri->iri_rsstype);
	m->m_pkthdr.csum_flags = ri->iri_csum_flags;
	m->m_pkthdr.csum_data = ri->iri_csum_data;
	return (m);
	}

	#if defined(INET6) \|\| defined(INET)
	static void
	iflib_get_ip_forwarding(struct lro_ctrl lc, bool v4, bool *v6)
	{
	CURVNET_SET(lc->ifp->if_vnet);
	#if defined(INET6)
	*v6 = V_ip6_forwarding;
	#endif
	#if defined(INET)
	*v4 = V_ipforwarding;
	#endif
	CURVNET_RESTORE();
	}

	/*
	* Returns true if it's possible this packet could be LROed.
	* if it returns false, it is guaranteed that tcp_lro_rx()
	* would not return zero.
	*/
	static bool
	iflib_check_lro_possible(struct mbuf *m, bool v4_forwarding, bool v6_forwarding)
	{
	struct ether_header *eh;

	eh = mtod(m, struct ether_header *);
	switch (eh->ether_type) {
	#if defined(INET6)
	case htons(ETHERTYPE_IPV6):
	return (!v6_forwarding);
	#endif
	#if defined (INET)
	case htons(ETHERTYPE_IP):
	return (!v4_forwarding);
	#endif
	}

	return false;
	}
	#else
	static void
	iflib_get_ip_forwarding(struct lro_ctrl lc __unused, bool v4 __unused, bool *v6 __unused)
	{
	}
	#endif

	static void
	_task_fn_rx_watchdog(void *context)
	{
	iflib_rxq_t rxq = context;

	GROUPTASK_ENQUEUE(&rxq->ifr_task);
	}

	static uint8_t
	iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
	{
	if_t ifp;
	if_ctx_t ctx = rxq->ifr_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	int avail, i;
	qidx_t *cidxp;
	struct if_rxd_info ri;
	int err, budget_left, rx_bytes, rx_pkts;
	iflib_fl_t fl;
	int lro_enabled;
	bool v4_forwarding, v6_forwarding, lro_possible;
	uint8_t retval = 0;

	/*
	* XXX early demux data packets so that if_input processing only handles
	* acks in interrupt context
	*/
	struct mbuf m, mh, mt, mf;

	NET_EPOCH_ASSERT();

	lro_possible = v4_forwarding = v6_forwarding = false;
	ifp = ctx->ifc_ifp;
	mh = mt = NULL;
	MPASS(budget > 0);
	rx_pkts = rx_bytes = 0;
	if (sctx->isc_flags & IFLIB_HAS_RXCQ)
	cidxp = &rxq->ifr_cq_cidx;
	else
	cidxp = &rxq->ifr_fl[0].ifl_cidx;
	if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) {
	for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
	retval \|= iflib_fl_refill_all(ctx, fl);
	DBG_COUNTER_INC(rx_unavail);
	return (retval);
	}

	/* pfil needs the vnet to be set */
	CURVNET_SET_QUIET(ifp->if_vnet);
	for (budget_left = budget; budget_left > 0 && avail > 0;) {
	if (__predict_false(!CTX_ACTIVE(ctx))) {
	DBG_COUNTER_INC(rx_ctx_inactive);
	break;
	}
	/*
	* Reset client set fields to their default values
	*/
	rxd_info_zero(&ri);
	ri.iri_qsidx = rxq->ifr_id;
	ri.iri_cidx = *cidxp;
	ri.iri_ifp = ifp;
	ri.iri_frags = rxq->ifr_frags;
	err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);

	if (err)
	goto err;
	rx_pkts += 1;
	rx_bytes += ri.iri_len;
	if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
	*cidxp = ri.iri_cidx;
	/* Update our consumer index */
	/* XXX NB: shurd - check if this is still safe */
	while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0])
	rxq->ifr_cq_cidx -= scctx->isc_nrxd[0];
	/* was this only a completion queue message? */
	if (__predict_false(ri.iri_nfrags == 0))
	continue;
	}
	MPASS(ri.iri_nfrags != 0);
	MPASS(ri.iri_len != 0);

	/* will advance the cidx on the corresponding free lists */
	m = iflib_rxd_pkt_get(rxq, &ri);
	avail--;
	budget_left--;
	if (avail == 0 && budget_left)
	avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left);

	if (__predict_false(m == NULL))
	continue;

	/* imm_pkt: -- cxgb */
	if (mh == NULL)
	mh = mt = m;
	else {
	mt->m_nextpkt = m;
	mt = m;
	}
	}
	CURVNET_RESTORE();
	/* make sure that we can refill faster than drain */
	for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
	retval \|= iflib_fl_refill_all(ctx, fl);

	lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
	if (lro_enabled)
	iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding);
	mt = mf = NULL;
	while (mh != NULL) {
	m = mh;
	mh = mh->m_nextpkt;
	m->m_nextpkt = NULL;
	#ifndef __NO_STRICT_ALIGNMENT
	if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL)
	continue;
	#endif
	rx_bytes += m->m_pkthdr.len;
	rx_pkts++;
	#if defined(INET6) \|\| defined(INET)
	if (lro_enabled) {
	if (!lro_possible) {
	lro_possible = iflib_check_lro_possible(m, v4_forwarding, v6_forwarding);
	if (lro_possible && mf != NULL) {
	ifp->if_input(ifp, mf);
	DBG_COUNTER_INC(rx_if_input);
	mt = mf = NULL;
	}
	}
	if ((m->m_pkthdr.csum_flags & (CSUM_L4_CALC\|CSUM_L4_VALID)) ==
	(CSUM_L4_CALC\|CSUM_L4_VALID)) {
	if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
	continue;
	}
	}
	#endif
	if (lro_possible) {
	ifp->if_input(ifp, m);
	DBG_COUNTER_INC(rx_if_input);
	continue;
	}

	if (mf == NULL)
	mf = m;
	if (mt != NULL)
	mt->m_nextpkt = m;
	mt = m;
	}
	if (mf != NULL) {
	ifp->if_input(ifp, mf);
	DBG_COUNTER_INC(rx_if_input);
	}

	if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
	if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts);

	/*
	* Flush any outstanding LRO work
	*/
	#if defined(INET6) \|\| defined(INET)
	tcp_lro_flush_all(&rxq->ifr_lc);
	#endif
	if (avail != 0 \|\| iflib_rxd_avail(ctx, rxq, *cidxp, 1) != 0)
	retval \|= IFLIB_RXEOF_MORE;
	return (retval);
	err:
	STATE_LOCK(ctx);
	ctx->ifc_flags \|= IFC_DO_RESET;
	iflib_admin_intr_deferred(ctx);
	STATE_UNLOCK(ctx);
	return (0);
	}

	#define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)
	static inline qidx_t
	txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
	{
	qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
	qidx_t minthresh = txq->ift_size / 8;
	if (in_use > 4*minthresh)
	return (notify_count);
	if (in_use > 2*minthresh)
	return (notify_count >> 1);
	if (in_use > minthresh)
	return (notify_count >> 3);
	return (0);
	}

	static inline qidx_t
	txq_max_rs_deferred(iflib_txq_t txq)
	{
	qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
	qidx_t minthresh = txq->ift_size / 8;
	if (txq->ift_in_use > 4*minthresh)
	return (notify_count);
	if (txq->ift_in_use > 2*minthresh)
	return (notify_count >> 1);
	if (txq->ift_in_use > minthresh)
	return (notify_count >> 2);
	return (2);
	}

	#define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
	#define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)

	#define TXQ_MAX_DB_DEFERRED(txq, in_use) txq_max_db_deferred((txq), (in_use))
	#define TXQ_MAX_RS_DEFERRED(txq) txq_max_rs_deferred(txq)
	#define TXQ_MAX_DB_CONSUMED(size) (size >> 4)

	/* forward compatibility for cxgb */
	#define FIRST_QSET(ctx) 0
	#define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
	#define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
	#define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
	#define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))

	/* XXX we should be setting this to something other than zero */
	#define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
	#define MAX_TX_DESC(ctx) max((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max, \
	(ctx)->ifc_softc_ctx.isc_tx_nsegments)

	static inline bool
	iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring, qidx_t in_use)
	{
	qidx_t dbval, max;
	bool rang;

	rang = false;
	max = TXQ_MAX_DB_DEFERRED(txq, in_use);
	if (ring \|\| txq->ift_db_pending >= max) {
	dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
	txq->ift_db_pending = txq->ift_npending = 0;
	rang = true;
	}
	return (rang);
	}

	#ifdef PKT_DEBUG
	static void
	print_pkt(if_pkt_info_t pi)
	{
	printf("pi len: %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n",
	pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx);
	printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n",
	pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag);
	printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n",
	pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto);
	}
	#endif

	#define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
	#define IS_TX_OFFLOAD4(pi) ((pi)->ipi_csum_flags & (CSUM_IP_TCP \| CSUM_IP_TSO))
	#define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
	#define IS_TX_OFFLOAD6(pi) ((pi)->ipi_csum_flags & (CSUM_IP6_TCP \| CSUM_IP6_TSO))

	static int
	iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
	{
	if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
	struct ether_vlan_header *eh;
	struct mbuf *m;

	m = *mp;
	if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) &&
	M_WRITABLE(m) == 0) {
	if ((m = m_dup(m, M_NOWAIT)) == NULL) {
	return (ENOMEM);
	} else {
	m_freem(*mp);
	DBG_COUNTER_INC(tx_frees);
	*mp = m;
	}
	}

	/*
	* Determine where frame payload starts.
	* Jump over vlan headers if already present,
	* helpful for QinQ too.
	*/
	if (__predict_false(m->m_len < sizeof(*eh))) {
	txq->ift_pullups++;
	if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL))
	return (ENOMEM);
	}
	eh = mtod(m, struct ether_vlan_header *);
	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
	pi->ipi_etype = ntohs(eh->evl_proto);
	pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
	} else {
	pi->ipi_etype = ntohs(eh->evl_encap_proto);
	pi->ipi_ehdrlen = ETHER_HDR_LEN;
	}

	switch (pi->ipi_etype) {
	#ifdef INET
	case ETHERTYPE_IP:
	{
	struct mbuf *n;
	struct ip *ip = NULL;
	struct tcphdr *th = NULL;
	int minthlen;

	minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(ip) + sizeof(th));
	if (__predict_false(m->m_len < minthlen)) {
	/*
	* if this code bloat is causing too much of a hit
	* move it to a separate function and mark it noinline
	*/
	if (m->m_len == pi->ipi_ehdrlen) {
	n = m->m_next;
	MPASS(n);
	if (n->m_len >= sizeof(*ip)) {
	ip = (struct ip *)n->m_data;
	if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th))
	th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
	} else {
	txq->ift_pullups++;
	if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
	return (ENOMEM);
	ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
	}
	} else {
	txq->ift_pullups++;
	if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
	return (ENOMEM);
	ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
	if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
	th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
	}
	} else {
	ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
	if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
	th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
	}
	pi->ipi_ip_hlen = ip->ip_hl << 2;
	pi->ipi_ipproto = ip->ip_p;
	pi->ipi_flags \|= IPI_TX_IPV4;

	/* TCP checksum offload may require TCP header length */
	if (IS_TX_OFFLOAD4(pi)) {
	if (__predict_true(pi->ipi_ipproto == IPPROTO_TCP)) {
	if (__predict_false(th == NULL)) {
	txq->ift_pullups++;
	if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
	return (ENOMEM);
	th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
	}
	pi->ipi_tcp_hflags = th->th_flags;
	pi->ipi_tcp_hlen = th->th_off << 2;
	pi->ipi_tcp_seq = th->th_seq;
	}
	if (IS_TSO4(pi)) {
	if (__predict_false(ip->ip_p != IPPROTO_TCP))
	return (ENXIO);
	/*
	* TSO always requires hardware checksum offload.
	*/
	pi->ipi_csum_flags \|= (CSUM_IP_TCP \| CSUM_IP);
	th->th_sum = in_pseudo(ip->ip_src.s_addr,
	ip->ip_dst.s_addr, htons(IPPROTO_TCP));
	pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
	if (sctx->isc_flags & IFLIB_TSO_INIT_IP) {
	ip->ip_sum = 0;
	ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz);
	}
	}
	}
	if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP))
	ip->ip_sum = 0;

	break;
	}
	#endif
	#ifdef INET6
	case ETHERTYPE_IPV6:
	{
	struct ip6_hdr ip6 = (struct ip6_hdr )(m->m_data + pi->ipi_ehdrlen);
	struct tcphdr *th;
	pi->ipi_ip_hlen = sizeof(struct ip6_hdr);

	if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) {
	txq->ift_pullups++;
	if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL))
	return (ENOMEM);
	}
	th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen);

	/* XXX-BZ this will go badly in case of ext hdrs. */
	pi->ipi_ipproto = ip6->ip6_nxt;
	pi->ipi_flags \|= IPI_TX_IPV6;

	/* TCP checksum offload may require TCP header length */
	if (IS_TX_OFFLOAD6(pi)) {
	if (pi->ipi_ipproto == IPPROTO_TCP) {
	if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
	txq->ift_pullups++;
	if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
	return (ENOMEM);
	}
	pi->ipi_tcp_hflags = th->th_flags;
	pi->ipi_tcp_hlen = th->th_off << 2;
	pi->ipi_tcp_seq = th->th_seq;
	}
	if (IS_TSO6(pi)) {
	if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP))
	return (ENXIO);
	/*
	* TSO always requires hardware checksum offload.
	*/
	pi->ipi_csum_flags \|= CSUM_IP6_TCP;
	th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
	pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
	}
	}
	break;
	}
	#endif
	default:
	pi->ipi_csum_flags &= ~CSUM_OFFLOAD;
	pi->ipi_ip_hlen = 0;
	break;
	}
	*mp = m;

	return (0);
	}

	/*
	* If dodgy hardware rejects the scatter gather chain we've handed it
	* we'll need to remove the mbuf chain from ifsg_m[] before we can add the
	* m_defrag'd mbufs
	*/
	static __noinline struct mbuf *
	iflib_remove_mbuf(iflib_txq_t txq)
	{
	int ntxd, pidx;
	struct mbuf m, *ifsd_m;

	ifsd_m = txq->ift_sds.ifsd_m;
	ntxd = txq->ift_size;
	pidx = txq->ift_pidx & (ntxd - 1);
	ifsd_m = txq->ift_sds.ifsd_m;
	m = ifsd_m[pidx];
	ifsd_m[pidx] = NULL;
	bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]);
	if (txq->ift_sds.ifsd_tso_map != NULL)
	bus_dmamap_unload(txq->ift_tso_buf_tag,
	txq->ift_sds.ifsd_tso_map[pidx]);
	#if MEMORY_LOGGING
	txq->ift_dequeued++;
	#endif
	return (m);
	}

	static inline caddr_t
	calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid)
	{
	qidx_t size;
	int ntxd;
	caddr_t start, end, cur, next;

	ntxd = txq->ift_size;
	size = txq->ift_txd_size[qid];
	start = txq->ift_ifdi[qid].idi_vaddr;

	if (__predict_false(size == 0))
	return (start);
	cur = start + size*cidx;
	end = start + size*ntxd;
	next = CACHE_PTR_NEXT(cur);
	return (next < end ? next : start);
	}

	/*
	* Pad an mbuf to ensure a minimum ethernet frame size.
	* min_frame_size is the frame size (less CRC) to pad the mbuf to
	*/
	static __noinline int
	iflib_ether_pad(device_t dev, struct mbuf **m_head, uint16_t min_frame_size)
	{
	/*
	* 18 is enough bytes to pad an ARP packet to 46 bytes, and
	* and ARP message is the smallest common payload I can think of
	*/
	static char pad[18]; /* just zeros */
	int n;
	struct mbuf *new_head;

	if (!M_WRITABLE(*m_head)) {
	new_head = m_dup(*m_head, M_NOWAIT);
	if (new_head == NULL) {
	m_freem(*m_head);
	device_printf(dev, "cannot pad short frame, m_dup() failed");
	DBG_COUNTER_INC(encap_pad_mbuf_fail);
	DBG_COUNTER_INC(tx_frees);
	return ENOMEM;
	}
	m_freem(*m_head);
	*m_head = new_head;
	}

	for (n = min_frame_size - (*m_head)->m_pkthdr.len;
	n > 0; n -= sizeof(pad))
	if (!m_append(*m_head, min(n, sizeof(pad)), pad))
	break;

	if (n > 0) {
	m_freem(*m_head);
	device_printf(dev, "cannot pad short frame\n");
	DBG_COUNTER_INC(encap_pad_mbuf_fail);
	DBG_COUNTER_INC(tx_frees);
	return (ENOBUFS);
	}

	return 0;
	}

	static int
	iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
	{
	if_ctx_t ctx;
	if_shared_ctx_t sctx;
	if_softc_ctx_t scctx;
	bus_dma_tag_t buf_tag;
	bus_dma_segment_t *segs;
	struct mbuf m_head, *ifsd_m;
	void *next_txd;
	bus_dmamap_t map;
	struct if_pkt_info pi;
	int remap = 0;
	int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd;

	ctx = txq->ift_ctx;
	sctx = ctx->ifc_sctx;
	scctx = &ctx->ifc_softc_ctx;
	segs = txq->ift_segs;
	ntxd = txq->ift_size;
	m_head = *m_headp;
	map = NULL;

	/*
	* If we're doing TSO the next descriptor to clean may be quite far ahead
	*/
	cidx = txq->ift_cidx;
	pidx = txq->ift_pidx;
	if (ctx->ifc_flags & IFC_PREFETCH) {
	next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
	if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) {
	next_txd = calc_next_txd(txq, cidx, 0);
	prefetch(next_txd);
	}

	/* prefetch the next cache line of mbuf pointers and flags */
	prefetch(&txq->ift_sds.ifsd_m[next]);
	prefetch(&txq->ift_sds.ifsd_map[next]);
	next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
	}
	map = txq->ift_sds.ifsd_map[pidx];
	ifsd_m = txq->ift_sds.ifsd_m;

	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
	buf_tag = txq->ift_tso_buf_tag;
	max_segs = scctx->isc_tx_tso_segments_max;
	map = txq->ift_sds.ifsd_tso_map[pidx];
	MPASS(buf_tag != NULL);
	MPASS(max_segs > 0);
	} else {
	buf_tag = txq->ift_buf_tag;
	max_segs = scctx->isc_tx_nsegments;
	map = txq->ift_sds.ifsd_map[pidx];
	}
	if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
	__predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
	err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size);
	if (err) {
	DBG_COUNTER_INC(encap_txd_encap_fail);
	return err;
	}
	}
	m_head = *m_headp;

	pkt_info_zero(&pi);
	pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG\|M_BCAST\|M_MCAST));
	pi.ipi_pidx = pidx;
	pi.ipi_qsidx = txq->ift_id;
	pi.ipi_len = m_head->m_pkthdr.len;
	pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
	pi.ipi_vtag = M_HAS_VLANTAG(m_head) ? m_head->m_pkthdr.ether_vtag : 0;

	/* deliberate bitwise OR to make one condition */
	if (__predict_true((pi.ipi_csum_flags \| pi.ipi_vtag))) {
	if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) {
	DBG_COUNTER_INC(encap_txd_encap_fail);
	return (err);
	}
	m_head = *m_headp;
	}

	retry:
	err = bus_dmamap_load_mbuf_sg(buf_tag, map, m_head, segs, &nsegs,
	BUS_DMA_NOWAIT);
	defrag:
	if (__predict_false(err)) {
	switch (err) {
	case EFBIG:
	/* try collapse once and defrag once */
	if (remap == 0) {
	m_head = m_collapse(*m_headp, M_NOWAIT, max_segs);
	/* try defrag if collapsing fails */
	if (m_head == NULL)
	remap++;
	}
	if (remap == 1) {
	txq->ift_mbuf_defrag++;
	m_head = m_defrag(*m_headp, M_NOWAIT);
	}
	/*
	* remap should never be >1 unless bus_dmamap_load_mbuf_sg
	* failed to map an mbuf that was run through m_defrag
	*/
	MPASS(remap <= 1);
	if (__predict_false(m_head == NULL \|\| remap > 1))
	goto defrag_failed;
	remap++;
	*m_headp = m_head;
	goto retry;
	break;
	case ENOMEM:
	txq->ift_no_tx_dma_setup++;
	break;
	default:
	txq->ift_no_tx_dma_setup++;
	m_freem(*m_headp);
	DBG_COUNTER_INC(tx_frees);
	*m_headp = NULL;
	break;
	}
	txq->ift_map_failed++;
	DBG_COUNTER_INC(encap_load_mbuf_fail);
	DBG_COUNTER_INC(encap_txd_encap_fail);
	return (err);
	}
	ifsd_m[pidx] = m_head;
	/*
	* XXX assumes a 1 to 1 relationship between segments and
	* descriptors - this does not hold true on all drivers, e.g.
	* cxgb
	*/
	if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
	txq->ift_no_desc_avail++;
	bus_dmamap_unload(buf_tag, map);
	DBG_COUNTER_INC(encap_txq_avail_fail);
	DBG_COUNTER_INC(encap_txd_encap_fail);
	if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)
	GROUPTASK_ENQUEUE(&txq->ift_task);
	return (ENOBUFS);
	}
	/*
	* On Intel cards we can greatly reduce the number of TX interrupts
	* we see by only setting report status on every Nth descriptor.
	* However, this also means that the driver will need to keep track
	* of the descriptors that RS was set on to check them for the DD bit.
	*/
	txq->ift_rs_pending += nsegs + 1;
	if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) \|\|
	iflib_no_tx_batch \|\| (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) {
	pi.ipi_flags \|= IPI_TX_INTR;
	txq->ift_rs_pending = 0;
	}

	pi.ipi_segs = segs;
	pi.ipi_nsegs = nsegs;

	MPASS(pidx >= 0 && pidx < txq->ift_size);
	#ifdef PKT_DEBUG
	print_pkt(&pi);
	#endif
	if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
	bus_dmamap_sync(buf_tag, map, BUS_DMASYNC_PREWRITE);
	DBG_COUNTER_INC(tx_encap);
	MPASS(pi.ipi_new_pidx < txq->ift_size);

	ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
	if (pi.ipi_new_pidx < pi.ipi_pidx) {
	ndesc += txq->ift_size;
	txq->ift_gen = 1;
	}
	/*
	* drivers can need as many as
	* two sentinels
	*/
	MPASS(ndesc <= pi.ipi_nsegs + 2);
	MPASS(pi.ipi_new_pidx != pidx);
	MPASS(ndesc > 0);
	txq->ift_in_use += ndesc;

	/*
	* We update the last software descriptor again here because there may
	* be a sentinel and/or there may be more mbufs than segments
	*/
	txq->ift_pidx = pi.ipi_new_pidx;
	txq->ift_npending += pi.ipi_ndescs;
	} else {
	*m_headp = m_head = iflib_remove_mbuf(txq);
	if (err == EFBIG) {
	txq->ift_txd_encap_efbig++;
	if (remap < 2) {
	remap = 1;
	goto defrag;
	}
	}
	goto defrag_failed;
	}
	/*
	* err can't possibly be non-zero here, so we don't neet to test it
	* to see if we need to DBG_COUNTER_INC(encap_txd_encap_fail).
	*/
	return (err);

	defrag_failed:
	txq->ift_mbuf_defrag_failed++;
	txq->ift_map_failed++;
	m_freem(*m_headp);
	DBG_COUNTER_INC(tx_frees);
	*m_headp = NULL;
	DBG_COUNTER_INC(encap_txd_encap_fail);
	return (ENOMEM);
	}

	static void
	iflib_tx_desc_free(iflib_txq_t txq, int n)
	{
	uint32_t qsize, cidx, mask, gen;
	struct mbuf m, *ifsd_m;
	bool do_prefetch;

	cidx = txq->ift_cidx;
	gen = txq->ift_gen;
	qsize = txq->ift_size;
	mask = qsize-1;
	ifsd_m = txq->ift_sds.ifsd_m;
	do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);

	while (n-- > 0) {
	if (do_prefetch) {
	prefetch(ifsd_m[(cidx + 3) & mask]);
	prefetch(ifsd_m[(cidx + 4) & mask]);
	}
	if ((m = ifsd_m[cidx]) != NULL) {
	prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
	bus_dmamap_sync(txq->ift_tso_buf_tag,
	txq->ift_sds.ifsd_tso_map[cidx],
	BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(txq->ift_tso_buf_tag,
	txq->ift_sds.ifsd_tso_map[cidx]);
	} else {
	bus_dmamap_sync(txq->ift_buf_tag,
	txq->ift_sds.ifsd_map[cidx],
	BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(txq->ift_buf_tag,
	txq->ift_sds.ifsd_map[cidx]);
	}
	/* XXX we don't support any drivers that batch packets yet */
	MPASS(m->m_nextpkt == NULL);
	m_freem(m);
	ifsd_m[cidx] = NULL;
	#if MEMORY_LOGGING
	txq->ift_dequeued++;
	#endif
	DBG_COUNTER_INC(tx_frees);
	}
	if (__predict_false(++cidx == qsize)) {
	cidx = 0;
	gen = 0;
	}
	}
	txq->ift_cidx = cidx;
	txq->ift_gen = gen;
	}

	static __inline int
	iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
	{
	int reclaim;
	if_ctx_t ctx = txq->ift_ctx;

	KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
	MPASS(thresh /+ MAX_TX_DESC(txq->ift_ctx) / < txq->ift_size);

	/*
	* Need a rate-limiting check so that this isn't called every time
	*/
	iflib_tx_credits_update(ctx, txq);
	reclaim = DESC_RECLAIMABLE(txq);

	if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
	#ifdef INVARIANTS
	if (iflib_verbose_debug) {
	printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__,
	txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
	reclaim, thresh);

	}
	#endif
	return (0);
	}
	iflib_tx_desc_free(txq, reclaim);
	txq->ift_cleaned += reclaim;
	txq->ift_in_use -= reclaim;

	return (reclaim);
	}

	static struct mbuf **
	_ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining)
	{
	int next, size;
	struct mbuf **items;

	size = r->size;
	next = (cidx + CACHE_PTR_INCREMENT) & (size-1);
	items = __DEVOLATILE(struct mbuf **, &r->items[0]);

	prefetch(items[(cidx + offset) & (size-1)]);
	if (remaining > 1) {
	prefetch2cachelines(&items[next]);
	prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]);
	prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]);
	prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]);
	}
	return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)]));
	}

	static void
	iflib_txq_check_drain(iflib_txq_t txq, int budget)
	{

	ifmp_ring_check_drainage(txq->ift_br, budget);
	}

	static uint32_t
	iflib_txq_can_drain(struct ifmp_ring *r)
	{
	iflib_txq_t txq = r->cookie;
	if_ctx_t ctx = txq->ift_ctx;

	if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2)
	return (1);
	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD);
	return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id,
	false));
	}

	static uint32_t
	iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
	{
	iflib_txq_t txq = r->cookie;
	if_ctx_t ctx = txq->ift_ctx;
	if_t ifp = ctx->ifc_ifp;
	struct mbuf m, *mp;
	int avail, bytes_sent, consumed, count, err, i, in_use_prev;
	int mcast_sent, pkt_sent, reclaimed, txq_avail;
	bool do_prefetch, rang, ring;

	if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) \|\|
	!LINK_ACTIVE(ctx))) {
	DBG_COUNTER_INC(txq_drain_notready);
	return (0);
	}
	reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
	rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use);
	avail = IDXDIFF(pidx, cidx, r->size);
	if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
	DBG_COUNTER_INC(txq_drain_flushing);
	for (i = 0; i < avail; i++) {
	if (__predict_true(r->items[(cidx + i) & (r->size-1)] != (void *)txq))
	m_free(r->items[(cidx + i) & (r->size-1)]);
	r->items[(cidx + i) & (r->size-1)] = NULL;
	}
	return (avail);
	}

	if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	CALLOUT_UNLOCK(txq);
	DBG_COUNTER_INC(txq_drain_oactive);
	return (0);
	}
	if (reclaimed)
	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	consumed = mcast_sent = bytes_sent = pkt_sent = 0;
	count = MIN(avail, TX_BATCH_SIZE);
	#ifdef INVARIANTS
	if (iflib_verbose_debug)
	printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__,
	avail, ctx->ifc_flags, TXQ_AVAIL(txq));
	#endif
	do_prefetch = (ctx->ifc_flags & IFC_PREFETCH);
	txq_avail = TXQ_AVAIL(txq);
	err = 0;
	for (i = 0; i < count && txq_avail > MAX_TX_DESC(ctx) + 2; i++) {
	int rem = do_prefetch ? count - i : 0;

	mp = _ring_peek_one(r, cidx, i, rem);
	MPASS(mp != NULL && *mp != NULL);
	if (__predict_false(mp == (struct mbuf )txq)) {
	consumed++;
	continue;
	}
	in_use_prev = txq->ift_in_use;
	err = iflib_encap(txq, mp);
	if (__predict_false(err)) {
	/* no room - bail out */
	if (err == ENOBUFS)
	break;
	consumed++;
	/* we can't send this packet - skip it */
	continue;
	}
	consumed++;
	pkt_sent++;
	m = *mp;
	DBG_COUNTER_INC(tx_sent);
	bytes_sent += m->m_pkthdr.len;
	mcast_sent += !!(m->m_flags & M_MCAST);
	txq_avail = TXQ_AVAIL(txq);

	txq->ift_db_pending += (txq->ift_in_use - in_use_prev);
	ETHER_BPF_MTAP(ifp, m);
	if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING)))
	break;
	rang = iflib_txd_db_check(ctx, txq, false, in_use_prev);
	}

	/* deliberate use of bitwise or to avoid gratuitous short-circuit */
	ring = rang ? false : (iflib_min_tx_latency \| err) \|\| (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx));
	iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use);
	if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
	if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
	if (mcast_sent)
	if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
	#ifdef INVARIANTS
	if (iflib_verbose_debug)
	printf("consumed=%d\n", consumed);
	#endif
	return (consumed);
	}

	static uint32_t
	iflib_txq_drain_always(struct ifmp_ring *r)
	{
	return (1);
	}

	static uint32_t
	iflib_txq_drain_free(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
	{
	int i, avail;
	struct mbuf **mp;
	iflib_txq_t txq;

	txq = r->cookie;

	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	CALLOUT_UNLOCK(txq);

	avail = IDXDIFF(pidx, cidx, r->size);
	for (i = 0; i < avail; i++) {
	mp = _ring_peek_one(r, cidx, i, avail - i);
	if (__predict_false(mp == (struct mbuf )txq))
	continue;
	m_freem(*mp);
	DBG_COUNTER_INC(tx_frees);
	}
	MPASS(ifmp_ring_is_stalled(r) == 0);
	return (avail);
	}

	static void
	iflib_ifmp_purge(iflib_txq_t txq)
	{
	struct ifmp_ring *r;

	r = txq->ift_br;
	r->drain = iflib_txq_drain_free;
	r->can_drain = iflib_txq_drain_always;

	ifmp_ring_check_drainage(r, r->size);

	r->drain = iflib_txq_drain;
	r->can_drain = iflib_txq_can_drain;
	}

	static void
	_task_fn_tx(void *context)
	{
	iflib_txq_t txq = context;
	if_ctx_t ctx = txq->ift_ctx;
	if_t ifp = ctx->ifc_ifp;
	int abdicate = ctx->ifc_sysctl_tx_abdicate;

	#ifdef IFLIB_DIAGNOSTICS
	txq->ift_cpu_exec_count[curcpu]++;
	#endif
	if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
	return;
	#ifdef DEV_NETMAP
	if ((if_getcapenable(ifp) & IFCAP_NETMAP) &&
	netmap_tx_irq(ifp, txq->ift_id))
	goto skip_ifmp;
	#endif
	#ifdef ALTQ
	if (ALTQ_IS_ENABLED(&ifp->if_snd))
	iflib_altq_if_start(ifp);
	#endif
	if (txq->ift_db_pending)
	ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate);
	else if (!abdicate)
	ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
	/*
	* When abdicating, we always need to check drainage, not just when we don't enqueue
	*/
	if (abdicate)
	ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
	#ifdef DEV_NETMAP
	skip_ifmp:
	#endif
	if (ctx->ifc_flags & IFC_LEGACY)
	IFDI_INTR_ENABLE(ctx);
	else
	IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
	}

	static void
	_task_fn_rx(void *context)
	{
	iflib_rxq_t rxq = context;
	if_ctx_t ctx = rxq->ifr_ctx;
	uint8_t more;
	uint16_t budget;
	#ifdef DEV_NETMAP
	u_int work = 0;
	int nmirq;
	#endif

	#ifdef IFLIB_DIAGNOSTICS
	rxq->ifr_cpu_exec_count[curcpu]++;
	#endif
	DBG_COUNTER_INC(task_fn_rxs);
	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
	return;
	#ifdef DEV_NETMAP
	nmirq = netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work);
	if (nmirq != NM_IRQ_PASS) {
	more = (nmirq == NM_IRQ_RESCHED) ? IFLIB_RXEOF_MORE : 0;
	goto skip_rxeof;
	}
	#endif
	budget = ctx->ifc_sysctl_rx_budget;
	if (budget == 0)
	budget = 16; /* XXX */
	more = iflib_rxeof(rxq, budget);
	#ifdef DEV_NETMAP
	skip_rxeof:
	#endif
	if ((more & IFLIB_RXEOF_MORE) == 0) {
	if (ctx->ifc_flags & IFC_LEGACY)
	IFDI_INTR_ENABLE(ctx);
	else
	IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
	DBG_COUNTER_INC(rx_intr_enables);
	}
	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
	return;

	if (more & IFLIB_RXEOF_MORE)
	GROUPTASK_ENQUEUE(&rxq->ifr_task);
	else if (more & IFLIB_RXEOF_EMPTY)
	callout_reset_curcpu(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq);
	}

	static void
	_task_fn_admin(void *context)
	{
	if_ctx_t ctx = context;
	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
	iflib_txq_t txq;
	int i;
	bool oactive, running, do_reset, do_watchdog, in_detach;
	uint32_t reset_on = hz / 2;

	STATE_LOCK(ctx);
	running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING);
	oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE);
	do_reset = (ctx->ifc_flags & IFC_DO_RESET);
	do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG);
	in_detach = (ctx->ifc_flags & IFC_IN_DETACH);
	ctx->ifc_flags &= ~(IFC_DO_RESET\|IFC_DO_WATCHDOG);
	STATE_UNLOCK(ctx);

	if ((!running && !oactive) && !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN))
	return;
	if (in_detach)
	return;

	CTX_LOCK(ctx);
	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	CALLOUT_UNLOCK(txq);
	}
	if (do_watchdog) {
	ctx->ifc_watchdog_events++;
	IFDI_WATCHDOG_RESET(ctx);
	}
	IFDI_UPDATE_ADMIN_STATUS(ctx);
	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
	#ifdef DEV_NETMAP
	reset_on = hz / 2;
	if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP)
	iflib_netmap_timer_adjust(ctx, txq, &reset_on);
	#endif
	callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu);
	}
	IFDI_LINK_INTR_ENABLE(ctx);
	if (do_reset)
	iflib_if_init_locked(ctx);
	CTX_UNLOCK(ctx);

	if (LINK_ACTIVE(ctx) == 0)
	return;
	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
	iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
	}


	static void
	_task_fn_iov(void *context)
	{
	if_ctx_t ctx = context;

	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) &&
	!(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN))
	return;

	CTX_LOCK(ctx);
	IFDI_VFLR_HANDLE(ctx);
	CTX_UNLOCK(ctx);
	}

	static int
	iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
	{
	int err;
	if_int_delay_info_t info;
	if_ctx_t ctx;

	info = (if_int_delay_info_t)arg1;
	ctx = info->iidi_ctx;
	info->iidi_req = req;
	info->iidi_oidp = oidp;
	CTX_LOCK(ctx);
	err = IFDI_SYSCTL_INT_DELAY(ctx, info);
	CTX_UNLOCK(ctx);
	return (err);
	}

	/*********************************************************************
	*
	* IFNET FUNCTIONS
	*
	**********************************************************************/

	static void
	iflib_if_init_locked(if_ctx_t ctx)
	{
	iflib_stop(ctx);
	iflib_init_locked(ctx);
	}


	static void
	iflib_if_init(void *arg)
	{
	if_ctx_t ctx = arg;

	CTX_LOCK(ctx);
	iflib_if_init_locked(ctx);
	CTX_UNLOCK(ctx);
	}

	static int
	iflib_if_transmit(if_t ifp, struct mbuf *m)
	{
	if_ctx_t ctx = if_getsoftc(ifp);

	iflib_txq_t txq;
	int err, qidx;
	int abdicate = ctx->ifc_sysctl_tx_abdicate;

	if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 \|\| !LINK_ACTIVE(ctx))) {
	DBG_COUNTER_INC(tx_frees);
	m_freem(m);
	return (ENETDOWN);
	}

	MPASS(m->m_nextpkt == NULL);
	/* ALTQ-enabled interfaces always use queue 0. */
	qidx = 0;
	if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m) && !ALTQ_IS_ENABLED(&ifp->if_snd))
	qidx = QIDX(ctx, m);
	/*
	* XXX calculate buf_ring based on flowid (divvy up bits?)
	*/
	txq = &ctx->ifc_txqs[qidx];

	#ifdef DRIVER_BACKPRESSURE
	if (txq->ift_closed) {
	while (m != NULL) {
	next = m->m_nextpkt;
	m->m_nextpkt = NULL;
	m_freem(m);
	DBG_COUNTER_INC(tx_frees);
	m = next;
	}
	return (ENOBUFS);
	}
	#endif
	#ifdef notyet
	qidx = count = 0;
	mp = marr;
	next = m;
	do {
	count++;
	next = next->m_nextpkt;
	} while (next != NULL);

	if (count > nitems(marr))
	if ((mp = malloc(countsizeof(struct mbuf ), M_IFLIB, M_NOWAIT)) == NULL) {
	/* XXX check nextpkt */
	m_freem(m);
	/* XXX simplify for now */
	DBG_COUNTER_INC(tx_frees);
	return (ENOBUFS);
	}
	for (next = m, i = 0; next != NULL; i++) {
	mp[i] = next;
	next = next->m_nextpkt;
	mp[i]->m_nextpkt = NULL;
	}
	#endif
	DBG_COUNTER_INC(tx_seen);
	err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate);

	if (abdicate)
	GROUPTASK_ENQUEUE(&txq->ift_task);
	if (err) {
	if (!abdicate)
	GROUPTASK_ENQUEUE(&txq->ift_task);
	/* support forthcoming later */
	#ifdef DRIVER_BACKPRESSURE
	txq->ift_closed = TRUE;
	#endif
	ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
	m_freem(m);
	DBG_COUNTER_INC(tx_frees);
	}

	return (err);
	}

	#ifdef ALTQ
	/*
	* The overall approach to integrating iflib with ALTQ is to continue to use
	* the iflib mp_ring machinery between the ALTQ queue(s) and the hardware
	* ring. Technically, when using ALTQ, queueing to an intermediate mp_ring
	* is redundant/unnecessary, but doing so minimizes the amount of
	* ALTQ-specific code required in iflib. It is assumed that the overhead of
	* redundantly queueing to an intermediate mp_ring is swamped by the
	* performance limitations inherent in using ALTQ.
	*
	* When ALTQ support is compiled in, all iflib drivers will use a transmit
	* routine, iflib_altq_if_transmit(), that checks if ALTQ is enabled for the
	* given interface. If ALTQ is enabled for an interface, then all
	* transmitted packets for that interface will be submitted to the ALTQ
	* subsystem via IFQ_ENQUEUE(). We don't use the legacy if_transmit()
	* implementation because it uses IFQ_HANDOFF(), which will duplicatively
	* update stats that the iflib machinery handles, and which is sensitve to
	* the disused IFF_DRV_OACTIVE flag. Additionally, iflib_altq_if_start()
	* will be installed as the start routine for use by ALTQ facilities that
	* need to trigger queue drains on a scheduled basis.
	*
	*/
	static void
	iflib_altq_if_start(if_t ifp)
	{
	struct ifaltq *ifq = &ifp->if_snd;
	struct mbuf *m;

	IFQ_LOCK(ifq);
	IFQ_DEQUEUE_NOLOCK(ifq, m);
	while (m != NULL) {
	iflib_if_transmit(ifp, m);
	IFQ_DEQUEUE_NOLOCK(ifq, m);
	}
	IFQ_UNLOCK(ifq);
	}

	static int
	iflib_altq_if_transmit(if_t ifp, struct mbuf *m)
	{
	int err;

	if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
	IFQ_ENQUEUE(&ifp->if_snd, m, err);
	if (err == 0)
	iflib_altq_if_start(ifp);
	} else
	err = iflib_if_transmit(ifp, m);

	return (err);
	}
	#endif /* ALTQ */

	static void
	iflib_if_qflush(if_t ifp)
	{
	if_ctx_t ctx = if_getsoftc(ifp);
	iflib_txq_t txq = ctx->ifc_txqs;
	int i;

	STATE_LOCK(ctx);
	ctx->ifc_flags \|= IFC_QFLUSH;
	STATE_UNLOCK(ctx);
	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
	while (!(ifmp_ring_is_idle(txq->ift_br) \|\| ifmp_ring_is_stalled(txq->ift_br)))
	iflib_txq_check_drain(txq, 0);
	STATE_LOCK(ctx);
	ctx->ifc_flags &= ~IFC_QFLUSH;
	STATE_UNLOCK(ctx);

	/*
	* When ALTQ is enabled, this will also take care of purging the
	* ALTQ queue(s).
	*/
	if_qflush(ifp);
	}


	#define IFCAP_FLAGS (IFCAP_HWCSUM_IPV6 \| IFCAP_HWCSUM \| IFCAP_LRO \| \
	IFCAP_TSO \| IFCAP_VLAN_HWTAGGING \| IFCAP_HWSTATS \| \
	IFCAP_VLAN_MTU \| IFCAP_VLAN_HWFILTER \| \
	IFCAP_VLAN_HWTSO \| IFCAP_VLAN_HWCSUM \| IFCAP_NOMAP)

	static int
	iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
	{
	if_ctx_t ctx = if_getsoftc(ifp);
	struct ifreq ifr = (struct ifreq )data;
	#if defined(INET) \|\| defined(INET6)
	struct ifaddr ifa = (struct ifaddr )data;
	#endif
	bool avoid_reset = false;
	int err = 0, reinit = 0, bits;

	switch (command) {
	case SIOCSIFADDR:
	#ifdef INET
	if (ifa->ifa_addr->sa_family == AF_INET)
	avoid_reset = true;
	#endif
	#ifdef INET6
	if (ifa->ifa_addr->sa_family == AF_INET6)
	avoid_reset = true;
	#endif
	/*
	** Calling init results in link renegotiation,
	** so we avoid doing it when possible.
	*/
	if (avoid_reset) {
	if_setflagbits(ifp, IFF_UP,0);
	if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
	reinit = 1;
	#ifdef INET
	if (!(if_getflags(ifp) & IFF_NOARP))
	arp_ifinit(ifp, ifa);
	#endif
	} else
	err = ether_ioctl(ifp, command, data);
	break;
	case SIOCSIFMTU:
	CTX_LOCK(ctx);
	if (ifr->ifr_mtu == if_getmtu(ifp)) {
	CTX_UNLOCK(ctx);
	break;
	}
	bits = if_getdrvflags(ifp);
	/* stop the driver and free any clusters before proceeding */
	iflib_stop(ctx);

	if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
	STATE_LOCK(ctx);
	if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)
	ctx->ifc_flags \|= IFC_MULTISEG;
	else
	ctx->ifc_flags &= ~IFC_MULTISEG;
	STATE_UNLOCK(ctx);
	err = if_setmtu(ifp, ifr->ifr_mtu);
	}
	iflib_init_locked(ctx);
	STATE_LOCK(ctx);
	if_setdrvflags(ifp, bits);
	STATE_UNLOCK(ctx);
	CTX_UNLOCK(ctx);
	break;
	case SIOCSIFFLAGS:
	CTX_LOCK(ctx);
	if (if_getflags(ifp) & IFF_UP) {
	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
	if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
	(IFF_PROMISC \| IFF_ALLMULTI)) {
	+ CTX_UNLOCK(ctx);
	err = IFDI_PROMISC_SET(ctx, if_getflags(ifp));
	+ CTX_LOCK(ctx);
	}
	} else
	reinit = 1;
	} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
	iflib_stop(ctx);
	}
	ctx->ifc_if_flags = if_getflags(ifp);
	CTX_UNLOCK(ctx);
	break;
	case SIOCADDMULTI:
	case SIOCDELMULTI:
	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
	CTX_LOCK(ctx);
	IFDI_INTR_DISABLE(ctx);
	IFDI_MULTI_SET(ctx);
	IFDI_INTR_ENABLE(ctx);
	CTX_UNLOCK(ctx);
	}
	break;
	case SIOCSIFMEDIA:
	CTX_LOCK(ctx);
	IFDI_MEDIA_SET(ctx);
	CTX_UNLOCK(ctx);
	/* FALLTHROUGH */
	case SIOCGIFMEDIA:
	case SIOCGIFXMEDIA:
	err = ifmedia_ioctl(ifp, ifr, ctx->ifc_mediap, command);
	break;
	case SIOCGI2C:
	{
	struct ifi2creq i2c;

	err = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
	if (err != 0)
	break;
	if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
	err = EINVAL;
	break;
	}
	if (i2c.len > sizeof(i2c.data)) {
	err = EINVAL;
	break;
	}

	if ((err = IFDI_I2C_REQ(ctx, &i2c)) == 0)
	err = copyout(&i2c, ifr_data_get_ptr(ifr),
	sizeof(i2c));
	break;
	}
	case SIOCSIFCAP:
	{
	int mask, setmask, oldmask;

	oldmask = if_getcapenable(ifp);
	mask = ifr->ifr_reqcap ^ oldmask;
	mask &= ctx->ifc_softc_ctx.isc_capabilities \| IFCAP_NOMAP;
	setmask = 0;
	#ifdef TCP_OFFLOAD
	setmask \|= mask & (IFCAP_TOE4\|IFCAP_TOE6);
	#endif
	setmask \|= (mask & IFCAP_FLAGS);
	setmask \|= (mask & IFCAP_WOL);

	/*
	* If any RX csum has changed, change all the ones that
	* are supported by the driver.
	*/
	if (setmask & (IFCAP_RXCSUM \| IFCAP_RXCSUM_IPV6)) {
	setmask \|= ctx->ifc_softc_ctx.isc_capabilities &
	(IFCAP_RXCSUM \| IFCAP_RXCSUM_IPV6);
	}

	/*
	* want to ensure that traffic has stopped before we change any of the flags
	*/
	if (setmask) {
	CTX_LOCK(ctx);
	bits = if_getdrvflags(ifp);
	if (bits & IFF_DRV_RUNNING && setmask & ~IFCAP_WOL)
	iflib_stop(ctx);
	STATE_LOCK(ctx);
	if_togglecapenable(ifp, setmask);
	STATE_UNLOCK(ctx);
	if (bits & IFF_DRV_RUNNING && setmask & ~IFCAP_WOL)
	iflib_init_locked(ctx);
	STATE_LOCK(ctx);
	if_setdrvflags(ifp, bits);
	STATE_UNLOCK(ctx);
	CTX_UNLOCK(ctx);
	}
	if_vlancap(ifp);
	break;
	}
	case SIOCGPRIVATE_0:
	case SIOCSDRVSPEC:
	case SIOCGDRVSPEC:
	CTX_LOCK(ctx);
	err = IFDI_PRIV_IOCTL(ctx, command, data);
	CTX_UNLOCK(ctx);
	break;
	default:
	err = ether_ioctl(ifp, command, data);
	break;
	}
	if (reinit)
	iflib_if_init(ctx);
	return (err);
	}

	static uint64_t
	iflib_if_get_counter(if_t ifp, ift_counter cnt)
	{
	if_ctx_t ctx = if_getsoftc(ifp);

	return (IFDI_GET_COUNTER(ctx, cnt));
	}

	/*********************************************************************
	*
	* OTHER FUNCTIONS EXPORTED TO THE STACK
	*
	**********************************************************************/

	static void
	iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag)
	{
	if_ctx_t ctx = if_getsoftc(ifp);

	if ((void *)ctx != arg)
	return;

	if ((vtag == 0) \|\| (vtag > 4095))
	return;

	if (iflib_in_detach(ctx))
	return;

	CTX_LOCK(ctx);
	/* Driver may need all untagged packets to be flushed */
	if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG))
	iflib_stop(ctx);
	IFDI_VLAN_REGISTER(ctx, vtag);
	/* Re-init to load the changes, if required */
	if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG))
	iflib_init_locked(ctx);
	CTX_UNLOCK(ctx);
	}

	static void
	iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag)
	{
	if_ctx_t ctx = if_getsoftc(ifp);

	if ((void *)ctx != arg)
	return;

	if ((vtag == 0) \|\| (vtag > 4095))
	return;

	CTX_LOCK(ctx);
	/* Driver may need all tagged packets to be flushed */
	if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG))
	iflib_stop(ctx);
	IFDI_VLAN_UNREGISTER(ctx, vtag);
	/* Re-init to load the changes, if required */
	if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG))
	iflib_init_locked(ctx);
	CTX_UNLOCK(ctx);
	}

	static void
	iflib_led_func(void *arg, int onoff)
	{
	if_ctx_t ctx = arg;

	CTX_LOCK(ctx);
	IFDI_LED_FUNC(ctx, onoff);
	CTX_UNLOCK(ctx);
	}

	/*********************************************************************
	*
	* BUS FUNCTION DEFINITIONS
	*
	**********************************************************************/

	int
	iflib_device_probe(device_t dev)
	{
	const pci_vendor_info_t *ent;
	if_shared_ctx_t sctx;
	uint16_t pci_device_id, pci_rev_id, pci_subdevice_id, pci_subvendor_id;
	uint16_t pci_vendor_id;

	if ((sctx = DEVICE_REGISTER(dev)) == NULL \|\| sctx->isc_magic != IFLIB_MAGIC)
	return (ENOTSUP);

	pci_vendor_id = pci_get_vendor(dev);
	pci_device_id = pci_get_device(dev);
	pci_subvendor_id = pci_get_subvendor(dev);
	pci_subdevice_id = pci_get_subdevice(dev);
	pci_rev_id = pci_get_revid(dev);
	if (sctx->isc_parse_devinfo != NULL)
	sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id);

	ent = sctx->isc_vendor_info;
	while (ent->pvi_vendor_id != 0) {
	if (pci_vendor_id != ent->pvi_vendor_id) {
	ent++;
	continue;
	}
	if ((pci_device_id == ent->pvi_device_id) &&
	((pci_subvendor_id == ent->pvi_subvendor_id) \|\|
	(ent->pvi_subvendor_id == 0)) &&
	((pci_subdevice_id == ent->pvi_subdevice_id) \|\|
	(ent->pvi_subdevice_id == 0)) &&
	((pci_rev_id == ent->pvi_rev_id) \|\|
	(ent->pvi_rev_id == 0))) {

	device_set_desc_copy(dev, ent->pvi_name);
	/* this needs to be changed to zero if the bus probing code
	* ever stops re-probing on best match because the sctx
	* may have its values over written by register calls
	* in subsequent probes
	*/
	return (BUS_PROBE_DEFAULT);
	}
	ent++;
	}
	return (ENXIO);
	}

	int
	iflib_device_probe_vendor(device_t dev)
	{
	int probe;

	probe = iflib_device_probe(dev);
	if (probe == BUS_PROBE_DEFAULT)
	return (BUS_PROBE_VENDOR);
	else
	return (probe);
	}

	static void
	iflib_reset_qvalues(if_ctx_t ctx)
	{
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	device_t dev = ctx->ifc_dev;
	int i;

	if (ctx->ifc_sysctl_ntxqs != 0)
	scctx->isc_ntxqsets = ctx->ifc_sysctl_ntxqs;
	if (ctx->ifc_sysctl_nrxqs != 0)
	scctx->isc_nrxqsets = ctx->ifc_sysctl_nrxqs;

	for (i = 0; i < sctx->isc_ntxqs; i++) {
	if (ctx->ifc_sysctl_ntxds[i] != 0)
	scctx->isc_ntxd[i] = ctx->ifc_sysctl_ntxds[i];
	else
	scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i];
	}

	for (i = 0; i < sctx->isc_nrxqs; i++) {
	if (ctx->ifc_sysctl_nrxds[i] != 0)
	scctx->isc_nrxd[i] = ctx->ifc_sysctl_nrxds[i];
	else
	scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i];
	}

	for (i = 0; i < sctx->isc_nrxqs; i++) {
	if (scctx->isc_nrxd[i] < sctx->isc_nrxd_min[i]) {
	device_printf(dev, "nrxd%d: %d less than nrxd_min %d - resetting to min\n",
	i, scctx->isc_nrxd[i], sctx->isc_nrxd_min[i]);
	scctx->isc_nrxd[i] = sctx->isc_nrxd_min[i];
	}
	if (scctx->isc_nrxd[i] > sctx->isc_nrxd_max[i]) {
	device_printf(dev, "nrxd%d: %d greater than nrxd_max %d - resetting to max\n",
	i, scctx->isc_nrxd[i], sctx->isc_nrxd_max[i]);
	scctx->isc_nrxd[i] = sctx->isc_nrxd_max[i];
	}
	if (!powerof2(scctx->isc_nrxd[i])) {
	device_printf(dev, "nrxd%d: %d is not a power of 2 - using default value of %d\n",
	i, scctx->isc_nrxd[i], sctx->isc_nrxd_default[i]);
	scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i];
	}
	}

	for (i = 0; i < sctx->isc_ntxqs; i++) {
	if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) {
	device_printf(dev, "ntxd%d: %d less than ntxd_min %d - resetting to min\n",
	i, scctx->isc_ntxd[i], sctx->isc_ntxd_min[i]);
	scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i];
	}
	if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) {
	device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n",
	i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]);
	scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i];
	}
	if (!powerof2(scctx->isc_ntxd[i])) {
	device_printf(dev, "ntxd%d: %d is not a power of 2 - using default value of %d\n",
	i, scctx->isc_ntxd[i], sctx->isc_ntxd_default[i]);
	scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i];
	}
	}
	}

	static void
	iflib_add_pfil(if_ctx_t ctx)
	{
	struct pfil_head *pfil;
	struct pfil_head_args pa;
	iflib_rxq_t rxq;
	int i;

	pa.pa_version = PFIL_VERSION;
	pa.pa_flags = PFIL_IN;
	pa.pa_type = PFIL_TYPE_ETHERNET;
	pa.pa_headname = ctx->ifc_ifp->if_xname;
	pfil = pfil_head_register(&pa);

	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
	rxq->pfil = pfil;
	}
	}

	static void
	iflib_rem_pfil(if_ctx_t ctx)
	{
	struct pfil_head *pfil;
	iflib_rxq_t rxq;
	int i;

	rxq = ctx->ifc_rxqs;
	pfil = rxq->pfil;
	for (i = 0; i < NRXQSETS(ctx); i++, rxq++) {
	rxq->pfil = NULL;
	}
	pfil_head_unregister(pfil);
	}

	static uint16_t
	get_ctx_core_offset(if_ctx_t ctx)
	{
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	struct cpu_offset *op;
	uint16_t qc;
	uint16_t ret = ctx->ifc_sysctl_core_offset;

	if (ret != CORE_OFFSET_UNSPECIFIED)
	return (ret);

	if (ctx->ifc_sysctl_separate_txrx)
	qc = scctx->isc_ntxqsets + scctx->isc_nrxqsets;
	else
	qc = max(scctx->isc_ntxqsets, scctx->isc_nrxqsets);

	mtx_lock(&cpu_offset_mtx);
	SLIST_FOREACH(op, &cpu_offsets, entries) {
	if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) {
	ret = op->offset;
	op->offset += qc;
	MPASS(op->refcount < UINT_MAX);
	op->refcount++;
	break;
	}
	}
	if (ret == CORE_OFFSET_UNSPECIFIED) {
	ret = 0;
	op = malloc(sizeof(struct cpu_offset), M_IFLIB,
	M_NOWAIT \| M_ZERO);
	if (op == NULL) {
	device_printf(ctx->ifc_dev,
	"allocation for cpu offset failed.\n");
	} else {
	op->offset = qc;
	op->refcount = 1;
	CPU_COPY(&ctx->ifc_cpus, &op->set);
	SLIST_INSERT_HEAD(&cpu_offsets, op, entries);
	}
	}
	mtx_unlock(&cpu_offset_mtx);

	return (ret);
	}

	static void
	unref_ctx_core_offset(if_ctx_t ctx)
	{
	struct cpu_offset op, top;

	mtx_lock(&cpu_offset_mtx);
	SLIST_FOREACH_SAFE(op, &cpu_offsets, entries, top) {
	if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) {
	MPASS(op->refcount > 0);
	op->refcount--;
	if (op->refcount == 0) {
	SLIST_REMOVE(&cpu_offsets, op, cpu_offset, entries);
	free(op, M_IFLIB);
	}
	break;
	}
	}
	mtx_unlock(&cpu_offset_mtx);
	}

	int
	iflib_device_register(device_t dev, void sc, if_shared_ctx_t sctx, if_ctx_t ctxp)
	{
	if_ctx_t ctx;
	if_t ifp;
	if_softc_ctx_t scctx;
	kobjop_desc_t kobj_desc;
	kobj_method_t *kobj_method;
	int err, msix, rid;
	int num_txd, num_rxd;

	ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK\|M_ZERO);

	if (sc == NULL) {
	sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK\|M_ZERO);
	device_set_softc(dev, ctx);
	ctx->ifc_flags \|= IFC_SC_ALLOCATED;
	}

	ctx->ifc_sctx = sctx;
	ctx->ifc_dev = dev;
	ctx->ifc_softc = sc;

	if ((err = iflib_register(ctx)) != 0) {
	device_printf(dev, "iflib_register failed %d\n", err);
	goto fail_ctx_free;
	}
	iflib_add_device_sysctl_pre(ctx);

	scctx = &ctx->ifc_softc_ctx;
	ifp = ctx->ifc_ifp;

	iflib_reset_qvalues(ctx);
	CTX_LOCK(ctx);
	if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
	device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
	goto fail_unlock;
	}
	_iflib_pre_assert(scctx);
	ctx->ifc_txrx = *scctx->isc_txrx;

	if (sctx->isc_flags & IFLIB_DRIVER_MEDIA)
	ctx->ifc_mediap = scctx->isc_media;

	#ifdef INVARIANTS
	if (scctx->isc_capabilities & IFCAP_TXCSUM)
	MPASS(scctx->isc_tx_csum_flags);
	#endif

	if_setcapabilities(ifp,
	scctx->isc_capabilities \| IFCAP_HWSTATS \| IFCAP_NOMAP);
	if_setcapenable(ifp,
	scctx->isc_capenable \| IFCAP_HWSTATS \| IFCAP_NOMAP);

	if (scctx->isc_ntxqsets == 0 \|\| (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
	scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
	if (scctx->isc_nrxqsets == 0 \|\| (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
	scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;

	num_txd = iflib_num_tx_descs(ctx);
	num_rxd = iflib_num_rx_descs(ctx);

	/* XXX change for per-queue sizes */
	device_printf(dev, "Using %d TX descriptors and %d RX descriptors\n",
	num_txd, num_rxd);

	if (scctx->isc_tx_nsegments > num_txd / MAX_SINGLE_PACKET_FRACTION)
	scctx->isc_tx_nsegments = max(1, num_txd /
	MAX_SINGLE_PACKET_FRACTION);
	if (scctx->isc_tx_tso_segments_max > num_txd /
	MAX_SINGLE_PACKET_FRACTION)
	scctx->isc_tx_tso_segments_max = max(1,
	num_txd / MAX_SINGLE_PACKET_FRACTION);

	/* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
	if (if_getcapabilities(ifp) & IFCAP_TSO) {
	/*
	* The stack can't handle a TSO size larger than IP_MAXPACKET,
	* but some MACs do.
	*/
	if_sethwtsomax(ifp, min(scctx->isc_tx_tso_size_max,
	IP_MAXPACKET));
	/*
	* Take maximum number of m_pullup(9)'s in iflib_parse_header()
	* into account. In the worst case, each of these calls will
	* add another mbuf and, thus, the requirement for another DMA
	* segment. So for best performance, it doesn't make sense to
	* advertize a maximum of TSO segments that typically will
	* require defragmentation in iflib_encap().
	*/
	if_sethwtsomaxsegcount(ifp, scctx->isc_tx_tso_segments_max - 3);
	if_sethwtsomaxsegsize(ifp, scctx->isc_tx_tso_segsize_max);
	}
	if (scctx->isc_rss_table_size == 0)
	scctx->isc_rss_table_size = 64;
	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;

	GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
	/* XXX format name */
	taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx,
	NULL, NULL, "admin");

	/* Set up cpu set. If it fails, use the set of all CPUs. */
	if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) {
	device_printf(dev, "Unable to fetch CPU list\n");
	CPU_COPY(&all_cpus, &ctx->ifc_cpus);
	}
	MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0);

	/*
	** Now set up MSI or MSI-X, should return us the number of supported
	** vectors (will be 1 for a legacy interrupt and MSI).
	*/
	if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
	msix = scctx->isc_vectors;
	} else if (scctx->isc_msix_bar != 0)
	/*
	* The simple fact that isc_msix_bar is not 0 does not mean we
	* we have a good value there that is known to work.
	*/
	msix = iflib_msix_init(ctx);
	else {
	scctx->isc_vectors = 1;
	scctx->isc_ntxqsets = 1;
	scctx->isc_nrxqsets = 1;
	scctx->isc_intr = IFLIB_INTR_LEGACY;
	msix = 0;
	}
	/* Get memory for the station queues */
	if ((err = iflib_queues_alloc(ctx))) {
	device_printf(dev, "Unable to allocate queue memory\n");
	goto fail_intr_free;
	}

	if ((err = iflib_qset_structures_setup(ctx)))
	goto fail_queues;

	/*
	* Now that we know how many queues there are, get the core offset.
	*/
	ctx->ifc_sysctl_core_offset = get_ctx_core_offset(ctx);

	if (msix > 1) {
	/*
	* When using MSI-X, ensure that ifdi_{r,t}x_queue_intr_enable
	* aren't the default NULL implementation.
	*/
	kobj_desc = &ifdi_rx_queue_intr_enable_desc;
	kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL,
	kobj_desc);
	if (kobj_method == &kobj_desc->deflt) {
	device_printf(dev,
	"MSI-X requires ifdi_rx_queue_intr_enable method");
	err = EOPNOTSUPP;
	goto fail_queues;
	}
	kobj_desc = &ifdi_tx_queue_intr_enable_desc;
	kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL,
	kobj_desc);
	if (kobj_method == &kobj_desc->deflt) {
	device_printf(dev,
	"MSI-X requires ifdi_tx_queue_intr_enable method");
	err = EOPNOTSUPP;
	goto fail_queues;
	}

	/*
	* Assign the MSI-X vectors.
	* Note that the default NULL ifdi_msix_intr_assign method will
	* fail here, too.
	*/
	err = IFDI_MSIX_INTR_ASSIGN(ctx, msix);
	if (err != 0) {
	device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n",
	err);
	goto fail_queues;
	}
	} else if (scctx->isc_intr != IFLIB_INTR_MSIX) {
	rid = 0;
	if (scctx->isc_intr == IFLIB_INTR_MSI) {
	MPASS(msix == 1);
	rid = 1;
	}
	if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) {
	device_printf(dev, "iflib_legacy_setup failed %d\n", err);
	goto fail_queues;
	}
	} else {
	device_printf(dev,
	"Cannot use iflib with only 1 MSI-X interrupt!\n");
	err = ENODEV;
	goto fail_intr_free;
	}

	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet);

	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
	device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
	goto fail_detach;
	}

	/*
	* Tell the upper layer(s) if IFCAP_VLAN_MTU is supported.
	* This must appear after the call to ether_ifattach() because
	* ether_ifattach() sets if_hdrlen to the default value.
	*/
	if (if_getcapabilities(ifp) & IFCAP_VLAN_MTU)
	if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));

	if ((err = iflib_netmap_attach(ctx))) {
	device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err);
	goto fail_detach;
	}
	*ctxp = ctx;

	DEBUGNET_SET(ctx->ifc_ifp, iflib);

	if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
	iflib_add_device_sysctl_post(ctx);
	iflib_add_pfil(ctx);
	ctx->ifc_flags \|= IFC_INIT_DONE;
	CTX_UNLOCK(ctx);

	return (0);

	fail_detach:
	ether_ifdetach(ctx->ifc_ifp);
	fail_intr_free:
	iflib_free_intr_mem(ctx);
	fail_queues:
	iflib_tx_structures_free(ctx);
	iflib_rx_structures_free(ctx);
	taskqgroup_detach(qgroup_if_config_tqg, &ctx->ifc_admin_task);
	IFDI_DETACH(ctx);
	fail_unlock:
	CTX_UNLOCK(ctx);
	iflib_deregister(ctx);
	fail_ctx_free:
	device_set_softc(ctx->ifc_dev, NULL);
	if (ctx->ifc_flags & IFC_SC_ALLOCATED)
	free(ctx->ifc_softc, M_IFLIB);
	free(ctx, M_IFLIB);
	return (err);
	}

	int
	iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp,
	struct iflib_cloneattach_ctx *clctx)
	{
	int num_txd, num_rxd;
	int err;
	if_ctx_t ctx;
	if_t ifp;
	if_softc_ctx_t scctx;
	int i;
	void *sc;

	ctx = malloc(sizeof(*ctx), M_IFLIB, M_WAITOK\|M_ZERO);
	sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK\|M_ZERO);
	ctx->ifc_flags \|= IFC_SC_ALLOCATED;
	if (sctx->isc_flags & (IFLIB_PSEUDO\|IFLIB_VIRTUAL))
	ctx->ifc_flags \|= IFC_PSEUDO;

	ctx->ifc_sctx = sctx;
	ctx->ifc_softc = sc;
	ctx->ifc_dev = dev;

	if ((err = iflib_register(ctx)) != 0) {
	device_printf(dev, "%s: iflib_register failed %d\n", __func__, err);
	goto fail_ctx_free;
	}
	iflib_add_device_sysctl_pre(ctx);

	scctx = &ctx->ifc_softc_ctx;
	ifp = ctx->ifc_ifp;

	iflib_reset_qvalues(ctx);
	CTX_LOCK(ctx);
	if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
	device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
	goto fail_unlock;
	}
	if (sctx->isc_flags & IFLIB_GEN_MAC)
	ether_gen_addr(ifp, &ctx->ifc_mac);
	if ((err = IFDI_CLONEATTACH(ctx, clctx->cc_ifc, clctx->cc_name,
	clctx->cc_params)) != 0) {
	device_printf(dev, "IFDI_CLONEATTACH failed %d\n", err);
	goto fail_unlock;
	}
	#ifdef INVARIANTS
	if (scctx->isc_capabilities & IFCAP_TXCSUM)
	MPASS(scctx->isc_tx_csum_flags);
	#endif

	if_setcapabilities(ifp, scctx->isc_capabilities \| IFCAP_HWSTATS \| IFCAP_LINKSTATE);
	if_setcapenable(ifp, scctx->isc_capenable \| IFCAP_HWSTATS \| IFCAP_LINKSTATE);

	ifp->if_flags \|= IFF_NOGROUP;
	if (sctx->isc_flags & IFLIB_PSEUDO) {
	ifmedia_add(ctx->ifc_mediap, IFM_ETHER \| IFM_AUTO, 0, NULL);
	ifmedia_set(ctx->ifc_mediap, IFM_ETHER \| IFM_AUTO);
	if (sctx->isc_flags & IFLIB_PSEUDO_ETHER) {
	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet);
	} else {
	if_attach(ctx->ifc_ifp);
	bpfattach(ctx->ifc_ifp, DLT_NULL, sizeof(u_int32_t));
	}

	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
	device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
	goto fail_detach;
	}
	*ctxp = ctx;

	/*
	* Tell the upper layer(s) if IFCAP_VLAN_MTU is supported.
	* This must appear after the call to ether_ifattach() because
	* ether_ifattach() sets if_hdrlen to the default value.
	*/
	if (if_getcapabilities(ifp) & IFCAP_VLAN_MTU)
	if_setifheaderlen(ifp,
	sizeof(struct ether_vlan_header));

	if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
	iflib_add_device_sysctl_post(ctx);
	ctx->ifc_flags \|= IFC_INIT_DONE;
	CTX_UNLOCK(ctx);
	return (0);
	}
	ifmedia_add(ctx->ifc_mediap, IFM_ETHER \| IFM_1000_T \| IFM_FDX, 0, NULL);
	ifmedia_add(ctx->ifc_mediap, IFM_ETHER \| IFM_AUTO, 0, NULL);
	ifmedia_set(ctx->ifc_mediap, IFM_ETHER \| IFM_AUTO);

	_iflib_pre_assert(scctx);
	ctx->ifc_txrx = *scctx->isc_txrx;

	if (scctx->isc_ntxqsets == 0 \|\| (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
	scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
	if (scctx->isc_nrxqsets == 0 \|\| (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
	scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;

	num_txd = iflib_num_tx_descs(ctx);
	num_rxd = iflib_num_rx_descs(ctx);

	/* XXX change for per-queue sizes */
	device_printf(dev, "Using %d TX descriptors and %d RX descriptors\n",
	num_txd, num_rxd);

	if (scctx->isc_tx_nsegments > num_txd / MAX_SINGLE_PACKET_FRACTION)
	scctx->isc_tx_nsegments = max(1, num_txd /
	MAX_SINGLE_PACKET_FRACTION);
	if (scctx->isc_tx_tso_segments_max > num_txd /
	MAX_SINGLE_PACKET_FRACTION)
	scctx->isc_tx_tso_segments_max = max(1,
	num_txd / MAX_SINGLE_PACKET_FRACTION);

	/* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
	if (if_getcapabilities(ifp) & IFCAP_TSO) {
	/*
	* The stack can't handle a TSO size larger than IP_MAXPACKET,
	* but some MACs do.
	*/
	if_sethwtsomax(ifp, min(scctx->isc_tx_tso_size_max,
	IP_MAXPACKET));
	/*
	* Take maximum number of m_pullup(9)'s in iflib_parse_header()
	* into account. In the worst case, each of these calls will
	* add another mbuf and, thus, the requirement for another DMA
	* segment. So for best performance, it doesn't make sense to
	* advertize a maximum of TSO segments that typically will
	* require defragmentation in iflib_encap().
	*/
	if_sethwtsomaxsegcount(ifp, scctx->isc_tx_tso_segments_max - 3);
	if_sethwtsomaxsegsize(ifp, scctx->isc_tx_tso_segsize_max);
	}
	if (scctx->isc_rss_table_size == 0)
	scctx->isc_rss_table_size = 64;
	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;

	GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
	/* XXX format name */
	taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx,
	NULL, NULL, "admin");

	/* XXX --- can support > 1 -- but keep it simple for now */
	scctx->isc_intr = IFLIB_INTR_LEGACY;

	/* Get memory for the station queues */
	if ((err = iflib_queues_alloc(ctx))) {
	device_printf(dev, "Unable to allocate queue memory\n");
	goto fail_iflib_detach;
	}

	if ((err = iflib_qset_structures_setup(ctx))) {
	device_printf(dev, "qset structure setup failed %d\n", err);
	goto fail_queues;
	}

	/*
	* XXX What if anything do we want to do about interrupts?
	*/
	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet);
	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
	device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
	goto fail_detach;
	}

	/*
	* Tell the upper layer(s) if IFCAP_VLAN_MTU is supported.
	* This must appear after the call to ether_ifattach() because
	* ether_ifattach() sets if_hdrlen to the default value.
	*/
	if (if_getcapabilities(ifp) & IFCAP_VLAN_MTU)
	if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));

	/* XXX handle more than one queue */
	for (i = 0; i < scctx->isc_nrxqsets; i++)
	IFDI_RX_CLSET(ctx, 0, i, ctx->ifc_rxqs[i].ifr_fl[0].ifl_sds.ifsd_cl);

	*ctxp = ctx;

	if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
	iflib_add_device_sysctl_post(ctx);
	ctx->ifc_flags \|= IFC_INIT_DONE;
	CTX_UNLOCK(ctx);

	return (0);
	fail_detach:
	ether_ifdetach(ctx->ifc_ifp);
	fail_queues:
	iflib_tx_structures_free(ctx);
	iflib_rx_structures_free(ctx);
	fail_iflib_detach:
	IFDI_DETACH(ctx);
	fail_unlock:
	CTX_UNLOCK(ctx);
	iflib_deregister(ctx);
	fail_ctx_free:
	free(ctx->ifc_softc, M_IFLIB);
	free(ctx, M_IFLIB);
	return (err);
	}

	int
	iflib_pseudo_deregister(if_ctx_t ctx)
	{
	if_t ifp = ctx->ifc_ifp;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	int i, j;
	struct taskqgroup *tqg;
	iflib_fl_t fl;

	/* Unregister VLAN event handlers early */
	iflib_unregister_vlan_handlers(ctx);

	if ((sctx->isc_flags & IFLIB_PSEUDO) &&
	(sctx->isc_flags & IFLIB_PSEUDO_ETHER) == 0) {
	bpfdetach(ifp);
	if_detach(ifp);
	} else {
	ether_ifdetach(ifp);
	}
	/* XXX drain any dependent tasks */
	tqg = qgroup_if_io_tqg;
	for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
	callout_drain(&txq->ift_timer);
	if (txq->ift_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &txq->ift_task);
	}
	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
	callout_drain(&rxq->ifr_watchdog);
	if (rxq->ifr_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &rxq->ifr_task);

	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	free(fl->ifl_rx_bitmap, M_IFLIB);
	}
	tqg = qgroup_if_config_tqg;
	if (ctx->ifc_admin_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &ctx->ifc_admin_task);
	if (ctx->ifc_vflr_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &ctx->ifc_vflr_task);

	iflib_tx_structures_free(ctx);
	iflib_rx_structures_free(ctx);

	iflib_deregister(ctx);

	if (ctx->ifc_flags & IFC_SC_ALLOCATED)
	free(ctx->ifc_softc, M_IFLIB);
	free(ctx, M_IFLIB);
	return (0);
	}

	int
	iflib_device_attach(device_t dev)
	{
	if_ctx_t ctx;
	if_shared_ctx_t sctx;

	if ((sctx = DEVICE_REGISTER(dev)) == NULL \|\| sctx->isc_magic != IFLIB_MAGIC)
	return (ENOTSUP);

	pci_enable_busmaster(dev);

	return (iflib_device_register(dev, NULL, sctx, &ctx));
	}

	int
	iflib_device_deregister(if_ctx_t ctx)
	{
	if_t ifp = ctx->ifc_ifp;
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	device_t dev = ctx->ifc_dev;
	int i, j;
	struct taskqgroup *tqg;
	iflib_fl_t fl;

	/* Make sure VLANS are not using driver */
	if (if_vlantrunkinuse(ifp)) {
	device_printf(dev, "Vlan in use, detach first\n");
	return (EBUSY);
	}
	#ifdef PCI_IOV
	if (!CTX_IS_VF(ctx) && pci_iov_detach(dev) != 0) {
	device_printf(dev, "SR-IOV in use; detach first.\n");
	return (EBUSY);
	}
	#endif

	STATE_LOCK(ctx);
	ctx->ifc_flags \|= IFC_IN_DETACH;
	STATE_UNLOCK(ctx);

	/* Unregister VLAN handlers before calling iflib_stop() */
	iflib_unregister_vlan_handlers(ctx);

	iflib_netmap_detach(ifp);
	ether_ifdetach(ifp);

	CTX_LOCK(ctx);
	iflib_stop(ctx);
	CTX_UNLOCK(ctx);

	iflib_rem_pfil(ctx);
	if (ctx->ifc_led_dev != NULL)
	led_destroy(ctx->ifc_led_dev);
	/* XXX drain any dependent tasks */
	tqg = qgroup_if_io_tqg;
	for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
	callout_drain(&txq->ift_timer);
	if (txq->ift_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &txq->ift_task);
	}
	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
	if (rxq->ifr_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &rxq->ifr_task);

	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	free(fl->ifl_rx_bitmap, M_IFLIB);
	}
	tqg = qgroup_if_config_tqg;
	if (ctx->ifc_admin_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &ctx->ifc_admin_task);
	if (ctx->ifc_vflr_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
	CTX_LOCK(ctx);
	IFDI_DETACH(ctx);
	CTX_UNLOCK(ctx);

	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
	iflib_free_intr_mem(ctx);

	bus_generic_detach(dev);

	iflib_tx_structures_free(ctx);
	iflib_rx_structures_free(ctx);

	iflib_deregister(ctx);

	device_set_softc(ctx->ifc_dev, NULL);
	if (ctx->ifc_flags & IFC_SC_ALLOCATED)
	free(ctx->ifc_softc, M_IFLIB);
	unref_ctx_core_offset(ctx);
	free(ctx, M_IFLIB);
	return (0);
	}

	static void
	iflib_free_intr_mem(if_ctx_t ctx)
	{

	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
	iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
	}
	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
	pci_release_msi(ctx->ifc_dev);
	}
	if (ctx->ifc_msix_mem != NULL) {
	bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
	rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem);
	ctx->ifc_msix_mem = NULL;
	}
	}

	int
	iflib_device_detach(device_t dev)
	{
	if_ctx_t ctx = device_get_softc(dev);

	return (iflib_device_deregister(ctx));
	}

	int
	iflib_device_suspend(device_t dev)
	{
	if_ctx_t ctx = device_get_softc(dev);

	CTX_LOCK(ctx);
	IFDI_SUSPEND(ctx);
	CTX_UNLOCK(ctx);

	return bus_generic_suspend(dev);
	}
	int
	iflib_device_shutdown(device_t dev)
	{
	if_ctx_t ctx = device_get_softc(dev);

	CTX_LOCK(ctx);
	IFDI_SHUTDOWN(ctx);
	CTX_UNLOCK(ctx);

	return bus_generic_suspend(dev);
	}


	int
	iflib_device_resume(device_t dev)
	{
	if_ctx_t ctx = device_get_softc(dev);
	iflib_txq_t txq = ctx->ifc_txqs;

	CTX_LOCK(ctx);
	IFDI_RESUME(ctx);
	iflib_if_init_locked(ctx);
	CTX_UNLOCK(ctx);
	for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
	iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);

	return (bus_generic_resume(dev));
	}

	int
	iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
	{
	int error;
	if_ctx_t ctx = device_get_softc(dev);

	CTX_LOCK(ctx);
	error = IFDI_IOV_INIT(ctx, num_vfs, params);
	CTX_UNLOCK(ctx);

	return (error);
	}

	void
	iflib_device_iov_uninit(device_t dev)
	{
	if_ctx_t ctx = device_get_softc(dev);

	CTX_LOCK(ctx);
	IFDI_IOV_UNINIT(ctx);
	CTX_UNLOCK(ctx);
	}

	int
	iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
	{
	int error;
	if_ctx_t ctx = device_get_softc(dev);

	CTX_LOCK(ctx);
	error = IFDI_IOV_VF_ADD(ctx, vfnum, params);
	CTX_UNLOCK(ctx);

	return (error);
	}

	/*********************************************************************
	*
	* MODULE FUNCTION DEFINITIONS
	*
	**********************************************************************/

	/*
	* - Start a fast taskqueue thread for each core
	* - Start a taskqueue for control operations
	*/
	static int
	iflib_module_init(void)
	{
	return (0);
	}

	static int
	iflib_module_event_handler(module_t mod, int what, void *arg)
	{
	int err;

	switch (what) {
	case MOD_LOAD:
	if ((err = iflib_module_init()) != 0)
	return (err);
	break;
	case MOD_UNLOAD:
	return (EBUSY);
	default:
	return (EOPNOTSUPP);
	}

	return (0);
	}

	/*********************************************************************
	*
	* PUBLIC FUNCTION DEFINITIONS
	* ordered as in iflib.h
	*
	**********************************************************************/


	static void
	_iflib_assert(if_shared_ctx_t sctx)
	{
	int i;

	MPASS(sctx->isc_tx_maxsize);
	MPASS(sctx->isc_tx_maxsegsize);

	MPASS(sctx->isc_rx_maxsize);
	MPASS(sctx->isc_rx_nsegments);
	MPASS(sctx->isc_rx_maxsegsize);

	MPASS(sctx->isc_nrxqs >= 1 && sctx->isc_nrxqs <= 8);
	for (i = 0; i < sctx->isc_nrxqs; i++) {
	MPASS(sctx->isc_nrxd_min[i]);
	MPASS(powerof2(sctx->isc_nrxd_min[i]));
	MPASS(sctx->isc_nrxd_max[i]);
	MPASS(powerof2(sctx->isc_nrxd_max[i]));
	MPASS(sctx->isc_nrxd_default[i]);
	MPASS(powerof2(sctx->isc_nrxd_default[i]));
	}

	MPASS(sctx->isc_ntxqs >= 1 && sctx->isc_ntxqs <= 8);
	for (i = 0; i < sctx->isc_ntxqs; i++) {
	MPASS(sctx->isc_ntxd_min[i]);
	MPASS(powerof2(sctx->isc_ntxd_min[i]));
	MPASS(sctx->isc_ntxd_max[i]);
	MPASS(powerof2(sctx->isc_ntxd_max[i]));
	MPASS(sctx->isc_ntxd_default[i]);
	MPASS(powerof2(sctx->isc_ntxd_default[i]));
	}
	}

	static void
	_iflib_pre_assert(if_softc_ctx_t scctx)
	{

	MPASS(scctx->isc_txrx->ift_txd_encap);
	MPASS(scctx->isc_txrx->ift_txd_flush);
	MPASS(scctx->isc_txrx->ift_txd_credits_update);
	MPASS(scctx->isc_txrx->ift_rxd_available);
	MPASS(scctx->isc_txrx->ift_rxd_pkt_get);
	MPASS(scctx->isc_txrx->ift_rxd_refill);
	MPASS(scctx->isc_txrx->ift_rxd_flush);
	}

	static int
	iflib_register(if_ctx_t ctx)
	{
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	driver_t *driver = sctx->isc_driver;
	device_t dev = ctx->ifc_dev;
	if_t ifp;
	u_char type;
	int iflags;

	if ((sctx->isc_flags & IFLIB_PSEUDO) == 0)
	_iflib_assert(sctx);

	CTX_LOCK_INIT(ctx);
	STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));
	if (sctx->isc_flags & IFLIB_PSEUDO) {
	if (sctx->isc_flags & IFLIB_PSEUDO_ETHER)
	type = IFT_ETHER;
	else
	type = IFT_PPP;
	} else
	type = IFT_ETHER;
	ifp = ctx->ifc_ifp = if_alloc(type);
	if (ifp == NULL) {
	device_printf(dev, "can not allocate ifnet structure\n");
	return (ENOMEM);
	}

	/*
	* Initialize our context's device specific methods
	*/
	kobj_init((kobj_t) ctx, (kobj_class_t) driver);
	kobj_class_compile((kobj_class_t) driver);

	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
	if_setsoftc(ifp, ctx);
	if_setdev(ifp, dev);
	if_setinitfn(ifp, iflib_if_init);
	if_setioctlfn(ifp, iflib_if_ioctl);
	#ifdef ALTQ
	if_setstartfn(ifp, iflib_altq_if_start);
	if_settransmitfn(ifp, iflib_altq_if_transmit);
	if_setsendqready(ifp);
	#else
	if_settransmitfn(ifp, iflib_if_transmit);
	#endif
	if_setqflushfn(ifp, iflib_if_qflush);
	iflags = IFF_MULTICAST \| IFF_KNOWSEPOCH;

	if ((sctx->isc_flags & IFLIB_PSEUDO) &&
	(sctx->isc_flags & IFLIB_PSEUDO_ETHER) == 0)
	iflags \|= IFF_POINTOPOINT;
	else
	iflags \|= IFF_BROADCAST \| IFF_SIMPLEX;
	if_setflags(ifp, iflags);
	ctx->ifc_vlan_attach_event =
	EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
	EVENTHANDLER_PRI_FIRST);
	ctx->ifc_vlan_detach_event =
	EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx,
	EVENTHANDLER_PRI_FIRST);

	if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) {
	ctx->ifc_mediap = &ctx->ifc_media;
	ifmedia_init(ctx->ifc_mediap, IFM_IMASK,
	iflib_media_change, iflib_media_status);
	}
	return (0);
	}

	static void
	iflib_unregister_vlan_handlers(if_ctx_t ctx)
	{
	/* Unregister VLAN events */
	if (ctx->ifc_vlan_attach_event != NULL) {
	EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
	ctx->ifc_vlan_attach_event = NULL;
	}
	if (ctx->ifc_vlan_detach_event != NULL) {
	EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
	ctx->ifc_vlan_detach_event = NULL;
	}

	}

	static void
	iflib_deregister(if_ctx_t ctx)
	{
	if_t ifp = ctx->ifc_ifp;

	/* Remove all media */
	ifmedia_removeall(&ctx->ifc_media);

	/* Ensure that VLAN event handlers are unregistered */
	iflib_unregister_vlan_handlers(ctx);

	/* Release kobject reference */
	kobj_delete((kobj_t) ctx, NULL);

	/* Free the ifnet structure */
	if_free(ifp);

	STATE_LOCK_DESTROY(ctx);

	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
	CTX_LOCK_DESTROY(ctx);
	}

	static int
	iflib_queues_alloc(if_ctx_t ctx)
	{
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	device_t dev = ctx->ifc_dev;
	int nrxqsets = scctx->isc_nrxqsets;
	int ntxqsets = scctx->isc_ntxqsets;
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	iflib_fl_t fl = NULL;
	int i, j, cpu, err, txconf, rxconf;
	iflib_dma_info_t ifdip;
	uint32_t *rxqsizes = scctx->isc_rxqsizes;
	uint32_t *txqsizes = scctx->isc_txqsizes;
	uint8_t nrxqs = sctx->isc_nrxqs;
	uint8_t ntxqs = sctx->isc_ntxqs;
	int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1;
	caddr_t *vaddrs;
	uint64_t *paddrs;

	KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1"));
	KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1"));

	/* Allocate the TX ring struct memory */
	if (!(ctx->ifc_txqs =
	(iflib_txq_t) malloc(sizeof(struct iflib_txq) *
	ntxqsets, M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev, "Unable to allocate TX ring memory\n");
	err = ENOMEM;
	goto fail;
	}

	/* Now allocate the RX */
	if (!(ctx->ifc_rxqs =
	(iflib_rxq_t) malloc(sizeof(struct iflib_rxq) *
	nrxqsets, M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev, "Unable to allocate RX ring memory\n");
	err = ENOMEM;
	goto rx_fail;
	}

	txq = ctx->ifc_txqs;
	rxq = ctx->ifc_rxqs;

	/*
	* XXX handle allocation failure
	*/
	for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) {
	/* Set up some basics */

	if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs,
	M_IFLIB, M_NOWAIT \| M_ZERO)) == NULL) {
	device_printf(dev,
	"Unable to allocate TX DMA info memory\n");
	err = ENOMEM;
	goto err_tx_desc;
	}
	txq->ift_ifdi = ifdip;
	for (j = 0; j < ntxqs; j++, ifdip++) {
	if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, 0)) {
	device_printf(dev,
	"Unable to allocate TX descriptors\n");
	err = ENOMEM;
	goto err_tx_desc;
	}
	txq->ift_txd_size[j] = scctx->isc_txd_size[j];
	bzero((void *)ifdip->idi_vaddr, txqsizes[j]);
	}
	txq->ift_ctx = ctx;
	txq->ift_id = i;
	if (sctx->isc_flags & IFLIB_HAS_TXCQ) {
	txq->ift_br_offset = 1;
	} else {
	txq->ift_br_offset = 0;
	}
	/* XXX fix this */
	txq->ift_timer.c_cpu = cpu;

	if (iflib_txsd_alloc(txq)) {
	device_printf(dev, "Critical Failure setting up TX buffers\n");
	err = ENOMEM;
	goto err_tx_desc;
	}

	/* Initialize the TX lock */
	snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:TX(%d):callout",
	device_get_nameunit(dev), txq->ift_id);
	mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
	callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);

	err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain,
	iflib_txq_can_drain, M_IFLIB, M_WAITOK);
	if (err) {
	/* XXX free any allocated rings */
	device_printf(dev, "Unable to allocate buf_ring\n");
	goto err_tx_desc;
	}
	}

	for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) {
	/* Set up some basics */
	callout_init(&rxq->ifr_watchdog, 1);

	if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs,
	M_IFLIB, M_NOWAIT \| M_ZERO)) == NULL) {
	device_printf(dev,
	"Unable to allocate RX DMA info memory\n");
	err = ENOMEM;
	goto err_tx_desc;
	}

	rxq->ifr_ifdi = ifdip;
	/* XXX this needs to be changed if #rx queues != #tx queues */
	rxq->ifr_ntxqirq = 1;
	rxq->ifr_txqid[0] = i;
	for (j = 0; j < nrxqs; j++, ifdip++) {
	if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, 0)) {
	device_printf(dev,
	"Unable to allocate RX descriptors\n");
	err = ENOMEM;
	goto err_tx_desc;
	}
	bzero((void *)ifdip->idi_vaddr, rxqsizes[j]);
	}
	rxq->ifr_ctx = ctx;
	rxq->ifr_id = i;
	if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
	rxq->ifr_fl_offset = 1;
	} else {
	rxq->ifr_fl_offset = 0;
	}
	rxq->ifr_nfl = nfree_lists;
	if (!(fl =
	(iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT \| M_ZERO))) {
	device_printf(dev, "Unable to allocate free list memory\n");
	err = ENOMEM;
	goto err_tx_desc;
	}
	rxq->ifr_fl = fl;
	for (j = 0; j < nfree_lists; j++) {
	fl[j].ifl_rxq = rxq;
	fl[j].ifl_id = j;
	fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
	fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];
	}
	/* Allocate receive buffers for the ring */
	if (iflib_rxsd_alloc(rxq)) {
	device_printf(dev,
	"Critical Failure setting up receive buffers\n");
	err = ENOMEM;
	goto err_rx_desc;
	}

	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB,
	M_WAITOK);
	}

	/* TXQs */
	vaddrs = malloc(sizeof(caddr_t)ntxqsetsntxqs, M_IFLIB, M_WAITOK);
	paddrs = malloc(sizeof(uint64_t)ntxqsetsntxqs, M_IFLIB, M_WAITOK);
	for (i = 0; i < ntxqsets; i++) {
	iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi;

	for (j = 0; j < ntxqs; j++, di++) {
	vaddrs[i*ntxqs + j] = di->idi_vaddr;
	paddrs[i*ntxqs + j] = di->idi_paddr;
	}
	}
	if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) {
	device_printf(ctx->ifc_dev,
	"Unable to allocate device TX queue\n");
	iflib_tx_structures_free(ctx);
	free(vaddrs, M_IFLIB);
	free(paddrs, M_IFLIB);
	goto err_rx_desc;
	}
	free(vaddrs, M_IFLIB);
	free(paddrs, M_IFLIB);

	/* RXQs */
	vaddrs = malloc(sizeof(caddr_t)nrxqsetsnrxqs, M_IFLIB, M_WAITOK);
	paddrs = malloc(sizeof(uint64_t)nrxqsetsnrxqs, M_IFLIB, M_WAITOK);
	for (i = 0; i < nrxqsets; i++) {
	iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi;

	for (j = 0; j < nrxqs; j++, di++) {
	vaddrs[i*nrxqs + j] = di->idi_vaddr;
	paddrs[i*nrxqs + j] = di->idi_paddr;
	}
	}
	if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) {
	device_printf(ctx->ifc_dev,
	"Unable to allocate device RX queue\n");
	iflib_tx_structures_free(ctx);
	free(vaddrs, M_IFLIB);
	free(paddrs, M_IFLIB);
	goto err_rx_desc;
	}
	free(vaddrs, M_IFLIB);
	free(paddrs, M_IFLIB);

	return (0);

	/* XXX handle allocation failure changes */
	err_rx_desc:
	err_tx_desc:
	rx_fail:
	if (ctx->ifc_rxqs != NULL)
	free(ctx->ifc_rxqs, M_IFLIB);
	ctx->ifc_rxqs = NULL;
	if (ctx->ifc_txqs != NULL)
	free(ctx->ifc_txqs, M_IFLIB);
	ctx->ifc_txqs = NULL;
	fail:
	return (err);
	}

	static int
	iflib_tx_structures_setup(if_ctx_t ctx)
	{
	iflib_txq_t txq = ctx->ifc_txqs;
	int i;

	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
	iflib_txq_setup(txq);

	return (0);
	}

	static void
	iflib_tx_structures_free(if_ctx_t ctx)
	{
	iflib_txq_t txq = ctx->ifc_txqs;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	int i, j;

	for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
	for (j = 0; j < sctx->isc_ntxqs; j++)
	iflib_dma_free(&txq->ift_ifdi[j]);
	iflib_txq_destroy(txq);
	}
	free(ctx->ifc_txqs, M_IFLIB);
	ctx->ifc_txqs = NULL;
	IFDI_QUEUES_FREE(ctx);
	}

	/*********************************************************************
	*
	* Initialize all receive rings.
	*
	**********************************************************************/
	static int
	iflib_rx_structures_setup(if_ctx_t ctx)
	{
	iflib_rxq_t rxq = ctx->ifc_rxqs;
	int q;
	#if defined(INET6) \|\| defined(INET)
	int err, i;
	#endif

	for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) {
	#if defined(INET6) \|\| defined(INET)
	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO) {
	err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp,
	TCP_LRO_ENTRIES, min(1024,
	ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset]));
	if (err != 0) {
	device_printf(ctx->ifc_dev,
	"LRO Initialization failed!\n");
	goto fail;
	}
	}
	#endif
	IFDI_RXQ_SETUP(ctx, rxq->ifr_id);
	}
	return (0);
	#if defined(INET6) \|\| defined(INET)
	fail:
	/*
	* Free LRO resources allocated so far, we will only handle
	* the rings that completed, the failing case will have
	* cleaned up for itself. 'q' failed, so its the terminus.
	*/
	rxq = ctx->ifc_rxqs;
	for (i = 0; i < q; ++i, rxq++) {
	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO)
	tcp_lro_free(&rxq->ifr_lc);
	}
	return (err);
	#endif
	}

	/*********************************************************************
	*
	* Free all receive rings.
	*
	**********************************************************************/
	static void
	iflib_rx_structures_free(if_ctx_t ctx)
	{
	iflib_rxq_t rxq = ctx->ifc_rxqs;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	int i, j;

	for (i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) {
	for (j = 0; j < sctx->isc_nrxqs; j++)
	iflib_dma_free(&rxq->ifr_ifdi[j]);
	iflib_rx_sds_free(rxq);
	#if defined(INET6) \|\| defined(INET)
	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO)
	tcp_lro_free(&rxq->ifr_lc);
	#endif
	}
	free(ctx->ifc_rxqs, M_IFLIB);
	ctx->ifc_rxqs = NULL;
	}

	static int
	iflib_qset_structures_setup(if_ctx_t ctx)
	{
	int err;

	/*
	* It is expected that the caller takes care of freeing queues if this
	* fails.
	*/
	if ((err = iflib_tx_structures_setup(ctx)) != 0) {
	device_printf(ctx->ifc_dev, "iflib_tx_structures_setup failed: %d\n", err);
	return (err);
	}

	if ((err = iflib_rx_structures_setup(ctx)) != 0)
	device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err);

	return (err);
	}

	int
	iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
	driver_filter_t filter, void filter_arg, driver_intr_t handler, void arg, const char *name)
	{

	return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
	}

	#ifdef SMP
	static int
	find_nth(if_ctx_t ctx, int qid)
	{
	cpuset_t cpus;
	int i, cpuid, eqid, count;

	CPU_COPY(&ctx->ifc_cpus, &cpus);
	count = CPU_COUNT(&cpus);
	eqid = qid % count;
	/* clear up to the qid'th bit */
	for (i = 0; i < eqid; i++) {
	cpuid = CPU_FFS(&cpus);
	MPASS(cpuid != 0);
	CPU_CLR(cpuid-1, &cpus);
	}
	cpuid = CPU_FFS(&cpus);
	MPASS(cpuid != 0);
	return (cpuid-1);
	}

	#ifdef SCHED_ULE
	extern struct cpu_group cpu_top; / CPU topology */

	static int
	find_child_with_core(int cpu, struct cpu_group *grp)
	{
	int i;

	if (grp->cg_children == 0)
	return -1;

	MPASS(grp->cg_child);
	for (i = 0; i < grp->cg_children; i++) {
	if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask))
	return i;
	}

	return -1;
	}

	/*
	* Find the nth "close" core to the specified core
	* "close" is defined as the deepest level that shares
	* at least an L2 cache. With threads, this will be
	* threads on the same core. If the shared cache is L3
	* or higher, simply returns the same core.
	*/
	static int
	find_close_core(int cpu, int core_offset)
	{
	struct cpu_group *grp;
	int i;
	int fcpu;
	cpuset_t cs;

	grp = cpu_top;
	if (grp == NULL)
	return cpu;
	i = 0;
	while ((i = find_child_with_core(cpu, grp)) != -1) {
	/* If the child only has one cpu, don't descend */
	if (grp->cg_child[i].cg_count <= 1)
	break;
	grp = &grp->cg_child[i];
	}

	/* If they don't share at least an L2 cache, use the same CPU */
	if (grp->cg_level > CG_SHARE_L2 \|\| grp->cg_level == CG_SHARE_NONE)
	return cpu;

	/* Now pick one */
	CPU_COPY(&grp->cg_mask, &cs);

	/* Add the selected CPU offset to core offset. */
	for (i = 0; (fcpu = CPU_FFS(&cs)) != 0; i++) {
	if (fcpu - 1 == cpu)
	break;
	CPU_CLR(fcpu - 1, &cs);
	}
	MPASS(fcpu);

	core_offset += i;

	CPU_COPY(&grp->cg_mask, &cs);
	for (i = core_offset % grp->cg_count; i > 0; i--) {
	MPASS(CPU_FFS(&cs));
	CPU_CLR(CPU_FFS(&cs) - 1, &cs);
	}
	MPASS(CPU_FFS(&cs));
	return CPU_FFS(&cs) - 1;
	}
	#else
	static int
	find_close_core(int cpu, int core_offset __unused)
	{
	return cpu;
	}
	#endif

	static int
	get_core_offset(if_ctx_t ctx, iflib_intr_type_t type, int qid)
	{
	switch (type) {
	case IFLIB_INTR_TX:
	/* TX queues get cores which share at least an L2 cache with the corresponding RX queue */
	/* XXX handle multiple RX threads per core and more than two core per L2 group */
	return qid / CPU_COUNT(&ctx->ifc_cpus) + 1;
	case IFLIB_INTR_RX:
	case IFLIB_INTR_RXTX:
	/* RX queues get the specified core */
	return qid / CPU_COUNT(&ctx->ifc_cpus);
	default:
	return -1;
	}
	}
	#else
	#define get_core_offset(ctx, type, qid) CPU_FIRST()
	#define find_close_core(cpuid, tid) CPU_FIRST()
	#define find_nth(ctx, gid) CPU_FIRST()
	#endif

	/* Just to avoid copy/paste */
	static inline int
	iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type,
	int qid, struct grouptask gtask, struct taskqgroup tqg, void *uniq,
	const char *name)
	{
	device_t dev;
	int co, cpuid, err, tid;

	dev = ctx->ifc_dev;
	co = ctx->ifc_sysctl_core_offset;
	if (ctx->ifc_sysctl_separate_txrx && type == IFLIB_INTR_TX)
	co += ctx->ifc_softc_ctx.isc_nrxqsets;
	cpuid = find_nth(ctx, qid + co);
	tid = get_core_offset(ctx, type, qid);
	if (tid < 0) {
	device_printf(dev, "get_core_offset failed\n");
	return (EOPNOTSUPP);
	}
	cpuid = find_close_core(cpuid, tid);
	err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res,
	name);
	if (err) {
	device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err);
	return (err);
	}
	#ifdef notyet
	if (cpuid > ctx->ifc_cpuid_highest)
	ctx->ifc_cpuid_highest = cpuid;
	#endif
	return (0);
	}

	int
	iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
	iflib_intr_type_t type, driver_filter_t *filter,
	void filter_arg, int qid, const char name)
	{
	device_t dev;
	struct grouptask *gtask;
	struct taskqgroup *tqg;
	iflib_filter_info_t info;
	gtask_fn_t *fn;
	int tqrid, err;
	driver_filter_t *intr_fast;
	void *q;

	info = &ctx->ifc_filter_info;
	tqrid = rid;

	switch (type) {
	/* XXX merge tx/rx for netmap? */
	case IFLIB_INTR_TX:
	q = &ctx->ifc_txqs[qid];
	info = &ctx->ifc_txqs[qid].ift_filter_info;
	gtask = &ctx->ifc_txqs[qid].ift_task;
	tqg = qgroup_if_io_tqg;
	fn = _task_fn_tx;
	intr_fast = iflib_fast_intr;
	GROUPTASK_INIT(gtask, 0, fn, q);
	ctx->ifc_flags \|= IFC_NETMAP_TX_IRQ;
	break;
	case IFLIB_INTR_RX:
	q = &ctx->ifc_rxqs[qid];
	info = &ctx->ifc_rxqs[qid].ifr_filter_info;
	gtask = &ctx->ifc_rxqs[qid].ifr_task;
	tqg = qgroup_if_io_tqg;
	fn = _task_fn_rx;
	intr_fast = iflib_fast_intr;
	NET_GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	case IFLIB_INTR_RXTX:
	q = &ctx->ifc_rxqs[qid];
	info = &ctx->ifc_rxqs[qid].ifr_filter_info;
	gtask = &ctx->ifc_rxqs[qid].ifr_task;
	tqg = qgroup_if_io_tqg;
	fn = _task_fn_rx;
	intr_fast = iflib_fast_intr_rxtx;
	NET_GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	case IFLIB_INTR_ADMIN:
	q = ctx;
	tqrid = -1;
	info = &ctx->ifc_filter_info;
	gtask = &ctx->ifc_admin_task;
	tqg = qgroup_if_config_tqg;
	fn = _task_fn_admin;
	intr_fast = iflib_fast_intr_ctx;
	break;
	default:
	device_printf(ctx->ifc_dev, "%s: unknown net intr type\n",
	__func__);
	return (EINVAL);
	}

	info->ifi_filter = filter;
	info->ifi_filter_arg = filter_arg;
	info->ifi_task = gtask;
	info->ifi_ctx = q;

	dev = ctx->ifc_dev;
	err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info, name);
	if (err != 0) {
	device_printf(dev, "_iflib_irq_alloc failed %d\n", err);
	return (err);
	}
	if (type == IFLIB_INTR_ADMIN)
	return (0);

	if (tqrid != -1) {
	err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg,
	q, name);
	if (err)
	return (err);
	} else {
	taskqgroup_attach(tqg, gtask, q, dev, irq->ii_res, name);
	}

	return (0);
	}

	void
	iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void arg, int qid, const char name)
	{
	struct grouptask *gtask;
	struct taskqgroup *tqg;
	gtask_fn_t *fn;
	void *q;
	int err;

	switch (type) {
	case IFLIB_INTR_TX:
	q = &ctx->ifc_txqs[qid];
	gtask = &ctx->ifc_txqs[qid].ift_task;
	tqg = qgroup_if_io_tqg;
	fn = _task_fn_tx;
	GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	case IFLIB_INTR_RX:
	q = &ctx->ifc_rxqs[qid];
	gtask = &ctx->ifc_rxqs[qid].ifr_task;
	tqg = qgroup_if_io_tqg;
	fn = _task_fn_rx;
	NET_GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	case IFLIB_INTR_IOV:
	q = ctx;
	gtask = &ctx->ifc_vflr_task;
	tqg = qgroup_if_config_tqg;
	fn = _task_fn_iov;
	GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	default:
	panic("unknown net intr type");
	}
	if (irq != NULL) {
	err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg,
	q, name);
	if (err)
	taskqgroup_attach(tqg, gtask, q, ctx->ifc_dev,
	irq->ii_res, name);
	} else {
	taskqgroup_attach(tqg, gtask, q, NULL, NULL, name);
	}
	}

	void
	iflib_irq_free(if_ctx_t ctx, if_irq_t irq)
	{

	if (irq->ii_tag)
	bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag);

	if (irq->ii_res)
	bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ,
	rman_get_rid(irq->ii_res), irq->ii_res);
	}

	static int
	iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void filter_arg, int rid, const char *name)
	{
	iflib_txq_t txq = ctx->ifc_txqs;
	iflib_rxq_t rxq = ctx->ifc_rxqs;
	if_irq_t irq = &ctx->ifc_legacy_irq;
	iflib_filter_info_t info;
	device_t dev;
	struct grouptask *gtask;
	struct resource *res;
	struct taskqgroup *tqg;
	void *q;
	int err, tqrid;
	bool rx_only;

	q = &ctx->ifc_rxqs[0];
	info = &rxq[0].ifr_filter_info;
	gtask = &rxq[0].ifr_task;
	tqg = qgroup_if_io_tqg;
	tqrid = *rid;
	rx_only = (ctx->ifc_sctx->isc_flags & IFLIB_SINGLE_IRQ_RX_ONLY) != 0;

	ctx->ifc_flags \|= IFC_LEGACY;
	info->ifi_filter = filter;
	info->ifi_filter_arg = filter_arg;
	info->ifi_task = gtask;
	info->ifi_ctx = rx_only ? ctx : q;

	dev = ctx->ifc_dev;
	/* We allocate a single interrupt resource */
	err = _iflib_irq_alloc(ctx, irq, tqrid, rx_only ? iflib_fast_intr_ctx :
	iflib_fast_intr_rxtx, NULL, info, name);
	if (err != 0)
	return (err);
	NET_GROUPTASK_INIT(gtask, 0, _task_fn_rx, q);
	res = irq->ii_res;
	taskqgroup_attach(tqg, gtask, q, dev, res, name);

	GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
	taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res,
	"tx");
	return (0);
	}

	void
	iflib_led_create(if_ctx_t ctx)
	{

	ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
	device_get_nameunit(ctx->ifc_dev));
	}

	void
	iflib_tx_intr_deferred(if_ctx_t ctx, int txqid)
	{

	GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
	}

	void
	iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid)
	{

	GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task);
	}

	void
	iflib_admin_intr_deferred(if_ctx_t ctx)
	{

	MPASS(ctx->ifc_admin_task.gt_taskqueue != NULL);
	GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
	}

	void
	iflib_iov_intr_deferred(if_ctx_t ctx)
	{

	GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task);
	}

	void
	iflib_io_tqg_attach(struct grouptask gt, void uniq, int cpu, const char *name)
	{

	taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, NULL, NULL,
	name);
	}

	void
	iflib_config_gtask_init(void ctx, struct grouptask gtask, gtask_fn_t *fn,
	const char *name)
	{

	GROUPTASK_INIT(gtask, 0, fn, ctx);
	taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, NULL, NULL,
	name);
	}

	void
	iflib_config_gtask_deinit(struct grouptask *gtask)
	{

	taskqgroup_detach(qgroup_if_config_tqg, gtask);
	}

	void
	iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate)
	{
	if_t ifp = ctx->ifc_ifp;
	iflib_txq_t txq = ctx->ifc_txqs;

	if_setbaudrate(ifp, baudrate);
	if (baudrate >= IF_Gbps(10)) {
	STATE_LOCK(ctx);
	ctx->ifc_flags \|= IFC_PREFETCH;
	STATE_UNLOCK(ctx);
	}
	/* If link down, disable watchdog */
	if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
	for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	}
	ctx->ifc_link_state = link_state;
	if_link_state_change(ifp, link_state);
	}

	static int
	iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
	{
	int credits;
	#ifdef INVARIANTS
	int credits_pre = txq->ift_cidx_processed;
	#endif

	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD);
	if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
	return (0);

	txq->ift_processed += credits;
	txq->ift_cidx_processed += credits;

	MPASS(credits_pre + credits == txq->ift_cidx_processed);
	if (txq->ift_cidx_processed >= txq->ift_size)
	txq->ift_cidx_processed -= txq->ift_size;
	return (credits);
	}

	static int
	iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget)
	{
	iflib_fl_t fl;
	u_int i;

	for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++)
	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx,
	budget));
	}

	void
	iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name,
	const char *description, if_int_delay_info_t info,
	int offset, int value)
	{
	info->iidi_ctx = ctx;
	info->iidi_offset = offset;
	info->iidi_value = value;
	SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
	SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)),
	OID_AUTO, name, CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_MPSAFE,
	info, 0, iflib_sysctl_int_delay, "I", description);
	}

	struct sx *
	iflib_ctx_lock_get(if_ctx_t ctx)
	{

	return (&ctx->ifc_ctx_sx);
	}

	static int
	iflib_msix_init(if_ctx_t ctx)
	{
	device_t dev = ctx->ifc_dev;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	int admincnt, bar, err, iflib_num_rx_queues, iflib_num_tx_queues;
	int msgs, queuemsgs, queues, rx_queues, tx_queues, vectors;

	iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs;
	iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs;

	if (bootverbose)
	device_printf(dev, "msix_init qsets capped at %d\n",
	imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets));

	/* Override by tuneable */
	if (scctx->isc_disable_msix)
	goto msi;

	/* First try MSI-X */
	if ((msgs = pci_msix_count(dev)) == 0) {
	if (bootverbose)
	device_printf(dev, "MSI-X not supported or disabled\n");
	goto msi;
	}

	bar = ctx->ifc_softc_ctx.isc_msix_bar;
	/*
	* bar == -1 => "trust me I know what I'm doing"
	* Some drivers are for hardware that is so shoddily
	* documented that no one knows which bars are which
	* so the developer has to map all bars. This hack
	* allows shoddy garbage to use MSI-X in this framework.
	*/
	if (bar != -1) {
	ctx->ifc_msix_mem = bus_alloc_resource_any(dev,
	SYS_RES_MEMORY, &bar, RF_ACTIVE);
	if (ctx->ifc_msix_mem == NULL) {
	device_printf(dev, "Unable to map MSI-X table\n");
	goto msi;
	}
	}

	admincnt = sctx->isc_admin_intrcnt;
	#if IFLIB_DEBUG
	/* use only 1 qset in debug mode */
	queuemsgs = min(msgs - admincnt, 1);
	#else
	queuemsgs = msgs - admincnt;
	#endif
	#ifdef RSS
	queues = imin(queuemsgs, rss_getnumbuckets());
	#else
	queues = queuemsgs;
	#endif
	queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
	if (bootverbose)
	device_printf(dev,
	"intr CPUs: %d queue msgs: %d admincnt: %d\n",
	CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
	#ifdef RSS
	/* If we're doing RSS, clamp at the number of RSS buckets */
	if (queues > rss_getnumbuckets())
	queues = rss_getnumbuckets();
	#endif
	if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt)
	rx_queues = iflib_num_rx_queues;
	else
	rx_queues = queues;

	if (rx_queues > scctx->isc_nrxqsets)
	rx_queues = scctx->isc_nrxqsets;

	/*
	* We want this to be all logical CPUs by default
	*/
	if (iflib_num_tx_queues > 0 && iflib_num_tx_queues < queues)
	tx_queues = iflib_num_tx_queues;
	else
	tx_queues = mp_ncpus;

	if (tx_queues > scctx->isc_ntxqsets)
	tx_queues = scctx->isc_ntxqsets;

	if (ctx->ifc_sysctl_qs_eq_override == 0) {
	#ifdef INVARIANTS
	if (tx_queues != rx_queues)
	device_printf(dev,
	"queue equality override not set, capping rx_queues at %d and tx_queues at %d\n",
	min(rx_queues, tx_queues), min(rx_queues, tx_queues));
	#endif
	tx_queues = min(rx_queues, tx_queues);
	rx_queues = min(rx_queues, tx_queues);
	}

	vectors = rx_queues + admincnt;
	if (msgs < vectors) {
	device_printf(dev,
	"insufficient number of MSI-X vectors "
	"(supported %d, need %d)\n", msgs, vectors);
	goto msi;
	}

	device_printf(dev, "Using %d RX queues %d TX queues\n", rx_queues,
	tx_queues);
	msgs = vectors;
	if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
	if (vectors != msgs) {
	device_printf(dev,
	"Unable to allocate sufficient MSI-X vectors "
	"(got %d, need %d)\n", vectors, msgs);
	pci_release_msi(dev);
	if (bar != -1) {
	bus_release_resource(dev, SYS_RES_MEMORY, bar,
	ctx->ifc_msix_mem);
	ctx->ifc_msix_mem = NULL;
	}
	goto msi;
	}
	device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
	vectors);
	scctx->isc_vectors = vectors;
	scctx->isc_nrxqsets = rx_queues;
	scctx->isc_ntxqsets = tx_queues;
	scctx->isc_intr = IFLIB_INTR_MSIX;

	return (vectors);
	} else {
	device_printf(dev,
	"failed to allocate %d MSI-X vectors, err: %d\n", vectors,
	err);
	if (bar != -1) {
	bus_release_resource(dev, SYS_RES_MEMORY, bar,
	ctx->ifc_msix_mem);
	ctx->ifc_msix_mem = NULL;
	}
	}

	msi:
	vectors = pci_msi_count(dev);
	scctx->isc_nrxqsets = 1;
	scctx->isc_ntxqsets = 1;
	scctx->isc_vectors = vectors;
	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
	device_printf(dev,"Using an MSI interrupt\n");
	scctx->isc_intr = IFLIB_INTR_MSI;
	} else {
	scctx->isc_vectors = 1;
	device_printf(dev,"Using a Legacy interrupt\n");
	scctx->isc_intr = IFLIB_INTR_LEGACY;
	}

	return (vectors);
	}

	static const char *ring_states[] = { "IDLE", "BUSY", "STALLED", "ABDICATED" };

	static int
	mp_ring_state_handler(SYSCTL_HANDLER_ARGS)
	{
	int rc;
	uint16_t state = ((uint16_t )oidp->oid_arg1);
	struct sbuf *sb;
	const char *ring_state = "UNKNOWN";

	/* XXX needed ? */
	rc = sysctl_wire_old_buffer(req, 0);
	MPASS(rc == 0);
	if (rc != 0)
	return (rc);
	sb = sbuf_new_for_sysctl(NULL, NULL, 80, req);
	MPASS(sb != NULL);
	if (sb == NULL)
	return (ENOMEM);
	if (state[3] <= 3)
	ring_state = ring_states[state[3]];

	sbuf_printf(sb, "pidx_head: %04hd pidx_tail: %04hd cidx: %04hd state: %s",
	state[0], state[1], state[2], ring_state);
	rc = sbuf_finish(sb);
	sbuf_delete(sb);
	return(rc);
	}

	enum iflib_ndesc_handler {
	IFLIB_NTXD_HANDLER,
	IFLIB_NRXD_HANDLER,
	};

	static int
	mp_ndesc_handler(SYSCTL_HANDLER_ARGS)
	{
	if_ctx_t ctx = (void *)arg1;
	enum iflib_ndesc_handler type = arg2;
	char buf[256] = {0};
	qidx_t *ndesc;
	char p, next;
	int nqs, rc, i;

	nqs = 8;
	switch(type) {
	case IFLIB_NTXD_HANDLER:
	ndesc = ctx->ifc_sysctl_ntxds;
	if (ctx->ifc_sctx)
	nqs = ctx->ifc_sctx->isc_ntxqs;
	break;
	case IFLIB_NRXD_HANDLER:
	ndesc = ctx->ifc_sysctl_nrxds;
	if (ctx->ifc_sctx)
	nqs = ctx->ifc_sctx->isc_nrxqs;
	break;
	default:
	printf("%s: unhandled type\n", __func__);
	return (EINVAL);
	}
	if (nqs == 0)
	nqs = 8;

	for (i=0; i<8; i++) {
	if (i >= nqs)
	break;
	if (i)
	strcat(buf, ",");
	sprintf(strchr(buf, 0), "%d", ndesc[i]);
	}

	rc = sysctl_handle_string(oidp, buf, sizeof(buf), req);
	if (rc \|\| req->newptr == NULL)
	return rc;

	for (i = 0, next = buf, p = strsep(&next, " ,"); i < 8 && p;
	i++, p = strsep(&next, " ,")) {
	ndesc[i] = strtoul(p, NULL, 10);
	}

	return(rc);
	}

	#define NAME_BUFLEN 32
	static void
	iflib_add_device_sysctl_pre(if_ctx_t ctx)
	{
	device_t dev = iflib_get_dev(ctx);
	struct sysctl_oid_list child, oid_list;
	struct sysctl_ctx_list *ctx_list;
	struct sysctl_oid *node;

	ctx_list = device_get_sysctl_ctx(dev);
	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
	ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "iflib",
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "IFLIB fields");
	oid_list = SYSCTL_CHILDREN(node);

	SYSCTL_ADD_CONST_STRING(ctx_list, oid_list, OID_AUTO, "driver_version",
	CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version,
	"driver version");

	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs",
	CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0,
	"# of txqs to use, 0 => use default #");
	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs",
	CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0,
	"# of rxqs to use, 0 => use default #");
	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable",
	CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0,
	"permit #txq != #rxq");
	SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix",
	CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0,
	"disable MSI-X (default 0)");
	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget",
	CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0,
	"set the RX budget");
	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate",
	CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0,
	"cause TX to abdicate instead of running to completion");
	ctx->ifc_sysctl_core_offset = CORE_OFFSET_UNSPECIFIED;
	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "core_offset",
	CTLFLAG_RDTUN, &ctx->ifc_sysctl_core_offset, 0,
	"offset to start using cores at");
	SYSCTL_ADD_U8(ctx_list, oid_list, OID_AUTO, "separate_txrx",
	CTLFLAG_RDTUN, &ctx->ifc_sysctl_separate_txrx, 0,
	"use separate cores for TX and RX");

	/* XXX change for per-queue sizes */
	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds",
	CTLTYPE_STRING \| CTLFLAG_RWTUN \| CTLFLAG_NEEDGIANT, ctx,
	IFLIB_NTXD_HANDLER, mp_ndesc_handler, "A",
	"list of # of TX descriptors to use, 0 = use default #");
	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds",
	CTLTYPE_STRING \| CTLFLAG_RWTUN \| CTLFLAG_NEEDGIANT, ctx,
	IFLIB_NRXD_HANDLER, mp_ndesc_handler, "A",
	"list of # of RX descriptors to use, 0 = use default #");
	}

	static void
	iflib_add_device_sysctl_post(if_ctx_t ctx)
	{
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	device_t dev = iflib_get_dev(ctx);
	struct sysctl_oid_list *child;
	struct sysctl_ctx_list *ctx_list;
	iflib_fl_t fl;
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	int i, j;
	char namebuf[NAME_BUFLEN];
	char *qfmt;
	struct sysctl_oid queue_node, fl_node, *node;
	struct sysctl_oid_list queue_list, fl_list;
	ctx_list = device_get_sysctl_ctx(dev);

	node = ctx->ifc_sysctl_node;
	child = SYSCTL_CHILDREN(node);

	if (scctx->isc_ntxqsets > 100)
	qfmt = "txq%03d";
	else if (scctx->isc_ntxqsets > 10)
	qfmt = "txq%02d";
	else
	qfmt = "txq%d";
	for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
	snprintf(namebuf, NAME_BUFLEN, qfmt, i);
	queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "Queue Name");
	queue_list = SYSCTL_CHILDREN(queue_node);
	#if MEMORY_LOGGING
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_dequeued",
	CTLFLAG_RD,
	&txq->ift_dequeued, "total mbufs freed");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_enqueued",
	CTLFLAG_RD,
	&txq->ift_enqueued, "total mbufs enqueued");
	#endif
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag",
	CTLFLAG_RD,
	&txq->ift_mbuf_defrag, "# of times m_defrag was called");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "m_pullups",
	CTLFLAG_RD,
	&txq->ift_pullups, "# of times m_pullup was called");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag_failed",
	CTLFLAG_RD,
	&txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_desc_avail",
	CTLFLAG_RD,
	&txq->ift_no_desc_avail, "# of times no descriptors were available");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "tx_map_failed",
	CTLFLAG_RD,
	&txq->ift_map_failed, "# of times DMA map failed");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txd_encap_efbig",
	CTLFLAG_RD,
	&txq->ift_txd_encap_efbig, "# of times txd_encap returned EFBIG");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_tx_dma_setup",
	CTLFLAG_RD,
	&txq->ift_no_tx_dma_setup, "# of times map failed for other than EFBIG");
	SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_pidx",
	CTLFLAG_RD,
	&txq->ift_pidx, 1, "Producer Index");
	SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx",
	CTLFLAG_RD,
	&txq->ift_cidx, 1, "Consumer Index");
	SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx_processed",
	CTLFLAG_RD,
	&txq->ift_cidx_processed, 1, "Consumer Index seen by credit update");
	SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_in_use",
	CTLFLAG_RD,
	&txq->ift_in_use, 1, "descriptors in use");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_processed",
	CTLFLAG_RD,
	&txq->ift_processed, "descriptors procesed for clean");
	SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_cleaned",
	CTLFLAG_RD,
	&txq->ift_cleaned, "total cleaned");
	SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state",
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	__DEVOLATILE(uint64_t *, &txq->ift_br->state), 0,
	mp_ring_state_handler, "A", "soft ring state");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues",
	CTLFLAG_RD, &txq->ift_br->enqueues,
	"# of enqueues to the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_drops",
	CTLFLAG_RD, &txq->ift_br->drops,
	"# of drops in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_starts",
	CTLFLAG_RD, &txq->ift_br->starts,
	"# of normal consumer starts in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_stalls",
	CTLFLAG_RD, &txq->ift_br->stalls,
	"# of consumer stalls in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_restarts",
	CTLFLAG_RD, &txq->ift_br->restarts,
	"# of consumer restarts in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_abdications",
	CTLFLAG_RD, &txq->ift_br->abdications,
	"# of consumer abdications in the mp_ring for this queue");
	}

	if (scctx->isc_nrxqsets > 100)
	qfmt = "rxq%03d";
	else if (scctx->isc_nrxqsets > 10)
	qfmt = "rxq%02d";
	else
	qfmt = "rxq%d";
	for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) {
	snprintf(namebuf, NAME_BUFLEN, qfmt, i);
	queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "Queue Name");
	queue_list = SYSCTL_CHILDREN(queue_node);
	if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
	SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_cidx",
	CTLFLAG_RD,
	&rxq->ifr_cq_cidx, 1, "Consumer Index");
	}

	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
	snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j);
	fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf,
	CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "freelist Name");
	fl_list = SYSCTL_CHILDREN(fl_node);
	SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "pidx",
	CTLFLAG_RD,
	&fl->ifl_pidx, 1, "Producer Index");
	SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "cidx",
	CTLFLAG_RD,
	&fl->ifl_cidx, 1, "Consumer Index");
	SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "credits",
	CTLFLAG_RD,
	&fl->ifl_credits, 1, "credits available");
	SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "buf_size",
	CTLFLAG_RD,
	&fl->ifl_buf_size, 1, "buffer size");
	#if MEMORY_LOGGING
	SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_enqueued",
	CTLFLAG_RD,
	&fl->ifl_m_enqueued, "mbufs allocated");
	SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_dequeued",
	CTLFLAG_RD,
	&fl->ifl_m_dequeued, "mbufs freed");
	SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_enqueued",
	CTLFLAG_RD,
	&fl->ifl_cl_enqueued, "clusters allocated");
	SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",
	CTLFLAG_RD,
	&fl->ifl_cl_dequeued, "clusters freed");
	#endif

	}
	}

	}

	void
	iflib_request_reset(if_ctx_t ctx)
	{

	STATE_LOCK(ctx);
	ctx->ifc_flags \|= IFC_DO_RESET;
	STATE_UNLOCK(ctx);
	}

	#ifndef __NO_STRICT_ALIGNMENT
	static struct mbuf *
	iflib_fixup_rx(struct mbuf *m)
	{
	struct mbuf *n;

	if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
	bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
	m->m_data += ETHER_HDR_LEN;
	n = m;
	} else {
	MGETHDR(n, M_NOWAIT, MT_DATA);
	if (n == NULL) {
	m_freem(m);
	return (NULL);
	}
	bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
	m->m_data += ETHER_HDR_LEN;
	m->m_len -= ETHER_HDR_LEN;
	n->m_len = ETHER_HDR_LEN;
	M_MOVE_PKTHDR(n, m);
	n->m_next = m;
	}
	return (n);
	}
	#endif

	#ifdef DEBUGNET
	static void
	iflib_debugnet_init(if_t ifp, int nrxr, int ncl, int *clsize)
	{
	if_ctx_t ctx;

	ctx = if_getsoftc(ifp);
	CTX_LOCK(ctx);
	*nrxr = NRXQSETS(ctx);
	*ncl = ctx->ifc_rxqs[0].ifr_fl->ifl_size;
	*clsize = ctx->ifc_rxqs[0].ifr_fl->ifl_buf_size;
	CTX_UNLOCK(ctx);
	}

	static void
	iflib_debugnet_event(if_t ifp, enum debugnet_ev event)
	{
	if_ctx_t ctx;
	if_softc_ctx_t scctx;
	iflib_fl_t fl;
	iflib_rxq_t rxq;
	int i, j;

	ctx = if_getsoftc(ifp);
	scctx = &ctx->ifc_softc_ctx;

	switch (event) {
	case DEBUGNET_START:
	for (i = 0; i < scctx->isc_nrxqsets; i++) {
	rxq = &ctx->ifc_rxqs[i];
	for (j = 0; j < rxq->ifr_nfl; j++) {
	fl = rxq->ifr_fl;
	fl->ifl_zone = m_getzone(fl->ifl_buf_size);
	}
	}
	iflib_no_tx_batch = 1;
	break;
	default:
	break;
	}
	}

	static int
	iflib_debugnet_transmit(if_t ifp, struct mbuf *m)
	{
	if_ctx_t ctx;
	iflib_txq_t txq;
	int error;

	ctx = if_getsoftc(ifp);
	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING \| IFF_DRV_OACTIVE)) !=
	IFF_DRV_RUNNING)
	return (EBUSY);

	txq = &ctx->ifc_txqs[0];
	error = iflib_encap(txq, &m);
	if (error == 0)
	(void)iflib_txd_db_check(ctx, txq, true, txq->ift_in_use);
	return (error);
	}

	static int
	iflib_debugnet_poll(if_t ifp, int count)
	{
	struct epoch_tracker et;
	if_ctx_t ctx;
	if_softc_ctx_t scctx;
	iflib_txq_t txq;
	int i;

	ctx = if_getsoftc(ifp);
	scctx = &ctx->ifc_softc_ctx;

	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING \| IFF_DRV_OACTIVE)) !=
	IFF_DRV_RUNNING)
	return (EBUSY);

	txq = &ctx->ifc_txqs[0];
	(void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));

	NET_EPOCH_ENTER(et);
	for (i = 0; i < scctx->isc_nrxqsets; i++)
	(void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */);
	NET_EPOCH_EXIT(et);
	return (0);
	}
	#endif /* DEBUGNET */
	diff --git a/sys/netgraph/ng_iface.c b/sys/netgraph/ng_iface.c
	index 5f4ce3c82f31..a91c202ef832 100644
	--- a/sys/netgraph/ng_iface.c
	+++ b/sys/netgraph/ng_iface.c
	@@ -1,820 +1,820 @@
	/*
	* ng_iface.c
	*/

	/*-
	* Copyright (c) 1996-1999 Whistle Communications, Inc.
	* All rights reserved.
	*
	* Subject to the following obligations and disclaimer of warranty, use and
	* redistribution of this software, in source or object code forms, with or
	* without modifications are expressly permitted by Whistle Communications;
	* provided, however, that:
	* 1. Any and all reproductions of the source or object code must include the
	* copyright notice above and the following disclaimer of warranties; and
	* 2. No rights are granted, in any manner or form, to use Whistle
	* Communications, Inc. trademarks, including the mark "WHISTLE
	* COMMUNICATIONS" on advertising, endorsements, or otherwise except as
	* such appears in the above copyright notice or in the software.
	*
	* THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
	* TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
	* REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
	* INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
	* WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
	* REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
	* SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
	* IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
	* RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
	* WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
	* PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
	* SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
	* OF SUCH DAMAGE.
	*
	* Author: Archie Cobbs <archie@freebsd.org>
	*
	* $FreeBSD$
	* $Whistle: ng_iface.c,v 1.33 1999/11/01 09:24:51 julian Exp $
	*/

	/*
	* This node is also a system networking interface. It has
	* a hook for each protocol (IP, AppleTalk, etc). Packets
	* are simply relayed between the interface and the hooks.
	*
	* Interfaces are named ng0, ng1, etc. New nodes take the
	* first available interface name.
	*
	* This node also includes Berkeley packet filter support.
	*/

	#include "opt_inet.h"
	#include "opt_inet6.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/errno.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/mbuf.h>
	#include <sys/errno.h>
	#include <sys/proc.h>
	#include <sys/random.h>
	#include <sys/rmlock.h>
	#include <sys/sockio.h>
	#include <sys/socket.h>
	#include <sys/sysctl.h>
	#include <sys/syslog.h>
	#include <sys/libkern.h>

	#include <net/if.h>
	#include <net/if_var.h>
	#include <net/if_types.h>
	#include <net/bpf.h>
	#include <net/netisr.h>
	#include <net/route.h>
	#include <net/vnet.h>

	#include <netinet/in.h>

	#include <netgraph/ng_message.h>
	#include <netgraph/netgraph.h>
	#include <netgraph/ng_parse.h>
	#include <netgraph/ng_iface.h>

	#ifdef NG_SEPARATE_MALLOC
	static MALLOC_DEFINE(M_NETGRAPH_IFACE, "netgraph_iface", "netgraph iface node");
	#else
	#define M_NETGRAPH_IFACE M_NETGRAPH
	#endif

	static SYSCTL_NODE(_net_graph, OID_AUTO, iface, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Point to point netgraph interface");
	VNET_DEFINE_STATIC(int, ng_iface_max_nest) = 2;
	#define V_ng_iface_max_nest VNET(ng_iface_max_nest)
	SYSCTL_INT(_net_graph_iface, OID_AUTO, max_nesting, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(ng_iface_max_nest), 0, "Max nested tunnels");

	/* This struct describes one address family */
	struct iffam {
	sa_family_t family; /* Address family */
	const char hookname; / Name for hook */
	};
	typedef const struct iffam *iffam_p;

	/* List of address families supported by our interface */
	const static struct iffam gFamilies[] = {
	{ AF_INET, NG_IFACE_HOOK_INET },
	{ AF_INET6, NG_IFACE_HOOK_INET6 },
	- { AF_ATM, NG_IFACE_HOOK_ATM },
	- { AF_NATM, NG_IFACE_HOOK_NATM },
	};
	#define NUM_FAMILIES nitems(gFamilies)

	/* Node private data */
	struct ng_iface_private {
	struct ifnet ifp; / Our interface */
	int unit; /* Interface unit number */
	node_p node; /* Our netgraph node */
	hook_p hooks[NUM_FAMILIES]; /* Hook for each address family */
	struct rmlock lock; /* Protect private data changes */
	};
	typedef struct ng_iface_private *priv_p;

	#define PRIV_RLOCK(priv, t) rm_rlock(&priv->lock, t)
	#define PRIV_RUNLOCK(priv, t) rm_runlock(&priv->lock, t)
	#define PRIV_WLOCK(priv) rm_wlock(&priv->lock)
	#define PRIV_WUNLOCK(priv) rm_wunlock(&priv->lock)

	/* Interface methods */
	static void ng_iface_start(struct ifnet *ifp);
	static int ng_iface_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
	static int ng_iface_output(struct ifnet ifp, struct mbuf m0,
	const struct sockaddr dst, struct route ro);
	static void ng_iface_bpftap(struct ifnet *ifp,
	struct mbuf *m, sa_family_t family);
	static int ng_iface_send(struct ifnet ifp, struct mbuf m,
	sa_family_t sa);
	#ifdef DEBUG
	static void ng_iface_print_ioctl(struct ifnet *ifp, int cmd, caddr_t data);
	#endif

	/* Netgraph methods */
	static int ng_iface_mod_event(module_t, int, void *);
	static ng_constructor_t ng_iface_constructor;
	static ng_rcvmsg_t ng_iface_rcvmsg;
	static ng_shutdown_t ng_iface_shutdown;
	static ng_newhook_t ng_iface_newhook;
	static ng_rcvdata_t ng_iface_rcvdata;
	static ng_disconnect_t ng_iface_disconnect;

	/* Helper stuff */
	static iffam_p get_iffam_from_af(sa_family_t family);
	static iffam_p get_iffam_from_hook(priv_p priv, hook_p hook);
	static iffam_p get_iffam_from_name(const char *name);
	static hook_p *get_hook_from_iffam(priv_p priv, iffam_p iffam);

	/* List of commands and how to convert arguments to/from ASCII */
	static const struct ng_cmdlist ng_iface_cmds[] = {
	{
	NGM_IFACE_COOKIE,
	NGM_IFACE_GET_IFNAME,
	"getifname",
	NULL,
	&ng_parse_string_type
	},
	{
	NGM_IFACE_COOKIE,
	NGM_IFACE_POINT2POINT,
	"point2point",
	NULL,
	NULL
	},
	{
	NGM_IFACE_COOKIE,
	NGM_IFACE_BROADCAST,
	"broadcast",
	NULL,
	NULL
	},
	{
	NGM_IFACE_COOKIE,
	NGM_IFACE_GET_IFINDEX,
	"getifindex",
	NULL,
	&ng_parse_uint32_type
	},
	{ 0 }
	};

	/* Node type descriptor */
	static struct ng_type typestruct = {
	.version = NG_ABI_VERSION,
	.name = NG_IFACE_NODE_TYPE,
	.mod_event = ng_iface_mod_event,
	.constructor = ng_iface_constructor,
	.rcvmsg = ng_iface_rcvmsg,
	.shutdown = ng_iface_shutdown,
	.newhook = ng_iface_newhook,
	.rcvdata = ng_iface_rcvdata,
	.disconnect = ng_iface_disconnect,
	.cmdlist = ng_iface_cmds,
	};
	NETGRAPH_INIT(iface, &typestruct);

	VNET_DEFINE_STATIC(struct unrhdr *, ng_iface_unit);
	#define V_ng_iface_unit VNET(ng_iface_unit)

	/************************************************************************
	HELPER STUFF
	************************************************************************/

	/*
	* Get the family descriptor from the family ID
	*/
	static __inline iffam_p
	get_iffam_from_af(sa_family_t family)
	{
	iffam_p iffam;
	int k;

	for (k = 0; k < NUM_FAMILIES; k++) {
	iffam = &gFamilies[k];
	if (iffam->family == family)
	return (iffam);
	}
	return (NULL);
	}

	/*
	* Get the family descriptor from the hook
	*/
	static __inline iffam_p
	get_iffam_from_hook(priv_p priv, hook_p hook)
	{
	int k;

	for (k = 0; k < NUM_FAMILIES; k++)
	if (priv->hooks[k] == hook)
	return (&gFamilies[k]);
	return (NULL);
	}

	/*
	* Get the hook from the iffam descriptor
	*/

	static __inline hook_p *
	get_hook_from_iffam(priv_p priv, iffam_p iffam)
	{
	return (&priv->hooks[iffam - gFamilies]);
	}

	/*
	* Get the iffam descriptor from the name
	*/
	static __inline iffam_p
	get_iffam_from_name(const char *name)
	{
	iffam_p iffam;
	int k;

	for (k = 0; k < NUM_FAMILIES; k++) {
	iffam = &gFamilies[k];
	if (!strcmp(iffam->hookname, name))
	return (iffam);
	}
	return (NULL);
	}

	/************************************************************************
	INTERFACE STUFF
	************************************************************************/

	/*
	* Process an ioctl for the virtual interface
	*/
	static int
	ng_iface_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
	{
	struct ifreq const ifr = (struct ifreq ) data;
	int error = 0;

	#ifdef DEBUG
	ng_iface_print_ioctl(ifp, command, data);
	#endif
	switch (command) {

	/* These two are mostly handled at a higher layer */
	case SIOCSIFADDR:
	ifp->if_flags \|= IFF_UP;
	ifp->if_drv_flags \|= IFF_DRV_RUNNING;
	ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
	break;
	case SIOCGIFADDR:
	break;

	/* Set flags */
	case SIOCSIFFLAGS:
	/*
	* If the interface is marked up and stopped, then start it.
	* If it is marked down and running, then stop it.
	*/
	if (ifr->ifr_flags & IFF_UP) {
	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
	ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
	ifp->if_drv_flags \|= IFF_DRV_RUNNING;
	}
	} else {
	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING \|
	IFF_DRV_OACTIVE);
	}
	break;

	/* Set the interface MTU */
	case SIOCSIFMTU:
	if (ifr->ifr_mtu > NG_IFACE_MTU_MAX
	\|\| ifr->ifr_mtu < NG_IFACE_MTU_MIN)
	error = EINVAL;
	else
	ifp->if_mtu = ifr->ifr_mtu;
	break;

	/* Stuff that's not supported */
	case SIOCADDMULTI:
	case SIOCDELMULTI:
	error = 0;
	break;
	case SIOCSIFPHYS:
	error = EOPNOTSUPP;
	break;

	default:
	error = EINVAL;
	break;
	}
	return (error);
	}

	/*
	* This routine is called to deliver a packet out the interface.
	* We simply look at the address family and relay the packet to
	* the corresponding hook, if it exists and is connected.
	*/

	static int
	ng_iface_output(struct ifnet ifp, struct mbuf m,
	const struct sockaddr dst, struct route ro)
	{
	uint32_t af;
	int error;

	/* Check interface flags */
	if (!((ifp->if_flags & IFF_UP) &&
	(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
	m_freem(m);
	return (ENETDOWN);
	}

	/* Protect from deadly infinite recursion. */
	error = if_tunnel_check_nesting(ifp, m, NGM_IFACE_COOKIE,
	V_ng_iface_max_nest);
	if (error) {
	m_freem(m);
	return (error);
	}

	/* BPF writes need to be handled specially. */
	if (dst->sa_family == AF_UNSPEC)
	bcopy(dst->sa_data, &af, sizeof(af));
	else
	af = dst->sa_family;

	/* Berkeley packet filter */
	ng_iface_bpftap(ifp, m, af);

	if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
	M_PREPEND(m, sizeof(sa_family_t), M_NOWAIT);
	if (m == NULL) {
	if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
	return (ENOBUFS);
	}
	(sa_family_t )m->m_data = af;
	error = (ifp->if_transmit)(ifp, m);
	} else
	error = ng_iface_send(ifp, m, af);

	return (error);
	}

	/*
	* Start method is used only when ALTQ is enabled.
	*/
	static void
	ng_iface_start(struct ifnet *ifp)
	{
	struct mbuf *m;
	sa_family_t sa;

	KASSERT(ALTQ_IS_ENABLED(&ifp->if_snd), ("%s without ALTQ", __func__));

	for(;;) {
	IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
	if (m == NULL)
	break;
	sa = mtod(m, sa_family_t );
	m_adj(m, sizeof(sa_family_t));
	ng_iface_send(ifp, m, sa);
	}
	}

	/*
	* Flash a packet by the BPF (requires prepending 4 byte AF header)
	* Note the phoney mbuf; this is OK because BPF treats it read-only.
	*/
	static void
	ng_iface_bpftap(struct ifnet ifp, struct mbuf m, sa_family_t family)
	{
	KASSERT(family != AF_UNSPEC, ("%s: family=AF_UNSPEC", __func__));
	if (bpf_peers_present(ifp->if_bpf)) {
	int32_t family4 = (int32_t)family;
	bpf_mtap2(ifp->if_bpf, &family4, sizeof(family4), m);
	}
	}

	/*
	* This routine does actual delivery of the packet into the
	* netgraph(4). It is called from ng_iface_start() and
	* ng_iface_output().
	*/
	static int
	ng_iface_send(struct ifnet ifp, struct mbuf m, sa_family_t sa)
	{
	struct rm_priotracker priv_tracker;
	const priv_p priv = (priv_p) ifp->if_softc;
	const iffam_p iffam = get_iffam_from_af(sa);
	hook_p hook;
	int error;
	int len;

	/* Check address family to determine hook (if known) */
	if (iffam == NULL) {
	m_freem(m);
	log(LOG_WARNING, "%s: can't handle af%d\n", ifp->if_xname, sa);
	return (EAFNOSUPPORT);
	}

	/* Copy length before the mbuf gets invalidated. */
	len = m->m_pkthdr.len;

	PRIV_RLOCK(priv, &priv_tracker);
	hook = *get_hook_from_iffam(priv, iffam);
	if (hook == NULL) {
	NG_FREE_M(m);
	PRIV_RUNLOCK(priv, &priv_tracker);
	return ENETDOWN;
	}
	NG_HOOK_REF(hook);
	PRIV_RUNLOCK(priv, &priv_tracker);

	NG_OUTBOUND_THREAD_REF();
	NG_SEND_DATA_ONLY(error, hook, m);
	NG_OUTBOUND_THREAD_UNREF();
	NG_HOOK_UNREF(hook);

	/* Update stats. */
	if (error == 0) {
	if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
	if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
	}

	return (error);
	}

	#ifdef DEBUG
	/*
	* Display an ioctl to the virtual interface
	*/

	static void
	ng_iface_print_ioctl(struct ifnet *ifp, int command, caddr_t data)
	{
	char *str;

	switch (command & IOC_DIRMASK) {
	case IOC_VOID:
	str = "IO";
	break;
	case IOC_OUT:
	str = "IOR";
	break;
	case IOC_IN:
	str = "IOW";
	break;
	case IOC_INOUT:
	str = "IORW";
	break;
	default:
	str = "IO??";
	}
	log(LOG_DEBUG, "%s: %s('%c', %d, char[%d])\n",
	ifp->if_xname,
	str,
	IOCGROUP(command),
	command & 0xff,
	IOCPARM_LEN(command));
	}
	#endif /* DEBUG */

	/************************************************************************
	NETGRAPH NODE STUFF
	************************************************************************/

	/*
	* Constructor for a node
	*/
	static int
	ng_iface_constructor(node_p node)
	{
	struct ifnet *ifp;
	priv_p priv;

	/* Allocate node and interface private structures */
	priv = malloc(sizeof(*priv), M_NETGRAPH_IFACE, M_WAITOK \| M_ZERO);
	ifp = if_alloc(IFT_PROPVIRTUAL);
	if (ifp == NULL) {
	free(priv, M_NETGRAPH_IFACE);
	return (ENOMEM);
	}

	rm_init(&priv->lock, "ng_iface private rmlock");

	/* Link them together */
	ifp->if_softc = priv;
	priv->ifp = ifp;

	/* Get an interface unit number */
	priv->unit = alloc_unr(V_ng_iface_unit);

	/* Link together node and private info */
	NG_NODE_SET_PRIVATE(node, priv);
	priv->node = node;

	/* Initialize interface structure */
	if_initname(ifp, NG_IFACE_IFACE_NAME, priv->unit);
	ifp->if_output = ng_iface_output;
	ifp->if_start = ng_iface_start;
	ifp->if_ioctl = ng_iface_ioctl;
	ifp->if_mtu = NG_IFACE_MTU_DEFAULT;
	ifp->if_flags = (IFF_SIMPLEX\|IFF_POINTOPOINT\|IFF_NOARP\|IFF_MULTICAST);
	ifp->if_type = IFT_PROPVIRTUAL; /* XXX */
	ifp->if_addrlen = 0; /* XXX */
	ifp->if_hdrlen = 0; /* XXX */
	ifp->if_baudrate = 64000; /* XXX */
	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
	IFQ_SET_READY(&ifp->if_snd);

	/* Give this node the same name as the interface (if possible) */
	if (ng_name_node(node, ifp->if_xname) != 0)
	log(LOG_WARNING, "%s: can't acquire netgraph name\n",
	ifp->if_xname);

	/* Attach the interface */
	if_attach(ifp);
	bpfattach(ifp, DLT_NULL, sizeof(u_int32_t));

	/* Done */
	return (0);
	}

	/*
	* Give our ok for a hook to be added
	*/
	static int
	ng_iface_newhook(node_p node, hook_p hook, const char *name)
	{
	const iffam_p iffam = get_iffam_from_name(name);
	const priv_p priv = NG_NODE_PRIVATE(node);
	hook_p *hookptr;

	if (iffam == NULL)
	return (EPFNOSUPPORT);
	PRIV_WLOCK(priv);
	hookptr = get_hook_from_iffam(priv, iffam);
	if (*hookptr != NULL) {
	PRIV_WUNLOCK(priv);
	return (EISCONN);
	}
	*hookptr = hook;
	NG_HOOK_HI_STACK(hook);
	NG_HOOK_SET_TO_INBOUND(hook);
	PRIV_WUNLOCK(priv);
	return (0);
	}

	/*
	* Receive a control message
	*/
	static int
	ng_iface_rcvmsg(node_p node, item_p item, hook_p lasthook)
	{
	const priv_p priv = NG_NODE_PRIVATE(node);
	struct ifnet *const ifp = priv->ifp;
	struct ng_mesg *resp = NULL;
	int error = 0;
	struct ng_mesg *msg;

	NGI_GET_MSG(item, msg);
	switch (msg->header.typecookie) {
	case NGM_IFACE_COOKIE:
	switch (msg->header.cmd) {
	case NGM_IFACE_GET_IFNAME:
	NG_MKRESPONSE(resp, msg, IFNAMSIZ, M_NOWAIT);
	if (resp == NULL) {
	error = ENOMEM;
	break;
	}
	strlcpy(resp->data, ifp->if_xname, IFNAMSIZ);
	break;

	case NGM_IFACE_POINT2POINT:
	case NGM_IFACE_BROADCAST:
	{

	/* Deny request if interface is UP */
	if ((ifp->if_flags & IFF_UP) != 0)
	return (EBUSY);

	/* Change flags */
	switch (msg->header.cmd) {
	case NGM_IFACE_POINT2POINT:
	ifp->if_flags \|= IFF_POINTOPOINT;
	ifp->if_flags &= ~IFF_BROADCAST;
	break;
	case NGM_IFACE_BROADCAST:
	ifp->if_flags &= ~IFF_POINTOPOINT;
	ifp->if_flags \|= IFF_BROADCAST;
	break;
	}
	break;
	}

	case NGM_IFACE_GET_IFINDEX:
	NG_MKRESPONSE(resp, msg, sizeof(uint32_t), M_NOWAIT);
	if (resp == NULL) {
	error = ENOMEM;
	break;
	}
	((uint32_t )resp->data) = priv->ifp->if_index;
	break;

	default:
	error = EINVAL;
	break;
	}
	break;
	case NGM_FLOW_COOKIE:
	switch (msg->header.cmd) {
	case NGM_LINK_IS_UP:
	if_link_state_change(ifp, LINK_STATE_UP);
	break;
	case NGM_LINK_IS_DOWN:
	if_link_state_change(ifp, LINK_STATE_DOWN);
	break;
	default:
	break;
	}
	break;
	default:
	error = EINVAL;
	break;
	}
	NG_RESPOND_MSG(error, node, item, resp);
	NG_FREE_MSG(msg);
	return (error);
	}

	/*
	* Recive data from a hook. Pass the packet to the correct input routine.
	*/
	static int
	ng_iface_rcvdata(hook_p hook, item_p item)
	{
	const priv_p priv = NG_NODE_PRIVATE(NG_HOOK_NODE(hook));
	const iffam_p iffam = get_iffam_from_hook(priv, hook);
	struct ifnet *const ifp = priv->ifp;
	struct epoch_tracker et;
	struct mbuf *m;
	int isr;

	NGI_GET_M(item, m);
	NG_FREE_ITEM(item);
	/* Sanity checks */
	KASSERT(iffam != NULL, ("%s: iffam", __func__));
	M_ASSERTPKTHDR(m);
	if ((ifp->if_flags & IFF_UP) == 0) {
	NG_FREE_M(m);
	return (ENETDOWN);
	}

	/* Update interface stats */
	if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
	if_inc_counter(ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len);

	/* Note receiving interface */
	m->m_pkthdr.rcvif = ifp;

	/* Berkeley packet filter */
	ng_iface_bpftap(ifp, m, iffam->family);

	/* Send packet */
	switch (iffam->family) {
	#ifdef INET
	case AF_INET:
	isr = NETISR_IP;
	break;
	#endif
	#ifdef INET6
	case AF_INET6:
	isr = NETISR_IPV6;
	break;
	#endif
	default:
	m_freem(m);
	return (EAFNOSUPPORT);
	}
	random_harvest_queue(m, sizeof(*m), RANDOM_NET_NG);
	M_SETFIB(m, ifp->if_fib);
	+ CURVNET_SET(ifp->if_vnet);
	NET_EPOCH_ENTER(et);
	netisr_dispatch(isr, m);
	NET_EPOCH_EXIT(et);
	+ CURVNET_RESTORE();
	return (0);
	}

	/*
	* Shutdown and remove the node and its associated interface.
	*/
	static int
	ng_iface_shutdown(node_p node)
	{
	const priv_p priv = NG_NODE_PRIVATE(node);

	/*
	* The ifnet may be in a different vnet than the netgraph node,
	* hence we have to change the current vnet context here.
	*/
	CURVNET_SET_QUIET(priv->ifp->if_vnet);
	bpfdetach(priv->ifp);
	if_detach(priv->ifp);
	if_free(priv->ifp);
	CURVNET_RESTORE();
	priv->ifp = NULL;
	free_unr(V_ng_iface_unit, priv->unit);
	rm_destroy(&priv->lock);
	free(priv, M_NETGRAPH_IFACE);
	NG_NODE_SET_PRIVATE(node, NULL);
	NG_NODE_UNREF(node);
	return (0);
	}

	/*
	* Hook disconnection. Note that we do not shutdown when all
	* hooks have been disconnected.
	*/
	static int
	ng_iface_disconnect(hook_p hook)
	{
	const priv_p priv = NG_NODE_PRIVATE(NG_HOOK_NODE(hook));
	const iffam_p iffam = get_iffam_from_hook(priv, hook);

	if (iffam == NULL)
	panic("%s", __func__);
	PRIV_WLOCK(priv);
	*get_hook_from_iffam(priv, iffam) = NULL;
	PRIV_WUNLOCK(priv);
	return (0);
	}

	/*
	* Handle loading and unloading for this node type.
	*/
	static int
	ng_iface_mod_event(module_t mod, int event, void *data)
	{
	int error = 0;

	switch (event) {
	case MOD_LOAD:
	case MOD_UNLOAD:
	break;
	default:
	error = EOPNOTSUPP;
	break;
	}
	return (error);
	}

	static void
	vnet_ng_iface_init(const void *unused)
	{

	V_ng_iface_unit = new_unrhdr(0, 0xffff, NULL);
	}
	VNET_SYSINIT(vnet_ng_iface_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
	vnet_ng_iface_init, NULL);

	static void
	vnet_ng_iface_uninit(const void *unused)
	{

	delete_unrhdr(V_ng_iface_unit);
	}
	VNET_SYSUNINIT(vnet_ng_iface_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
	vnet_ng_iface_uninit, NULL);
	diff --git a/sys/netgraph/ng_iface.h b/sys/netgraph/ng_iface.h
	index 891422b4b359..49c75832384d 100644
	--- a/sys/netgraph/ng_iface.h
	+++ b/sys/netgraph/ng_iface.h
	@@ -1,73 +1,71 @@
	/*
	* ng_iface.h
	*/

	/*-
	* Copyright (c) 1996-1999 Whistle Communications, Inc.
	* All rights reserved.
	*
	* Subject to the following obligations and disclaimer of warranty, use and
	* redistribution of this software, in source or object code forms, with or
	* without modifications are expressly permitted by Whistle Communications;
	* provided, however, that:
	* 1. Any and all reproductions of the source or object code must include the
	* copyright notice above and the following disclaimer of warranties; and
	* 2. No rights are granted, in any manner or form, to use Whistle
	* Communications, Inc. trademarks, including the mark "WHISTLE
	* COMMUNICATIONS" on advertising, endorsements, or otherwise except as
	* such appears in the above copyright notice or in the software.
	*
	* THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
	* TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
	* REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
	* INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT.
	* WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY
	* REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS
	* SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE.
	* IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES
	* RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING
	* WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
	* PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR
	* SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
	* OF SUCH DAMAGE.
	*
	* Author: Archie Cobbs <archie@freebsd.org>
	*
	* $FreeBSD$
	* $Whistle: ng_iface.h,v 1.5 1999/01/20 00:22:13 archie Exp $
	*/

	#ifndef _NETGRAPH_NG_IFACE_H_
	#define _NETGRAPH_NG_IFACE_H_

	/* Node type name and magic cookie */
	#define NG_IFACE_NODE_TYPE "iface"
	#define NGM_IFACE_COOKIE 1108312559

	/* Interface base name */
	#define NG_IFACE_IFACE_NAME "ng"

	/* My hook names */
	#define NG_IFACE_HOOK_INET "inet"
	#define NG_IFACE_HOOK_INET6 "inet6"
	-#define NG_IFACE_HOOK_ATM "atm"
	-#define NG_IFACE_HOOK_NATM "natm"

	/* MTU bounds */
	#define NG_IFACE_MTU_MIN 72
	#define NG_IFACE_MTU_MAX 65535
	#define NG_IFACE_MTU_DEFAULT 1500

	/* Netgraph commands */
	enum {
	NGM_IFACE_GET_IFNAME = 1,
	NGM_IFACE_POINT2POINT,
	NGM_IFACE_BROADCAST,
	NGM_IFACE_GET_IFINDEX,
	};

	#endif /* _NETGRAPH_NG_IFACE_H_ */
	diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
	index 777fd7200ccd..bf4b413bd4de 100644
	--- a/sys/netinet/tcp_subr.c
	+++ b/sys/netinet/tcp_subr.c
	@@ -1,3494 +1,3501 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
	* The Regents of the University of California. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)tcp_subr.c 8.2 (Berkeley) 5/24/95
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_ipsec.h"
	#include "opt_kern_tls.h"
	#include "opt_tcpdebug.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/arb.h>
	#include <sys/callout.h>
	#include <sys/eventhandler.h>
	#ifdef TCP_HHOOK
	#include <sys/hhook.h>
	#endif
	#include <sys/kernel.h>
	#ifdef TCP_HHOOK
	#include <sys/khelp.h>
	#endif
	#ifdef KERN_TLS
	#include <sys/ktls.h>
	#endif
	#include <sys/qmath.h>
	#include <sys/stats.h>
	#include <sys/sysctl.h>
	#include <sys/jail.h>
	#include <sys/malloc.h>
	#include <sys/refcount.h>
	#include <sys/mbuf.h>
	#ifdef INET6
	#include <sys/domain.h>
	#endif
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/sdt.h>
	#include <sys/socket.h>
	#include <sys/socketvar.h>
	#include <sys/protosw.h>
	#include <sys/random.h>

	#include <vm/uma.h>

	#include <net/route.h>
	#include <net/route/nhop.h>
	#include <net/if.h>
	#include <net/if_var.h>
	#include <net/vnet.h>

	#include <netinet/in.h>
	#include <netinet/in_fib.h>
	#include <netinet/in_kdtrace.h>
	#include <netinet/in_pcb.h>
	#include <netinet/in_systm.h>
	#include <netinet/in_var.h>
	#include <netinet/ip.h>
	#include <netinet/ip_icmp.h>
	#include <netinet/ip_var.h>
	#ifdef INET6
	#include <netinet/icmp6.h>
	#include <netinet/ip6.h>
	#include <netinet6/in6_fib.h>
	#include <netinet6/in6_pcb.h>
	#include <netinet6/ip6_var.h>
	#include <netinet6/scope6_var.h>
	#include <netinet6/nd6.h>
	#endif

	#include <netinet/tcp.h>
	#include <netinet/tcp_fsm.h>
	#include <netinet/tcp_seq.h>
	#include <netinet/tcp_timer.h>
	#include <netinet/tcp_var.h>
	#include <netinet/tcp_log_buf.h>
	#include <netinet/tcp_syncache.h>
	#include <netinet/tcp_hpts.h>
	#include <netinet/cc/cc.h>
	#ifdef INET6
	#include <netinet6/tcp6_var.h>
	#endif
	#include <netinet/tcpip.h>
	#include <netinet/tcp_fastopen.h>
	#ifdef TCPPCAP
	#include <netinet/tcp_pcap.h>
	#endif
	#ifdef TCPDEBUG
	#include <netinet/tcp_debug.h>
	#endif
	#ifdef INET6
	#include <netinet6/ip6protosw.h>
	#endif
	#ifdef TCP_OFFLOAD
	#include <netinet/tcp_offload.h>
	#endif

	#include <netipsec/ipsec_support.h>

	#include <machine/in_cksum.h>
	#include <crypto/siphash/siphash.h>

	#include <security/mac/mac_framework.h>

	VNET_DEFINE(int, tcp_mssdflt) = TCP_MSS;
	#ifdef INET6
	VNET_DEFINE(int, tcp_v6mssdflt) = TCP6_MSS;
	#endif

	#ifdef NETFLIX_EXP_DETECTION
	/* Sack attack detection thresholds and such */
	SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack_attack,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Sack Attack detection thresholds");
	int32_t tcp_force_detection = 0;
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, force_detection,
	CTLFLAG_RW,
	&tcp_force_detection, 0,
	"Do we force detection even if the INP has it off?");
	int32_t tcp_sack_to_ack_thresh = 700; /* 70 % */
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sack_to_ack_thresh,
	CTLFLAG_RW,
	&tcp_sack_to_ack_thresh, 700,
	"Percentage of sacks to acks we must see above (10.1 percent is 101)?");
	int32_t tcp_sack_to_move_thresh = 600; /* 60 % */
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, move_thresh,
	CTLFLAG_RW,
	&tcp_sack_to_move_thresh, 600,
	"Percentage of sack moves we must see above (10.1 percent is 101)");
	int32_t tcp_restoral_thresh = 650; /* 65 % (sack:2:ack -5%) */
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, restore_thresh,
	CTLFLAG_RW,
	&tcp_restoral_thresh, 550,
	"Percentage of sack to ack percentage we must see below to restore(10.1 percent is 101)");
	int32_t tcp_sad_decay_val = 800;
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, decay_per,
	CTLFLAG_RW,
	&tcp_sad_decay_val, 800,
	"The decay percentage (10.1 percent equals 101 )");
	int32_t tcp_map_minimum = 500;
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, nummaps,
	CTLFLAG_RW,
	&tcp_map_minimum, 500,
	"Number of Map enteries before we start detection");
	int32_t tcp_attack_on_turns_on_logging = 0;
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, attacks_logged,
	CTLFLAG_RW,
	&tcp_attack_on_turns_on_logging, 0,
	"When we have a positive hit on attack, do we turn on logging?");
	int32_t tcp_sad_pacing_interval = 2000;
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_pacing_int,
	CTLFLAG_RW,
	&tcp_sad_pacing_interval, 2000,
	"What is the minimum pacing interval for a classified attacker?");

	int32_t tcp_sad_low_pps = 100;
	SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_low_pps,
	CTLFLAG_RW,
	&tcp_sad_low_pps, 100,
	"What is the input pps that below which we do not decay?");
	#endif

	struct rwlock tcp_function_lock;

	static int
	sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)
	{
	int error, new;

	new = V_tcp_mssdflt;
	error = sysctl_handle_int(oidp, &new, 0, req);
	if (error == 0 && req->newptr) {
	if (new < TCP_MINMSS)
	error = EINVAL;
	else
	V_tcp_mssdflt = new;
	}
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt,
	CTLFLAG_VNET \| CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&VNET_NAME(tcp_mssdflt), 0, &sysctl_net_inet_tcp_mss_check, "I",
	"Default TCP Maximum Segment Size");

	#ifdef INET6
	static int
	sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)
	{
	int error, new;

	new = V_tcp_v6mssdflt;
	error = sysctl_handle_int(oidp, &new, 0, req);
	if (error == 0 && req->newptr) {
	if (new < TCP_MINMSS)
	error = EINVAL;
	else
	V_tcp_v6mssdflt = new;
	}
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt,
	CTLFLAG_VNET \| CTLTYPE_INT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&VNET_NAME(tcp_v6mssdflt), 0, &sysctl_net_inet_tcp_mss_v6_check, "I",
	"Default TCP Maximum Segment Size for IPv6");
	#endif /* INET6 */

	/*
	* Minimum MSS we accept and use. This prevents DoS attacks where
	* we are forced to a ridiculous low MSS like 20 and send hundreds
	* of packets instead of one. The effect scales with the available
	* bandwidth and quickly saturates the CPU and network interface
	* with packet generation and sending. Set to zero to disable MINMSS
	* checking. This setting prevents us from sending too small packets.
	*/
	VNET_DEFINE(int, tcp_minmss) = TCP_MINMSS;
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(tcp_minmss), 0,
	"Minimum TCP Maximum Segment Size");

	VNET_DEFINE(int, tcp_do_rfc1323) = 1;
	SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(tcp_do_rfc1323), 0,
	"Enable rfc1323 (high performance TCP) extensions");

	VNET_DEFINE(int, tcp_ts_offset_per_conn) = 1;
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, ts_offset_per_conn, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(tcp_ts_offset_per_conn), 0,
	"Initialize TCP timestamps per connection instead of per host pair");

	static int tcp_log_debug = 0;
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_debug, CTLFLAG_RW,
	&tcp_log_debug, 0, "Log errors caused by incoming TCP segments");

	static int tcp_tcbhashsize;
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN \| CTLFLAG_NOFETCH,
	&tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");

	static int do_tcpdrain = 1;
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW, &do_tcpdrain, 0,
	"Enable tcp_drain routine for extra help when low on mbufs");

	SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_VNET \| CTLFLAG_RD,
	&VNET_NAME(tcbinfo.ipi_count), 0, "Number of active PCBs");

	VNET_DEFINE_STATIC(int, icmp_may_rst) = 1;
	#define V_icmp_may_rst VNET(icmp_may_rst)
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(icmp_may_rst), 0,
	"Certain ICMP unreachable messages may abort connections in SYN_SENT");

	VNET_DEFINE_STATIC(int, tcp_isn_reseed_interval) = 0;
	#define V_tcp_isn_reseed_interval VNET(tcp_isn_reseed_interval)
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(tcp_isn_reseed_interval), 0,
	"Seconds between reseeding of ISN secret");

	static int tcp_soreceive_stream;
	SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
	&tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");

	VNET_DEFINE(uma_zone_t, sack_hole_zone);
	#define V_sack_hole_zone VNET(sack_hole_zone)
	VNET_DEFINE(uint32_t, tcp_map_entries_limit) = 0; /* unlimited */
	static int
	sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)
	{
	int error;
	uint32_t new;

	new = V_tcp_map_entries_limit;
	error = sysctl_handle_int(oidp, &new, 0, req);
	if (error == 0 && req->newptr) {
	/* only allow "0" and value > minimum */
	if (new > 0 && new < TCP_MIN_MAP_ENTRIES_LIMIT)
	error = EINVAL;
	else
	V_tcp_map_entries_limit = new;
	}
	return (error);
	}
	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, map_limit,
	CTLFLAG_VNET \| CTLTYPE_UINT \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	&VNET_NAME(tcp_map_entries_limit), 0,
	&sysctl_net_inet_tcp_map_limit_check, "IU",
	"Total sendmap entries limit");

	VNET_DEFINE(uint32_t, tcp_map_split_limit) = 0; /* unlimited */
	SYSCTL_UINT(_net_inet_tcp, OID_AUTO, split_limit, CTLFLAG_VNET \| CTLFLAG_RW,
	&VNET_NAME(tcp_map_split_limit), 0,
	"Total sendmap split entries limit");

	#ifdef TCP_HHOOK
	VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
	#endif

	#define TS_OFFSET_SECRET_LENGTH SIPHASH_KEY_LENGTH
	VNET_DEFINE_STATIC(u_char, ts_offset_secret[TS_OFFSET_SECRET_LENGTH]);
	#define V_ts_offset_secret VNET(ts_offset_secret)

	static int tcp_default_fb_init(struct tcpcb *tp);
	static void tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged);
	static int tcp_default_handoff_ok(struct tcpcb *tp);
	static struct inpcb tcp_notify(struct inpcb , int);
	static struct inpcb tcp_mtudisc_notify(struct inpcb , int);
	static void tcp_mtudisc(struct inpcb *, int);
	static char * tcp_log_addr(struct in_conninfo inc, struct tcphdr th,
	void ip4hdr, const void ip6hdr);


	static struct tcp_function_block tcp_def_funcblk = {
	.tfb_tcp_block_name = "freebsd",
	.tfb_tcp_output = tcp_output,
	.tfb_tcp_do_segment = tcp_do_segment,
	.tfb_tcp_ctloutput = tcp_default_ctloutput,
	.tfb_tcp_handoff_ok = tcp_default_handoff_ok,
	.tfb_tcp_fb_init = tcp_default_fb_init,
	.tfb_tcp_fb_fini = tcp_default_fb_fini,
	};

	static int tcp_fb_cnt = 0;
	struct tcp_funchead t_functions;
	static struct tcp_function_block *tcp_func_set_ptr = &tcp_def_funcblk;

	static struct tcp_function_block *
	find_tcp_functions_locked(struct tcp_function_set *fs)
	{
	struct tcp_function *f;
	struct tcp_function_block *blk=NULL;

	TAILQ_FOREACH(f, &t_functions, tf_next) {
	if (strcmp(f->tf_name, fs->function_set_name) == 0) {
	blk = f->tf_fb;
	break;
	}
	}
	return(blk);
	}

	static struct tcp_function_block *
	find_tcp_fb_locked(struct tcp_function_block blk, struct tcp_function *s)
	{
	struct tcp_function_block *rblk=NULL;
	struct tcp_function *f;

	TAILQ_FOREACH(f, &t_functions, tf_next) {
	if (f->tf_fb == blk) {
	rblk = blk;
	if (s) {
	*s = f;
	}
	break;
	}
	}
	return (rblk);
	}

	struct tcp_function_block *
	find_and_ref_tcp_functions(struct tcp_function_set *fs)
	{
	struct tcp_function_block *blk;

	rw_rlock(&tcp_function_lock);
	blk = find_tcp_functions_locked(fs);
	if (blk)
	refcount_acquire(&blk->tfb_refcnt);
	rw_runlock(&tcp_function_lock);
	return(blk);
	}

	struct tcp_function_block *
	find_and_ref_tcp_fb(struct tcp_function_block *blk)
	{
	struct tcp_function_block *rblk;

	rw_rlock(&tcp_function_lock);
	rblk = find_tcp_fb_locked(blk, NULL);
	if (rblk)
	refcount_acquire(&rblk->tfb_refcnt);
	rw_runlock(&tcp_function_lock);
	return(rblk);
	}

	static struct tcp_function_block *
	find_and_ref_tcp_default_fb(void)
	{
	struct tcp_function_block *rblk;

	rw_rlock(&tcp_function_lock);
	rblk = tcp_func_set_ptr;
	refcount_acquire(&rblk->tfb_refcnt);
	rw_runlock(&tcp_function_lock);
	return (rblk);
	}

	void
	tcp_switch_back_to_default(struct tcpcb *tp)
	{
	struct tcp_function_block *tfb;

	KASSERT(tp->t_fb != &tcp_def_funcblk,
	("%s: called by the built-in default stack", __func__));

	/*
	* Release the old stack. This function will either find a new one
	* or panic.
	*/
	if (tp->t_fb->tfb_tcp_fb_fini != NULL)
	(*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
	refcount_release(&tp->t_fb->tfb_refcnt);

	/*
	* Now, we'll find a new function block to use.
	* Start by trying the current user-selected
	* default, unless this stack is the user-selected
	* default.
	*/
	tfb = find_and_ref_tcp_default_fb();
	if (tfb == tp->t_fb) {
	refcount_release(&tfb->tfb_refcnt);
	tfb = NULL;
	}
	/* Does the stack accept this connection? */
	if (tfb != NULL && tfb->tfb_tcp_handoff_ok != NULL &&
	(*tfb->tfb_tcp_handoff_ok)(tp)) {
	refcount_release(&tfb->tfb_refcnt);
	tfb = NULL;
	}
	/* Try to use that stack. */
	if (tfb != NULL) {
	/* Initialize the new stack. If it succeeds, we are done. */
	tp->t_fb = tfb;
	if (tp->t_fb->tfb_tcp_fb_init == NULL \|\|
	(*tp->t_fb->tfb_tcp_fb_init)(tp) == 0)
	return;

	/*
	* Initialization failed. Release the reference count on
	* the stack.
	*/
	refcount_release(&tfb->tfb_refcnt);
	}

	/*
	* If that wasn't feasible, use the built-in default
	* stack which is not allowed to reject anyone.
	*/
	tfb = find_and_ref_tcp_fb(&tcp_def_funcblk);
	if (tfb == NULL) {
	/* there always should be a default */
	panic("Can't refer to tcp_def_funcblk");
	}
	if (tfb->tfb_tcp_handoff_ok != NULL) {
	if ((*tfb->tfb_tcp_handoff_ok) (tp)) {
	/* The default stack cannot say no */
	panic("Default stack rejects a new session?");
	}
	}
	tp->t_fb = tfb;
	if (tp->t_fb->tfb_tcp_fb_init != NULL &&
	(*tp->t_fb->tfb_tcp_fb_init)(tp)) {
	/* The default stack cannot fail */
	panic("Default stack initialization failed");
	}
	}

	static int
	sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)
	{
	int error=ENOENT;
	struct tcp_function_set fs;
	struct tcp_function_block *blk;

	memset(&fs, 0, sizeof(fs));
	rw_rlock(&tcp_function_lock);
	blk = find_tcp_fb_locked(tcp_func_set_ptr, NULL);
	if (blk) {
	/* Found him */
	strcpy(fs.function_set_name, blk->tfb_tcp_block_name);
	fs.pcbcnt = blk->tfb_refcnt;
	}
	rw_runlock(&tcp_function_lock);
	error = sysctl_handle_string(oidp, fs.function_set_name,
	sizeof(fs.function_set_name), req);

	/* Check for error or no change */
	if (error != 0 \|\| req->newptr == NULL)
	return(error);

	rw_wlock(&tcp_function_lock);
	blk = find_tcp_functions_locked(&fs);
	if ((blk == NULL) \|\|
	(blk->tfb_flags & TCP_FUNC_BEING_REMOVED)) {
	error = ENOENT;
	goto done;
	}
	tcp_func_set_ptr = blk;
	done:
	rw_wunlock(&tcp_function_lock);
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_default,
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	NULL, 0, sysctl_net_inet_default_tcp_functions, "A",
	"Set/get the default TCP functions");

	static int
	sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)
	{
	int error, cnt, linesz;
	struct tcp_function *f;
	char buffer, cp;
	size_t bufsz, outsz;
	bool alias;

	cnt = 0;
	rw_rlock(&tcp_function_lock);
	TAILQ_FOREACH(f, &t_functions, tf_next) {
	cnt++;
	}
	rw_runlock(&tcp_function_lock);

	bufsz = (cnt+2) * ((TCP_FUNCTION_NAME_LEN_MAX * 2) + 13) + 1;
	buffer = malloc(bufsz, M_TEMP, M_WAITOK);

	error = 0;
	cp = buffer;

	linesz = snprintf(cp, bufsz, "\n%-32s%c %-32s %s\n", "Stack", 'D',
	"Alias", "PCB count");
	cp += linesz;
	bufsz -= linesz;
	outsz = linesz;

	rw_rlock(&tcp_function_lock);
	TAILQ_FOREACH(f, &t_functions, tf_next) {
	alias = (f->tf_name != f->tf_fb->tfb_tcp_block_name);
	linesz = snprintf(cp, bufsz, "%-32s%c %-32s %u\n",
	f->tf_fb->tfb_tcp_block_name,
	(f->tf_fb == tcp_func_set_ptr) ? '*' : ' ',
	alias ? f->tf_name : "-",
	f->tf_fb->tfb_refcnt);
	if (linesz >= bufsz) {
	error = EOVERFLOW;
	break;
	}
	cp += linesz;
	bufsz -= linesz;
	outsz += linesz;
	}
	rw_runlock(&tcp_function_lock);
	if (error == 0)
	error = sysctl_handle_string(oidp, buffer, outsz + 1, req);
	free(buffer, M_TEMP);
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available,
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	NULL, 0, sysctl_net_inet_list_available, "A",
	"list available TCP Function sets");

	/*
	* Exports one (struct tcp_function_info) for each alias/name.
	*/
	static int
	sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)
	{
	int cnt, error;
	struct tcp_function *f;
	struct tcp_function_info tfi;

	/*
	* We don't allow writes.
	*/
	if (req->newptr != NULL)
	return (EINVAL);

	/*
	* Wire the old buffer so we can directly copy the functions to
	* user space without dropping the lock.
	*/
	if (req->oldptr != NULL) {
	error = sysctl_wire_old_buffer(req, 0);
	if (error)
	return (error);
	}

	/*
	* Walk the list and copy out matching entries. If INVARIANTS
	* is compiled in, also walk the list to verify the length of
	* the list matches what we have recorded.
	*/
	rw_rlock(&tcp_function_lock);

	cnt = 0;
	#ifndef INVARIANTS
	if (req->oldptr == NULL) {
	cnt = tcp_fb_cnt;
	goto skip_loop;
	}
	#endif
	TAILQ_FOREACH(f, &t_functions, tf_next) {
	#ifdef INVARIANTS
	cnt++;
	#endif
	if (req->oldptr != NULL) {
	bzero(&tfi, sizeof(tfi));
	tfi.tfi_refcnt = f->tf_fb->tfb_refcnt;
	tfi.tfi_id = f->tf_fb->tfb_id;
	(void)strlcpy(tfi.tfi_alias, f->tf_name,
	sizeof(tfi.tfi_alias));
	(void)strlcpy(tfi.tfi_name,
	f->tf_fb->tfb_tcp_block_name, sizeof(tfi.tfi_name));
	error = SYSCTL_OUT(req, &tfi, sizeof(tfi));
	/*
	* Don't stop on error, as that is the
	* mechanism we use to accumulate length
	* information if the buffer was too short.
	*/
	}
	}
	KASSERT(cnt == tcp_fb_cnt,
	("%s: cnt (%d) != tcp_fb_cnt (%d)", __func__, cnt, tcp_fb_cnt));
	#ifndef INVARIANTS
	skip_loop:
	#endif
	rw_runlock(&tcp_function_lock);
	if (req->oldptr == NULL)
	error = SYSCTL_OUT(req, NULL,
	(cnt + 1) * sizeof(struct tcp_function_info));

	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_info,
	CTLTYPE_OPAQUE \| CTLFLAG_SKIP \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
	NULL, 0, sysctl_net_inet_list_func_info, "S,tcp_function_info",
	"List TCP function block name-to-ID mappings");

	/*
	* tfb_tcp_handoff_ok() function for the default stack.
	* Note that we'll basically try to take all comers.
	*/
	static int
	tcp_default_handoff_ok(struct tcpcb *tp)
	{

	return (0);
	}

	/*
	* tfb_tcp_fb_init() function for the default stack.
	*
	* This handles making sure we have appropriate timers set if you are
	* transitioning a socket that has some amount of setup done.
	*
	* The init() fuction from the default can never return non-zero i.e.
	* it is required to always succeed since it is the stack of last resort!
	*/
	static int
	tcp_default_fb_init(struct tcpcb *tp)
	{

	struct socket *so;

	INP_WLOCK_ASSERT(tp->t_inpcb);

	KASSERT(tp->t_state >= 0 && tp->t_state < TCPS_TIME_WAIT,
	("%s: connection %p in unexpected state %d", __func__, tp,
	tp->t_state));

	/*
	* Nothing to do for ESTABLISHED or LISTEN states. And, we don't
	* know what to do for unexpected states (which includes TIME_WAIT).
	*/
	if (tp->t_state <= TCPS_LISTEN \|\| tp->t_state >= TCPS_TIME_WAIT)
	return (0);

	/*
	* Make sure some kind of transmission timer is set if there is
	* outstanding data.
	*/
	so = tp->t_inpcb->inp_socket;
	if ((!TCPS_HAVEESTABLISHED(tp->t_state) \|\| sbavail(&so->so_snd) \|\|
	tp->snd_una != tp->snd_max) && !(tcp_timer_active(tp, TT_REXMT) \|\|
	tcp_timer_active(tp, TT_PERSIST))) {
	/*
	* If the session has established and it looks like it should
	* be in the persist state, set the persist timer. Otherwise,
	* set the retransmit timer.
	*/
	if (TCPS_HAVEESTABLISHED(tp->t_state) && tp->snd_wnd == 0 &&
	(int32_t)(tp->snd_nxt - tp->snd_una) <
	(int32_t)sbavail(&so->so_snd))
	tcp_setpersist(tp);
	else
	tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur);
	}

	/* All non-embryonic sessions get a keepalive timer. */
	if (!tcp_timer_active(tp, TT_KEEP))
	tcp_timer_activate(tp, TT_KEEP,
	TCPS_HAVEESTABLISHED(tp->t_state) ? TP_KEEPIDLE(tp) :
	TP_KEEPINIT(tp));

	return (0);
	}

	/*
	* tfb_tcp_fb_fini() function for the default stack.
	*
	* This changes state as necessary (or prudent) to prepare for another stack
	* to assume responsibility for the connection.
	*/
	static void
	tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged)
	{

	INP_WLOCK_ASSERT(tp->t_inpcb);
	return;
	}

	/*
	* Target size of TCP PCB hash tables. Must be a power of two.
	*
	* Note that this can be overridden by the kernel environment
	* variable net.inet.tcp.tcbhashsize
	*/
	#ifndef TCBHASHSIZE
	#define TCBHASHSIZE 0
	#endif

	/*
	* XXX
	* Callouts should be moved into struct tcp directly. They are currently
	* separate because the tcpcb structure is exported to userland for sysctl
	* parsing purposes, which do not know about callouts.
	*/
	struct tcpcb_mem {
	struct tcpcb tcb;
	struct tcp_timer tt;
	struct cc_var ccv;
	#ifdef TCP_HHOOK
	struct osd osd;
	#endif
	};

	VNET_DEFINE_STATIC(uma_zone_t, tcpcb_zone);
	#define V_tcpcb_zone VNET(tcpcb_zone)

	MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
	MALLOC_DEFINE(M_TCPFUNCTIONS, "tcpfunc", "TCP function set memory");

	static struct mtx isn_mtx;

	#define ISN_LOCK_INIT() mtx_init(&isn_mtx, "isn_mtx", NULL, MTX_DEF)
	#define ISN_LOCK() mtx_lock(&isn_mtx)
	#define ISN_UNLOCK() mtx_unlock(&isn_mtx)

	/*
	* TCP initialization.
	*/
	static void
	tcp_zone_change(void *tag)
	{

	uma_zone_set_max(V_tcbinfo.ipi_zone, maxsockets);
	uma_zone_set_max(V_tcpcb_zone, maxsockets);
	tcp_tw_zone_change();
	}

	static int
	tcp_inpcb_init(void *mem, int size, int flags)
	{
	struct inpcb *inp = mem;

	INP_LOCK_INIT(inp, "inp", "tcpinp");
	return (0);
	}

	/*
	* Take a value and get the next power of 2 that doesn't overflow.
	* Used to size the tcp_inpcb hash buckets.
	*/
	static int
	maketcp_hashsize(int size)
	{
	int hashsize;

	/*
	* auto tune.
	* get the next power of 2 higher than maxsockets.
	*/
	hashsize = 1 << fls(size);
	/* catch overflow, and just go one power of 2 smaller */
	if (hashsize < size) {
	hashsize = 1 << (fls(size) - 1);
	}
	return (hashsize);
	}

	static volatile int next_tcp_stack_id = 1;

	/*
	* Register a TCP function block with the name provided in the names
	* array. (Note that this function does NOT automatically register
	* blk->tfb_tcp_block_name as a stack name. Therefore, you should
	* explicitly include blk->tfb_tcp_block_name in the list of names if
	* you wish to register the stack with that name.)
	*
	* Either all name registrations will succeed or all will fail. If
	* a name registration fails, the function will update the num_names
	* argument to point to the array index of the name that encountered
	* the failure.
	*
	* Returns 0 on success, or an error code on failure.
	*/
	int
	register_tcp_functions_as_names(struct tcp_function_block *blk, int wait,
	const char names[], int num_names)
	{
	struct tcp_function *n;
	struct tcp_function_set fs;
	int error, i;

	KASSERT(names != NULL && *num_names > 0,
	("%s: Called with 0-length name list", __func__));
	KASSERT(names != NULL, ("%s: Called with NULL name list", __func__));
	KASSERT(rw_initialized(&tcp_function_lock),
	("%s: called too early", __func__));

	if ((blk->tfb_tcp_output == NULL) \|\|
	(blk->tfb_tcp_do_segment == NULL) \|\|
	(blk->tfb_tcp_ctloutput == NULL) \|\|
	(strlen(blk->tfb_tcp_block_name) == 0)) {
	/*
	* These functions are required and you
	* need a name.
	*/
	*num_names = 0;
	return (EINVAL);
	}
	if (blk->tfb_tcp_timer_stop_all \|\|
	blk->tfb_tcp_timer_activate \|\|
	blk->tfb_tcp_timer_active \|\|
	blk->tfb_tcp_timer_stop) {
	/*
	* If you define one timer function you
	* must have them all.
	*/
	if ((blk->tfb_tcp_timer_stop_all == NULL) \|\|
	(blk->tfb_tcp_timer_activate == NULL) \|\|
	(blk->tfb_tcp_timer_active == NULL) \|\|
	(blk->tfb_tcp_timer_stop == NULL)) {
	*num_names = 0;
	return (EINVAL);
	}
	}

	if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
	*num_names = 0;
	return (EINVAL);
	}

	refcount_init(&blk->tfb_refcnt, 0);
	blk->tfb_id = atomic_fetchadd_int(&next_tcp_stack_id, 1);
	for (i = 0; i < *num_names; i++) {
	n = malloc(sizeof(struct tcp_function), M_TCPFUNCTIONS, wait);
	if (n == NULL) {
	error = ENOMEM;
	goto cleanup;
	}
	n->tf_fb = blk;

	(void)strlcpy(fs.function_set_name, names[i],
	sizeof(fs.function_set_name));
	rw_wlock(&tcp_function_lock);
	if (find_tcp_functions_locked(&fs) != NULL) {
	/* Duplicate name space not allowed */
	rw_wunlock(&tcp_function_lock);
	free(n, M_TCPFUNCTIONS);
	error = EALREADY;
	goto cleanup;
	}
	(void)strlcpy(n->tf_name, names[i], sizeof(n->tf_name));
	TAILQ_INSERT_TAIL(&t_functions, n, tf_next);
	tcp_fb_cnt++;
	rw_wunlock(&tcp_function_lock);
	}
	return(0);

	cleanup:
	/*
	* Deregister the names we just added. Because registration failed
	* for names[i], we don't need to deregister that name.
	*/
	*num_names = i;
	rw_wlock(&tcp_function_lock);
	while (--i >= 0) {
	TAILQ_FOREACH(n, &t_functions, tf_next) {
	if (!strncmp(n->tf_name, names[i],
	TCP_FUNCTION_NAME_LEN_MAX)) {
	TAILQ_REMOVE(&t_functions, n, tf_next);
	tcp_fb_cnt--;
	n->tf_fb = NULL;
	free(n, M_TCPFUNCTIONS);
	break;
	}
	}
	}
	rw_wunlock(&tcp_function_lock);
	return (error);
	}

	/*
	* Register a TCP function block using the name provided in the name
	* argument.
	*
	* Returns 0 on success, or an error code on failure.
	*/
	int
	register_tcp_functions_as_name(struct tcp_function_block blk, const char name,
	int wait)
	{
	const char *name_list[1];
	int num_names, rv;

	num_names = 1;
	if (name != NULL)
	name_list[0] = name;
	else
	name_list[0] = blk->tfb_tcp_block_name;
	rv = register_tcp_functions_as_names(blk, wait, name_list, &num_names);
	return (rv);
	}

	/*
	* Register a TCP function block using the name defined in
	* blk->tfb_tcp_block_name.
	*
	* Returns 0 on success, or an error code on failure.
	*/
	int
	register_tcp_functions(struct tcp_function_block *blk, int wait)
	{

	return (register_tcp_functions_as_name(blk, NULL, wait));
	}

	/*
	* Deregister all names associated with a function block. This
	* functionally removes the function block from use within the system.
	*
	* When called with a true quiesce argument, mark the function block
	* as being removed so no more stacks will use it and determine
	* whether the removal would succeed.
	*
	* When called with a false quiesce argument, actually attempt the
	* removal.
	*
	* When called with a force argument, attempt to switch all TCBs to
	* use the default stack instead of returning EBUSY.
	*
	* Returns 0 on success (or if the removal would succeed, or an error
	* code on failure.
	*/
	int
	deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce,
	bool force)
	{
	struct tcp_function *f;

	if (blk == &tcp_def_funcblk) {
	/* You can't un-register the default */
	return (EPERM);
	}
	rw_wlock(&tcp_function_lock);
	if (blk == tcp_func_set_ptr) {
	/* You can't free the current default */
	rw_wunlock(&tcp_function_lock);
	return (EBUSY);
	}
	/* Mark the block so no more stacks can use it. */
	blk->tfb_flags \|= TCP_FUNC_BEING_REMOVED;
	/*
	* If TCBs are still attached to the stack, attempt to switch them
	* to the default stack.
	*/
	if (force && blk->tfb_refcnt) {
	struct inpcb *inp;
	struct tcpcb *tp;
	VNET_ITERATOR_DECL(vnet_iter);

	rw_wunlock(&tcp_function_lock);

	VNET_LIST_RLOCK();
	VNET_FOREACH(vnet_iter) {
	CURVNET_SET(vnet_iter);
	INP_INFO_WLOCK(&V_tcbinfo);
	CK_LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
	INP_WLOCK(inp);
	if (inp->inp_flags & INP_TIMEWAIT) {
	INP_WUNLOCK(inp);
	continue;
	}
	tp = intotcpcb(inp);
	if (tp == NULL \|\| tp->t_fb != blk) {
	INP_WUNLOCK(inp);
	continue;
	}
	tcp_switch_back_to_default(tp);
	INP_WUNLOCK(inp);
	}
	INP_INFO_WUNLOCK(&V_tcbinfo);
	CURVNET_RESTORE();
	}
	VNET_LIST_RUNLOCK();

	rw_wlock(&tcp_function_lock);
	}
	if (blk->tfb_refcnt) {
	/* TCBs still attached. */
	rw_wunlock(&tcp_function_lock);
	return (EBUSY);
	}
	if (quiesce) {
	/* Skip removal. */
	rw_wunlock(&tcp_function_lock);
	return (0);
	}
	/* Remove any function names that map to this function block. */
	while (find_tcp_fb_locked(blk, &f) != NULL) {
	TAILQ_REMOVE(&t_functions, f, tf_next);
	tcp_fb_cnt--;
	f->tf_fb = NULL;
	free(f, M_TCPFUNCTIONS);
	}
	rw_wunlock(&tcp_function_lock);
	return (0);
	}

	void
	tcp_init(void)
	{
	const char *tcbhash_tuneable;
	int hashsize;

	tcbhash_tuneable = "net.inet.tcp.tcbhashsize";

	#ifdef TCP_HHOOK
	if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN,
	&V_tcp_hhh[HHOOK_TCP_EST_IN], HHOOK_NOWAIT\|HHOOK_HEADISINVNET) != 0)
	printf("%s: WARNING: unable to register helper hook\n", __func__);
	if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT,
	&V_tcp_hhh[HHOOK_TCP_EST_OUT], HHOOK_NOWAIT\|HHOOK_HEADISINVNET) != 0)
	printf("%s: WARNING: unable to register helper hook\n", __func__);
	#endif
	#ifdef STATS
	if (tcp_stats_init())
	printf("%s: WARNING: unable to initialise TCP stats\n",
	__func__);
	#endif
	hashsize = TCBHASHSIZE;
	TUNABLE_INT_FETCH(tcbhash_tuneable, &hashsize);
	if (hashsize == 0) {
	/*
	* Auto tune the hash size based on maxsockets.
	* A perfect hash would have a 1:1 mapping
	* (hashsize = maxsockets) however it's been
	* suggested that O(2) average is better.
	*/
	hashsize = maketcp_hashsize(maxsockets / 4);
	/*
	* Our historical default is 512,
	* do not autotune lower than this.
	*/
	if (hashsize < 512)
	hashsize = 512;
	if (bootverbose && IS_DEFAULT_VNET(curvnet))
	printf("%s: %s auto tuned to %d\n", __func__,
	tcbhash_tuneable, hashsize);
	}
	/*
	* We require a hashsize to be a power of two.
	* Previously if it was not a power of two we would just reset it
	* back to 512, which could be a nasty surprise if you did not notice
	* the error message.
	* Instead what we do is clip it to the closest power of two lower
	* than the specified hash value.
	*/
	if (!powerof2(hashsize)) {
	int oldhashsize = hashsize;

	hashsize = maketcp_hashsize(hashsize);
	/* prevent absurdly low value */
	if (hashsize < 16)
	hashsize = 16;
	printf("%s: WARNING: TCB hash size not a power of 2, "
	"clipped from %d to %d.\n", __func__, oldhashsize,
	hashsize);
	}
	in_pcbinfo_init(&V_tcbinfo, "tcp", &V_tcb, hashsize, hashsize,
	"tcp_inpcb", tcp_inpcb_init, IPI_HASHFIELDS_4TUPLE);

	/*
	* These have to be type stable for the benefit of the timers.
	*/
	V_tcpcb_zone = uma_zcreate("tcpcb", sizeof(struct tcpcb_mem),
	NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
	uma_zone_set_max(V_tcpcb_zone, maxsockets);
	uma_zone_set_warning(V_tcpcb_zone, "kern.ipc.maxsockets limit reached");

	tcp_tw_init();
	syncache_init();
	tcp_hc_init();

	TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
	V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
	NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);

	tcp_fastopen_init();

	/* Skip initialization of globals for non-default instances. */
	if (!IS_DEFAULT_VNET(curvnet))
	return;

	tcp_reass_global_init();

	/* XXX virtualize those bellow? */
	tcp_delacktime = TCPTV_DELACK;
	tcp_keepinit = TCPTV_KEEP_INIT;
	tcp_keepidle = TCPTV_KEEP_IDLE;
	tcp_keepintvl = TCPTV_KEEPINTVL;
	tcp_maxpersistidle = TCPTV_KEEP_IDLE;
	tcp_msl = TCPTV_MSL;
	tcp_rexmit_initial = TCPTV_RTOBASE;
	if (tcp_rexmit_initial < 1)
	tcp_rexmit_initial = 1;
	tcp_rexmit_min = TCPTV_MIN;
	if (tcp_rexmit_min < 1)
	tcp_rexmit_min = 1;
	tcp_persmin = TCPTV_PERSMIN;
	tcp_persmax = TCPTV_PERSMAX;
	tcp_rexmit_slop = TCPTV_CPU_VAR;
	tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
	tcp_tcbhashsize = hashsize;

	/* Setup the tcp function block list */
	TAILQ_INIT(&t_functions);
	rw_init(&tcp_function_lock, "tcp_func_lock");
	register_tcp_functions(&tcp_def_funcblk, M_WAITOK);
	#ifdef TCP_BLACKBOX
	/* Initialize the TCP logging data. */
	tcp_log_init();
	#endif
	arc4rand(&V_ts_offset_secret, sizeof(V_ts_offset_secret), 0);

	if (tcp_soreceive_stream) {
	#ifdef INET
	tcp_usrreqs.pru_soreceive = soreceive_stream;
	#endif
	#ifdef INET6
	tcp6_usrreqs.pru_soreceive = soreceive_stream;
	#endif /* INET6 */
	}

	#ifdef INET6
	#define TCP_MINPROTOHDR (sizeof(struct ip6_hdr) + sizeof(struct tcphdr))
	#else /* INET6 */
	#define TCP_MINPROTOHDR (sizeof(struct tcpiphdr))
	#endif /* INET6 */
	if (max_protohdr < TCP_MINPROTOHDR)
	max_protohdr = TCP_MINPROTOHDR;
	if (max_linkhdr + TCP_MINPROTOHDR > MHLEN)
	panic("tcp_init");
	#undef TCP_MINPROTOHDR

	ISN_LOCK_INIT();
	EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
	SHUTDOWN_PRI_DEFAULT);
	EVENTHANDLER_REGISTER(maxsockets_change, tcp_zone_change, NULL,
	EVENTHANDLER_PRI_ANY);

	tcp_inp_lro_direct_queue = counter_u64_alloc(M_WAITOK);
	tcp_inp_lro_wokeup_queue = counter_u64_alloc(M_WAITOK);
	tcp_inp_lro_compressed = counter_u64_alloc(M_WAITOK);
	tcp_inp_lro_single_push = counter_u64_alloc(M_WAITOK);
	tcp_inp_lro_locks_taken = counter_u64_alloc(M_WAITOK);
	tcp_inp_lro_sack_wake = counter_u64_alloc(M_WAITOK);
	#ifdef TCPPCAP
	tcp_pcap_init();
	#endif
	}

	#ifdef VIMAGE
	static void
	tcp_destroy(void *unused __unused)
	{
	int n;
	#ifdef TCP_HHOOK
	int error;
	#endif

	/*
	* All our processes are gone, all our sockets should be cleaned
	* up, which means, we should be past the tcp_discardcb() calls.
	* Sleep to let all tcpcb timers really disappear and cleanup.
	*/
	for (;;) {
	INP_LIST_RLOCK(&V_tcbinfo);
	n = V_tcbinfo.ipi_count;
	INP_LIST_RUNLOCK(&V_tcbinfo);
	if (n == 0)
	break;
	pause("tcpdes", hz / 10);
	}
	tcp_hc_destroy();
	syncache_destroy();
	tcp_tw_destroy();
	in_pcbinfo_destroy(&V_tcbinfo);
	/* tcp_discardcb() clears the sack_holes up. */
	uma_zdestroy(V_sack_hole_zone);
	uma_zdestroy(V_tcpcb_zone);

	/*
	* Cannot free the zone until all tcpcbs are released as we attach
	* the allocations to them.
	*/
	tcp_fastopen_destroy();

	#ifdef TCP_HHOOK
	error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]);
	if (error != 0) {
	printf("%s: WARNING: unable to deregister helper hook "
	"type=%d, id=%d: error %d returned\n", __func__,
	HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN, error);
	}
	error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_OUT]);
	if (error != 0) {
	printf("%s: WARNING: unable to deregister helper hook "
	"type=%d, id=%d: error %d returned\n", __func__,
	HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT, error);
	}
	#endif
	}
	VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_destroy, NULL);
	#endif

	void
	tcp_fini(void *xtp)
	{

	}

	/*
	* Fill in the IP and TCP headers for an outgoing packet, given the tcpcb.
	* tcp_template used to store this data in mbufs, but we now recopy it out
	* of the tcpcb each time to conserve mbufs.
	*/
	void
	tcpip_fillheaders(struct inpcb inp, void ip_ptr, void *tcp_ptr)
	{
	struct tcphdr th = (struct tcphdr )tcp_ptr;

	INP_WLOCK_ASSERT(inp);

	#ifdef INET6
	if ((inp->inp_vflag & INP_IPV6) != 0) {
	struct ip6_hdr *ip6;

	ip6 = (struct ip6_hdr *)ip_ptr;
	ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) \|
	(inp->inp_flow & IPV6_FLOWINFO_MASK);
	ip6->ip6_vfc = (ip6->ip6_vfc & ~IPV6_VERSION_MASK) \|
	(IPV6_VERSION & IPV6_VERSION_MASK);
	ip6->ip6_nxt = IPPROTO_TCP;
	ip6->ip6_plen = htons(sizeof(struct tcphdr));
	ip6->ip6_src = inp->in6p_laddr;
	ip6->ip6_dst = inp->in6p_faddr;
	}
	#endif /* INET6 */
	#if defined(INET6) && defined(INET)
	else
	#endif
	#ifdef INET
	{
	struct ip *ip;

	ip = (struct ip *)ip_ptr;
	ip->ip_v = IPVERSION;
	ip->ip_hl = 5;
	ip->ip_tos = inp->inp_ip_tos;
	ip->ip_len = 0;
	ip->ip_id = 0;
	ip->ip_off = 0;
	ip->ip_ttl = inp->inp_ip_ttl;
	ip->ip_sum = 0;
	ip->ip_p = IPPROTO_TCP;
	ip->ip_src = inp->inp_laddr;
	ip->ip_dst = inp->inp_faddr;
	}
	#endif /* INET */
	th->th_sport = inp->inp_lport;
	th->th_dport = inp->inp_fport;
	th->th_seq = 0;
	th->th_ack = 0;
	th->th_x2 = 0;
	th->th_off = 5;
	th->th_flags = 0;
	th->th_win = 0;
	th->th_urp = 0;
	th->th_sum = 0; /* in_pseudo() is called later for ipv4 */
	}

	/*
	* Create template to be used to send tcp packets on a connection.
	* Allocates an mbuf and fills in a skeletal tcp/ip header. The only
	* use for this function is in keepalives, which use tcp_respond.
	*/
	struct tcptemp *
	tcpip_maketemplate(struct inpcb *inp)
	{
	struct tcptemp *t;

	t = malloc(sizeof(*t), M_TEMP, M_NOWAIT);
	if (t == NULL)
	return (NULL);
	tcpip_fillheaders(inp, (void )&t->tt_ipgen, (void )&t->tt_t);
	return (t);
	}

	/*
	* Send a single message to the TCP at address specified by
	* the given TCP/IP header. If m == NULL, then we make a copy
	* of the tcpiphdr at th and send directly to the addressed host.
	* This is used to force keep alive messages out using the TCP
	* template for a connection. If flags are given then we send
	* a message back to the TCP which originated the segment th,
	* and discard the mbuf containing it and any other attached mbufs.
	*
	* In any case the ack and sequence number of the transmitted
	* segment are as specified by the parameters.
	*
	* NOTE: If m != NULL, then th must point to inside the mbuf.
	*/
	void
	tcp_respond(struct tcpcb tp, void ipgen, struct tcphdr th, struct mbuf m,
	tcp_seq ack, tcp_seq seq, int flags)
	{
	struct tcpopt to;
	struct inpcb *inp;
	struct ip *ip;
	struct mbuf *optm;
	struct tcphdr *nth;
	u_char *optp;
	#ifdef INET6
	struct ip6_hdr *ip6;
	int isipv6;
	#endif /* INET6 */
	int optlen, tlen, win;
	bool incl_opts;

	KASSERT(tp != NULL \|\| m != NULL, ("tcp_respond: tp and m both NULL"));
	NET_EPOCH_ASSERT();

	#ifdef INET6
	isipv6 = ((struct ip *)ipgen)->ip_v == (IPV6_VERSION >> 4);
	ip6 = ipgen;
	#endif /* INET6 */
	ip = ipgen;

	if (tp != NULL) {
	inp = tp->t_inpcb;
	KASSERT(inp != NULL, ("tcp control block w/o inpcb"));
	INP_WLOCK_ASSERT(inp);
	} else
	inp = NULL;

	incl_opts = false;
	win = 0;
	if (tp != NULL) {
	if (!(flags & TH_RST)) {
	win = sbspace(&inp->inp_socket->so_rcv);
	if (win > TCP_MAXWIN << tp->rcv_scale)
	win = TCP_MAXWIN << tp->rcv_scale;
	}
	if ((tp->t_flags & TF_NOOPT) == 0)
	incl_opts = true;
	}
	if (m == NULL) {
	m = m_gethdr(M_NOWAIT, MT_DATA);
	if (m == NULL)
	return;
	m->m_data += max_linkhdr;
	#ifdef INET6
	if (isipv6) {
	bcopy((caddr_t)ip6, mtod(m, caddr_t),
	sizeof(struct ip6_hdr));
	ip6 = mtod(m, struct ip6_hdr *);
	nth = (struct tcphdr *)(ip6 + 1);
	} else
	#endif /* INET6 */
	{
	bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip));
	ip = mtod(m, struct ip *);
	nth = (struct tcphdr *)(ip + 1);
	}
	bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
	flags = TH_ACK;
	} else if (!M_WRITABLE(m)) {
	struct mbuf *n;

	/* Can't reuse 'm', allocate a new mbuf. */
	n = m_gethdr(M_NOWAIT, MT_DATA);
	if (n == NULL) {
	m_freem(m);
	return;
	}

	if (!m_dup_pkthdr(n, m, M_NOWAIT)) {
	m_freem(m);
	m_freem(n);
	return;
	}

	n->m_data += max_linkhdr;
	/* m_len is set later */
	#define xchg(a,b,type) { type t; t=a; a=b; b=t; }
	#ifdef INET6
	if (isipv6) {
	bcopy((caddr_t)ip6, mtod(n, caddr_t),
	sizeof(struct ip6_hdr));
	ip6 = mtod(n, struct ip6_hdr *);
	xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
	nth = (struct tcphdr *)(ip6 + 1);
	} else
	#endif /* INET6 */
	{
	bcopy((caddr_t)ip, mtod(n, caddr_t), sizeof(struct ip));
	ip = mtod(n, struct ip *);
	xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
	nth = (struct tcphdr *)(ip + 1);
	}
	bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
	xchg(nth->th_dport, nth->th_sport, uint16_t);
	th = nth;
	m_freem(m);
	m = n;
	} else {
	/*
	* reuse the mbuf.
	* XXX MRT We inherit the FIB, which is lucky.
	*/
	m_freem(m->m_next);
	m->m_next = NULL;
	m->m_data = (caddr_t)ipgen;
	/* m_len is set later */
	#ifdef INET6
	if (isipv6) {
	xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
	nth = (struct tcphdr *)(ip6 + 1);
	} else
	#endif /* INET6 */
	{
	xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
	nth = (struct tcphdr *)(ip + 1);
	}
	if (th != nth) {
	/*
	* this is usually a case when an extension header
	* exists between the IPv6 header and the
	* TCP header.
	*/
	nth->th_sport = th->th_sport;
	nth->th_dport = th->th_dport;
	}
	xchg(nth->th_dport, nth->th_sport, uint16_t);
	#undef xchg
	}
	tlen = 0;
	#ifdef INET6
	if (isipv6)
	tlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
	#endif
	#if defined(INET) && defined(INET6)
	else
	#endif
	#ifdef INET
	tlen = sizeof (struct tcpiphdr);
	#endif
	#ifdef INVARIANTS
	m->m_len = 0;
	KASSERT(M_TRAILINGSPACE(m) >= tlen,
	("Not enough trailing space for message (m=%p, need=%d, have=%ld)",
	m, tlen, (long)M_TRAILINGSPACE(m)));
	#endif
	m->m_len = tlen;
	to.to_flags = 0;
	if (incl_opts) {
	/* Make sure we have room. */
	if (M_TRAILINGSPACE(m) < TCP_MAXOLEN) {
	m->m_next = m_get(M_NOWAIT, MT_DATA);
	if (m->m_next) {
	optp = mtod(m->m_next, u_char *);
	optm = m->m_next;
	} else
	incl_opts = false;
	} else {
	optp = (u_char *) (nth + 1);
	optm = m;
	}
	}
	if (incl_opts) {
	/* Timestamps. */
	if (tp->t_flags & TF_RCVD_TSTMP) {
	to.to_tsval = tcp_ts_getticks() + tp->ts_offset;
	to.to_tsecr = tp->ts_recent;
	to.to_flags \|= TOF_TS;
	}
	#if defined(IPSEC_SUPPORT) \|\| defined(TCP_SIGNATURE)
	/* TCP-MD5 (RFC2385). */
	if (tp->t_flags & TF_SIGNATURE)
	to.to_flags \|= TOF_SIGNATURE;
	#endif
	/* Add the options. */
	tlen += optlen = tcp_addoptions(&to, optp);

	/* Update m_len in the correct mbuf. */
	optm->m_len += optlen;
	} else
	optlen = 0;
	#ifdef INET6
	if (isipv6) {
	ip6->ip6_flow = 0;
	ip6->ip6_vfc = IPV6_VERSION;
	ip6->ip6_nxt = IPPROTO_TCP;
	ip6->ip6_plen = htons(tlen - sizeof(*ip6));
	}
	#endif
	#if defined(INET) && defined(INET6)
	else
	#endif
	#ifdef INET
	{
	ip->ip_len = htons(tlen);
	ip->ip_ttl = V_ip_defttl;
	if (V_path_mtu_discovery)
	ip->ip_off \|= htons(IP_DF);
	}
	#endif
	m->m_pkthdr.len = tlen;
	m->m_pkthdr.rcvif = NULL;
	#ifdef MAC
	if (inp != NULL) {
	/*
	* Packet is associated with a socket, so allow the
	* label of the response to reflect the socket label.
	*/
	INP_WLOCK_ASSERT(inp);
	mac_inpcb_create_mbuf(inp, m);
	} else {
	/*
	* Packet is not associated with a socket, so possibly
	* update the label in place.
	*/
	mac_netinet_tcp_reply(m);
	}
	#endif
	nth->th_seq = htonl(seq);
	nth->th_ack = htonl(ack);
	nth->th_x2 = 0;
	nth->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
	nth->th_flags = flags;
	if (tp != NULL)
	nth->th_win = htons((u_short) (win >> tp->rcv_scale));
	else
	nth->th_win = htons((u_short)win);
	nth->th_urp = 0;

	#if defined(IPSEC_SUPPORT) \|\| defined(TCP_SIGNATURE)
	if (to.to_flags & TOF_SIGNATURE) {
	if (!TCPMD5_ENABLED() \|\|
	TCPMD5_OUTPUT(m, nth, to.to_signature) != 0) {
	m_freem(m);
	return;
	}
	}
	#endif

	m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
	#ifdef INET6
	if (isipv6) {
	m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
	nth->th_sum = in6_cksum_pseudo(ip6,
	tlen - sizeof(struct ip6_hdr), IPPROTO_TCP, 0);
	ip6->ip6_hlim = in6_selecthlim(tp != NULL ? tp->t_inpcb :
	NULL, NULL);
	}
	#endif /* INET6 */
	#if defined(INET6) && defined(INET)
	else
	#endif
	#ifdef INET
	{
	m->m_pkthdr.csum_flags = CSUM_TCP;
	nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
	htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p)));
	}
	#endif /* INET */
	#ifdef TCPDEBUG
	if (tp == NULL \|\| (inp->inp_socket->so_options & SO_DEBUG))
	tcp_trace(TA_OUTPUT, 0, tp, mtod(m, void *), th, 0);
	#endif
	TCP_PROBE3(debug__output, tp, th, m);
	if (flags & TH_RST)
	TCP_PROBE5(accept__refused, NULL, NULL, m, tp, nth);

	#ifdef INET6
	if (isipv6) {
	TCP_PROBE5(send, NULL, tp, ip6, tp, nth);
	(void)ip6_output(m, NULL, NULL, 0, NULL, NULL, inp);
	}
	#endif /* INET6 */
	#if defined(INET) && defined(INET6)
	else
	#endif
	#ifdef INET
	{
	TCP_PROBE5(send, NULL, tp, ip, tp, nth);
	(void)ip_output(m, NULL, NULL, 0, NULL, inp);
	}
	#endif
	}

	/*
	* Create a new TCP control block, making an
	* empty reassembly queue and hooking it to the argument
	* protocol control block. The `inp' parameter must have
	* come from the zone allocator set up in tcp_init().
	*/
	struct tcpcb *
	tcp_newtcpcb(struct inpcb *inp)
	{
	struct tcpcb_mem *tm;
	struct tcpcb *tp;
	#ifdef INET6
	int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
	#endif /* INET6 */

	tm = uma_zalloc(V_tcpcb_zone, M_NOWAIT \| M_ZERO);
	if (tm == NULL)
	return (NULL);
	tp = &tm->tcb;

	/* Initialise cc_var struct for this tcpcb. */
	tp->ccv = &tm->ccv;
	tp->ccv->type = IPPROTO_TCP;
	tp->ccv->ccvc.tcp = tp;
	rw_rlock(&tcp_function_lock);
	tp->t_fb = tcp_func_set_ptr;
	refcount_acquire(&tp->t_fb->tfb_refcnt);
	rw_runlock(&tcp_function_lock);
	/*
	* Use the current system default CC algorithm.
	*/
	CC_LIST_RLOCK();
	KASSERT(!STAILQ_EMPTY(&cc_list), ("cc_list is empty!"));
	CC_ALGO(tp) = CC_DEFAULT();
	CC_LIST_RUNLOCK();
	/*
	* The tcpcb will hold a reference on its inpcb until tcp_discardcb()
	* is called.
	*/
	in_pcbref(inp); /* Reference for tcpcb */
	tp->t_inpcb = inp;

	if (CC_ALGO(tp)->cb_init != NULL)
	if (CC_ALGO(tp)->cb_init(tp->ccv) > 0) {
	if (tp->t_fb->tfb_tcp_fb_fini)
	(*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
	+ in_pcbrele_wlocked(inp);
	refcount_release(&tp->t_fb->tfb_refcnt);
	uma_zfree(V_tcpcb_zone, tm);
	return (NULL);
	}

	#ifdef TCP_HHOOK
	tp->osd = &tm->osd;
	if (khelp_init_osd(HELPER_CLASS_TCP, tp->osd)) {
	if (tp->t_fb->tfb_tcp_fb_fini)
	(*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
	+ in_pcbrele_wlocked(inp);
	refcount_release(&tp->t_fb->tfb_refcnt);
	uma_zfree(V_tcpcb_zone, tm);
	return (NULL);
	}
	#endif

	#ifdef VIMAGE
	tp->t_vnet = inp->inp_vnet;
	#endif
	tp->t_timers = &tm->tt;
	TAILQ_INIT(&tp->t_segq);
	tp->t_maxseg =
	#ifdef INET6
	isipv6 ? V_tcp_v6mssdflt :
	#endif /* INET6 */
	V_tcp_mssdflt;

	/* Set up our timeouts. */
	callout_init(&tp->t_timers->tt_rexmt, 1);
	callout_init(&tp->t_timers->tt_persist, 1);
	callout_init(&tp->t_timers->tt_keep, 1);
	callout_init(&tp->t_timers->tt_2msl, 1);
	callout_init(&tp->t_timers->tt_delack, 1);

	if (V_tcp_do_rfc1323)
	tp->t_flags = (TF_REQ_SCALE\|TF_REQ_TSTMP);
	if (V_tcp_do_sack)
	tp->t_flags \|= TF_SACK_PERMIT;
	TAILQ_INIT(&tp->snd_holes);

	/*
	* Init srtt to TCPTV_SRTTBASE (0), so we can tell that we have no
	* rtt estimate. Set rttvar so that srtt + 4 * rttvar gives
	* reasonable initial retransmit time.
	*/
	tp->t_srtt = TCPTV_SRTTBASE;
	tp->t_rttvar = ((tcp_rexmit_initial - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
	tp->t_rttmin = tcp_rexmit_min;
	tp->t_rxtcur = tcp_rexmit_initial;
	tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
	tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
	tp->t_rcvtime = ticks;
	/*
	* IPv4 TTL initialization is necessary for an IPv6 socket as well,
	* because the socket may be bound to an IPv6 wildcard address,
	* which may match an IPv4-mapped IPv6 address.
	*/
	inp->inp_ip_ttl = V_ip_defttl;
	inp->inp_ppcb = tp;
	#ifdef TCPPCAP
	/*
	* Init the TCP PCAP queues.
	*/
	tcp_pcap_tcpcb_init(tp);
	#endif
	#ifdef TCP_BLACKBOX
	/* Initialize the per-TCPCB log data. */
	tcp_log_tcpcbinit(tp);
	#endif
	if (tp->t_fb->tfb_tcp_fb_init) {
	- (*tp->t_fb->tfb_tcp_fb_init)(tp);
	+ if ((*tp->t_fb->tfb_tcp_fb_init)(tp)) {
	+ refcount_release(&tp->t_fb->tfb_refcnt);
	+ in_pcbrele_wlocked(inp);
	+ uma_zfree(V_tcpcb_zone, tm);
	+ return (NULL);
	+ }
	}
	#ifdef STATS
	if (V_tcp_perconn_stats_enable == 1)
	tp->t_stats = stats_blob_alloc(V_tcp_perconn_stats_dflt_tpl, 0);
	#endif
	return (tp); /* XXX */
	}

	/*
	* Switch the congestion control algorithm back to NewReno for any active
	* control blocks using an algorithm which is about to go away.
	* This ensures the CC framework can allow the unload to proceed without leaving
	* any dangling pointers which would trigger a panic.
	* Returning non-zero would inform the CC framework that something went wrong
	* and it would be unsafe to allow the unload to proceed. However, there is no
	* way for this to occur with this implementation so we always return zero.
	*/
	int
	tcp_ccalgounload(struct cc_algo *unload_algo)
	{
	struct cc_algo *tmpalgo;
	struct inpcb *inp;
	struct tcpcb *tp;
	VNET_ITERATOR_DECL(vnet_iter);

	/*
	* Check all active control blocks across all network stacks and change
	* any that are using "unload_algo" back to NewReno. If "unload_algo"
	* requires cleanup code to be run, call it.
	*/
	VNET_LIST_RLOCK();
	VNET_FOREACH(vnet_iter) {
	CURVNET_SET(vnet_iter);
	INP_INFO_WLOCK(&V_tcbinfo);
	/*
	* New connections already part way through being initialised
	* with the CC algo we're removing will not race with this code
	* because the INP_INFO_WLOCK is held during initialisation. We
	* therefore don't enter the loop below until the connection
	* list has stabilised.
	*/
	CK_LIST_FOREACH(inp, &V_tcb, inp_list) {
	INP_WLOCK(inp);
	/* Important to skip tcptw structs. */
	if (!(inp->inp_flags & INP_TIMEWAIT) &&
	(tp = intotcpcb(inp)) != NULL) {
	/*
	* By holding INP_WLOCK here, we are assured
	* that the connection is not currently
	* executing inside the CC module's functions
	* i.e. it is safe to make the switch back to
	* NewReno.
	*/
	if (CC_ALGO(tp) == unload_algo) {
	tmpalgo = CC_ALGO(tp);
	if (tmpalgo->cb_destroy != NULL)
	tmpalgo->cb_destroy(tp->ccv);
	CC_DATA(tp) = NULL;
	/*
	* NewReno may allocate memory on
	* demand for certain stateful
	* configuration as needed, but is
	* coded to never fail on memory
	* allocation failure so it is a safe
	* fallback.
	*/
	CC_ALGO(tp) = &newreno_cc_algo;
	}
	}
	INP_WUNLOCK(inp);
	}
	INP_INFO_WUNLOCK(&V_tcbinfo);
	CURVNET_RESTORE();
	}
	VNET_LIST_RUNLOCK();

	return (0);
	}

	/*
	* Drop a TCP connection, reporting
	* the specified error. If connection is synchronized,
	* then send a RST to peer.
	*/
	struct tcpcb *
	tcp_drop(struct tcpcb *tp, int errno)
	{
	struct socket *so = tp->t_inpcb->inp_socket;

	NET_EPOCH_ASSERT();
	INP_INFO_LOCK_ASSERT(&V_tcbinfo);
	INP_WLOCK_ASSERT(tp->t_inpcb);

	if (TCPS_HAVERCVDSYN(tp->t_state)) {
	tcp_state_change(tp, TCPS_CLOSED);
	(void) tp->t_fb->tfb_tcp_output(tp);
	TCPSTAT_INC(tcps_drops);
	} else
	TCPSTAT_INC(tcps_conndrops);
	if (errno == ETIMEDOUT && tp->t_softerror)
	errno = tp->t_softerror;
	so->so_error = errno;
	return (tcp_close(tp));
	}

	void
	tcp_discardcb(struct tcpcb *tp)
	{
	struct inpcb *inp = tp->t_inpcb;
	struct socket *so = inp->inp_socket;
	#ifdef INET6
	int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
	#endif /* INET6 */
	int released __unused;

	INP_WLOCK_ASSERT(inp);

	/*
	* Make sure that all of our timers are stopped before we delete the
	* PCB.
	*
	* If stopping a timer fails, we schedule a discard function in same
	* callout, and the last discard function called will take care of
	* deleting the tcpcb.
	*/
	tp->t_timers->tt_draincnt = 0;
	tcp_timer_stop(tp, TT_REXMT);
	tcp_timer_stop(tp, TT_PERSIST);
	tcp_timer_stop(tp, TT_KEEP);
	tcp_timer_stop(tp, TT_2MSL);
	tcp_timer_stop(tp, TT_DELACK);
	if (tp->t_fb->tfb_tcp_timer_stop_all) {
	/*
	* Call the stop-all function of the methods,
	* this function should call the tcp_timer_stop()
	* method with each of the function specific timeouts.
	* That stop will be called via the tfb_tcp_timer_stop()
	* which should use the async drain function of the
	* callout system (see tcp_var.h).
	*/
	tp->t_fb->tfb_tcp_timer_stop_all(tp);
	}

	/*
	* If we got enough samples through the srtt filter,
	* save the rtt and rttvar in the routing entry.
	* 'Enough' is arbitrarily defined as 4 rtt samples.
	* 4 samples is enough for the srtt filter to converge
	* to within enough % of the correct value; fewer samples
	* and we could save a bogus rtt. The danger is not high
	* as tcp quickly recovers from everything.
	* XXX: Works very well but needs some more statistics!
	*/
	if (tp->t_rttupdated >= 4) {
	struct hc_metrics_lite metrics;
	uint32_t ssthresh;

	bzero(&metrics, sizeof(metrics));
	/*
	* Update the ssthresh always when the conditions below
	* are satisfied. This gives us better new start value
	* for the congestion avoidance for new connections.
	* ssthresh is only set if packet loss occurred on a session.
	*
	* XXXRW: 'so' may be NULL here, and/or socket buffer may be
	* being torn down. Ideally this code would not use 'so'.
	*/
	ssthresh = tp->snd_ssthresh;
	if (ssthresh != 0 && ssthresh < so->so_snd.sb_hiwat / 2) {
	/*
	* convert the limit from user data bytes to
	* packets then to packet data bytes.
	*/
	ssthresh = (ssthresh + tp->t_maxseg / 2) / tp->t_maxseg;
	if (ssthresh < 2)
	ssthresh = 2;
	ssthresh *= (tp->t_maxseg +
	#ifdef INET6
	(isipv6 ? sizeof (struct ip6_hdr) +
	sizeof (struct tcphdr) :
	#endif
	sizeof (struct tcpiphdr)
	#ifdef INET6
	)
	#endif
	);
	} else
	ssthresh = 0;
	metrics.rmx_ssthresh = ssthresh;

	metrics.rmx_rtt = tp->t_srtt;
	metrics.rmx_rttvar = tp->t_rttvar;
	metrics.rmx_cwnd = tp->snd_cwnd;
	metrics.rmx_sendpipe = 0;
	metrics.rmx_recvpipe = 0;

	tcp_hc_update(&inp->inp_inc, &metrics);
	}

	/* free the reassembly queue, if any */
	tcp_reass_flush(tp);

	#ifdef TCP_OFFLOAD
	/* Disconnect offload device, if any. */
	if (tp->t_flags & TF_TOE)
	tcp_offload_detach(tp);
	#endif

	tcp_free_sackholes(tp);

	#ifdef TCPPCAP
	/* Free the TCP PCAP queues. */
	tcp_pcap_drain(&(tp->t_inpkts));
	tcp_pcap_drain(&(tp->t_outpkts));
	#endif

	/* Allow the CC algorithm to clean up after itself. */
	if (CC_ALGO(tp)->cb_destroy != NULL)
	CC_ALGO(tp)->cb_destroy(tp->ccv);
	CC_DATA(tp) = NULL;

	#ifdef TCP_HHOOK
	khelp_destroy_osd(tp->osd);
	#endif
	#ifdef STATS
	stats_blob_destroy(tp->t_stats);
	#endif

	CC_ALGO(tp) = NULL;
	inp->inp_ppcb = NULL;
	if (tp->t_timers->tt_draincnt == 0) {
	/* We own the last reference on tcpcb, let's free it. */
	#ifdef TCP_BLACKBOX
	tcp_log_tcpcbfini(tp);
	#endif
	TCPSTATES_DEC(tp->t_state);
	if (tp->t_fb->tfb_tcp_fb_fini)
	(*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
	refcount_release(&tp->t_fb->tfb_refcnt);
	tp->t_inpcb = NULL;
	uma_zfree(V_tcpcb_zone, tp);
	released = in_pcbrele_wlocked(inp);
	KASSERT(!released, ("%s: inp %p should not have been released "
	"here", __func__, inp));
	}
	}

	void
	tcp_timer_discard(void *ptp)
	{
	struct inpcb *inp;
	struct tcpcb *tp;
	struct epoch_tracker et;

	tp = (struct tcpcb *)ptp;
	CURVNET_SET(tp->t_vnet);
	NET_EPOCH_ENTER(et);
	inp = tp->t_inpcb;
	KASSERT(inp != NULL, ("%s: tp %p tp->t_inpcb == NULL",
	__func__, tp));
	INP_WLOCK(inp);
	KASSERT((tp->t_timers->tt_flags & TT_STOPPED) != 0,
	("%s: tcpcb has to be stopped here", __func__));
	tp->t_timers->tt_draincnt--;
	if (tp->t_timers->tt_draincnt == 0) {
	/* We own the last reference on this tcpcb, let's free it. */
	#ifdef TCP_BLACKBOX
	tcp_log_tcpcbfini(tp);
	#endif
	TCPSTATES_DEC(tp->t_state);
	if (tp->t_fb->tfb_tcp_fb_fini)
	(*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
	refcount_release(&tp->t_fb->tfb_refcnt);
	tp->t_inpcb = NULL;
	uma_zfree(V_tcpcb_zone, tp);
	if (in_pcbrele_wlocked(inp)) {
	NET_EPOCH_EXIT(et);
	CURVNET_RESTORE();
	return;
	}
	}
	INP_WUNLOCK(inp);
	NET_EPOCH_EXIT(et);
	CURVNET_RESTORE();
	}

	/*
	* Attempt to close a TCP control block, marking it as dropped, and freeing
	* the socket if we hold the only reference.
	*/
	struct tcpcb *
	tcp_close(struct tcpcb *tp)
	{
	struct inpcb *inp = tp->t_inpcb;
	struct socket *so;

	INP_INFO_LOCK_ASSERT(&V_tcbinfo);
	INP_WLOCK_ASSERT(inp);

	#ifdef TCP_OFFLOAD
	if (tp->t_state == TCPS_LISTEN)
	tcp_offload_listen_stop(tp);
	#endif
	/*
	* This releases the TFO pending counter resource for TFO listen
	* sockets as well as passively-created TFO sockets that transition
	* from SYN_RECEIVED to CLOSED.
	*/
	if (tp->t_tfo_pending) {
	tcp_fastopen_decrement_counter(tp->t_tfo_pending);
	tp->t_tfo_pending = NULL;
	}
	in_pcbdrop(inp);
	TCPSTAT_INC(tcps_closed);
	if (tp->t_state != TCPS_CLOSED)
	tcp_state_change(tp, TCPS_CLOSED);
	KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
	so = inp->inp_socket;
	soisdisconnected(so);
	if (inp->inp_flags & INP_SOCKREF) {
	KASSERT(so->so_state & SS_PROTOREF,
	("tcp_close: !SS_PROTOREF"));
	inp->inp_flags &= ~INP_SOCKREF;
	INP_WUNLOCK(inp);
	SOCK_LOCK(so);
	so->so_state &= ~SS_PROTOREF;
	sofree(so);
	return (NULL);
	}
	return (tp);
	}

	void
	tcp_drain(void)
	{
	VNET_ITERATOR_DECL(vnet_iter);

	if (!do_tcpdrain)
	return;

	VNET_LIST_RLOCK_NOSLEEP();
	VNET_FOREACH(vnet_iter) {
	CURVNET_SET(vnet_iter);
	struct inpcb *inpb;
	struct tcpcb *tcpb;

	/*
	* Walk the tcpbs, if existing, and flush the reassembly queue,
	* if there is one...
	* XXX: The "Net/3" implementation doesn't imply that the TCP
	* reassembly queue should be flushed, but in a situation
	* where we're really low on mbufs, this is potentially
	* useful.
	*/
	INP_INFO_WLOCK(&V_tcbinfo);
	CK_LIST_FOREACH(inpb, V_tcbinfo.ipi_listhead, inp_list) {
	INP_WLOCK(inpb);
	if (inpb->inp_flags & INP_TIMEWAIT) {
	INP_WUNLOCK(inpb);
	continue;
	}
	if ((tcpb = intotcpcb(inpb)) != NULL) {
	tcp_reass_flush(tcpb);
	tcp_clean_sackreport(tcpb);
	#ifdef TCP_BLACKBOX
	tcp_log_drain(tcpb);
	#endif
	#ifdef TCPPCAP
	if (tcp_pcap_aggressive_free) {
	/* Free the TCP PCAP queues. */
	tcp_pcap_drain(&(tcpb->t_inpkts));
	tcp_pcap_drain(&(tcpb->t_outpkts));
	}
	#endif
	}
	INP_WUNLOCK(inpb);
	}
	INP_INFO_WUNLOCK(&V_tcbinfo);
	CURVNET_RESTORE();
	}
	VNET_LIST_RUNLOCK_NOSLEEP();
	}

	/*
	* Notify a tcp user of an asynchronous error;
	* store error as soft error, but wake up user
	* (for now, won't do anything until can select for soft error).
	*
	* Do not wake up user since there currently is no mechanism for
	* reporting soft errors (yet - a kqueue filter may be added).
	*/
	static struct inpcb *
	tcp_notify(struct inpcb *inp, int error)
	{
	struct tcpcb *tp;

	INP_INFO_LOCK_ASSERT(&V_tcbinfo);
	INP_WLOCK_ASSERT(inp);

	if ((inp->inp_flags & INP_TIMEWAIT) \|\|
	(inp->inp_flags & INP_DROPPED))
	return (inp);

	tp = intotcpcb(inp);
	KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));

	/*
	* Ignore some errors if we are hooked up.
	* If connection hasn't completed, has retransmitted several times,
	* and receives a second error, give up now. This is better
	* than waiting a long time to establish a connection that
	* can never complete.
	*/
	if (tp->t_state == TCPS_ESTABLISHED &&
	(error == EHOSTUNREACH \|\| error == ENETUNREACH \|\|
	error == EHOSTDOWN)) {
	if (inp->inp_route.ro_nh) {
	NH_FREE(inp->inp_route.ro_nh);
	inp->inp_route.ro_nh = (struct nhop_object *)NULL;
	}
	return (inp);
	} else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
	tp->t_softerror) {
	tp = tcp_drop(tp, error);
	if (tp != NULL)
	return (inp);
	else
	return (NULL);
	} else {
	tp->t_softerror = error;
	return (inp);
	}
	#if 0
	wakeup( &so->so_timeo);
	sorwakeup(so);
	sowwakeup(so);
	#endif
	}

	static int
	tcp_pcblist(SYSCTL_HANDLER_ARGS)
	{
	struct epoch_tracker et;
	struct inpcb *inp;
	struct xinpgen xig;
	int error;

	if (req->newptr != NULL)
	return (EPERM);

	if (req->oldptr == NULL) {
	int n;

	n = V_tcbinfo.ipi_count +
	counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
	n += imax(n / 8, 10);
	req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
	return (0);
	}

	if ((error = sysctl_wire_old_buffer(req, 0)) != 0)
	return (error);

	bzero(&xig, sizeof(xig));
	xig.xig_len = sizeof xig;
	xig.xig_count = V_tcbinfo.ipi_count +
	counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
	xig.xig_gen = V_tcbinfo.ipi_gencnt;
	xig.xig_sogen = so_gencnt;
	error = SYSCTL_OUT(req, &xig, sizeof xig);
	if (error)
	return (error);

	error = syncache_pcblist(req);
	if (error)
	return (error);

	NET_EPOCH_ENTER(et);
	for (inp = CK_LIST_FIRST(V_tcbinfo.ipi_listhead);
	inp != NULL;
	inp = CK_LIST_NEXT(inp, inp_list)) {
	INP_RLOCK(inp);
	if (inp->inp_gencnt <= xig.xig_gen) {
	int crerr;

	/*
	* XXX: This use of cr_cansee(), introduced with
	* TCP state changes, is not quite right, but for
	* now, better than nothing.
	*/
	if (inp->inp_flags & INP_TIMEWAIT) {
	if (intotw(inp) != NULL)
	crerr = cr_cansee(req->td->td_ucred,
	intotw(inp)->tw_cred);
	else
	crerr = EINVAL; /* Skip this inp. */
	} else
	crerr = cr_canseeinpcb(req->td->td_ucred, inp);
	if (crerr == 0) {
	struct xtcpcb xt;

	tcp_inptoxtp(inp, &xt);
	INP_RUNLOCK(inp);
	error = SYSCTL_OUT(req, &xt, sizeof xt);
	if (error)
	break;
	else
	continue;
	}
	}
	INP_RUNLOCK(inp);
	}
	NET_EPOCH_EXIT(et);

	if (!error) {
	/*
	* Give the user an updated idea of our state.
	* If the generation differs from what we told
	* her before, she knows that something happened
	* while we were processing this request, and it
	* might be necessary to retry.
	*/
	xig.xig_gen = V_tcbinfo.ipi_gencnt;
	xig.xig_sogen = so_gencnt;
	xig.xig_count = V_tcbinfo.ipi_count +
	counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
	error = SYSCTL_OUT(req, &xig, sizeof xig);
	}

	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
	CTLTYPE_OPAQUE \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	NULL, 0, tcp_pcblist, "S,xtcpcb",
	"List of active TCP connections");

	#ifdef INET
	static int
	tcp_getcred(SYSCTL_HANDLER_ARGS)
	{
	struct xucred xuc;
	struct sockaddr_in addrs[2];
	struct epoch_tracker et;
	struct inpcb *inp;
	int error;

	error = priv_check(req->td, PRIV_NETINET_GETCRED);
	if (error)
	return (error);
	error = SYSCTL_IN(req, addrs, sizeof(addrs));
	if (error)
	return (error);
	NET_EPOCH_ENTER(et);
	inp = in_pcblookup(&V_tcbinfo, addrs[1].sin_addr, addrs[1].sin_port,
	addrs[0].sin_addr, addrs[0].sin_port, INPLOOKUP_RLOCKPCB, NULL);
	NET_EPOCH_EXIT(et);
	if (inp != NULL) {
	if (inp->inp_socket == NULL)
	error = ENOENT;
	if (error == 0)
	error = cr_canseeinpcb(req->td->td_ucred, inp);
	if (error == 0)
	cru2x(inp->inp_cred, &xuc);
	INP_RUNLOCK(inp);
	} else
	error = ENOENT;
	if (error == 0)
	error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, getcred,
	CTLTYPE_OPAQUE \| CTLFLAG_RW \| CTLFLAG_PRISON \| CTLFLAG_NEEDGIANT,
	0, 0, tcp_getcred, "S,xucred",
	"Get the xucred of a TCP connection");
	#endif /* INET */

	#ifdef INET6
	static int
	tcp6_getcred(SYSCTL_HANDLER_ARGS)
	{
	struct epoch_tracker et;
	struct xucred xuc;
	struct sockaddr_in6 addrs[2];
	struct inpcb *inp;
	int error;
	#ifdef INET
	int mapped = 0;
	#endif

	error = priv_check(req->td, PRIV_NETINET_GETCRED);
	if (error)
	return (error);
	error = SYSCTL_IN(req, addrs, sizeof(addrs));
	if (error)
	return (error);
	if ((error = sa6_embedscope(&addrs[0], V_ip6_use_defzone)) != 0 \|\|
	(error = sa6_embedscope(&addrs[1], V_ip6_use_defzone)) != 0) {
	return (error);
	}
	if (IN6_IS_ADDR_V4MAPPED(&addrs[0].sin6_addr)) {
	#ifdef INET
	if (IN6_IS_ADDR_V4MAPPED(&addrs[1].sin6_addr))
	mapped = 1;
	else
	#endif
	return (EINVAL);
	}

	NET_EPOCH_ENTER(et);
	#ifdef INET
	if (mapped == 1)
	inp = in_pcblookup(&V_tcbinfo,
	(struct in_addr )&addrs[1].sin6_addr.s6_addr[12],
	addrs[1].sin6_port,
	(struct in_addr )&addrs[0].sin6_addr.s6_addr[12],
	addrs[0].sin6_port, INPLOOKUP_RLOCKPCB, NULL);
	else
	#endif
	inp = in6_pcblookup(&V_tcbinfo,
	&addrs[1].sin6_addr, addrs[1].sin6_port,
	&addrs[0].sin6_addr, addrs[0].sin6_port,
	INPLOOKUP_RLOCKPCB, NULL);
	NET_EPOCH_EXIT(et);
	if (inp != NULL) {
	if (inp->inp_socket == NULL)
	error = ENOENT;
	if (error == 0)
	error = cr_canseeinpcb(req->td->td_ucred, inp);
	if (error == 0)
	cru2x(inp->inp_cred, &xuc);
	INP_RUNLOCK(inp);
	} else
	error = ENOENT;
	if (error == 0)
	error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
	return (error);
	}

	SYSCTL_PROC(_net_inet6_tcp6, OID_AUTO, getcred,
	CTLTYPE_OPAQUE \| CTLFLAG_RW \| CTLFLAG_PRISON \| CTLFLAG_NEEDGIANT,
	0, 0, tcp6_getcred, "S,xucred",
	"Get the xucred of a TCP6 connection");
	#endif /* INET6 */


	#ifdef INET
	void
	tcp_ctlinput(int cmd, struct sockaddr sa, void vip)
	{
	struct ip *ip = vip;
	struct tcphdr *th;
	struct in_addr faddr;
	struct inpcb *inp;
	struct tcpcb *tp;
	struct inpcb (notify)(struct inpcb *, int) = tcp_notify;
	struct icmp *icp;
	struct in_conninfo inc;
	tcp_seq icmp_tcp_seq;
	int mtu;

	faddr = ((struct sockaddr_in *)sa)->sin_addr;
	if (sa->sa_family != AF_INET \|\| faddr.s_addr == INADDR_ANY)
	return;

	if (cmd == PRC_MSGSIZE)
	notify = tcp_mtudisc_notify;
	else if (V_icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB \|\|
	cmd == PRC_UNREACH_PORT \|\| cmd == PRC_UNREACH_PROTOCOL \|\|
	cmd == PRC_TIMXCEED_INTRANS) && ip)
	notify = tcp_drop_syn_sent;

	/*
	* Hostdead is ugly because it goes linearly through all PCBs.
	* XXX: We never get this from ICMP, otherwise it makes an
	* excellent DoS attack on machines with many connections.
	*/
	else if (cmd == PRC_HOSTDEAD)
	ip = NULL;
	else if ((unsigned)cmd >= PRC_NCMDS \|\| inetctlerrmap[cmd] == 0)
	return;

	if (ip == NULL) {
	in_pcbnotifyall(&V_tcbinfo, faddr, inetctlerrmap[cmd], notify);
	return;
	}

	icp = (struct icmp *)((caddr_t)ip - offsetof(struct icmp, icmp_ip));
	th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
	inp = in_pcblookup(&V_tcbinfo, faddr, th->th_dport, ip->ip_src,
	th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
	if (inp != NULL && PRC_IS_REDIRECT(cmd)) {
	/* signal EHOSTDOWN, as it flushes the cached route */
	inp = (*notify)(inp, EHOSTDOWN);
	goto out;
	}
	icmp_tcp_seq = th->th_seq;
	if (inp != NULL) {
	if (!(inp->inp_flags & INP_TIMEWAIT) &&
	!(inp->inp_flags & INP_DROPPED) &&
	!(inp->inp_socket == NULL)) {
	tp = intotcpcb(inp);
	if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
	SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
	if (cmd == PRC_MSGSIZE) {
	/*
	* MTU discovery:
	* If we got a needfrag set the MTU
	* in the route to the suggested new
	* value (if given) and then notify.
	*/
	mtu = ntohs(icp->icmp_nextmtu);
	/*
	* If no alternative MTU was
	* proposed, try the next smaller
	* one.
	*/
	if (!mtu)
	mtu = ip_next_mtu(
	ntohs(ip->ip_len), 1);
	if (mtu < V_tcp_minmss +
	sizeof(struct tcpiphdr))
	mtu = V_tcp_minmss +
	sizeof(struct tcpiphdr);
	/*
	* Only process the offered MTU if it
	* is smaller than the current one.
	*/
	if (mtu < tp->t_maxseg +
	sizeof(struct tcpiphdr)) {
	bzero(&inc, sizeof(inc));
	inc.inc_faddr = faddr;
	inc.inc_fibnum =
	inp->inp_inc.inc_fibnum;
	tcp_hc_updatemtu(&inc, mtu);
	tcp_mtudisc(inp, mtu);
	}
	} else
	inp = (*notify)(inp,
	inetctlerrmap[cmd]);
	}
	}
	} else {
	bzero(&inc, sizeof(inc));
	inc.inc_fport = th->th_dport;
	inc.inc_lport = th->th_sport;
	inc.inc_faddr = faddr;
	inc.inc_laddr = ip->ip_src;
	syncache_unreach(&inc, icmp_tcp_seq);
	}
	out:
	if (inp != NULL)
	INP_WUNLOCK(inp);
	}
	#endif /* INET */

	#ifdef INET6
	void
	tcp6_ctlinput(int cmd, struct sockaddr sa, void d)
	{
	struct in6_addr *dst;
	struct inpcb (notify)(struct inpcb *, int) = tcp_notify;
	struct ip6_hdr *ip6;
	struct mbuf *m;
	struct inpcb *inp;
	struct tcpcb *tp;
	struct icmp6_hdr *icmp6;
	struct ip6ctlparam *ip6cp = NULL;
	const struct sockaddr_in6 *sa6_src = NULL;
	struct in_conninfo inc;
	struct tcp_ports {
	uint16_t th_sport;
	uint16_t th_dport;
	} t_ports;
	tcp_seq icmp_tcp_seq;
	unsigned int mtu;
	unsigned int off;

	if (sa->sa_family != AF_INET6 \|\|
	sa->sa_len != sizeof(struct sockaddr_in6))
	return;

	/* if the parameter is from icmp6, decode it. */
	if (d != NULL) {
	ip6cp = (struct ip6ctlparam *)d;
	icmp6 = ip6cp->ip6c_icmp6;
	m = ip6cp->ip6c_m;
	ip6 = ip6cp->ip6c_ip6;
	off = ip6cp->ip6c_off;
	sa6_src = ip6cp->ip6c_src;
	dst = ip6cp->ip6c_finaldst;
	} else {
	m = NULL;
	ip6 = NULL;
	off = 0; /* fool gcc */
	sa6_src = &sa6_any;
	dst = NULL;
	}

	if (cmd == PRC_MSGSIZE)
	notify = tcp_mtudisc_notify;
	else if (V_icmp_may_rst && (cmd == PRC_UNREACH_ADMIN_PROHIB \|\|
	cmd == PRC_UNREACH_PORT \|\| cmd == PRC_UNREACH_PROTOCOL \|\|
	cmd == PRC_TIMXCEED_INTRANS) && ip6 != NULL)
	notify = tcp_drop_syn_sent;

	/*
	* Hostdead is ugly because it goes linearly through all PCBs.
	* XXX: We never get this from ICMP, otherwise it makes an
	* excellent DoS attack on machines with many connections.
	*/
	else if (cmd == PRC_HOSTDEAD)
	ip6 = NULL;
	else if ((unsigned)cmd >= PRC_NCMDS \|\| inet6ctlerrmap[cmd] == 0)
	return;

	if (ip6 == NULL) {
	in6_pcbnotify(&V_tcbinfo, sa, 0,
	(const struct sockaddr *)sa6_src,
	0, cmd, NULL, notify);
	return;
	}

	/* Check if we can safely get the ports from the tcp hdr */
	if (m == NULL \|\|
	(m->m_pkthdr.len <
	(int32_t) (off + sizeof(struct tcp_ports)))) {
	return;
	}
	bzero(&t_ports, sizeof(struct tcp_ports));
	m_copydata(m, off, sizeof(struct tcp_ports), (caddr_t)&t_ports);
	inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_dst, t_ports.th_dport,
	&ip6->ip6_src, t_ports.th_sport, INPLOOKUP_WLOCKPCB, NULL);
	if (inp != NULL && PRC_IS_REDIRECT(cmd)) {
	/* signal EHOSTDOWN, as it flushes the cached route */
	inp = (*notify)(inp, EHOSTDOWN);
	goto out;
	}
	off += sizeof(struct tcp_ports);
	if (m->m_pkthdr.len < (int32_t) (off + sizeof(tcp_seq))) {
	goto out;
	}
	m_copydata(m, off, sizeof(tcp_seq), (caddr_t)&icmp_tcp_seq);
	if (inp != NULL) {
	if (!(inp->inp_flags & INP_TIMEWAIT) &&
	!(inp->inp_flags & INP_DROPPED) &&
	!(inp->inp_socket == NULL)) {
	tp = intotcpcb(inp);
	if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
	SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
	if (cmd == PRC_MSGSIZE) {
	/*
	* MTU discovery:
	* If we got a needfrag set the MTU
	* in the route to the suggested new
	* value (if given) and then notify.
	*/
	mtu = ntohl(icmp6->icmp6_mtu);
	/*
	* If no alternative MTU was
	* proposed, or the proposed
	* MTU was too small, set to
	* the min.
	*/
	if (mtu < IPV6_MMTU)
	mtu = IPV6_MMTU - 8;
	bzero(&inc, sizeof(inc));
	inc.inc_fibnum = M_GETFIB(m);
	inc.inc_flags \|= INC_ISIPV6;
	inc.inc6_faddr = *dst;
	if (in6_setscope(&inc.inc6_faddr,
	m->m_pkthdr.rcvif, NULL))
	goto out;
	/*
	* Only process the offered MTU if it
	* is smaller than the current one.
	*/
	if (mtu < tp->t_maxseg +
	sizeof (struct tcphdr) +
	sizeof (struct ip6_hdr)) {
	tcp_hc_updatemtu(&inc, mtu);
	tcp_mtudisc(inp, mtu);
	ICMP6STAT_INC(icp6s_pmtuchg);
	}
	} else
	inp = (*notify)(inp,
	inet6ctlerrmap[cmd]);
	}
	}
	} else {
	bzero(&inc, sizeof(inc));
	inc.inc_fibnum = M_GETFIB(m);
	inc.inc_flags \|= INC_ISIPV6;
	inc.inc_fport = t_ports.th_dport;
	inc.inc_lport = t_ports.th_sport;
	inc.inc6_faddr = *dst;
	inc.inc6_laddr = ip6->ip6_src;
	syncache_unreach(&inc, icmp_tcp_seq);
	}
	out:
	if (inp != NULL)
	INP_WUNLOCK(inp);
	}
	#endif /* INET6 */

	static uint32_t
	tcp_keyed_hash(struct in_conninfo inc, u_char key, u_int len)
	{
	SIPHASH_CTX ctx;
	uint32_t hash[2];

	KASSERT(len >= SIPHASH_KEY_LENGTH,
	("%s: keylen %u too short ", __func__, len));
	SipHash24_Init(&ctx);
	SipHash_SetKey(&ctx, (uint8_t *)key);
	SipHash_Update(&ctx, &inc->inc_fport, sizeof(uint16_t));
	SipHash_Update(&ctx, &inc->inc_lport, sizeof(uint16_t));
	switch (inc->inc_flags & INC_ISIPV6) {
	#ifdef INET
	case 0:
	SipHash_Update(&ctx, &inc->inc_faddr, sizeof(struct in_addr));
	SipHash_Update(&ctx, &inc->inc_laddr, sizeof(struct in_addr));
	break;
	#endif
	#ifdef INET6
	case INC_ISIPV6:
	SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(struct in6_addr));
	SipHash_Update(&ctx, &inc->inc6_laddr, sizeof(struct in6_addr));
	break;
	#endif
	}
	SipHash_Final((uint8_t *)hash, &ctx);

	return (hash[0] ^ hash[1]);
	}

	uint32_t
	tcp_new_ts_offset(struct in_conninfo *inc)
	{
	struct in_conninfo inc_store, *local_inc;

	if (!V_tcp_ts_offset_per_conn) {
	memcpy(&inc_store, inc, sizeof(struct in_conninfo));
	inc_store.inc_lport = 0;
	inc_store.inc_fport = 0;
	local_inc = &inc_store;
	} else {
	local_inc = inc;
	}
	return (tcp_keyed_hash(local_inc, V_ts_offset_secret,
	sizeof(V_ts_offset_secret)));
	}

	/*
	* Following is where TCP initial sequence number generation occurs.
	*
	* There are two places where we must use initial sequence numbers:
	* 1. In SYN-ACK packets.
	* 2. In SYN packets.
	*
	* All ISNs for SYN-ACK packets are generated by the syncache. See
	* tcp_syncache.c for details.
	*
	* The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
	* depends on this property. In addition, these ISNs should be
	* unguessable so as to prevent connection hijacking. To satisfy
	* the requirements of this situation, the algorithm outlined in
	* RFC 1948 is used, with only small modifications.
	*
	* Implementation details:
	*
	* Time is based off the system timer, and is corrected so that it
	* increases by one megabyte per second. This allows for proper
	* recycling on high speed LANs while still leaving over an hour
	* before rollover.
	*
	* As reading the exact system time is too expensive to be done
	* whenever setting up a TCP connection, we increment the time
	* offset in two ways. First, a small random positive increment
	* is added to isn_offset for each connection that is set up.
	* Second, the function tcp_isn_tick fires once per clock tick
	* and increments isn_offset as necessary so that sequence numbers
	* are incremented at approximately ISN_BYTES_PER_SECOND. The
	* random positive increments serve only to ensure that the same
	* exact sequence number is never sent out twice (as could otherwise
	* happen when a port is recycled in less than the system tick
	* interval.)
	*
	* net.inet.tcp.isn_reseed_interval controls the number of seconds
	* between seeding of isn_secret. This is normally set to zero,
	* as reseeding should not be necessary.
	*
	* Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
	* isn_offset_old, and isn_ctx is performed using the ISN lock. In
	* general, this means holding an exclusive (write) lock.
	*/

	#define ISN_BYTES_PER_SECOND 1048576
	#define ISN_STATIC_INCREMENT 4096
	#define ISN_RANDOM_INCREMENT (4096 - 1)
	#define ISN_SECRET_LENGTH SIPHASH_KEY_LENGTH

	VNET_DEFINE_STATIC(u_char, isn_secret[ISN_SECRET_LENGTH]);
	VNET_DEFINE_STATIC(int, isn_last);
	VNET_DEFINE_STATIC(int, isn_last_reseed);
	VNET_DEFINE_STATIC(u_int32_t, isn_offset);
	VNET_DEFINE_STATIC(u_int32_t, isn_offset_old);

	#define V_isn_secret VNET(isn_secret)
	#define V_isn_last VNET(isn_last)
	#define V_isn_last_reseed VNET(isn_last_reseed)
	#define V_isn_offset VNET(isn_offset)
	#define V_isn_offset_old VNET(isn_offset_old)

	tcp_seq
	tcp_new_isn(struct in_conninfo *inc)
	{
	tcp_seq new_isn;
	u_int32_t projected_offset;

	ISN_LOCK();
	/* Seed if this is the first use, reseed if requested. */
	if ((V_isn_last_reseed == 0) \|\| ((V_tcp_isn_reseed_interval > 0) &&
	(((u_int)V_isn_last_reseed + (u_int)V_tcp_isn_reseed_interval*hz)
	< (u_int)ticks))) {
	arc4rand(&V_isn_secret, sizeof(V_isn_secret), 0);
	V_isn_last_reseed = ticks;
	}

	/* Compute the hash and return the ISN. */
	new_isn = (tcp_seq)tcp_keyed_hash(inc, V_isn_secret,
	sizeof(V_isn_secret));
	V_isn_offset += ISN_STATIC_INCREMENT +
	(arc4random() & ISN_RANDOM_INCREMENT);
	if (ticks != V_isn_last) {
	projected_offset = V_isn_offset_old +
	ISN_BYTES_PER_SECOND / hz * (ticks - V_isn_last);
	if (SEQ_GT(projected_offset, V_isn_offset))
	V_isn_offset = projected_offset;
	V_isn_offset_old = V_isn_offset;
	V_isn_last = ticks;
	}
	new_isn += V_isn_offset;
	ISN_UNLOCK();
	return (new_isn);
	}

	/*
	* When a specific ICMP unreachable message is received and the
	* connection state is SYN-SENT, drop the connection. This behavior
	* is controlled by the icmp_may_rst sysctl.
	*/
	struct inpcb *
	tcp_drop_syn_sent(struct inpcb *inp, int errno)
	{
	struct tcpcb *tp;

	NET_EPOCH_ASSERT();
	INP_WLOCK_ASSERT(inp);

	if ((inp->inp_flags & INP_TIMEWAIT) \|\|
	(inp->inp_flags & INP_DROPPED))
	return (inp);

	tp = intotcpcb(inp);
	if (tp->t_state != TCPS_SYN_SENT)
	return (inp);

	if (IS_FASTOPEN(tp->t_flags))
	tcp_fastopen_disable_path(tp);

	tp = tcp_drop(tp, errno);
	if (tp != NULL)
	return (inp);
	else
	return (NULL);
	}

	/*
	* When `need fragmentation' ICMP is received, update our idea of the MSS
	* based on the new value. Also nudge TCP to send something, since we
	* know the packet we just sent was dropped.
	* This duplicates some code in the tcp_mss() function in tcp_input.c.
	*/
	static struct inpcb *
	tcp_mtudisc_notify(struct inpcb *inp, int error)
	{

	tcp_mtudisc(inp, -1);
	return (inp);
	}

	static void
	tcp_mtudisc(struct inpcb *inp, int mtuoffer)
	{
	struct tcpcb *tp;
	struct socket *so;

	INP_WLOCK_ASSERT(inp);
	if ((inp->inp_flags & INP_TIMEWAIT) \|\|
	(inp->inp_flags & INP_DROPPED))
	return;

	tp = intotcpcb(inp);
	KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));

	tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);

	so = inp->inp_socket;
	SOCKBUF_LOCK(&so->so_snd);
	/* If the mss is larger than the socket buffer, decrease the mss. */
	if (so->so_snd.sb_hiwat < tp->t_maxseg)
	tp->t_maxseg = so->so_snd.sb_hiwat;
	SOCKBUF_UNLOCK(&so->so_snd);

	TCPSTAT_INC(tcps_mturesent);
	tp->t_rtttime = 0;
	tp->snd_nxt = tp->snd_una;
	tcp_free_sackholes(tp);
	tp->snd_recover = tp->snd_max;
	if (tp->t_flags & TF_SACK_PERMIT)
	EXIT_FASTRECOVERY(tp->t_flags);
	tp->t_fb->tfb_tcp_output(tp);
	}

	#ifdef INET
	/*
	* Look-up the routing entry to the peer of this inpcb. If no route
	* is found and it cannot be allocated, then return 0. This routine
	* is called by TCP routines that access the rmx structure and by
	* tcp_mss_update to get the peer/interface MTU.
	*/
	uint32_t
	tcp_maxmtu(struct in_conninfo inc, struct tcp_ifcap cap)
	{
	struct nhop_object *nh;
	struct ifnet *ifp;
	uint32_t maxmtu = 0;

	KASSERT(inc != NULL, ("tcp_maxmtu with NULL in_conninfo pointer"));

	if (inc->inc_faddr.s_addr != INADDR_ANY) {

	nh = fib4_lookup(inc->inc_fibnum, inc->inc_faddr, 0, NHR_NONE, 0);
	if (nh == NULL)
	return (0);

	ifp = nh->nh_ifp;
	maxmtu = nh->nh_mtu;

	/* Report additional interface capabilities. */
	if (cap != NULL) {
	if (ifp->if_capenable & IFCAP_TSO4 &&
	ifp->if_hwassist & CSUM_TSO) {
	cap->ifcap \|= CSUM_TSO;
	cap->tsomax = ifp->if_hw_tsomax;
	cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
	cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
	}
	}
	}
	return (maxmtu);
	}
	#endif /* INET */

	#ifdef INET6
	uint32_t
	tcp_maxmtu6(struct in_conninfo inc, struct tcp_ifcap cap)
	{
	struct nhop_object *nh;
	struct in6_addr dst6;
	uint32_t scopeid;
	struct ifnet *ifp;
	uint32_t maxmtu = 0;

	KASSERT(inc != NULL, ("tcp_maxmtu6 with NULL in_conninfo pointer"));

	if (inc->inc_flags & INC_IPV6MINMTU)
	return (IPV6_MMTU);

	if (!IN6_IS_ADDR_UNSPECIFIED(&inc->inc6_faddr)) {
	in6_splitscope(&inc->inc6_faddr, &dst6, &scopeid);
	nh = fib6_lookup(inc->inc_fibnum, &dst6, scopeid, NHR_NONE, 0);
	if (nh == NULL)
	return (0);

	ifp = nh->nh_ifp;
	maxmtu = nh->nh_mtu;

	/* Report additional interface capabilities. */
	if (cap != NULL) {
	if (ifp->if_capenable & IFCAP_TSO6 &&
	ifp->if_hwassist & CSUM_TSO) {
	cap->ifcap \|= CSUM_TSO;
	cap->tsomax = ifp->if_hw_tsomax;
	cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
	cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
	}
	}
	}

	return (maxmtu);
	}
	#endif /* INET6 */

	/*
	* Calculate effective SMSS per RFC5681 definition for a given TCP
	* connection at its current state, taking into account SACK and etc.
	*/
	u_int
	tcp_maxseg(const struct tcpcb *tp)
	{
	u_int optlen;

	if (tp->t_flags & TF_NOOPT)
	return (tp->t_maxseg);

	/*
	* Here we have a simplified code from tcp_addoptions(),
	* without a proper loop, and having most of paddings hardcoded.
	* We might make mistakes with padding here in some edge cases,
	* but this is harmless, since result of tcp_maxseg() is used
	* only in cwnd and ssthresh estimations.
	*/
	#define PAD(len) ((((len) / 4) + !!((len) % 4)) * 4)
	if (TCPS_HAVEESTABLISHED(tp->t_state)) {
	if (tp->t_flags & TF_RCVD_TSTMP)
	optlen = TCPOLEN_TSTAMP_APPA;
	else
	optlen = 0;
	#if defined(IPSEC_SUPPORT) \|\| defined(TCP_SIGNATURE)
	if (tp->t_flags & TF_SIGNATURE)
	optlen += PAD(TCPOLEN_SIGNATURE);
	#endif
	if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0) {
	optlen += TCPOLEN_SACKHDR;
	optlen += tp->rcv_numsacks * TCPOLEN_SACK;
	optlen = PAD(optlen);
	}
	} else {
	if (tp->t_flags & TF_REQ_TSTMP)
	optlen = TCPOLEN_TSTAMP_APPA;
	else
	optlen = PAD(TCPOLEN_MAXSEG);
	if (tp->t_flags & TF_REQ_SCALE)
	optlen += PAD(TCPOLEN_WINDOW);
	#if defined(IPSEC_SUPPORT) \|\| defined(TCP_SIGNATURE)
	if (tp->t_flags & TF_SIGNATURE)
	optlen += PAD(TCPOLEN_SIGNATURE);
	#endif
	if (tp->t_flags & TF_SACK_PERMIT)
	optlen += PAD(TCPOLEN_SACK_PERMITTED);
	}
	#undef PAD
	optlen = min(optlen, TCP_MAXOLEN);
	return (tp->t_maxseg - optlen);
	}

	static int
	sysctl_drop(SYSCTL_HANDLER_ARGS)
	{
	/* addrs[0] is a foreign socket, addrs[1] is a local one. */
	struct sockaddr_storage addrs[2];
	struct inpcb *inp;
	struct tcpcb *tp;
	struct tcptw *tw;
	struct sockaddr_in fin, lin;
	struct epoch_tracker et;
	#ifdef INET6
	struct sockaddr_in6 fin6, lin6;
	#endif
	int error;

	inp = NULL;
	fin = lin = NULL;
	#ifdef INET6
	fin6 = lin6 = NULL;
	#endif
	error = 0;

	if (req->oldptr != NULL \|\| req->oldlen != 0)
	return (EINVAL);
	if (req->newptr == NULL)
	return (EPERM);
	if (req->newlen < sizeof(addrs))
	return (ENOMEM);
	error = SYSCTL_IN(req, &addrs, sizeof(addrs));
	if (error)
	return (error);

	switch (addrs[0].ss_family) {
	#ifdef INET6
	case AF_INET6:
	fin6 = (struct sockaddr_in6 *)&addrs[0];
	lin6 = (struct sockaddr_in6 *)&addrs[1];
	if (fin6->sin6_len != sizeof(struct sockaddr_in6) \|\|
	lin6->sin6_len != sizeof(struct sockaddr_in6))
	return (EINVAL);
	if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
	if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
	return (EINVAL);
	in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
	in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
	fin = (struct sockaddr_in *)&addrs[0];
	lin = (struct sockaddr_in *)&addrs[1];
	break;
	}
	error = sa6_embedscope(fin6, V_ip6_use_defzone);
	if (error)
	return (error);
	error = sa6_embedscope(lin6, V_ip6_use_defzone);
	if (error)
	return (error);
	break;
	#endif
	#ifdef INET
	case AF_INET:
	fin = (struct sockaddr_in *)&addrs[0];
	lin = (struct sockaddr_in *)&addrs[1];
	if (fin->sin_len != sizeof(struct sockaddr_in) \|\|
	lin->sin_len != sizeof(struct sockaddr_in))
	return (EINVAL);
	break;
	#endif
	default:
	return (EINVAL);
	}
	NET_EPOCH_ENTER(et);
	switch (addrs[0].ss_family) {
	#ifdef INET6
	case AF_INET6:
	inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
	fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
	INPLOOKUP_WLOCKPCB, NULL);
	break;
	#endif
	#ifdef INET
	case AF_INET:
	inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
	lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
	break;
	#endif
	}
	if (inp != NULL) {
	if (inp->inp_flags & INP_TIMEWAIT) {
	/*
	* XXXRW: There currently exists a state where an
	* inpcb is present, but its timewait state has been
	* discarded. For now, don't allow dropping of this
	* type of inpcb.
	*/
	tw = intotw(inp);
	if (tw != NULL)
	tcp_twclose(tw, 0);
	else
	INP_WUNLOCK(inp);
	} else if (!(inp->inp_flags & INP_DROPPED) &&
	!(inp->inp_socket->so_options & SO_ACCEPTCONN)) {
	tp = intotcpcb(inp);
	tp = tcp_drop(tp, ECONNABORTED);
	if (tp != NULL)
	INP_WUNLOCK(inp);
	} else
	INP_WUNLOCK(inp);
	} else
	error = ESRCH;
	NET_EPOCH_EXIT(et);
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, TCPCTL_DROP, drop,
	CTLFLAG_VNET \| CTLTYPE_STRUCT \| CTLFLAG_WR \| CTLFLAG_SKIP \|
	CTLFLAG_NEEDGIANT, NULL, 0, sysctl_drop, "",
	"Drop TCP connection");

	#ifdef KERN_TLS
	static int
	sysctl_switch_tls(SYSCTL_HANDLER_ARGS)
	{
	/* addrs[0] is a foreign socket, addrs[1] is a local one. */
	struct sockaddr_storage addrs[2];
	struct inpcb *inp;
	struct sockaddr_in fin, lin;
	struct epoch_tracker et;
	#ifdef INET6
	struct sockaddr_in6 fin6, lin6;
	#endif
	int error;

	inp = NULL;
	fin = lin = NULL;
	#ifdef INET6
	fin6 = lin6 = NULL;
	#endif
	error = 0;

	if (req->oldptr != NULL \|\| req->oldlen != 0)
	return (EINVAL);
	if (req->newptr == NULL)
	return (EPERM);
	if (req->newlen < sizeof(addrs))
	return (ENOMEM);
	error = SYSCTL_IN(req, &addrs, sizeof(addrs));
	if (error)
	return (error);

	switch (addrs[0].ss_family) {
	#ifdef INET6
	case AF_INET6:
	fin6 = (struct sockaddr_in6 *)&addrs[0];
	lin6 = (struct sockaddr_in6 *)&addrs[1];
	if (fin6->sin6_len != sizeof(struct sockaddr_in6) \|\|
	lin6->sin6_len != sizeof(struct sockaddr_in6))
	return (EINVAL);
	if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
	if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
	return (EINVAL);
	in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
	in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
	fin = (struct sockaddr_in *)&addrs[0];
	lin = (struct sockaddr_in *)&addrs[1];
	break;
	}
	error = sa6_embedscope(fin6, V_ip6_use_defzone);
	if (error)
	return (error);
	error = sa6_embedscope(lin6, V_ip6_use_defzone);
	if (error)
	return (error);
	break;
	#endif
	#ifdef INET
	case AF_INET:
	fin = (struct sockaddr_in *)&addrs[0];
	lin = (struct sockaddr_in *)&addrs[1];
	if (fin->sin_len != sizeof(struct sockaddr_in) \|\|
	lin->sin_len != sizeof(struct sockaddr_in))
	return (EINVAL);
	break;
	#endif
	default:
	return (EINVAL);
	}
	NET_EPOCH_ENTER(et);
	switch (addrs[0].ss_family) {
	#ifdef INET6
	case AF_INET6:
	inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
	fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
	INPLOOKUP_WLOCKPCB, NULL);
	break;
	#endif
	#ifdef INET
	case AF_INET:
	inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
	lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
	break;
	#endif
	}
	NET_EPOCH_EXIT(et);
	if (inp != NULL) {
	if ((inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) != 0 \|\|
	inp->inp_socket == NULL) {
	error = ECONNRESET;
	INP_WUNLOCK(inp);
	} else {
	struct socket *so;

	so = inp->inp_socket;
	soref(so);
	error = ktls_set_tx_mode(so,
	arg2 == 0 ? TCP_TLS_MODE_SW : TCP_TLS_MODE_IFNET);
	INP_WUNLOCK(inp);
	SOCK_LOCK(so);
	sorele(so);
	}
	} else
	error = ESRCH;
	return (error);
	}

	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_sw_tls,
	CTLFLAG_VNET \| CTLTYPE_STRUCT \| CTLFLAG_WR \| CTLFLAG_SKIP \|
	CTLFLAG_NEEDGIANT, NULL, 0, sysctl_switch_tls, "",
	"Switch TCP connection to SW TLS");
	SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_ifnet_tls,
	CTLFLAG_VNET \| CTLTYPE_STRUCT \| CTLFLAG_WR \| CTLFLAG_SKIP \|
	CTLFLAG_NEEDGIANT, NULL, 1, sysctl_switch_tls, "",
	"Switch TCP connection to ifnet TLS");
	#endif

	/*
	* Generate a standardized TCP log line for use throughout the
	* tcp subsystem. Memory allocation is done with M_NOWAIT to
	* allow use in the interrupt context.
	*
	* NB: The caller MUST free(s, M_TCPLOG) the returned string.
	* NB: The function may return NULL if memory allocation failed.
	*
	* Due to header inclusion and ordering limitations the struct ip
	* and ip6_hdr pointers have to be passed as void pointers.
	*/
	char *
	tcp_log_vain(struct in_conninfo inc, struct tcphdr th, void *ip4hdr,
	const void *ip6hdr)
	{

	/* Is logging enabled? */
	if (V_tcp_log_in_vain == 0)
	return (NULL);

	return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
	}

	char *
	tcp_log_addrs(struct in_conninfo inc, struct tcphdr th, void *ip4hdr,
	const void *ip6hdr)
	{

	/* Is logging enabled? */
	if (tcp_log_debug == 0)
	return (NULL);

	return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
	}

	static char *
	tcp_log_addr(struct in_conninfo inc, struct tcphdr th, void *ip4hdr,
	const void *ip6hdr)
	{
	char s, sp;
	size_t size;
	struct ip *ip;
	#ifdef INET6
	const struct ip6_hdr *ip6;

	ip6 = (const struct ip6_hdr *)ip6hdr;
	#endif /* INET6 */
	ip = (struct ip *)ip4hdr;

	/*
	* The log line looks like this:
	* "TCP: [1.2.3.4]:50332 to [1.2.3.4]:80 tcpflags 0x2<SYN>"
	*/
	size = sizeof("TCP: []:12345 to []:12345 tcpflags 0x2<>") +
	sizeof(PRINT_TH_FLAGS) + 1 +
	#ifdef INET6
	2 * INET6_ADDRSTRLEN;
	#else
	2 * INET_ADDRSTRLEN;
	#endif /* INET6 */

	s = malloc(size, M_TCPLOG, M_ZERO\|M_NOWAIT);
	if (s == NULL)
	return (NULL);

	strcat(s, "TCP: [");
	sp = s + strlen(s);

	if (inc && ((inc->inc_flags & INC_ISIPV6) == 0)) {
	inet_ntoa_r(inc->inc_faddr, sp);
	sp = s + strlen(s);
	sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
	sp = s + strlen(s);
	inet_ntoa_r(inc->inc_laddr, sp);
	sp = s + strlen(s);
	sprintf(sp, "]:%i", ntohs(inc->inc_lport));
	#ifdef INET6
	} else if (inc) {
	ip6_sprintf(sp, &inc->inc6_faddr);
	sp = s + strlen(s);
	sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
	sp = s + strlen(s);
	ip6_sprintf(sp, &inc->inc6_laddr);
	sp = s + strlen(s);
	sprintf(sp, "]:%i", ntohs(inc->inc_lport));
	} else if (ip6 && th) {
	ip6_sprintf(sp, &ip6->ip6_src);
	sp = s + strlen(s);
	sprintf(sp, "]:%i to [", ntohs(th->th_sport));
	sp = s + strlen(s);
	ip6_sprintf(sp, &ip6->ip6_dst);
	sp = s + strlen(s);
	sprintf(sp, "]:%i", ntohs(th->th_dport));
	#endif /* INET6 */
	#ifdef INET
	} else if (ip && th) {
	inet_ntoa_r(ip->ip_src, sp);
	sp = s + strlen(s);
	sprintf(sp, "]:%i to [", ntohs(th->th_sport));
	sp = s + strlen(s);
	inet_ntoa_r(ip->ip_dst, sp);
	sp = s + strlen(s);
	sprintf(sp, "]:%i", ntohs(th->th_dport));
	#endif /* INET */
	} else {
	free(s, M_TCPLOG);
	return (NULL);
	}
	sp = s + strlen(s);
	if (th)
	sprintf(sp, " tcpflags 0x%b", th->th_flags, PRINT_TH_FLAGS);
	if (*(s + size - 1) != '\0')
	panic("%s: string too long", __func__);
	return (s);
	}

	/*
	* A subroutine which makes it easy to track TCP state changes with DTrace.
	* This function shouldn't be called for t_state initializations that don't
	* correspond to actual TCP state transitions.
	*/
	void
	tcp_state_change(struct tcpcb *tp, int newstate)
	{
	#if defined(KDTRACE_HOOKS)
	int pstate = tp->t_state;
	#endif

	TCPSTATES_DEC(tp->t_state);
	TCPSTATES_INC(newstate);
	tp->t_state = newstate;
	TCP_PROBE6(state__change, NULL, tp, NULL, tp, NULL, pstate);
	}

	/*
	* Create an external-format (``xtcpcb'') structure using the information in
	* the kernel-format tcpcb structure pointed to by tp. This is done to
	* reduce the spew of irrelevant information over this interface, to isolate
	* user code from changes in the kernel structure, and potentially to provide
	* information-hiding if we decide that some of this information should be
	* hidden from users.
	*/
	void
	tcp_inptoxtp(const struct inpcb inp, struct xtcpcb xt)
	{
	struct tcpcb *tp = intotcpcb(inp);
	sbintime_t now;

	bzero(xt, sizeof(*xt));
	if (inp->inp_flags & INP_TIMEWAIT) {
	xt->t_state = TCPS_TIME_WAIT;
	} else {
	xt->t_state = tp->t_state;
	xt->t_logstate = tp->t_logstate;
	xt->t_flags = tp->t_flags;
	xt->t_sndzerowin = tp->t_sndzerowin;
	xt->t_sndrexmitpack = tp->t_sndrexmitpack;
	xt->t_rcvoopack = tp->t_rcvoopack;

	now = getsbinuptime();
	#define COPYTIMER(ttt) do { \
	if (callout_active(&tp->t_timers->ttt)) \
	xt->ttt = (tp->t_timers->ttt.c_time - now) / \
	SBT_1MS; \
	else \
	xt->ttt = 0; \
	} while (0)
	COPYTIMER(tt_delack);
	COPYTIMER(tt_rexmt);
	COPYTIMER(tt_persist);
	COPYTIMER(tt_keep);
	COPYTIMER(tt_2msl);
	#undef COPYTIMER
	xt->t_rcvtime = 1000 * (ticks - tp->t_rcvtime) / hz;

	bcopy(tp->t_fb->tfb_tcp_block_name, xt->xt_stack,
	TCP_FUNCTION_NAME_LEN_MAX);
	#ifdef TCP_BLACKBOX
	(void)tcp_log_get_id(tp, xt->xt_logid);
	#endif
	}

	xt->xt_len = sizeof(struct xtcpcb);
	in_pcbtoxinpcb(inp, &xt->xt_inp);
	if (inp->inp_socket == NULL)
	xt->xt_inp.xi_socket.xso_protocol = IPPROTO_TCP;
	}

	void
	tcp_log_end_status(struct tcpcb *tp, uint8_t status)
	{
	uint32_t bit, i;

	if ((tp == NULL) \|\|
	(status > TCP_EI_STATUS_MAX_VALUE) \|\|
	(status == 0)) {
	/* Invalid */
	return;
	}
	if (status > (sizeof(uint32_t) * 8)) {
	/* Should this be a KASSERT? */
	return;
	}
	bit = 1U << (status - 1);
	if (bit & tp->t_end_info_status) {
	/* already logged */
	return;
	}
	for (i = 0; i < TCP_END_BYTE_INFO; i++) {
	if (tp->t_end_info_bytes[i] == TCP_EI_EMPTY_SLOT) {
	tp->t_end_info_bytes[i] = status;
	tp->t_end_info_status \|= bit;
	break;
	}
	}
	}
	diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
	index 941af58d367a..975a000f1c81 100644
	--- a/sys/netinet6/ip6_output.c
	+++ b/sys/netinet6/ip6_output.c
	@@ -1,3335 +1,3339 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the project nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $KAME: ip6_output.c,v 1.279 2002/01/26 06:12:30 jinmei Exp $
	*/

	/*-
	* Copyright (c) 1982, 1986, 1988, 1990, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)ip_output.c 8.3 (Berkeley) 1/21/94
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_ipsec.h"
	#include "opt_kern_tls.h"
	#include "opt_ratelimit.h"
	#include "opt_route.h"
	#include "opt_rss.h"
	#include "opt_sctp.h"

	#include <sys/param.h>
	#include <sys/kernel.h>
	#include <sys/ktls.h>
	#include <sys/malloc.h>
	#include <sys/mbuf.h>
	#include <sys/errno.h>
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/protosw.h>
	#include <sys/socket.h>
	#include <sys/socketvar.h>
	#include <sys/syslog.h>
	#include <sys/ucred.h>

	#include <machine/in_cksum.h>

	#include <net/if.h>
	#include <net/if_var.h>
	#include <net/if_llatbl.h>
	#include <net/netisr.h>
	#include <net/route.h>
	#include <net/route/nhop.h>
	#include <net/pfil.h>
	#include <net/rss_config.h>
	#include <net/vnet.h>

	#include <netinet/in.h>
	#include <netinet/in_var.h>
	#include <netinet/ip_var.h>
	#include <netinet6/in6_fib.h>
	#include <netinet6/in6_var.h>
	#include <netinet/ip6.h>
	#include <netinet/icmp6.h>
	#include <netinet6/ip6_var.h>
	#include <netinet/in_pcb.h>
	#include <netinet/tcp_var.h>
	#include <netinet6/nd6.h>
	#include <netinet6/in6_rss.h>

	#include <netipsec/ipsec_support.h>
	#if defined(SCTP) \|\| defined(SCTP_SUPPORT)
	#include <netinet/sctp.h>
	#include <netinet/sctp_crc32.h>
	#endif

	#include <netinet6/ip6protosw.h>
	#include <netinet6/scope6_var.h>

	extern int in6_mcast_loop;

	struct ip6_exthdrs {
	struct mbuf *ip6e_ip6;
	struct mbuf *ip6e_hbh;
	struct mbuf *ip6e_dest1;
	struct mbuf *ip6e_rthdr;
	struct mbuf *ip6e_dest2;
	};

	static MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options");

	static int ip6_pcbopt(int, u_char , int, struct ip6_pktopts *,
	struct ucred *, int);
	static int ip6_pcbopts(struct ip6_pktopts *, struct mbuf ,
	struct socket , struct sockopt );
	static int ip6_getpcbopt(struct inpcb , int, struct sockopt );
	static int ip6_setpktopt(int, u_char , int, struct ip6_pktopts ,
	struct ucred *, int, int, int);

	static int ip6_copyexthdr(struct mbuf **, caddr_t, int);
	static int ip6_insertfraghdr(struct mbuf , struct mbuf , int,
	struct ip6_frag **);
	static int ip6_insert_jumboopt(struct ip6_exthdrs *, u_int32_t);
	static int ip6_splithdr(struct mbuf , struct ip6_exthdrs );
	static int ip6_getpmtu(struct route_in6 *, int,
	struct ifnet , const struct in6_addr , u_long , int , u_int,
	u_int);
	static int ip6_calcmtu(struct ifnet , const struct in6_addr , u_long,
	u_long , int , u_int);
	static int ip6_getpmtu_ctl(u_int, const struct in6_addr , u_long );
	static int copypktopts(struct ip6_pktopts , struct ip6_pktopts , int);


	/*
	* Make an extension header from option data. hp is the source,
	* mp is the destination, and _ol is the optlen.
	*/
	#define MAKE_EXTHDR(hp, mp, _ol) \
	do { \
	if (hp) { \
	struct ip6_ext eh = (struct ip6_ext )(hp); \
	error = ip6_copyexthdr((mp), (caddr_t)(hp), \
	((eh)->ip6e_len + 1) << 3); \
	if (error) \
	goto freehdrs; \
	(_ol) += (*(mp))->m_len; \
	} \
	} while (/CONSTCOND/ 0)

	/*
	* Form a chain of extension headers.
	* m is the extension header mbuf
	* mp is the previous mbuf in the chain
	* p is the next header
	* i is the type of option.
	*/
	#define MAKE_CHAIN(m, mp, p, i)\
	do {\
	if (m) {\
	if (!hdrsplit) \
	panic("%s:%d: assumption failed: "\
	"hdr not split: hdrsplit %d exthdrs %p",\
	__func__, __LINE__, hdrsplit, &exthdrs);\
	mtod((m), u_char ) = *(p);\
	*(p) = (i);\
	p = mtod((m), u_char *);\
	(m)->m_next = (mp)->m_next;\
	(mp)->m_next = (m);\
	(mp) = (m);\
	}\
	} while (/CONSTCOND/ 0)

	void
	in6_delayed_cksum(struct mbuf *m, uint32_t plen, u_short offset)
	{
	u_short csum;

	csum = in_cksum_skip(m, offset + plen, offset);
	if (m->m_pkthdr.csum_flags & CSUM_UDP_IPV6 && csum == 0)
	csum = 0xffff;
	offset += m->m_pkthdr.csum_data; /* checksum offset */

	if (offset + sizeof(csum) > m->m_len)
	m_copyback(m, offset, sizeof(csum), (caddr_t)&csum);
	else
	(u_short )mtodo(m, offset) = csum;
	}

	static int
	ip6_output_delayed_csum(struct mbuf m, struct ifnet ifp, int csum_flags,
	int plen, int optlen, bool frag)
	{

	KASSERT((plen >= optlen), ("%s:%d: plen %d < optlen %d, m %p, ifp %p "
	"csum_flags %#x frag %d\n",
	__func__, __LINE__, plen, optlen, m, ifp, csum_flags, frag));

	if ((csum_flags & CSUM_DELAY_DATA_IPV6) \|\|
	#if defined(SCTP) \|\| defined(SCTP_SUPPORT)
	(csum_flags & CSUM_SCTP_IPV6) \|\|
	#endif
	(!frag && (ifp->if_capenable & IFCAP_NOMAP) == 0)) {
	m = mb_unmapped_to_ext(m);
	if (m == NULL) {
	if (frag)
	in6_ifstat_inc(ifp, ifs6_out_fragfail);
	else
	IP6STAT_INC(ip6s_odropped);
	return (ENOBUFS);
	}
	if (csum_flags & CSUM_DELAY_DATA_IPV6) {
	in6_delayed_cksum(m, plen - optlen,
	sizeof(struct ip6_hdr) + optlen);
	m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA_IPV6;
	}
	#if defined(SCTP) \|\| defined(SCTP_SUPPORT)
	if (csum_flags & CSUM_SCTP_IPV6) {
	sctp_delayed_cksum(m, sizeof(struct ip6_hdr) + optlen);
	m->m_pkthdr.csum_flags &= ~CSUM_SCTP_IPV6;
	}
	#endif
	}

	return (0);
	}

	int
	ip6_fragment(struct ifnet ifp, struct mbuf m0, int hlen, u_char nextproto,
	int fraglen , uint32_t id)
	{
	struct mbuf m, mnext, m_frgpart;
	struct ip6_hdr ip6, mhip6;
	struct ip6_frag *ip6f;
	int off;
	int error;
	int tlen = m0->m_pkthdr.len;

	KASSERT((fraglen % 8 == 0), ("Fragment length must be a multiple of 8"));

	m = m0;
	ip6 = mtod(m, struct ip6_hdr *);
	mnext = &m->m_nextpkt;

	for (off = hlen; off < tlen; off += fraglen) {
	m = m_gethdr(M_NOWAIT, MT_DATA);
	if (!m) {
	IP6STAT_INC(ip6s_odropped);
	return (ENOBUFS);
	}

	/*
	* Make sure the complete packet header gets copied
	* from the originating mbuf to the newly created
	* mbuf. This also ensures that existing firewall
	* classification(s), VLAN tags and so on get copied
	* to the resulting fragmented packet(s):
	*/
	if (m_dup_pkthdr(m, m0, M_NOWAIT) == 0) {
	m_free(m);
	IP6STAT_INC(ip6s_odropped);
	return (ENOBUFS);
	}

	*mnext = m;
	mnext = &m->m_nextpkt;
	m->m_data += max_linkhdr;
	mhip6 = mtod(m, struct ip6_hdr *);
	mhip6 = ip6;
	m->m_len = sizeof(*mhip6);
	error = ip6_insertfraghdr(m0, m, hlen, &ip6f);
	if (error) {
	IP6STAT_INC(ip6s_odropped);
	return (error);
	}
	ip6f->ip6f_offlg = htons((u_short)((off - hlen) & ~7));
	if (off + fraglen >= tlen)
	fraglen = tlen - off;
	else
	ip6f->ip6f_offlg \|= IP6F_MORE_FRAG;
	mhip6->ip6_plen = htons((u_short)(fraglen + hlen +
	sizeof(*ip6f) - sizeof(struct ip6_hdr)));
	if ((m_frgpart = m_copym(m0, off, fraglen, M_NOWAIT)) == NULL) {
	IP6STAT_INC(ip6s_odropped);
	return (ENOBUFS);
	}
	m_cat(m, m_frgpart);
	m->m_pkthdr.len = fraglen + hlen + sizeof(*ip6f);
	ip6f->ip6f_reserved = 0;
	ip6f->ip6f_ident = id;
	ip6f->ip6f_nxt = nextproto;
	IP6STAT_INC(ip6s_ofragments);
	in6_ifstat_inc(ifp, ifs6_out_fragcreat);
	}

	return (0);
	}

	static int
	ip6_output_send(struct inpcb inp, struct ifnet ifp, struct ifnet *origifp,
	struct mbuf m, struct sockaddr_in6 dst, struct route_in6 *ro,
	bool stamp_tag)
	{
	#ifdef KERN_TLS
	struct ktls_session *tls = NULL;
	#endif
	struct m_snd_tag *mst;
	int error;

	MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
	mst = NULL;

	#ifdef KERN_TLS
	/*
	* If this is an unencrypted TLS record, save a reference to
	* the record. This local reference is used to call
	* ktls_output_eagain after the mbuf has been freed (thus
	* dropping the mbuf's reference) in if_output.
	*/
	if (m->m_next != NULL && mbuf_has_tls_session(m->m_next)) {
	tls = ktls_hold(m->m_next->m_epg_tls);
	mst = tls->snd_tag;

	/*
	* If a TLS session doesn't have a valid tag, it must
	* have had an earlier ifp mismatch, so drop this
	* packet.
	*/
	if (mst == NULL) {
	error = EAGAIN;
	goto done;
	}
	/*
	* Always stamp tags that include NIC ktls.
	*/
	stamp_tag = true;
	}
	#endif
	#ifdef RATELIMIT
	if (inp != NULL && mst == NULL) {
	if ((inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) != 0 \|\|
	(inp->inp_snd_tag != NULL &&
	inp->inp_snd_tag->ifp != ifp))
	in_pcboutput_txrtlmt(inp, ifp, m);

	if (inp->inp_snd_tag != NULL)
	mst = inp->inp_snd_tag;
	}
	#endif
	if (stamp_tag && mst != NULL) {
	KASSERT(m->m_pkthdr.rcvif == NULL,
	("trying to add a send tag to a forwarded packet"));
	if (mst->ifp != ifp) {
	error = EAGAIN;
	goto done;
	}

	/* stamp send tag on mbuf */
	m->m_pkthdr.snd_tag = m_snd_tag_ref(mst);
	m->m_pkthdr.csum_flags \|= CSUM_SND_TAG;
	}

	error = nd6_output_ifp(ifp, origifp, m, dst, (struct route *)ro);

	done:
	/* Check for route change invalidating send tags. */
	#ifdef KERN_TLS
	if (tls != NULL) {
	if (error == EAGAIN)
	error = ktls_output_eagain(inp, tls);
	ktls_free(tls);
	}
	#endif
	#ifdef RATELIMIT
	if (error == EAGAIN)
	in_pcboutput_eagain(inp);
	#endif
	return (error);
	}

	/*
	* IP6 output.
	* The packet in mbuf chain m contains a skeletal IP6 header (with pri, len,
	* nxt, hlim, src, dst).
	* This function may modify ver and hlim only.
	* The mbuf chain containing the packet will be freed.
	* The mbuf opt, if present, will not be freed.
	* If route_in6 ro is present and has ro_nh initialized, route lookup would be
	* skipped and ro->ro_nh would be used. If ro is present but ro->ro_nh is NULL,
	* then result of route lookup is stored in ro->ro_nh.
	*
	* Type of "mtu": rt_mtu is u_long, ifnet.ifr_mtu is int, and nd_ifinfo.linkmtu
	* is uint32_t. So we use u_long to hold largest one, which is rt_mtu.
	*
	* ifpp - XXX: just for statistics
	*/
	/*
	* XXX TODO: no flowid is assigned for outbound flows?
	*/
	int
	ip6_output(struct mbuf m0, struct ip6_pktopts opt,
	struct route_in6 ro, int flags, struct ip6_moptions im6o,
	struct ifnet *ifpp, struct inpcb inp)
	{
	struct ip6_hdr *ip6;
	struct ifnet ifp, origifp;
	struct mbuf *m = m0;
	struct mbuf *mprev;
	struct route_in6 *ro_pmtu;
	struct nhop_object *nh;
	struct sockaddr_in6 *dst, sin6, src_sa, dst_sa;
	struct in6_addr odst;
	u_char *nexthdrp;
	int tlen, len;
	int error = 0;
	struct in6_ifaddr *ia = NULL;
	u_long mtu;
	int alwaysfrag, dontfrag;
	u_int32_t optlen, plen = 0, unfragpartlen;
	struct ip6_exthdrs exthdrs;
	struct in6_addr src0, dst0;
	u_int32_t zone;
	bool hdrsplit;
	int sw_csum, tso;
	int needfiblookup;
	uint32_t fibnum;
	struct m_tag *fwd_tag = NULL;
	uint32_t id;

	NET_EPOCH_ASSERT();

	if (inp != NULL) {
	INP_LOCK_ASSERT(inp);
	M_SETFIB(m, inp->inp_inc.inc_fibnum);
	if ((flags & IP_NODEFAULTFLOWID) == 0) {
	/* Unconditionally set flowid. */
	m->m_pkthdr.flowid = inp->inp_flowid;
	M_HASHTYPE_SET(m, inp->inp_flowtype);
	}
	#ifdef NUMA
	m->m_pkthdr.numa_domain = inp->inp_numa_domain;
	#endif
	}

	#if defined(IPSEC) \|\| defined(IPSEC_SUPPORT)
	/*
	* IPSec checking which handles several cases.
	* FAST IPSEC: We re-injected the packet.
	* XXX: need scope argument.
	*/
	if (IPSEC_ENABLED(ipv6)) {
	if ((error = IPSEC_OUTPUT(ipv6, m, inp)) != 0) {
	if (error == EINPROGRESS)
	error = 0;
	goto done;
	}
	}
	#endif /* IPSEC */

	/* Source address validation. */
	ip6 = mtod(m, struct ip6_hdr *);
	if (IN6_IS_ADDR_UNSPECIFIED(&ip6->ip6_src) &&
	(flags & IPV6_UNSPECSRC) == 0) {
	error = EOPNOTSUPP;
	IP6STAT_INC(ip6s_badscope);
	goto bad;
	}
	if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_src)) {
	error = EOPNOTSUPP;
	IP6STAT_INC(ip6s_badscope);
	goto bad;
	}

	/*
	* If we are given packet options to add extension headers prepare them.
	* Calculate the total length of the extension header chain.
	* Keep the length of the unfragmentable part for fragmentation.
	*/
	bzero(&exthdrs, sizeof(exthdrs));
	optlen = 0;
	unfragpartlen = sizeof(struct ip6_hdr);
	if (opt) {
	/* Hop-by-Hop options header. */
	MAKE_EXTHDR(opt->ip6po_hbh, &exthdrs.ip6e_hbh, optlen);

	/* Destination options header (1st part). */
	if (opt->ip6po_rthdr) {
	#ifndef RTHDR_SUPPORT_IMPLEMENTED
	/*
	* If there is a routing header, discard the packet
	* right away here. RH0/1 are obsolete and we do not
	* currently support RH2/3/4.
	* People trying to use RH253/254 may want to disable
	* this check.
	* The moment we do support any routing header (again)
	* this block should check the routing type more
	* selectively.
	*/
	error = EINVAL;
	goto bad;
	#endif

	/*
	* Destination options header (1st part).
	* This only makes sense with a routing header.
	* See Section 9.2 of RFC 3542.
	* Disabling this part just for MIP6 convenience is
	* a bad idea. We need to think carefully about a
	* way to make the advanced API coexist with MIP6
	* options, which might automatically be inserted in
	* the kernel.
	*/
	MAKE_EXTHDR(opt->ip6po_dest1, &exthdrs.ip6e_dest1,
	optlen);
	}
	/* Routing header. */
	MAKE_EXTHDR(opt->ip6po_rthdr, &exthdrs.ip6e_rthdr, optlen);

	unfragpartlen += optlen;

	/*
	* NOTE: we don't add AH/ESP length here (done in
	* ip6_ipsec_output()).
	*/

	/* Destination options header (2nd part). */
	MAKE_EXTHDR(opt->ip6po_dest2, &exthdrs.ip6e_dest2, optlen);
	}

	/*
	* If there is at least one extension header,
	* separate IP6 header from the payload.
	*/
	hdrsplit = false;
	if (optlen) {
	if ((error = ip6_splithdr(m, &exthdrs)) != 0) {
	m = NULL;
	goto freehdrs;
	}
	m = exthdrs.ip6e_ip6;
	ip6 = mtod(m, struct ip6_hdr *);
	hdrsplit = true;
	}

	/* Adjust mbuf packet header length. */
	m->m_pkthdr.len += optlen;
	plen = m->m_pkthdr.len - sizeof(*ip6);

	/* If this is a jumbo payload, insert a jumbo payload option. */
	if (plen > IPV6_MAXPACKET) {
	if (!hdrsplit) {
	if ((error = ip6_splithdr(m, &exthdrs)) != 0) {
	m = NULL;
	goto freehdrs;
	}
	m = exthdrs.ip6e_ip6;
	ip6 = mtod(m, struct ip6_hdr *);
	hdrsplit = true;
	}
	if ((error = ip6_insert_jumboopt(&exthdrs, plen)) != 0)
	goto freehdrs;
	ip6->ip6_plen = 0;
	} else
	ip6->ip6_plen = htons(plen);
	nexthdrp = &ip6->ip6_nxt;

	if (optlen) {
	/*
	* Concatenate headers and fill in next header fields.
	* Here we have, on "m"
	* IPv6 payload
	* and we insert headers accordingly.
	* Finally, we should be getting:
	* IPv6 hbh dest1 rthdr ah* [esp* dest2 payload].
	*
	* During the header composing process "m" points to IPv6
	* header. "mprev" points to an extension header prior to esp.
	*/
	mprev = m;

	/*
	* We treat dest2 specially. This makes IPsec processing
	* much easier. The goal here is to make mprev point the
	* mbuf prior to dest2.
	*
	* Result: IPv6 dest2 payload.
	* m and mprev will point to IPv6 header.
	*/
	if (exthdrs.ip6e_dest2) {
	if (!hdrsplit)
	panic("%s:%d: assumption failed: "
	"hdr not split: hdrsplit %d exthdrs %p",
	__func__, __LINE__, hdrsplit, &exthdrs);
	exthdrs.ip6e_dest2->m_next = m->m_next;
	m->m_next = exthdrs.ip6e_dest2;
	mtod(exthdrs.ip6e_dest2, u_char ) = ip6->ip6_nxt;
	ip6->ip6_nxt = IPPROTO_DSTOPTS;
	}

	/*
	* Result: IPv6 hbh dest1 rthdr dest2 payload.
	* m will point to IPv6 header. mprev will point to the
	* extension header prior to dest2 (rthdr in the above case).
	*/
	MAKE_CHAIN(exthdrs.ip6e_hbh, mprev, nexthdrp, IPPROTO_HOPOPTS);
	MAKE_CHAIN(exthdrs.ip6e_dest1, mprev, nexthdrp,
	IPPROTO_DSTOPTS);
	MAKE_CHAIN(exthdrs.ip6e_rthdr, mprev, nexthdrp,
	IPPROTO_ROUTING);
	}

	IP6STAT_INC(ip6s_localout);

	/* Route packet. */
	ro_pmtu = ro;
	if (opt && opt->ip6po_rthdr)
	ro = &opt->ip6po_route;
	if (ro != NULL)
	dst = (struct sockaddr_in6 *)&ro->ro_dst;
	else
	dst = &sin6;
	fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : M_GETFIB(m);

	again:
	/*
	* If specified, try to fill in the traffic class field.
	* Do not override if a non-zero value is already set.
	* We check the diffserv field and the ECN field separately.
	*/
	if (opt && opt->ip6po_tclass >= 0) {
	int mask = 0;

	if ((ip6->ip6_flow & htonl(0xfc << 20)) == 0)
	mask \|= 0xfc;
	if ((ip6->ip6_flow & htonl(0x03 << 20)) == 0)
	mask \|= 0x03;
	if (mask != 0)
	ip6->ip6_flow \|= htonl((opt->ip6po_tclass & mask) << 20);
	}

	/* Fill in or override the hop limit field, if necessary. */
	if (opt && opt->ip6po_hlim != -1)
	ip6->ip6_hlim = opt->ip6po_hlim & 0xff;
	else if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst)) {
	if (im6o != NULL)
	ip6->ip6_hlim = im6o->im6o_multicast_hlim;
	else
	ip6->ip6_hlim = V_ip6_defmcasthlim;
	}

	if (ro == NULL \|\| ro->ro_nh == NULL) {
	bzero(dst, sizeof(*dst));
	dst->sin6_family = AF_INET6;
	dst->sin6_len = sizeof(*dst);
	dst->sin6_addr = ip6->ip6_dst;
	}
	/*
	* Validate route against routing table changes.
	* Make sure that the address family is set in route.
	*/
	nh = NULL;
	ifp = NULL;
	mtu = 0;
	if (ro != NULL) {
	if (ro->ro_nh != NULL && inp != NULL) {
	ro->ro_dst.sin6_family = AF_INET6; /* XXX KASSERT? */
	NH_VALIDATE((struct route *)ro, &inp->inp_rt_cookie,
	fibnum);
	}
	if (ro->ro_nh != NULL && fwd_tag == NULL &&
	(!NH_IS_VALID(ro->ro_nh) \|\|
	ro->ro_dst.sin6_family != AF_INET6 \|\|
	!IN6_ARE_ADDR_EQUAL(&ro->ro_dst.sin6_addr, &ip6->ip6_dst)))
	RO_INVALIDATE_CACHE(ro);

	if (ro->ro_nh != NULL && fwd_tag == NULL &&
	ro->ro_dst.sin6_family == AF_INET6 &&
	IN6_ARE_ADDR_EQUAL(&ro->ro_dst.sin6_addr, &ip6->ip6_dst)) {
	nh = ro->ro_nh;
	ifp = nh->nh_ifp;
	} else {
	if (ro->ro_lle)
	LLE_FREE(ro->ro_lle); /* zeros ro_lle */
	ro->ro_lle = NULL;
	if (fwd_tag == NULL) {
	bzero(&dst_sa, sizeof(dst_sa));
	dst_sa.sin6_family = AF_INET6;
	dst_sa.sin6_len = sizeof(dst_sa);
	dst_sa.sin6_addr = ip6->ip6_dst;
	}
	error = in6_selectroute(&dst_sa, opt, im6o, ro, &ifp,
	&nh, fibnum, m->m_pkthdr.flowid);
	if (error != 0) {
	IP6STAT_INC(ip6s_noroute);
	if (ifp != NULL)
	in6_ifstat_inc(ifp, ifs6_out_discard);
	goto bad;
	}
	if (ifp != NULL)
	mtu = ifp->if_mtu;
	}
	if (nh == NULL) {
	/*
	* If in6_selectroute() does not return a nexthop
	* dst may not have been updated.
	*/
	dst = dst_sa; / XXX */
	} else {
	if (nh->nh_flags & NHF_HOST)
	mtu = nh->nh_mtu;
	ia = (struct in6_ifaddr *)(nh->nh_ifa);
	counter_u64_add(nh->nh_pksent, 1);
	}
	} else {
	struct nhop_object *nh;
	struct in6_addr kdst;
	uint32_t scopeid;

	if (fwd_tag == NULL) {
	bzero(&dst_sa, sizeof(dst_sa));
	dst_sa.sin6_family = AF_INET6;
	dst_sa.sin6_len = sizeof(dst_sa);
	dst_sa.sin6_addr = ip6->ip6_dst;
	}

	if (IN6_IS_ADDR_MULTICAST(&dst_sa.sin6_addr) &&
	im6o != NULL &&
	(ifp = im6o->im6o_multicast_ifp) != NULL) {
	/* We do not need a route lookup. */
	dst = dst_sa; / XXX */
	goto nonh6lookup;
	}

	in6_splitscope(&dst_sa.sin6_addr, &kdst, &scopeid);

	if (IN6_IS_ADDR_MC_LINKLOCAL(&dst_sa.sin6_addr) \|\|
	IN6_IS_ADDR_MC_NODELOCAL(&dst_sa.sin6_addr)) {
	if (scopeid > 0) {
	ifp = in6_getlinkifnet(scopeid);
	+ if (ifp == NULL) {
	+ error = EHOSTUNREACH;
	+ goto bad;
	+ }
	dst = dst_sa; / XXX */
	goto nonh6lookup;
	}
	}

	nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE, 0);
	if (nh == NULL) {
	IP6STAT_INC(ip6s_noroute);
	/* No ifp in6_ifstat_inc(ifp, ifs6_out_discard); */
	error = EHOSTUNREACH;;
	goto bad;
	}

	ifp = nh->nh_ifp;
	mtu = nh->nh_mtu;
	ia = ifatoia6(nh->nh_ifa);
	if (nh->nh_flags & NHF_GATEWAY)
	dst->sin6_addr = nh->gw6_sa.sin6_addr;
	nonh6lookup:
	;
	}

	/* Then nh (for unicast) and ifp must be non-NULL valid values. */
	if ((flags & IPV6_FORWARDING) == 0) {
	/* XXX: the FORWARDING flag can be set for mrouting. */
	in6_ifstat_inc(ifp, ifs6_out_request);
	}

	/* Setup data structures for scope ID checks. */
	src0 = ip6->ip6_src;
	bzero(&src_sa, sizeof(src_sa));
	src_sa.sin6_family = AF_INET6;
	src_sa.sin6_len = sizeof(src_sa);
	src_sa.sin6_addr = ip6->ip6_src;

	dst0 = ip6->ip6_dst;
	/* Re-initialize to be sure. */
	bzero(&dst_sa, sizeof(dst_sa));
	dst_sa.sin6_family = AF_INET6;
	dst_sa.sin6_len = sizeof(dst_sa);
	dst_sa.sin6_addr = ip6->ip6_dst;

	/* Check for valid scope ID. */
	if (in6_setscope(&src0, ifp, &zone) == 0 &&
	sa6_recoverscope(&src_sa) == 0 && zone == src_sa.sin6_scope_id &&
	in6_setscope(&dst0, ifp, &zone) == 0 &&
	sa6_recoverscope(&dst_sa) == 0 && zone == dst_sa.sin6_scope_id) {
	/*
	* The outgoing interface is in the zone of the source
	* and destination addresses.
	*
	* Because the loopback interface cannot receive
	* packets with a different scope ID than its own,
	* there is a trick to pretend the outgoing packet
	* was received by the real network interface, by
	* setting "origifp" different from "ifp". This is
	* only allowed when "ifp" is a loopback network
	* interface. Refer to code in nd6_output_ifp() for
	* more details.
	*/
	origifp = ifp;

	/*
	* We should use ia_ifp to support the case of sending
	* packets to an address of our own.
	*/
	if (ia != NULL && ia->ia_ifp)
	ifp = ia->ia_ifp;

	} else if ((ifp->if_flags & IFF_LOOPBACK) == 0 \|\|
	sa6_recoverscope(&src_sa) != 0 \|\|
	sa6_recoverscope(&dst_sa) != 0 \|\|
	dst_sa.sin6_scope_id == 0 \|\|
	(src_sa.sin6_scope_id != 0 &&
	src_sa.sin6_scope_id != dst_sa.sin6_scope_id) \|\|
	(origifp = ifnet_byindex(dst_sa.sin6_scope_id)) == NULL) {
	/*
	* If the destination network interface is not a
	* loopback interface, or the destination network
	* address has no scope ID, or the source address has
	* a scope ID set which is different from the
	* destination address one, or there is no network
	* interface representing this scope ID, the address
	* pair is considered invalid.
	*/
	IP6STAT_INC(ip6s_badscope);
	in6_ifstat_inc(ifp, ifs6_out_discard);
	if (error == 0)
	error = EHOSTUNREACH; /* XXX */
	goto bad;
	}
	/* All scope ID checks are successful. */

	if (nh && !IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst)) {
	if (opt && opt->ip6po_nextroute.ro_nh) {
	/*
	* The nexthop is explicitly specified by the
	* application. We assume the next hop is an IPv6
	* address.
	*/
	dst = (struct sockaddr_in6 *)opt->ip6po_nexthop;
	}
	else if ((nh->nh_flags & NHF_GATEWAY))
	dst = &nh->gw6_sa;
	}

	if (!IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst)) {
	m->m_flags &= ~(M_BCAST \| M_MCAST); /* Just in case. */
	} else {
	m->m_flags = (m->m_flags & ~M_BCAST) \| M_MCAST;
	in6_ifstat_inc(ifp, ifs6_out_mcast);

	/* Confirm that the outgoing interface supports multicast. */
	if (!(ifp->if_flags & IFF_MULTICAST)) {
	IP6STAT_INC(ip6s_noroute);
	in6_ifstat_inc(ifp, ifs6_out_discard);
	error = ENETUNREACH;
	goto bad;
	}
	if ((im6o == NULL && in6_mcast_loop) \|\|
	(im6o && im6o->im6o_multicast_loop)) {
	/*
	* Loop back multicast datagram if not expressly
	* forbidden to do so, even if we have not joined
	* the address; protocols will filter it later,
	* thus deferring a hash lookup and lock acquisition
	* at the expense of an m_copym().
	*/
	ip6_mloopback(ifp, m);
	} else {
	/*
	* If we are acting as a multicast router, perform
	* multicast forwarding as if the packet had just
	* arrived on the interface to which we are about
	* to send. The multicast forwarding function
	* recursively calls this function, using the
	* IPV6_FORWARDING flag to prevent infinite recursion.
	*
	* Multicasts that are looped back by ip6_mloopback(),
	* above, will be forwarded by the ip6_input() routine,
	* if necessary.
	*/
	if (V_ip6_mrouter && (flags & IPV6_FORWARDING) == 0) {
	/*
	* XXX: ip6_mforward expects that rcvif is NULL
	* when it is called from the originating path.
	* However, it may not always be the case.
	*/
	m->m_pkthdr.rcvif = NULL;
	if (ip6_mforward(ip6, ifp, m) != 0) {
	m_freem(m);
	goto done;
	}
	}
	}
	/*
	* Multicasts with a hoplimit of zero may be looped back,
	* above, but must not be transmitted on a network.
	* Also, multicasts addressed to the loopback interface
	* are not sent -- the above call to ip6_mloopback() will
	* loop back a copy if this host actually belongs to the
	* destination group on the loopback interface.
	*/
	if (ip6->ip6_hlim == 0 \|\| (ifp->if_flags & IFF_LOOPBACK) \|\|
	IN6_IS_ADDR_MC_INTFACELOCAL(&ip6->ip6_dst)) {
	m_freem(m);
	goto done;
	}
	}

	/*
	* Fill the outgoing inteface to tell the upper layer
	* to increment per-interface statistics.
	*/
	if (ifpp)
	*ifpp = ifp;

	/* Determine path MTU. */
	if ((error = ip6_getpmtu(ro_pmtu, ro != ro_pmtu, ifp, &ip6->ip6_dst,
	&mtu, &alwaysfrag, fibnum, *nexthdrp)) != 0)
	goto bad;
	KASSERT(mtu > 0, ("%s:%d: mtu %ld, ro_pmtu %p ro %p ifp %p "
	"alwaysfrag %d fibnum %u\n", __func__, __LINE__, mtu, ro_pmtu, ro,
	ifp, alwaysfrag, fibnum));

	/*
	* The caller of this function may specify to use the minimum MTU
	* in some cases.
	* An advanced API option (IPV6_USE_MIN_MTU) can also override MTU
	* setting. The logic is a bit complicated; by default, unicast
	* packets will follow path MTU while multicast packets will be sent at
	* the minimum MTU. If IP6PO_MINMTU_ALL is specified, all packets
	* including unicast ones will be sent at the minimum MTU. Multicast
	* packets will always be sent at the minimum MTU unless
	* IP6PO_MINMTU_DISABLE is explicitly specified.
	* See RFC 3542 for more details.
	*/
	if (mtu > IPV6_MMTU) {
	if ((flags & IPV6_MINMTU))
	mtu = IPV6_MMTU;
	else if (opt && opt->ip6po_minmtu == IP6PO_MINMTU_ALL)
	mtu = IPV6_MMTU;
	else if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst) &&
	(opt == NULL \|\|
	opt->ip6po_minmtu != IP6PO_MINMTU_DISABLE)) {
	mtu = IPV6_MMTU;
	}
	}

	/*
	* Clear embedded scope identifiers if necessary.
	* in6_clearscope() will touch the addresses only when necessary.
	*/
	in6_clearscope(&ip6->ip6_src);
	in6_clearscope(&ip6->ip6_dst);

	/*
	* If the outgoing packet contains a hop-by-hop options header,
	* it must be examined and processed even by the source node.
	* (RFC 2460, section 4.)
	*/
	if (exthdrs.ip6e_hbh) {
	struct ip6_hbh hbh = mtod(exthdrs.ip6e_hbh, struct ip6_hbh );
	u_int32_t dummy; /* XXX unused */
	u_int32_t plen = 0; /* XXX: ip6_process will check the value */

	#ifdef DIAGNOSTIC
	if ((hbh->ip6h_len + 1) << 3 > exthdrs.ip6e_hbh->m_len)
	panic("ip6e_hbh is not contiguous");
	#endif
	/*
	* XXX: if we have to send an ICMPv6 error to the sender,
	* we need the M_LOOP flag since icmp6_error() expects
	* the IPv6 and the hop-by-hop options header are
	* contiguous unless the flag is set.
	*/
	m->m_flags \|= M_LOOP;
	m->m_pkthdr.rcvif = ifp;
	if (ip6_process_hopopts(m, (u_int8_t *)(hbh + 1),
	((hbh->ip6h_len + 1) << 3) - sizeof(struct ip6_hbh),
	&dummy, &plen) < 0) {
	/* m was already freed at this point. */
	error = EINVAL;/* better error? */
	goto done;
	}
	m->m_flags &= ~M_LOOP; /* XXX */
	m->m_pkthdr.rcvif = NULL;
	}

	/* Jump over all PFIL processing if hooks are not active. */
	if (!PFIL_HOOKED_OUT(V_inet6_pfil_head))
	goto passout;

	odst = ip6->ip6_dst;
	/* Run through list of hooks for output packets. */
	switch (pfil_run_hooks(V_inet6_pfil_head, &m, ifp, PFIL_OUT, inp)) {
	case PFIL_PASS:
	ip6 = mtod(m, struct ip6_hdr *);
	break;
	case PFIL_DROPPED:
	error = EACCES;
	/* FALLTHROUGH */
	case PFIL_CONSUMED:
	goto done;
	}

	needfiblookup = 0;
	/* See if destination IP address was changed by packet filter. */
	if (!IN6_ARE_ADDR_EQUAL(&odst, &ip6->ip6_dst)) {
	m->m_flags \|= M_SKIP_FIREWALL;
	/* If destination is now ourself drop to ip6_input(). */
	if (in6_localip(&ip6->ip6_dst)) {
	m->m_flags \|= M_FASTFWD_OURS;
	if (m->m_pkthdr.rcvif == NULL)
	m->m_pkthdr.rcvif = V_loif;
	if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
	m->m_pkthdr.csum_flags \|=
	CSUM_DATA_VALID_IPV6 \| CSUM_PSEUDO_HDR;
	m->m_pkthdr.csum_data = 0xffff;
	}
	#if defined(SCTP) \|\| defined(SCTP_SUPPORT)
	if (m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6)
	m->m_pkthdr.csum_flags \|= CSUM_SCTP_VALID;
	#endif
	error = netisr_queue(NETISR_IPV6, m);
	goto done;
	} else {
	if (ro != NULL)
	RO_INVALIDATE_CACHE(ro);
	needfiblookup = 1; /* Redo the routing table lookup. */
	}
	}
	/* See if fib was changed by packet filter. */
	if (fibnum != M_GETFIB(m)) {
	m->m_flags \|= M_SKIP_FIREWALL;
	fibnum = M_GETFIB(m);
	if (ro != NULL)
	RO_INVALIDATE_CACHE(ro);
	needfiblookup = 1;
	}
	if (needfiblookup)
	goto again;

	/* See if local, if yes, send it to netisr. */
	if (m->m_flags & M_FASTFWD_OURS) {
	if (m->m_pkthdr.rcvif == NULL)
	m->m_pkthdr.rcvif = V_loif;
	if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
	m->m_pkthdr.csum_flags \|=
	CSUM_DATA_VALID_IPV6 \| CSUM_PSEUDO_HDR;
	m->m_pkthdr.csum_data = 0xffff;
	}
	#if defined(SCTP) \|\| defined(SCTP_SUPPORT)
	if (m->m_pkthdr.csum_flags & CSUM_SCTP_IPV6)
	m->m_pkthdr.csum_flags \|= CSUM_SCTP_VALID;
	#endif
	error = netisr_queue(NETISR_IPV6, m);
	goto done;
	}
	/* Or forward to some other address? */
	if ((m->m_flags & M_IP6_NEXTHOP) &&
	(fwd_tag = m_tag_find(m, PACKET_TAG_IPFORWARD, NULL)) != NULL) {
	if (ro != NULL)
	dst = (struct sockaddr_in6 *)&ro->ro_dst;
	else
	dst = &sin6;
	bcopy((fwd_tag+1), &dst_sa, sizeof(struct sockaddr_in6));
	m->m_flags \|= M_SKIP_FIREWALL;
	m->m_flags &= ~M_IP6_NEXTHOP;
	m_tag_delete(m, fwd_tag);
	goto again;
	}

	passout:
	/*
	* Send the packet to the outgoing interface.
	* If necessary, do IPv6 fragmentation before sending.
	*
	* The logic here is rather complex:
	* 1: normal case (dontfrag == 0, alwaysfrag == 0)
	* 1-a: send as is if tlen <= path mtu
	* 1-b: fragment if tlen > path mtu
	*
	* 2: if user asks us not to fragment (dontfrag == 1)
	* 2-a: send as is if tlen <= interface mtu
	* 2-b: error if tlen > interface mtu
	*
	* 3: if we always need to attach fragment header (alwaysfrag == 1)
	* always fragment
	*
	* 4: if dontfrag == 1 && alwaysfrag == 1
	* error, as we cannot handle this conflicting request.
	*/
	sw_csum = m->m_pkthdr.csum_flags;
	if (!hdrsplit) {
	tso = ((sw_csum & ifp->if_hwassist & CSUM_TSO) != 0) ? 1 : 0;
	sw_csum &= ~ifp->if_hwassist;
	} else
	tso = 0;
	/*
	* If we added extension headers, we will not do TSO and calculate the
	* checksums ourselves for now.
	* XXX-BZ Need a framework to know when the NIC can handle it, even
	* with ext. hdrs.
	*/
	error = ip6_output_delayed_csum(m, ifp, sw_csum, plen, optlen, false);
	if (error != 0)
	goto bad;
	/* XXX-BZ m->m_pkthdr.csum_flags &= ~ifp->if_hwassist; */
	tlen = m->m_pkthdr.len;

	if ((opt && (opt->ip6po_flags & IP6PO_DONTFRAG)) \|\| tso)
	dontfrag = 1;
	else
	dontfrag = 0;
	if (dontfrag && alwaysfrag) { /* Case 4. */
	/* Conflicting request - can't transmit. */
	error = EMSGSIZE;
	goto bad;
	}
	if (dontfrag && tlen > IN6_LINKMTU(ifp) && !tso) { /* Case 2-b. */
	/*
	* Even if the DONTFRAG option is specified, we cannot send the
	* packet when the data length is larger than the MTU of the
	* outgoing interface.
	* Notify the error by sending IPV6_PATHMTU ancillary data if
	* application wanted to know the MTU value. Also return an
	* error code (this is not described in the API spec).
	*/
	if (inp != NULL)
	ip6_notify_pmtu(inp, &dst_sa, (u_int32_t)mtu);
	error = EMSGSIZE;
	goto bad;
	}

	/* Transmit packet without fragmentation. */
	if (dontfrag \|\| (!alwaysfrag && tlen <= mtu)) { /* Cases 1-a and 2-a. */
	struct in6_ifaddr *ia6;

	ip6 = mtod(m, struct ip6_hdr *);
	ia6 = in6_ifawithifp(ifp, &ip6->ip6_src);
	if (ia6) {
	/* Record statistics for this interface address. */
	counter_u64_add(ia6->ia_ifa.ifa_opackets, 1);
	counter_u64_add(ia6->ia_ifa.ifa_obytes,
	m->m_pkthdr.len);
	ifa_free(&ia6->ia_ifa);
	}
	error = ip6_output_send(inp, ifp, origifp, m, dst, ro,
	(flags & IP_NO_SND_TAG_RL) ? false : true);
	goto done;
	}

	/* Try to fragment the packet. Cases 1-b and 3. */
	if (mtu < IPV6_MMTU) {
	/* Path MTU cannot be less than IPV6_MMTU. */
	error = EMSGSIZE;
	in6_ifstat_inc(ifp, ifs6_out_fragfail);
	goto bad;
	} else if (ip6->ip6_plen == 0) {
	/* Jumbo payload cannot be fragmented. */
	error = EMSGSIZE;
	in6_ifstat_inc(ifp, ifs6_out_fragfail);
	goto bad;
	} else {
	u_char nextproto;

	/*
	* Too large for the destination or interface;
	* fragment if possible.
	* Must be able to put at least 8 bytes per fragment.
	*/
	if (mtu > IPV6_MAXPACKET)
	mtu = IPV6_MAXPACKET;

	len = (mtu - unfragpartlen - sizeof(struct ip6_frag)) & ~7;
	if (len < 8) {
	error = EMSGSIZE;
	in6_ifstat_inc(ifp, ifs6_out_fragfail);
	goto bad;
	}

	/*
	* If the interface will not calculate checksums on
	* fragmented packets, then do it here.
	* XXX-BZ handle the hw offloading case. Need flags.
	*/
	error = ip6_output_delayed_csum(m, ifp, m->m_pkthdr.csum_flags,
	plen, optlen, true);
	if (error != 0)
	goto bad;

	/*
	* Change the next header field of the last header in the
	* unfragmentable part.
	*/
	if (exthdrs.ip6e_rthdr) {
	nextproto = mtod(exthdrs.ip6e_rthdr, u_char );
	mtod(exthdrs.ip6e_rthdr, u_char ) = IPPROTO_FRAGMENT;
	} else if (exthdrs.ip6e_dest1) {
	nextproto = mtod(exthdrs.ip6e_dest1, u_char );
	mtod(exthdrs.ip6e_dest1, u_char ) = IPPROTO_FRAGMENT;
	} else if (exthdrs.ip6e_hbh) {
	nextproto = mtod(exthdrs.ip6e_hbh, u_char );
	mtod(exthdrs.ip6e_hbh, u_char ) = IPPROTO_FRAGMENT;
	} else {
	ip6 = mtod(m, struct ip6_hdr *);
	nextproto = ip6->ip6_nxt;
	ip6->ip6_nxt = IPPROTO_FRAGMENT;
	}

	/*
	* Loop through length of segment after first fragment,
	* make new header and copy data of each part and link onto
	* chain.
	*/
	m0 = m;
	id = htonl(ip6_randomid());
	error = ip6_fragment(ifp, m, unfragpartlen, nextproto,len, id);
	if (error != 0)
	goto sendorfree;

	in6_ifstat_inc(ifp, ifs6_out_fragok);
	}

	/* Remove leading garbage. */
	sendorfree:
	m = m0->m_nextpkt;
	m0->m_nextpkt = 0;
	m_freem(m0);
	for (; m; m = m0) {
	m0 = m->m_nextpkt;
	m->m_nextpkt = 0;
	if (error == 0) {
	/* Record statistics for this interface address. */
	if (ia) {
	counter_u64_add(ia->ia_ifa.ifa_opackets, 1);
	counter_u64_add(ia->ia_ifa.ifa_obytes,
	m->m_pkthdr.len);
	}
	error = ip6_output_send(inp, ifp, origifp, m, dst, ro,
	true);
	} else
	m_freem(m);
	}

	if (error == 0)
	IP6STAT_INC(ip6s_fragmented);

	done:
	return (error);

	freehdrs:
	m_freem(exthdrs.ip6e_hbh); /* m_freem() checks if mbuf is NULL. */
	m_freem(exthdrs.ip6e_dest1);
	m_freem(exthdrs.ip6e_rthdr);
	m_freem(exthdrs.ip6e_dest2);
	/* FALLTHROUGH */
	bad:
	if (m)
	m_freem(m);
	goto done;
	}

	static int
	ip6_copyexthdr(struct mbuf **mp, caddr_t hdr, int hlen)
	{
	struct mbuf *m;

	if (hlen > MCLBYTES)
	return (ENOBUFS); /* XXX */

	if (hlen > MLEN)
	m = m_getcl(M_NOWAIT, MT_DATA, 0);
	else
	m = m_get(M_NOWAIT, MT_DATA);
	if (m == NULL)
	return (ENOBUFS);
	m->m_len = hlen;
	if (hdr)
	bcopy(hdr, mtod(m, caddr_t), hlen);

	*mp = m;
	return (0);
	}

	/*
	* Insert jumbo payload option.
	*/
	static int
	ip6_insert_jumboopt(struct ip6_exthdrs *exthdrs, u_int32_t plen)
	{
	struct mbuf *mopt;
	u_char *optbuf;
	u_int32_t v;

	#define JUMBOOPTLEN 8 /* length of jumbo payload option and padding */

	/*
	* If there is no hop-by-hop options header, allocate new one.
	* If there is one but it doesn't have enough space to store the
	* jumbo payload option, allocate a cluster to store the whole options.
	* Otherwise, use it to store the options.
	*/
	if (exthdrs->ip6e_hbh == NULL) {
	mopt = m_get(M_NOWAIT, MT_DATA);
	if (mopt == NULL)
	return (ENOBUFS);
	mopt->m_len = JUMBOOPTLEN;
	optbuf = mtod(mopt, u_char *);
	optbuf[1] = 0; /* = ((JUMBOOPTLEN) >> 3) - 1 */
	exthdrs->ip6e_hbh = mopt;
	} else {
	struct ip6_hbh *hbh;

	mopt = exthdrs->ip6e_hbh;
	if (M_TRAILINGSPACE(mopt) < JUMBOOPTLEN) {
	/*
	* XXX assumption:
	* - exthdrs->ip6e_hbh is not referenced from places
	* other than exthdrs.
	* - exthdrs->ip6e_hbh is not an mbuf chain.
	*/
	int oldoptlen = mopt->m_len;
	struct mbuf *n;

	/*
	* XXX: give up if the whole (new) hbh header does
	* not fit even in an mbuf cluster.
	*/
	if (oldoptlen + JUMBOOPTLEN > MCLBYTES)
	return (ENOBUFS);

	/*
	* As a consequence, we must always prepare a cluster
	* at this point.
	*/
	n = m_getcl(M_NOWAIT, MT_DATA, 0);
	if (n == NULL)
	return (ENOBUFS);
	n->m_len = oldoptlen + JUMBOOPTLEN;
	bcopy(mtod(mopt, caddr_t), mtod(n, caddr_t),
	oldoptlen);
	optbuf = mtod(n, caddr_t) + oldoptlen;
	m_freem(mopt);
	mopt = exthdrs->ip6e_hbh = n;
	} else {
	optbuf = mtod(mopt, u_char *) + mopt->m_len;
	mopt->m_len += JUMBOOPTLEN;
	}
	optbuf[0] = IP6OPT_PADN;
	optbuf[1] = 1;

	/*
	* Adjust the header length according to the pad and
	* the jumbo payload option.
	*/
	hbh = mtod(mopt, struct ip6_hbh *);
	hbh->ip6h_len += (JUMBOOPTLEN >> 3);
	}

	/* fill in the option. */
	optbuf[2] = IP6OPT_JUMBO;
	optbuf[3] = 4;
	v = (u_int32_t)htonl(plen + JUMBOOPTLEN);
	bcopy(&v, &optbuf[4], sizeof(u_int32_t));

	/* finally, adjust the packet header length */
	exthdrs->ip6e_ip6->m_pkthdr.len += JUMBOOPTLEN;

	return (0);
	#undef JUMBOOPTLEN
	}

	/*
	* Insert fragment header and copy unfragmentable header portions.
	*/
	static int
	ip6_insertfraghdr(struct mbuf m0, struct mbuf m, int hlen,
	struct ip6_frag **frghdrp)
	{
	struct mbuf n, mlast;

	if (hlen > sizeof(struct ip6_hdr)) {
	n = m_copym(m0, sizeof(struct ip6_hdr),
	hlen - sizeof(struct ip6_hdr), M_NOWAIT);
	if (n == NULL)
	return (ENOBUFS);
	m->m_next = n;
	} else
	n = m;

	/* Search for the last mbuf of unfragmentable part. */
	for (mlast = n; mlast->m_next; mlast = mlast->m_next)
	;

	if (M_WRITABLE(mlast) &&
	M_TRAILINGSPACE(mlast) >= sizeof(struct ip6_frag)) {
	/* use the trailing space of the last mbuf for the fragment hdr */
	frghdrp = (struct ip6_frag )(mtod(mlast, caddr_t) +
	mlast->m_len);
	mlast->m_len += sizeof(struct ip6_frag);
	m->m_pkthdr.len += sizeof(struct ip6_frag);
	} else {
	/* allocate a new mbuf for the fragment header */
	struct mbuf *mfrg;

	mfrg = m_get(M_NOWAIT, MT_DATA);
	if (mfrg == NULL)
	return (ENOBUFS);
	mfrg->m_len = sizeof(struct ip6_frag);
	frghdrp = mtod(mfrg, struct ip6_frag );
	mlast->m_next = mfrg;
	}

	return (0);
	}

	/*
	* Calculates IPv6 path mtu for destination @dst.
	* Resulting MTU is stored in @mtup.
	*
	* Returns 0 on success.
	*/
	static int
	ip6_getpmtu_ctl(u_int fibnum, const struct in6_addr dst, u_long mtup)
	{
	struct epoch_tracker et;
	struct nhop_object *nh;
	struct in6_addr kdst;
	uint32_t scopeid;
	int error;

	in6_splitscope(dst, &kdst, &scopeid);

	NET_EPOCH_ENTER(et);
	nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE, 0);
	if (nh != NULL)
	error = ip6_calcmtu(nh->nh_ifp, dst, nh->nh_mtu, mtup, NULL, 0);
	else
	error = EHOSTUNREACH;
	NET_EPOCH_EXIT(et);

	return (error);
	}

	/*
	* Calculates IPv6 path MTU for @dst based on transmit @ifp,
	* and cached data in @ro_pmtu.
	* MTU from (successful) route lookup is saved (along with dst)
	* inside @ro_pmtu to avoid subsequent route lookups after packet
	* filter processing.
	*
	* Stores mtu and always-frag value into @mtup and @alwaysfragp.
	* Returns 0 on success.
	*/
	static int
	ip6_getpmtu(struct route_in6 *ro_pmtu, int do_lookup,
	struct ifnet ifp, const struct in6_addr dst, u_long *mtup,
	int *alwaysfragp, u_int fibnum, u_int proto)
	{
	struct nhop_object *nh;
	struct in6_addr kdst;
	uint32_t scopeid;
	struct sockaddr_in6 *sa6_dst, sin6;
	u_long mtu;

	NET_EPOCH_ASSERT();

	mtu = 0;
	if (ro_pmtu == NULL \|\| do_lookup) {

	/*
	* Here ro_pmtu has final destination address, while
	* ro might represent immediate destination.
	* Use ro_pmtu destination since mtu might differ.
	*/
	if (ro_pmtu != NULL) {
	sa6_dst = (struct sockaddr_in6 *)&ro_pmtu->ro_dst;
	if (!IN6_ARE_ADDR_EQUAL(&sa6_dst->sin6_addr, dst))
	ro_pmtu->ro_mtu = 0;
	} else
	sa6_dst = &sin6;

	if (ro_pmtu == NULL \|\| ro_pmtu->ro_mtu == 0) {
	bzero(sa6_dst, sizeof(*sa6_dst));
	sa6_dst->sin6_family = AF_INET6;
	sa6_dst->sin6_len = sizeof(struct sockaddr_in6);
	sa6_dst->sin6_addr = *dst;

	in6_splitscope(dst, &kdst, &scopeid);
	nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE, 0);
	if (nh != NULL) {
	mtu = nh->nh_mtu;
	if (ro_pmtu != NULL)
	ro_pmtu->ro_mtu = mtu;
	}
	} else
	mtu = ro_pmtu->ro_mtu;
	}

	if (ro_pmtu != NULL && ro_pmtu->ro_nh != NULL)
	mtu = ro_pmtu->ro_nh->nh_mtu;

	return (ip6_calcmtu(ifp, dst, mtu, mtup, alwaysfragp, proto));
	}

	/*
	* Calculate MTU based on transmit @ifp, route mtu @rt_mtu and
	* hostcache data for @dst.
	* Stores mtu and always-frag value into @mtup and @alwaysfragp.
	*
	* Returns 0 on success.
	*/
	static int
	ip6_calcmtu(struct ifnet ifp, const struct in6_addr dst, u_long rt_mtu,
	u_long mtup, int alwaysfragp, u_int proto)
	{
	u_long mtu = 0;
	int alwaysfrag = 0;
	int error = 0;

	if (rt_mtu > 0) {
	u_int32_t ifmtu;
	struct in_conninfo inc;

	bzero(&inc, sizeof(inc));
	inc.inc_flags \|= INC_ISIPV6;
	inc.inc6_faddr = *dst;

	ifmtu = IN6_LINKMTU(ifp);

	/* TCP is known to react to pmtu changes so skip hc */
	if (proto != IPPROTO_TCP)
	mtu = tcp_hc_getmtu(&inc);

	if (mtu)
	mtu = min(mtu, rt_mtu);
	else
	mtu = rt_mtu;
	if (mtu == 0)
	mtu = ifmtu;
	else if (mtu < IPV6_MMTU) {
	/*
	* RFC2460 section 5, last paragraph:
	* if we record ICMPv6 too big message with
	* mtu < IPV6_MMTU, transmit packets sized IPV6_MMTU
	* or smaller, with framgent header attached.
	* (fragment header is needed regardless from the
	* packet size, for translators to identify packets)
	*/
	alwaysfrag = 1;
	mtu = IPV6_MMTU;
	}
	} else if (ifp) {
	mtu = IN6_LINKMTU(ifp);
	} else
	error = EHOSTUNREACH; /* XXX */

	*mtup = mtu;
	if (alwaysfragp)
	*alwaysfragp = alwaysfrag;
	return (error);
	}

	/*
	* IP6 socket option processing.
	*/
	int
	ip6_ctloutput(struct socket so, struct sockopt sopt)
	{
	int optdatalen, uproto;
	void *optdata;
	struct inpcb *inp = sotoinpcb(so);
	int error, optval;
	int level, op, optname;
	int optlen;
	struct thread *td;
	#ifdef RSS
	uint32_t rss_bucket;
	int retval;
	#endif

	/*
	* Don't use more than a quarter of mbuf clusters. N.B.:
	* nmbclusters is an int, but nmbclusters * MCLBYTES may overflow
	* on LP64 architectures, so cast to u_long to avoid undefined
	* behavior. ILP32 architectures cannot have nmbclusters
	* large enough to overflow for other reasons.
	*/
	#define IPV6_PKTOPTIONS_MBUF_LIMIT ((u_long)nmbclusters * MCLBYTES / 4)

	level = sopt->sopt_level;
	op = sopt->sopt_dir;
	optname = sopt->sopt_name;
	optlen = sopt->sopt_valsize;
	td = sopt->sopt_td;
	error = 0;
	optval = 0;
	uproto = (int)so->so_proto->pr_protocol;

	if (level != IPPROTO_IPV6) {
	error = EINVAL;

	if (sopt->sopt_level == SOL_SOCKET &&
	sopt->sopt_dir == SOPT_SET) {
	switch (sopt->sopt_name) {
	case SO_REUSEADDR:
	INP_WLOCK(inp);
	if ((so->so_options & SO_REUSEADDR) != 0)
	inp->inp_flags2 \|= INP_REUSEADDR;
	else
	inp->inp_flags2 &= ~INP_REUSEADDR;
	INP_WUNLOCK(inp);
	error = 0;
	break;
	case SO_REUSEPORT:
	INP_WLOCK(inp);
	if ((so->so_options & SO_REUSEPORT) != 0)
	inp->inp_flags2 \|= INP_REUSEPORT;
	else
	inp->inp_flags2 &= ~INP_REUSEPORT;
	INP_WUNLOCK(inp);
	error = 0;
	break;
	case SO_REUSEPORT_LB:
	INP_WLOCK(inp);
	if ((so->so_options & SO_REUSEPORT_LB) != 0)
	inp->inp_flags2 \|= INP_REUSEPORT_LB;
	else
	inp->inp_flags2 &= ~INP_REUSEPORT_LB;
	INP_WUNLOCK(inp);
	error = 0;
	break;
	case SO_SETFIB:
	INP_WLOCK(inp);
	inp->inp_inc.inc_fibnum = so->so_fibnum;
	INP_WUNLOCK(inp);
	error = 0;
	break;
	case SO_MAX_PACING_RATE:
	#ifdef RATELIMIT
	INP_WLOCK(inp);
	inp->inp_flags2 \|= INP_RATE_LIMIT_CHANGED;
	INP_WUNLOCK(inp);
	error = 0;
	#else
	error = EOPNOTSUPP;
	#endif
	break;
	default:
	break;
	}
	}
	} else { /* level == IPPROTO_IPV6 */
	switch (op) {

	case SOPT_SET:
	switch (optname) {
	case IPV6_2292PKTOPTIONS:
	#ifdef IPV6_PKTOPTIONS
	case IPV6_PKTOPTIONS:
	#endif
	{
	struct mbuf *m;

	if (optlen > IPV6_PKTOPTIONS_MBUF_LIMIT) {
	printf("ip6_ctloutput: mbuf limit hit\n");
	error = ENOBUFS;
	break;
	}

	error = soopt_getm(sopt, &m); /* XXX */
	if (error != 0)
	break;
	error = soopt_mcopyin(sopt, m); /* XXX */
	if (error != 0)
	break;
	INP_WLOCK(inp);
	error = ip6_pcbopts(&inp->in6p_outputopts, m,
	so, sopt);
	INP_WUNLOCK(inp);
	m_freem(m); /* XXX */
	break;
	}

	/*
	* Use of some Hop-by-Hop options or some
	* Destination options, might require special
	* privilege. That is, normal applications
	* (without special privilege) might be forbidden
	* from setting certain options in outgoing packets,
	* and might never see certain options in received
	* packets. [RFC 2292 Section 6]
	* KAME specific note:
	* KAME prevents non-privileged users from sending or
	* receiving ANY hbh/dst options in order to avoid
	* overhead of parsing options in the kernel.
	*/
	case IPV6_RECVHOPOPTS:
	case IPV6_RECVDSTOPTS:
	case IPV6_RECVRTHDRDSTOPTS:
	if (td != NULL) {
	error = priv_check(td,
	PRIV_NETINET_SETHDROPTS);
	if (error)
	break;
	}
	/* FALLTHROUGH */
	case IPV6_UNICAST_HOPS:
	case IPV6_HOPLIMIT:

	case IPV6_RECVPKTINFO:
	case IPV6_RECVHOPLIMIT:
	case IPV6_RECVRTHDR:
	case IPV6_RECVPATHMTU:
	case IPV6_RECVTCLASS:
	case IPV6_RECVFLOWID:
	#ifdef RSS
	case IPV6_RECVRSSBUCKETID:
	#endif
	case IPV6_V6ONLY:
	case IPV6_AUTOFLOWLABEL:
	case IPV6_ORIGDSTADDR:
	case IPV6_BINDANY:
	case IPV6_BINDMULTI:
	#ifdef RSS
	case IPV6_RSS_LISTEN_BUCKET:
	#endif
	if (optname == IPV6_BINDANY && td != NULL) {
	error = priv_check(td,
	PRIV_NETINET_BINDANY);
	if (error)
	break;
	}

	if (optlen != sizeof(int)) {
	error = EINVAL;
	break;
	}
	error = sooptcopyin(sopt, &optval,
	sizeof optval, sizeof optval);
	if (error)
	break;
	switch (optname) {

	case IPV6_UNICAST_HOPS:
	if (optval < -1 \|\| optval >= 256)
	error = EINVAL;
	else {
	/* -1 = kernel default */
	inp->in6p_hops = optval;
	if ((inp->inp_vflag &
	INP_IPV4) != 0)
	inp->inp_ip_ttl = optval;
	}
	break;
	#define OPTSET(bit) \
	do { \
	INP_WLOCK(inp); \
	if (optval) \
	inp->inp_flags \|= (bit); \
	else \
	inp->inp_flags &= ~(bit); \
	INP_WUNLOCK(inp); \
	} while (/CONSTCOND/ 0)
	#define OPTSET2292(bit) \
	do { \
	INP_WLOCK(inp); \
	inp->inp_flags \|= IN6P_RFC2292; \
	if (optval) \
	inp->inp_flags \|= (bit); \
	else \
	inp->inp_flags &= ~(bit); \
	INP_WUNLOCK(inp); \
	} while (/CONSTCOND/ 0)
	#define OPTBIT(bit) (inp->inp_flags & (bit) ? 1 : 0)

	#define OPTSET2_N(bit, val) do { \
	if (val) \
	inp->inp_flags2 \|= bit; \
	else \
	inp->inp_flags2 &= ~bit; \
	} while (0)
	#define OPTSET2(bit, val) do { \
	INP_WLOCK(inp); \
	OPTSET2_N(bit, val); \
	INP_WUNLOCK(inp); \
	} while (0)
	#define OPTBIT2(bit) (inp->inp_flags2 & (bit) ? 1 : 0)
	#define OPTSET2292_EXCLUSIVE(bit) \
	do { \
	INP_WLOCK(inp); \
	if (OPTBIT(IN6P_RFC2292)) { \
	error = EINVAL; \
	} else { \
	if (optval) \
	inp->inp_flags \|= (bit); \
	else \
	inp->inp_flags &= ~(bit); \
	} \
	INP_WUNLOCK(inp); \
	} while (/CONSTCOND/ 0)

	case IPV6_RECVPKTINFO:
	OPTSET2292_EXCLUSIVE(IN6P_PKTINFO);
	break;

	case IPV6_HOPLIMIT:
	{
	struct ip6_pktopts **optp;

	/* cannot mix with RFC2292 */
	if (OPTBIT(IN6P_RFC2292)) {
	error = EINVAL;
	break;
	}
	INP_WLOCK(inp);
	if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) {
	INP_WUNLOCK(inp);
	return (ECONNRESET);
	}
	optp = &inp->in6p_outputopts;
	error = ip6_pcbopt(IPV6_HOPLIMIT,
	(u_char *)&optval, sizeof(optval),
	optp, (td != NULL) ? td->td_ucred :
	NULL, uproto);
	INP_WUNLOCK(inp);
	break;
	}

	case IPV6_RECVHOPLIMIT:
	OPTSET2292_EXCLUSIVE(IN6P_HOPLIMIT);
	break;

	case IPV6_RECVHOPOPTS:
	OPTSET2292_EXCLUSIVE(IN6P_HOPOPTS);
	break;

	case IPV6_RECVDSTOPTS:
	OPTSET2292_EXCLUSIVE(IN6P_DSTOPTS);
	break;

	case IPV6_RECVRTHDRDSTOPTS:
	OPTSET2292_EXCLUSIVE(IN6P_RTHDRDSTOPTS);
	break;

	case IPV6_RECVRTHDR:
	OPTSET2292_EXCLUSIVE(IN6P_RTHDR);
	break;

	case IPV6_RECVPATHMTU:
	/*
	* We ignore this option for TCP
	* sockets.
	* (RFC3542 leaves this case
	* unspecified.)
	*/
	if (uproto != IPPROTO_TCP)
	OPTSET(IN6P_MTU);
	break;

	case IPV6_RECVFLOWID:
	OPTSET2(INP_RECVFLOWID, optval);
	break;

	#ifdef RSS
	case IPV6_RECVRSSBUCKETID:
	OPTSET2(INP_RECVRSSBUCKETID, optval);
	break;
	#endif

	case IPV6_V6ONLY:
	INP_WLOCK(inp);
	if (inp->inp_lport \|\|
	!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
	/*
	* The socket is already bound.
	*/
	INP_WUNLOCK(inp);
	error = EINVAL;
	break;
	}
	if (optval) {
	inp->inp_flags \|= IN6P_IPV6_V6ONLY;
	inp->inp_vflag &= ~INP_IPV4;
	} else {
	inp->inp_flags &= ~IN6P_IPV6_V6ONLY;
	inp->inp_vflag \|= INP_IPV4;
	}
	INP_WUNLOCK(inp);
	break;
	case IPV6_RECVTCLASS:
	/* cannot mix with RFC2292 XXX */
	OPTSET2292_EXCLUSIVE(IN6P_TCLASS);
	break;
	case IPV6_AUTOFLOWLABEL:
	OPTSET(IN6P_AUTOFLOWLABEL);
	break;

	case IPV6_ORIGDSTADDR:
	OPTSET2(INP_ORIGDSTADDR, optval);
	break;
	case IPV6_BINDANY:
	OPTSET(INP_BINDANY);
	break;

	case IPV6_BINDMULTI:
	OPTSET2(INP_BINDMULTI, optval);
	break;
	#ifdef RSS
	case IPV6_RSS_LISTEN_BUCKET:
	if ((optval >= 0) &&
	(optval < rss_getnumbuckets())) {
	INP_WLOCK(inp);
	inp->inp_rss_listen_bucket = optval;
	OPTSET2_N(INP_RSS_BUCKET_SET, 1);
	INP_WUNLOCK(inp);
	} else {
	error = EINVAL;
	}
	break;
	#endif
	}
	break;

	case IPV6_TCLASS:
	case IPV6_DONTFRAG:
	case IPV6_USE_MIN_MTU:
	case IPV6_PREFER_TEMPADDR:
	if (optlen != sizeof(optval)) {
	error = EINVAL;
	break;
	}
	error = sooptcopyin(sopt, &optval,
	sizeof optval, sizeof optval);
	if (error)
	break;
	{
	struct ip6_pktopts **optp;
	INP_WLOCK(inp);
	if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) {
	INP_WUNLOCK(inp);
	return (ECONNRESET);
	}
	optp = &inp->in6p_outputopts;
	error = ip6_pcbopt(optname,
	(u_char *)&optval, sizeof(optval),
	optp, (td != NULL) ? td->td_ucred :
	NULL, uproto);
	INP_WUNLOCK(inp);
	break;
	}

	case IPV6_2292PKTINFO:
	case IPV6_2292HOPLIMIT:
	case IPV6_2292HOPOPTS:
	case IPV6_2292DSTOPTS:
	case IPV6_2292RTHDR:
	/* RFC 2292 */
	if (optlen != sizeof(int)) {
	error = EINVAL;
	break;
	}
	error = sooptcopyin(sopt, &optval,
	sizeof optval, sizeof optval);
	if (error)
	break;
	switch (optname) {
	case IPV6_2292PKTINFO:
	OPTSET2292(IN6P_PKTINFO);
	break;
	case IPV6_2292HOPLIMIT:
	OPTSET2292(IN6P_HOPLIMIT);
	break;
	case IPV6_2292HOPOPTS:
	/*
	* Check super-user privilege.
	* See comments for IPV6_RECVHOPOPTS.
	*/
	if (td != NULL) {
	error = priv_check(td,
	PRIV_NETINET_SETHDROPTS);
	if (error)
	return (error);
	}
	OPTSET2292(IN6P_HOPOPTS);
	break;
	case IPV6_2292DSTOPTS:
	if (td != NULL) {
	error = priv_check(td,
	PRIV_NETINET_SETHDROPTS);
	if (error)
	return (error);
	}
	OPTSET2292(IN6P_DSTOPTS\|IN6P_RTHDRDSTOPTS); /* XXX */
	break;
	case IPV6_2292RTHDR:
	OPTSET2292(IN6P_RTHDR);
	break;
	}
	break;
	case IPV6_PKTINFO:
	case IPV6_HOPOPTS:
	case IPV6_RTHDR:
	case IPV6_DSTOPTS:
	case IPV6_RTHDRDSTOPTS:
	case IPV6_NEXTHOP:
	{
	/* new advanced API (RFC3542) */
	u_char *optbuf;
	u_char optbuf_storage[MCLBYTES];
	int optlen;
	struct ip6_pktopts **optp;

	/* cannot mix with RFC2292 */
	if (OPTBIT(IN6P_RFC2292)) {
	error = EINVAL;
	break;
	}

	/*
	* We only ensure valsize is not too large
	* here. Further validation will be done
	* later.
	*/
	error = sooptcopyin(sopt, optbuf_storage,
	sizeof(optbuf_storage), 0);
	if (error)
	break;
	optlen = sopt->sopt_valsize;
	optbuf = optbuf_storage;
	INP_WLOCK(inp);
	if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) {
	INP_WUNLOCK(inp);
	return (ECONNRESET);
	}
	optp = &inp->in6p_outputopts;
	error = ip6_pcbopt(optname, optbuf, optlen,
	optp, (td != NULL) ? td->td_ucred : NULL,
	uproto);
	INP_WUNLOCK(inp);
	break;
	}
	#undef OPTSET

	case IPV6_MULTICAST_IF:
	case IPV6_MULTICAST_HOPS:
	case IPV6_MULTICAST_LOOP:
	case IPV6_JOIN_GROUP:
	case IPV6_LEAVE_GROUP:
	case IPV6_MSFILTER:
	case MCAST_BLOCK_SOURCE:
	case MCAST_UNBLOCK_SOURCE:
	case MCAST_JOIN_GROUP:
	case MCAST_LEAVE_GROUP:
	case MCAST_JOIN_SOURCE_GROUP:
	case MCAST_LEAVE_SOURCE_GROUP:
	error = ip6_setmoptions(inp, sopt);
	break;

	case IPV6_PORTRANGE:
	error = sooptcopyin(sopt, &optval,
	sizeof optval, sizeof optval);
	if (error)
	break;

	INP_WLOCK(inp);
	switch (optval) {
	case IPV6_PORTRANGE_DEFAULT:
	inp->inp_flags &= ~(INP_LOWPORT);
	inp->inp_flags &= ~(INP_HIGHPORT);
	break;

	case IPV6_PORTRANGE_HIGH:
	inp->inp_flags &= ~(INP_LOWPORT);
	inp->inp_flags \|= INP_HIGHPORT;
	break;

	case IPV6_PORTRANGE_LOW:
	inp->inp_flags &= ~(INP_HIGHPORT);
	inp->inp_flags \|= INP_LOWPORT;
	break;

	default:
	error = EINVAL;
	break;
	}
	INP_WUNLOCK(inp);
	break;

	#if defined(IPSEC) \|\| defined(IPSEC_SUPPORT)
	case IPV6_IPSEC_POLICY:
	if (IPSEC_ENABLED(ipv6)) {
	error = IPSEC_PCBCTL(ipv6, inp, sopt);
	break;
	}
	/* FALLTHROUGH */
	#endif /* IPSEC */

	default:
	error = ENOPROTOOPT;
	break;
	}
	break;

	case SOPT_GET:
	switch (optname) {

	case IPV6_2292PKTOPTIONS:
	#ifdef IPV6_PKTOPTIONS
	case IPV6_PKTOPTIONS:
	#endif
	/*
	* RFC3542 (effectively) deprecated the
	* semantics of the 2292-style pktoptions.
	* Since it was not reliable in nature (i.e.,
	* applications had to expect the lack of some
	* information after all), it would make sense
	* to simplify this part by always returning
	* empty data.
	*/
	sopt->sopt_valsize = 0;
	break;

	case IPV6_RECVHOPOPTS:
	case IPV6_RECVDSTOPTS:
	case IPV6_RECVRTHDRDSTOPTS:
	case IPV6_UNICAST_HOPS:
	case IPV6_RECVPKTINFO:
	case IPV6_RECVHOPLIMIT:
	case IPV6_RECVRTHDR:
	case IPV6_RECVPATHMTU:

	case IPV6_V6ONLY:
	case IPV6_PORTRANGE:
	case IPV6_RECVTCLASS:
	case IPV6_AUTOFLOWLABEL:
	case IPV6_BINDANY:
	case IPV6_FLOWID:
	case IPV6_FLOWTYPE:
	case IPV6_RECVFLOWID:
	#ifdef RSS
	case IPV6_RSSBUCKETID:
	case IPV6_RECVRSSBUCKETID:
	#endif
	case IPV6_BINDMULTI:
	switch (optname) {

	case IPV6_RECVHOPOPTS:
	optval = OPTBIT(IN6P_HOPOPTS);
	break;

	case IPV6_RECVDSTOPTS:
	optval = OPTBIT(IN6P_DSTOPTS);
	break;

	case IPV6_RECVRTHDRDSTOPTS:
	optval = OPTBIT(IN6P_RTHDRDSTOPTS);
	break;

	case IPV6_UNICAST_HOPS:
	optval = inp->in6p_hops;
	break;

	case IPV6_RECVPKTINFO:
	optval = OPTBIT(IN6P_PKTINFO);
	break;

	case IPV6_RECVHOPLIMIT:
	optval = OPTBIT(IN6P_HOPLIMIT);
	break;

	case IPV6_RECVRTHDR:
	optval = OPTBIT(IN6P_RTHDR);
	break;

	case IPV6_RECVPATHMTU:
	optval = OPTBIT(IN6P_MTU);
	break;

	case IPV6_V6ONLY:
	optval = OPTBIT(IN6P_IPV6_V6ONLY);
	break;

	case IPV6_PORTRANGE:
	{
	int flags;
	flags = inp->inp_flags;
	if (flags & INP_HIGHPORT)
	optval = IPV6_PORTRANGE_HIGH;
	else if (flags & INP_LOWPORT)
	optval = IPV6_PORTRANGE_LOW;
	else
	optval = 0;
	break;
	}
	case IPV6_RECVTCLASS:
	optval = OPTBIT(IN6P_TCLASS);
	break;

	case IPV6_AUTOFLOWLABEL:
	optval = OPTBIT(IN6P_AUTOFLOWLABEL);
	break;

	case IPV6_ORIGDSTADDR:
	optval = OPTBIT2(INP_ORIGDSTADDR);
	break;

	case IPV6_BINDANY:
	optval = OPTBIT(INP_BINDANY);
	break;

	case IPV6_FLOWID:
	optval = inp->inp_flowid;
	break;

	case IPV6_FLOWTYPE:
	optval = inp->inp_flowtype;
	break;

	case IPV6_RECVFLOWID:
	optval = OPTBIT2(INP_RECVFLOWID);
	break;
	#ifdef RSS
	case IPV6_RSSBUCKETID:
	retval =
	rss_hash2bucket(inp->inp_flowid,
	inp->inp_flowtype,
	&rss_bucket);
	if (retval == 0)
	optval = rss_bucket;
	else
	error = EINVAL;
	break;

	case IPV6_RECVRSSBUCKETID:
	optval = OPTBIT2(INP_RECVRSSBUCKETID);
	break;
	#endif

	case IPV6_BINDMULTI:
	optval = OPTBIT2(INP_BINDMULTI);
	break;

	}
	if (error)
	break;
	error = sooptcopyout(sopt, &optval,
	sizeof optval);
	break;

	case IPV6_PATHMTU:
	{
	u_long pmtu = 0;
	struct ip6_mtuinfo mtuinfo;
	struct in6_addr addr;

	if (!(so->so_state & SS_ISCONNECTED))
	return (ENOTCONN);
	/*
	* XXX: we dot not consider the case of source
	* routing, or optional information to specify
	* the outgoing interface.
	* Copy faddr out of inp to avoid holding lock
	* on inp during route lookup.
	*/
	INP_RLOCK(inp);
	bcopy(&inp->in6p_faddr, &addr, sizeof(addr));
	INP_RUNLOCK(inp);
	error = ip6_getpmtu_ctl(so->so_fibnum,
	&addr, &pmtu);
	if (error)
	break;
	if (pmtu > IPV6_MAXPACKET)
	pmtu = IPV6_MAXPACKET;

	bzero(&mtuinfo, sizeof(mtuinfo));
	mtuinfo.ip6m_mtu = (u_int32_t)pmtu;
	optdata = (void *)&mtuinfo;
	optdatalen = sizeof(mtuinfo);
	error = sooptcopyout(sopt, optdata,
	optdatalen);
	break;
	}

	case IPV6_2292PKTINFO:
	case IPV6_2292HOPLIMIT:
	case IPV6_2292HOPOPTS:
	case IPV6_2292RTHDR:
	case IPV6_2292DSTOPTS:
	switch (optname) {
	case IPV6_2292PKTINFO:
	optval = OPTBIT(IN6P_PKTINFO);
	break;
	case IPV6_2292HOPLIMIT:
	optval = OPTBIT(IN6P_HOPLIMIT);
	break;
	case IPV6_2292HOPOPTS:
	optval = OPTBIT(IN6P_HOPOPTS);
	break;
	case IPV6_2292RTHDR:
	optval = OPTBIT(IN6P_RTHDR);
	break;
	case IPV6_2292DSTOPTS:
	optval = OPTBIT(IN6P_DSTOPTS\|IN6P_RTHDRDSTOPTS);
	break;
	}
	error = sooptcopyout(sopt, &optval,
	sizeof optval);
	break;
	case IPV6_PKTINFO:
	case IPV6_HOPOPTS:
	case IPV6_RTHDR:
	case IPV6_DSTOPTS:
	case IPV6_RTHDRDSTOPTS:
	case IPV6_NEXTHOP:
	case IPV6_TCLASS:
	case IPV6_DONTFRAG:
	case IPV6_USE_MIN_MTU:
	case IPV6_PREFER_TEMPADDR:
	error = ip6_getpcbopt(inp, optname, sopt);
	break;

	case IPV6_MULTICAST_IF:
	case IPV6_MULTICAST_HOPS:
	case IPV6_MULTICAST_LOOP:
	case IPV6_MSFILTER:
	error = ip6_getmoptions(inp, sopt);
	break;

	#if defined(IPSEC) \|\| defined(IPSEC_SUPPORT)
	case IPV6_IPSEC_POLICY:
	if (IPSEC_ENABLED(ipv6)) {
	error = IPSEC_PCBCTL(ipv6, inp, sopt);
	break;
	}
	/* FALLTHROUGH */
	#endif /* IPSEC */
	default:
	error = ENOPROTOOPT;
	break;
	}
	break;
	}
	}
	return (error);
	}

	int
	ip6_raw_ctloutput(struct socket so, struct sockopt sopt)
	{
	int error = 0, optval, optlen;
	const int icmp6off = offsetof(struct icmp6_hdr, icmp6_cksum);
	struct inpcb *inp = sotoinpcb(so);
	int level, op, optname;

	level = sopt->sopt_level;
	op = sopt->sopt_dir;
	optname = sopt->sopt_name;
	optlen = sopt->sopt_valsize;

	if (level != IPPROTO_IPV6) {
	return (EINVAL);
	}

	switch (optname) {
	case IPV6_CHECKSUM:
	/*
	* For ICMPv6 sockets, no modification allowed for checksum
	* offset, permit "no change" values to help existing apps.
	*
	* RFC3542 says: "An attempt to set IPV6_CHECKSUM
	* for an ICMPv6 socket will fail."
	* The current behavior does not meet RFC3542.
	*/
	switch (op) {
	case SOPT_SET:
	if (optlen != sizeof(int)) {
	error = EINVAL;
	break;
	}
	error = sooptcopyin(sopt, &optval, sizeof(optval),
	sizeof(optval));
	if (error)
	break;
	if (optval < -1 \|\| (optval % 2) != 0) {
	/*
	* The API assumes non-negative even offset
	* values or -1 as a special value.
	*/
	error = EINVAL;
	} else if (so->so_proto->pr_protocol ==
	IPPROTO_ICMPV6) {
	if (optval != icmp6off)
	error = EINVAL;
	} else
	inp->in6p_cksum = optval;
	break;

	case SOPT_GET:
	if (so->so_proto->pr_protocol == IPPROTO_ICMPV6)
	optval = icmp6off;
	else
	optval = inp->in6p_cksum;

	error = sooptcopyout(sopt, &optval, sizeof(optval));
	break;

	default:
	error = EINVAL;
	break;
	}
	break;

	default:
	error = ENOPROTOOPT;
	break;
	}

	return (error);
	}

	/*
	* Set up IP6 options in pcb for insertion in output packets or
	* specifying behavior of outgoing packets.
	*/
	static int
	ip6_pcbopts(struct ip6_pktopts *pktopt, struct mbuf m,
	struct socket so, struct sockopt sopt)
	{
	struct ip6_pktopts opt = pktopt;
	int error = 0;
	struct thread *td = sopt->sopt_td;

	/* turn off any old options. */
	if (opt) {
	#ifdef DIAGNOSTIC
	if (opt->ip6po_pktinfo \|\| opt->ip6po_nexthop \|\|
	opt->ip6po_hbh \|\| opt->ip6po_dest1 \|\| opt->ip6po_dest2 \|\|
	opt->ip6po_rhinfo.ip6po_rhi_rthdr)
	printf("ip6_pcbopts: all specified options are cleared.\n");
	#endif
	ip6_clearpktopts(opt, -1);
	} else {
	opt = malloc(sizeof(*opt), M_IP6OPT, M_NOWAIT);
	if (opt == NULL)
	return (ENOMEM);
	}
	*pktopt = NULL;

	if (!m \|\| m->m_len == 0) {
	/*
	* Only turning off any previous options, regardless of
	* whether the opt is just created or given.
	*/
	free(opt, M_IP6OPT);
	return (0);
	}

	/* set options specified by user. */
	if ((error = ip6_setpktopts(m, opt, NULL, (td != NULL) ?
	td->td_ucred : NULL, so->so_proto->pr_protocol)) != 0) {
	ip6_clearpktopts(opt, -1); /* XXX: discard all options */
	free(opt, M_IP6OPT);
	return (error);
	}
	*pktopt = opt;
	return (0);
	}

	/*
	* initialize ip6_pktopts. beware that there are non-zero default values in
	* the struct.
	*/
	void
	ip6_initpktopts(struct ip6_pktopts *opt)
	{

	bzero(opt, sizeof(*opt));
	opt->ip6po_hlim = -1; /* -1 means default hop limit */
	opt->ip6po_tclass = -1; /* -1 means default traffic class */
	opt->ip6po_minmtu = IP6PO_MINMTU_MCASTONLY;
	opt->ip6po_prefer_tempaddr = IP6PO_TEMPADDR_SYSTEM;
	}

	static int
	ip6_pcbopt(int optname, u_char buf, int len, struct ip6_pktopts *pktopt,
	struct ucred *cred, int uproto)
	{
	struct ip6_pktopts *opt;

	if (*pktopt == NULL) {
	*pktopt = malloc(sizeof(struct ip6_pktopts), M_IP6OPT,
	M_NOWAIT);
	if (*pktopt == NULL)
	return (ENOBUFS);
	ip6_initpktopts(*pktopt);
	}
	opt = *pktopt;

	return (ip6_setpktopt(optname, buf, len, opt, cred, 1, 0, uproto));
	}

	#define GET_PKTOPT_VAR(field, lenexpr) do { \
	if (pktopt && pktopt->field) { \
	INP_RUNLOCK(inp); \
	optdata = malloc(sopt->sopt_valsize, M_TEMP, M_WAITOK); \
	malloc_optdata = true; \
	INP_RLOCK(inp); \
	if (inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) { \
	INP_RUNLOCK(inp); \
	free(optdata, M_TEMP); \
	return (ECONNRESET); \
	} \
	pktopt = inp->in6p_outputopts; \
	if (pktopt && pktopt->field) { \
	optdatalen = min(lenexpr, sopt->sopt_valsize); \
	bcopy(&pktopt->field, optdata, optdatalen); \
	} else { \
	free(optdata, M_TEMP); \
	optdata = NULL; \
	malloc_optdata = false; \
	} \
	} \
	} while(0)

	#define GET_PKTOPT_EXT_HDR(field) GET_PKTOPT_VAR(field, \
	(((struct ip6_ext *)pktopt->field)->ip6e_len + 1) << 3)

	#define GET_PKTOPT_SOCKADDR(field) GET_PKTOPT_VAR(field, \
	pktopt->field->sa_len)

	static int
	ip6_getpcbopt(struct inpcb inp, int optname, struct sockopt sopt)
	{
	void *optdata = NULL;
	bool malloc_optdata = false;
	int optdatalen = 0;
	int error = 0;
	struct in6_pktinfo null_pktinfo;
	int deftclass = 0, on;
	int defminmtu = IP6PO_MINMTU_MCASTONLY;
	int defpreftemp = IP6PO_TEMPADDR_SYSTEM;
	struct ip6_pktopts *pktopt;

	INP_RLOCK(inp);
	pktopt = inp->in6p_outputopts;

	switch (optname) {
	case IPV6_PKTINFO:
	optdata = (void *)&null_pktinfo;
	if (pktopt && pktopt->ip6po_pktinfo) {
	bcopy(pktopt->ip6po_pktinfo, &null_pktinfo,
	sizeof(null_pktinfo));
	in6_clearscope(&null_pktinfo.ipi6_addr);
	} else {
	/* XXX: we don't have to do this every time... */
	bzero(&null_pktinfo, sizeof(null_pktinfo));
	}
	optdatalen = sizeof(struct in6_pktinfo);
	break;
	case IPV6_TCLASS:
	if (pktopt && pktopt->ip6po_tclass >= 0)
	deftclass = pktopt->ip6po_tclass;
	optdata = (void *)&deftclass;
	optdatalen = sizeof(int);
	break;
	case IPV6_HOPOPTS:
	GET_PKTOPT_EXT_HDR(ip6po_hbh);
	break;
	case IPV6_RTHDR:
	GET_PKTOPT_EXT_HDR(ip6po_rthdr);
	break;
	case IPV6_RTHDRDSTOPTS:
	GET_PKTOPT_EXT_HDR(ip6po_dest1);
	break;
	case IPV6_DSTOPTS:
	GET_PKTOPT_EXT_HDR(ip6po_dest2);
	break;
	case IPV6_NEXTHOP:
	GET_PKTOPT_SOCKADDR(ip6po_nexthop);
	break;
	case IPV6_USE_MIN_MTU:
	if (pktopt)
	defminmtu = pktopt->ip6po_minmtu;
	optdata = (void *)&defminmtu;
	optdatalen = sizeof(int);
	break;
	case IPV6_DONTFRAG:
	if (pktopt && ((pktopt->ip6po_flags) & IP6PO_DONTFRAG))
	on = 1;
	else
	on = 0;
	optdata = (void *)&on;
	optdatalen = sizeof(on);
	break;
	case IPV6_PREFER_TEMPADDR:
	if (pktopt)
	defpreftemp = pktopt->ip6po_prefer_tempaddr;
	optdata = (void *)&defpreftemp;
	optdatalen = sizeof(int);
	break;
	default: /* should not happen */
	#ifdef DIAGNOSTIC
	panic("ip6_getpcbopt: unexpected option\n");
	#endif
	INP_RUNLOCK(inp);
	return (ENOPROTOOPT);
	}
	INP_RUNLOCK(inp);

	error = sooptcopyout(sopt, optdata, optdatalen);
	if (malloc_optdata)
	free(optdata, M_TEMP);

	return (error);
	}

	void
	ip6_clearpktopts(struct ip6_pktopts *pktopt, int optname)
	{
	if (pktopt == NULL)
	return;

	if (optname == -1 \|\| optname == IPV6_PKTINFO) {
	if (pktopt->ip6po_pktinfo)
	free(pktopt->ip6po_pktinfo, M_IP6OPT);
	pktopt->ip6po_pktinfo = NULL;
	}
	if (optname == -1 \|\| optname == IPV6_HOPLIMIT)
	pktopt->ip6po_hlim = -1;
	if (optname == -1 \|\| optname == IPV6_TCLASS)
	pktopt->ip6po_tclass = -1;
	if (optname == -1 \|\| optname == IPV6_NEXTHOP) {
	if (pktopt->ip6po_nextroute.ro_nh) {
	NH_FREE(pktopt->ip6po_nextroute.ro_nh);
	pktopt->ip6po_nextroute.ro_nh = NULL;
	}
	if (pktopt->ip6po_nexthop)
	free(pktopt->ip6po_nexthop, M_IP6OPT);
	pktopt->ip6po_nexthop = NULL;
	}
	if (optname == -1 \|\| optname == IPV6_HOPOPTS) {
	if (pktopt->ip6po_hbh)
	free(pktopt->ip6po_hbh, M_IP6OPT);
	pktopt->ip6po_hbh = NULL;
	}
	if (optname == -1 \|\| optname == IPV6_RTHDRDSTOPTS) {
	if (pktopt->ip6po_dest1)
	free(pktopt->ip6po_dest1, M_IP6OPT);
	pktopt->ip6po_dest1 = NULL;
	}
	if (optname == -1 \|\| optname == IPV6_RTHDR) {
	if (pktopt->ip6po_rhinfo.ip6po_rhi_rthdr)
	free(pktopt->ip6po_rhinfo.ip6po_rhi_rthdr, M_IP6OPT);
	pktopt->ip6po_rhinfo.ip6po_rhi_rthdr = NULL;
	if (pktopt->ip6po_route.ro_nh) {
	NH_FREE(pktopt->ip6po_route.ro_nh);
	pktopt->ip6po_route.ro_nh = NULL;
	}
	}
	if (optname == -1 \|\| optname == IPV6_DSTOPTS) {
	if (pktopt->ip6po_dest2)
	free(pktopt->ip6po_dest2, M_IP6OPT);
	pktopt->ip6po_dest2 = NULL;
	}
	}

	#define PKTOPT_EXTHDRCPY(type) \
	do {\
	if (src->type) {\
	int hlen = (((struct ip6_ext *)src->type)->ip6e_len + 1) << 3;\
	dst->type = malloc(hlen, M_IP6OPT, canwait);\
	if (dst->type == NULL)\
	goto bad;\
	bcopy(src->type, dst->type, hlen);\
	}\
	} while (/CONSTCOND/ 0)

	static int
	copypktopts(struct ip6_pktopts dst, struct ip6_pktopts src, int canwait)
	{
	if (dst == NULL \|\| src == NULL) {
	printf("ip6_clearpktopts: invalid argument\n");
	return (EINVAL);
	}

	dst->ip6po_hlim = src->ip6po_hlim;
	dst->ip6po_tclass = src->ip6po_tclass;
	dst->ip6po_flags = src->ip6po_flags;
	dst->ip6po_minmtu = src->ip6po_minmtu;
	dst->ip6po_prefer_tempaddr = src->ip6po_prefer_tempaddr;
	if (src->ip6po_pktinfo) {
	dst->ip6po_pktinfo = malloc(sizeof(*dst->ip6po_pktinfo),
	M_IP6OPT, canwait);
	if (dst->ip6po_pktinfo == NULL)
	goto bad;
	dst->ip6po_pktinfo = src->ip6po_pktinfo;
	}
	if (src->ip6po_nexthop) {
	dst->ip6po_nexthop = malloc(src->ip6po_nexthop->sa_len,
	M_IP6OPT, canwait);
	if (dst->ip6po_nexthop == NULL)
	goto bad;
	bcopy(src->ip6po_nexthop, dst->ip6po_nexthop,
	src->ip6po_nexthop->sa_len);
	}
	PKTOPT_EXTHDRCPY(ip6po_hbh);
	PKTOPT_EXTHDRCPY(ip6po_dest1);
	PKTOPT_EXTHDRCPY(ip6po_dest2);
	PKTOPT_EXTHDRCPY(ip6po_rthdr); /* not copy the cached route */
	return (0);

	bad:
	ip6_clearpktopts(dst, -1);
	return (ENOBUFS);
	}
	#undef PKTOPT_EXTHDRCPY

	struct ip6_pktopts *
	ip6_copypktopts(struct ip6_pktopts *src, int canwait)
	{
	int error;
	struct ip6_pktopts *dst;

	dst = malloc(sizeof(*dst), M_IP6OPT, canwait);
	if (dst == NULL)
	return (NULL);
	ip6_initpktopts(dst);

	if ((error = copypktopts(dst, src, canwait)) != 0) {
	free(dst, M_IP6OPT);
	return (NULL);
	}

	return (dst);
	}

	void
	ip6_freepcbopts(struct ip6_pktopts *pktopt)
	{
	if (pktopt == NULL)
	return;

	ip6_clearpktopts(pktopt, -1);

	free(pktopt, M_IP6OPT);
	}

	/*
	* Set IPv6 outgoing packet options based on advanced API.
	*/
	int
	ip6_setpktopts(struct mbuf control, struct ip6_pktopts opt,
	struct ip6_pktopts stickyopt, struct ucred cred, int uproto)
	{
	struct cmsghdr *cm = NULL;

	if (control == NULL \|\| opt == NULL)
	return (EINVAL);

	ip6_initpktopts(opt);
	if (stickyopt) {
	int error;

	/*
	* If stickyopt is provided, make a local copy of the options
	* for this particular packet, then override them by ancillary
	* objects.
	* XXX: copypktopts() does not copy the cached route to a next
	* hop (if any). This is not very good in terms of efficiency,
	* but we can allow this since this option should be rarely
	* used.
	*/
	if ((error = copypktopts(opt, stickyopt, M_NOWAIT)) != 0)
	return (error);
	}

	/*
	* XXX: Currently, we assume all the optional information is stored
	* in a single mbuf.
	*/
	if (control->m_next)
	return (EINVAL);

	for (; control->m_len > 0; control->m_data += CMSG_ALIGN(cm->cmsg_len),
	control->m_len -= CMSG_ALIGN(cm->cmsg_len)) {
	int error;

	if (control->m_len < CMSG_LEN(0))
	return (EINVAL);

	cm = mtod(control, struct cmsghdr *);
	if (cm->cmsg_len == 0 \|\| cm->cmsg_len > control->m_len)
	return (EINVAL);
	if (cm->cmsg_level != IPPROTO_IPV6)
	continue;

	error = ip6_setpktopt(cm->cmsg_type, CMSG_DATA(cm),
	cm->cmsg_len - CMSG_LEN(0), opt, cred, 0, 1, uproto);
	if (error)
	return (error);
	}

	return (0);
	}

	/*
	* Set a particular packet option, as a sticky option or an ancillary data
	* item. "len" can be 0 only when it's a sticky option.
	* We have 4 cases of combination of "sticky" and "cmsg":
	* "sticky=0, cmsg=0": impossible
	* "sticky=0, cmsg=1": RFC2292 or RFC3542 ancillary data
	* "sticky=1, cmsg=0": RFC3542 socket option
	* "sticky=1, cmsg=1": RFC2292 socket option
	*/
	static int
	ip6_setpktopt(int optname, u_char buf, int len, struct ip6_pktopts opt,
	struct ucred *cred, int sticky, int cmsg, int uproto)
	{
	int minmtupolicy, preftemp;
	int error;

	if (!sticky && !cmsg) {
	#ifdef DIAGNOSTIC
	printf("ip6_setpktopt: impossible case\n");
	#endif
	return (EINVAL);
	}

	/*
	* IPV6_2292xxx is for backward compatibility to RFC2292, and should
	* not be specified in the context of RFC3542. Conversely,
	* RFC3542 types should not be specified in the context of RFC2292.
	*/
	if (!cmsg) {
	switch (optname) {
	case IPV6_2292PKTINFO:
	case IPV6_2292HOPLIMIT:
	case IPV6_2292NEXTHOP:
	case IPV6_2292HOPOPTS:
	case IPV6_2292DSTOPTS:
	case IPV6_2292RTHDR:
	case IPV6_2292PKTOPTIONS:
	return (ENOPROTOOPT);
	}
	}
	if (sticky && cmsg) {
	switch (optname) {
	case IPV6_PKTINFO:
	case IPV6_HOPLIMIT:
	case IPV6_NEXTHOP:
	case IPV6_HOPOPTS:
	case IPV6_DSTOPTS:
	case IPV6_RTHDRDSTOPTS:
	case IPV6_RTHDR:
	case IPV6_USE_MIN_MTU:
	case IPV6_DONTFRAG:
	case IPV6_TCLASS:
	case IPV6_PREFER_TEMPADDR: /* XXX: not an RFC3542 option */
	return (ENOPROTOOPT);
	}
	}

	switch (optname) {
	case IPV6_2292PKTINFO:
	case IPV6_PKTINFO:
	{
	struct ifnet *ifp = NULL;
	struct in6_pktinfo *pktinfo;

	if (len != sizeof(struct in6_pktinfo))
	return (EINVAL);

	pktinfo = (struct in6_pktinfo *)buf;

	/*
	* An application can clear any sticky IPV6_PKTINFO option by
	* doing a "regular" setsockopt with ipi6_addr being
	* in6addr_any and ipi6_ifindex being zero.
	* [RFC 3542, Section 6]
	*/
	if (optname == IPV6_PKTINFO && opt->ip6po_pktinfo &&
	pktinfo->ipi6_ifindex == 0 &&
	IN6_IS_ADDR_UNSPECIFIED(&pktinfo->ipi6_addr)) {
	ip6_clearpktopts(opt, optname);
	break;
	}

	if (uproto == IPPROTO_TCP && optname == IPV6_PKTINFO &&
	sticky && !IN6_IS_ADDR_UNSPECIFIED(&pktinfo->ipi6_addr)) {
	return (EINVAL);
	}
	if (IN6_IS_ADDR_MULTICAST(&pktinfo->ipi6_addr))
	return (EINVAL);
	/* validate the interface index if specified. */
	if (pktinfo->ipi6_ifindex > V_if_index)
	return (ENXIO);
	if (pktinfo->ipi6_ifindex) {
	ifp = ifnet_byindex(pktinfo->ipi6_ifindex);
	if (ifp == NULL)
	return (ENXIO);
	}
	if (ifp != NULL && (ifp->if_afdata[AF_INET6] == NULL \|\|
	(ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) != 0))
	return (ENETDOWN);

	if (ifp != NULL &&
	!IN6_IS_ADDR_UNSPECIFIED(&pktinfo->ipi6_addr)) {
	struct in6_ifaddr *ia;

	in6_setscope(&pktinfo->ipi6_addr, ifp, NULL);
	ia = in6ifa_ifpwithaddr(ifp, &pktinfo->ipi6_addr);
	if (ia == NULL)
	return (EADDRNOTAVAIL);
	ifa_free(&ia->ia_ifa);
	}
	/*
	* We store the address anyway, and let in6_selectsrc()
	* validate the specified address. This is because ipi6_addr
	* may not have enough information about its scope zone, and
	* we may need additional information (such as outgoing
	* interface or the scope zone of a destination address) to
	* disambiguate the scope.
	* XXX: the delay of the validation may confuse the
	* application when it is used as a sticky option.
	*/
	if (opt->ip6po_pktinfo == NULL) {
	opt->ip6po_pktinfo = malloc(sizeof(*pktinfo),
	M_IP6OPT, M_NOWAIT);
	if (opt->ip6po_pktinfo == NULL)
	return (ENOBUFS);
	}
	bcopy(pktinfo, opt->ip6po_pktinfo, sizeof(*pktinfo));
	break;
	}

	case IPV6_2292HOPLIMIT:
	case IPV6_HOPLIMIT:
	{
	int *hlimp;

	/*
	* RFC 3542 deprecated the usage of sticky IPV6_HOPLIMIT
	* to simplify the ordering among hoplimit options.
	*/
	if (optname == IPV6_HOPLIMIT && sticky)
	return (ENOPROTOOPT);

	if (len != sizeof(int))
	return (EINVAL);
	hlimp = (int *)buf;
	if (hlimp < -1 \|\| hlimp > 255)
	return (EINVAL);

	opt->ip6po_hlim = *hlimp;
	break;
	}

	case IPV6_TCLASS:
	{
	int tclass;

	if (len != sizeof(int))
	return (EINVAL);
	tclass = (int )buf;
	if (tclass < -1 \|\| tclass > 255)
	return (EINVAL);

	opt->ip6po_tclass = tclass;
	break;
	}

	case IPV6_2292NEXTHOP:
	case IPV6_NEXTHOP:
	if (cred != NULL) {
	error = priv_check_cred(cred, PRIV_NETINET_SETHDROPTS);
	if (error)
	return (error);
	}

	if (len == 0) { /* just remove the option */
	ip6_clearpktopts(opt, IPV6_NEXTHOP);
	break;
	}

	/* check if cmsg_len is large enough for sa_len */
	if (len < sizeof(struct sockaddr) \|\| len < *buf)
	return (EINVAL);

	switch (((struct sockaddr *)buf)->sa_family) {
	case AF_INET6:
	{
	struct sockaddr_in6 sa6 = (struct sockaddr_in6 )buf;
	int error;

	if (sa6->sin6_len != sizeof(struct sockaddr_in6))
	return (EINVAL);

	if (IN6_IS_ADDR_UNSPECIFIED(&sa6->sin6_addr) \|\|
	IN6_IS_ADDR_MULTICAST(&sa6->sin6_addr)) {
	return (EINVAL);
	}
	if ((error = sa6_embedscope(sa6, V_ip6_use_defzone))
	!= 0) {
	return (error);
	}
	break;
	}
	case AF_LINK: /* should eventually be supported */
	default:
	return (EAFNOSUPPORT);
	}

	/* turn off the previous option, then set the new option. */
	ip6_clearpktopts(opt, IPV6_NEXTHOP);
	opt->ip6po_nexthop = malloc(*buf, M_IP6OPT, M_NOWAIT);
	if (opt->ip6po_nexthop == NULL)
	return (ENOBUFS);
	bcopy(buf, opt->ip6po_nexthop, *buf);
	break;

	case IPV6_2292HOPOPTS:
	case IPV6_HOPOPTS:
	{
	struct ip6_hbh *hbh;
	int hbhlen;

	/*
	* XXX: We don't allow a non-privileged user to set ANY HbH
	* options, since per-option restriction has too much
	* overhead.
	*/
	if (cred != NULL) {
	error = priv_check_cred(cred, PRIV_NETINET_SETHDROPTS);
	if (error)
	return (error);
	}

	if (len == 0) {
	ip6_clearpktopts(opt, IPV6_HOPOPTS);
	break; /* just remove the option */
	}

	/* message length validation */
	if (len < sizeof(struct ip6_hbh))
	return (EINVAL);
	hbh = (struct ip6_hbh *)buf;
	hbhlen = (hbh->ip6h_len + 1) << 3;
	if (len != hbhlen)
	return (EINVAL);

	/* turn off the previous option, then set the new option. */
	ip6_clearpktopts(opt, IPV6_HOPOPTS);
	opt->ip6po_hbh = malloc(hbhlen, M_IP6OPT, M_NOWAIT);
	if (opt->ip6po_hbh == NULL)
	return (ENOBUFS);
	bcopy(hbh, opt->ip6po_hbh, hbhlen);

	break;
	}

	case IPV6_2292DSTOPTS:
	case IPV6_DSTOPTS:
	case IPV6_RTHDRDSTOPTS:
	{
	struct ip6_dest dest, *newdest = NULL;
	int destlen;

	if (cred != NULL) { /* XXX: see the comment for IPV6_HOPOPTS */
	error = priv_check_cred(cred, PRIV_NETINET_SETHDROPTS);
	if (error)
	return (error);
	}

	if (len == 0) {
	ip6_clearpktopts(opt, optname);
	break; /* just remove the option */
	}

	/* message length validation */
	if (len < sizeof(struct ip6_dest))
	return (EINVAL);
	dest = (struct ip6_dest *)buf;
	destlen = (dest->ip6d_len + 1) << 3;
	if (len != destlen)
	return (EINVAL);

	/*
	* Determine the position that the destination options header
	* should be inserted; before or after the routing header.
	*/
	switch (optname) {
	case IPV6_2292DSTOPTS:
	/*
	* The old advacned API is ambiguous on this point.
	* Our approach is to determine the position based
	* according to the existence of a routing header.
	* Note, however, that this depends on the order of the
	* extension headers in the ancillary data; the 1st
	* part of the destination options header must appear
	* before the routing header in the ancillary data,
	* too.
	* RFC3542 solved the ambiguity by introducing
	* separate ancillary data or option types.
	*/
	if (opt->ip6po_rthdr == NULL)
	newdest = &opt->ip6po_dest1;
	else
	newdest = &opt->ip6po_dest2;
	break;
	case IPV6_RTHDRDSTOPTS:
	newdest = &opt->ip6po_dest1;
	break;
	case IPV6_DSTOPTS:
	newdest = &opt->ip6po_dest2;
	break;
	}

	/* turn off the previous option, then set the new option. */
	ip6_clearpktopts(opt, optname);
	*newdest = malloc(destlen, M_IP6OPT, M_NOWAIT);
	if (*newdest == NULL)
	return (ENOBUFS);
	bcopy(dest, *newdest, destlen);

	break;
	}

	case IPV6_2292RTHDR:
	case IPV6_RTHDR:
	{
	struct ip6_rthdr *rth;
	int rthlen;

	if (len == 0) {
	ip6_clearpktopts(opt, IPV6_RTHDR);
	break; /* just remove the option */
	}

	/* message length validation */
	if (len < sizeof(struct ip6_rthdr))
	return (EINVAL);
	rth = (struct ip6_rthdr *)buf;
	rthlen = (rth->ip6r_len + 1) << 3;
	if (len != rthlen)
	return (EINVAL);

	switch (rth->ip6r_type) {
	case IPV6_RTHDR_TYPE_0:
	if (rth->ip6r_len == 0) /* must contain one addr */
	return (EINVAL);
	if (rth->ip6r_len % 2) /* length must be even */
	return (EINVAL);
	if (rth->ip6r_len / 2 != rth->ip6r_segleft)
	return (EINVAL);
	break;
	default:
	return (EINVAL); /* not supported */
	}

	/* turn off the previous option */
	ip6_clearpktopts(opt, IPV6_RTHDR);
	opt->ip6po_rthdr = malloc(rthlen, M_IP6OPT, M_NOWAIT);
	if (opt->ip6po_rthdr == NULL)
	return (ENOBUFS);
	bcopy(rth, opt->ip6po_rthdr, rthlen);

	break;
	}

	case IPV6_USE_MIN_MTU:
	if (len != sizeof(int))
	return (EINVAL);
	minmtupolicy = (int )buf;
	if (minmtupolicy != IP6PO_MINMTU_MCASTONLY &&
	minmtupolicy != IP6PO_MINMTU_DISABLE &&
	minmtupolicy != IP6PO_MINMTU_ALL) {
	return (EINVAL);
	}
	opt->ip6po_minmtu = minmtupolicy;
	break;

	case IPV6_DONTFRAG:
	if (len != sizeof(int))
	return (EINVAL);

	if (uproto == IPPROTO_TCP \|\| (int )buf == 0) {
	/*
	* we ignore this option for TCP sockets.
	* (RFC3542 leaves this case unspecified.)
	*/
	opt->ip6po_flags &= ~IP6PO_DONTFRAG;
	} else
	opt->ip6po_flags \|= IP6PO_DONTFRAG;
	break;

	case IPV6_PREFER_TEMPADDR:
	if (len != sizeof(int))
	return (EINVAL);
	preftemp = (int )buf;
	if (preftemp != IP6PO_TEMPADDR_SYSTEM &&
	preftemp != IP6PO_TEMPADDR_NOTPREFER &&
	preftemp != IP6PO_TEMPADDR_PREFER) {
	return (EINVAL);
	}
	opt->ip6po_prefer_tempaddr = preftemp;
	break;

	default:
	return (ENOPROTOOPT);
	} /* end of switch */

	return (0);
	}

	/*
	* Routine called from ip6_output() to loop back a copy of an IP6 multicast
	* packet to the input queue of a specified interface. Note that this
	* calls the output routine of the loopback "driver", but with an interface
	* pointer that might NOT be &loif -- easier than replicating that code here.
	*/
	void
	ip6_mloopback(struct ifnet ifp, struct mbuf m)
	{
	struct mbuf *copym;
	struct ip6_hdr *ip6;

	copym = m_copym(m, 0, M_COPYALL, M_NOWAIT);
	if (copym == NULL)
	return;

	/*
	* Make sure to deep-copy IPv6 header portion in case the data
	* is in an mbuf cluster, so that we can safely override the IPv6
	* header portion later.
	*/
	if (!M_WRITABLE(copym) \|\|
	copym->m_len < sizeof(struct ip6_hdr)) {
	copym = m_pullup(copym, sizeof(struct ip6_hdr));
	if (copym == NULL)
	return;
	}
	ip6 = mtod(copym, struct ip6_hdr *);
	/*
	* clear embedded scope identifiers if necessary.
	* in6_clearscope will touch the addresses only when necessary.
	*/
	in6_clearscope(&ip6->ip6_src);
	in6_clearscope(&ip6->ip6_dst);
	if (copym->m_pkthdr.csum_flags & CSUM_DELAY_DATA_IPV6) {
	copym->m_pkthdr.csum_flags \|= CSUM_DATA_VALID_IPV6 \|
	CSUM_PSEUDO_HDR;
	copym->m_pkthdr.csum_data = 0xffff;
	}
	if_simloop(ifp, copym, AF_INET6, 0);
	}

	/*
	* Chop IPv6 header off from the payload.
	*/
	static int
	ip6_splithdr(struct mbuf m, struct ip6_exthdrs exthdrs)
	{
	struct mbuf *mh;
	struct ip6_hdr *ip6;

	ip6 = mtod(m, struct ip6_hdr *);
	if (m->m_len > sizeof(*ip6)) {
	mh = m_gethdr(M_NOWAIT, MT_DATA);
	if (mh == NULL) {
	m_freem(m);
	return ENOBUFS;
	}
	m_move_pkthdr(mh, m);
	M_ALIGN(mh, sizeof(*ip6));
	m->m_len -= sizeof(*ip6);
	m->m_data += sizeof(*ip6);
	mh->m_next = m;
	m = mh;
	m->m_len = sizeof(*ip6);
	bcopy((caddr_t)ip6, mtod(m, caddr_t), sizeof(*ip6));
	}
	exthdrs->ip6e_ip6 = m;
	return 0;
	}

	/*
	* Compute IPv6 extension header length.
	*/
	int
	ip6_optlen(struct inpcb *inp)
	{
	int len;

	if (!inp->in6p_outputopts)
	return 0;

	len = 0;
	#define elen(x) \
	(((struct ip6_ext )(x)) ? (((struct ip6_ext )(x))->ip6e_len + 1) << 3 : 0)

	len += elen(inp->in6p_outputopts->ip6po_hbh);
	if (inp->in6p_outputopts->ip6po_rthdr)
	/* dest1 is valid with rthdr only */
	len += elen(inp->in6p_outputopts->ip6po_dest1);
	len += elen(inp->in6p_outputopts->ip6po_rthdr);
	len += elen(inp->in6p_outputopts->ip6po_dest2);
	return len;
	#undef elen
	}
	diff --git a/sys/powerpc/powerpc/trap.c b/sys/powerpc/powerpc/trap.c
	index d6579ecfc9cf..743299c09fdb 100644
	--- a/sys/powerpc/powerpc/trap.c
	+++ b/sys/powerpc/powerpc/trap.c
	@@ -1,988 +1,995 @@
	/*-
	* Copyright (C) 1995, 1996 Wolfgang Solfrank.
	* Copyright (C) 1995, 1996 TooLs GmbH.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. All advertising materials mentioning features or use of this software
	* must display the following acknowledgement:
	* This product includes software developed by TooLs GmbH.
	* 4. The name of TooLs GmbH may not be used to endorse or promote products
	* derived from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
	* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
	* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	* $NetBSD: trap.c,v 1.58 2002/03/04 04:07:35 dbj Exp $
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/kdb.h>
	#include <sys/proc.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/ptrace.h>
	#include <sys/reboot.h>
	#include <sys/syscall.h>
	#include <sys/sysent.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/uio.h>
	#include <sys/signalvar.h>
	#include <sys/vmmeter.h>

	#include <security/audit/audit.h>

	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_param.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_map.h>
	#include <vm/vm_page.h>

	#include <machine/_inttypes.h>
	#include <machine/altivec.h>
	#include <machine/cpu.h>
	#include <machine/db_machdep.h>
	#include <machine/fpu.h>
	#include <machine/frame.h>
	#include <machine/pcb.h>
	#include <machine/psl.h>
	#include <machine/slb.h>
	#include <machine/spr.h>
	#include <machine/sr.h>
	#include <machine/trap.h>

	/* Below matches setjmp.S */
	#define FAULTBUF_LR 21
	#define FAULTBUF_R1 1
	#define FAULTBUF_R2 2
	#define FAULTBUF_CR 22
	#define FAULTBUF_R14 3

	#define MOREARGS(sp) ((caddr_t)((uintptr_t)(sp) + \
	sizeof(struct callframe) - 3sizeof(register_t))) / more args go here */

	static void trap_fatal(struct trapframe *frame);
	static void printtrap(u_int vector, struct trapframe *frame, int isfatal,
	int user);
	static bool trap_pfault(struct trapframe frame, bool user, int signo,
	int *ucode);
	static int fix_unaligned(struct thread td, struct trapframe frame);
	static int handle_onfault(struct trapframe *frame);
	static void syscall(struct trapframe *frame);

	#if defined(__powerpc64__) && defined(AIM)
	static void normalize_inputs(void);
	#endif

	extern vm_offset_t __startkernel;

	extern int copy_fault(void);
	extern int fusufault(void);

	#ifdef KDB
	int db_trap_glue(struct trapframe ); / Called from trap_subr.S */
	#endif

	struct powerpc_exception {
	u_int vector;
	char *name;
	};

	#ifdef KDTRACE_HOOKS
	#include <sys/dtrace_bsd.h>

	int (dtrace_invop_jump_addr)(struct trapframe );
	#endif

	static struct powerpc_exception powerpc_exceptions[] = {
	{ EXC_CRIT, "critical input" },
	{ EXC_RST, "system reset" },
	{ EXC_MCHK, "machine check" },
	{ EXC_DSI, "data storage interrupt" },
	{ EXC_DSE, "data segment exception" },
	{ EXC_ISI, "instruction storage interrupt" },
	{ EXC_ISE, "instruction segment exception" },
	{ EXC_EXI, "external interrupt" },
	{ EXC_ALI, "alignment" },
	{ EXC_PGM, "program" },
	{ EXC_HEA, "hypervisor emulation assistance" },
	{ EXC_FPU, "floating-point unavailable" },
	{ EXC_APU, "auxiliary proc unavailable" },
	{ EXC_DECR, "decrementer" },
	{ EXC_FIT, "fixed-interval timer" },
	{ EXC_WDOG, "watchdog timer" },
	{ EXC_SC, "system call" },
	{ EXC_TRC, "trace" },
	{ EXC_FPA, "floating-point assist" },
	{ EXC_DEBUG, "debug" },
	{ EXC_PERF, "performance monitoring" },
	{ EXC_VEC, "altivec unavailable" },
	{ EXC_VSX, "vsx unavailable" },
	{ EXC_FAC, "facility unavailable" },
	{ EXC_ITMISS, "instruction tlb miss" },
	{ EXC_DLMISS, "data load tlb miss" },
	{ EXC_DSMISS, "data store tlb miss" },
	{ EXC_BPT, "instruction breakpoint" },
	{ EXC_SMI, "system management" },
	{ EXC_VECAST_G4, "altivec assist" },
	{ EXC_THRM, "thermal management" },
	{ EXC_RUNMODETRC, "run mode/trace" },
	{ EXC_SOFT_PATCH, "soft patch exception" },
	{ EXC_LAST, NULL }
	};

	#define ESR_BITMASK \
	"\20" \
	"\040b0\037b1\036b2\035b3\034PIL\033PRR\032PTR\031FP" \
	"\030ST\027b9\026DLK\025ILK\024b12\023b13\022BO\021PIE" \
	"\020b16\017b17\016b18\015b19\014b20\013b21\012b22\011b23" \
	"\010SPE\007EPID\006b26\005b27\004b28\003b29\002b30\001b31"
	#define MCSR_BITMASK \
	"\20" \
	"\040MCP\037ICERR\036DCERR\035TLBPERR\034L2MMU_MHIT\033b5\032b6\031b7" \
	"\030b8\027b9\026b10\025NMI\024MAV\023MEA\022b14\021IF" \
	"\020LD\017ST\016LDG\015b19\014b20\013b21\012b22\011b23" \
	"\010b24\007b25\006b26\005b27\004b28\003b29\002TLBSYNC\001BSL2_ERR"
	#define MSSSR_BITMASK \
	"\20" \
	"\040b0\037b1\036b2\035b3\034b4\033b5\032b6\031b7" \
	"\030b8\027b9\026b10\025b11\024b12\023L2TAG\022L2DAT\021L3TAG" \
	"\020L3DAT\017APE\016DPE\015TEA\014b20\013b21\012b22\011b23" \
	"\010b24\007b25\006b26\005b27\004b28\003b29\002b30\001b31"


	static const char *
	trapname(u_int vector)
	{
	struct powerpc_exception *pe;

	for (pe = powerpc_exceptions; pe->vector != EXC_LAST; pe++) {
	if (pe->vector == vector)
	return (pe->name);
	}

	return ("unknown");
	}

	static inline bool
	frame_is_trap_inst(struct trapframe *frame)
	{
	#ifdef AIM
	return (frame->exc == EXC_PGM && frame->srr1 & EXC_PGM_TRAP);
	#else
	return ((frame->cpu.booke.esr & ESR_PTR) != 0);
	#endif
	}

	void
	trap(struct trapframe *frame)
	{
	struct thread *td;
	struct proc *p;
	#ifdef KDTRACE_HOOKS
	uint32_t inst;
	#endif
	int sig, type, user;
	u_int ucode;
	ksiginfo_t ksi;
	- register_t fscr;
	+ register_t addr, fscr;

	VM_CNT_INC(v_trap);

	#ifdef KDB
	if (kdb_active) {
	kdb_reenter();
	return;
	}
	#endif

	td = curthread;
	p = td->td_proc;

	type = ucode = frame->exc;
	sig = 0;
	user = frame->srr1 & PSL_PR;
	+ addr = 0;

	CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name,
	trapname(type), user ? "user" : "kernel");

	#ifdef KDTRACE_HOOKS
	/*
	* A trap can occur while DTrace executes a probe. Before
	* executing the probe, DTrace blocks re-scheduling and sets
	* a flag in its per-cpu flags to indicate that it doesn't
	* want to fault. On returning from the probe, the no-fault
	* flag is cleared and finally re-scheduling is enabled.
	*
	* If the DTrace kernel module has registered a trap handler,
	* call it and if it returns non-zero, assume that it has
	* handled the trap and modified the trap frame so that this
	* function can return normally.
	*/
	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type) != 0)
	return;
	#endif

	if (user) {
	td->td_pticks = 0;
	td->td_frame = frame;
	+ addr = frame->srr0;
	if (td->td_cowgen != p->p_cowgen)
	thread_cow_update(td);

	/* User Mode Traps */
	switch (type) {
	case EXC_RUNMODETRC:
	case EXC_TRC:
	frame->srr1 &= ~PSL_SE;
	sig = SIGTRAP;
	ucode = TRAP_TRACE;
	break;

	#if defined(__powerpc64__) && defined(AIM)
	- case EXC_ISE:
	case EXC_DSE:
	+ addr = frame->dar;
	+ /* FALLTHROUGH */
	+ case EXC_ISE:
	/* DSE/ISE are automatically fatal with radix pmap. */
	if (radix_mmu \|\|
	handle_user_slb_spill(&p->p_vmspace->vm_pmap,
	- (type == EXC_ISE) ? frame->srr0 : frame->dar) != 0){
	+ addr) != 0){
	sig = SIGSEGV;
	ucode = SEGV_MAPERR;
	}
	break;
	#endif
	case EXC_DSI:
	+ addr = frame->dar;
	+ /* FALLTHROUGH */
	case EXC_ISI:
	if (trap_pfault(frame, true, &sig, &ucode))
	sig = 0;
	break;

	case EXC_SC:
	syscall(frame);
	break;

	case EXC_FPU:
	KASSERT((td->td_pcb->pcb_flags & PCB_FPU) != PCB_FPU,
	("FPU already enabled for thread"));
	enable_fpu(td);
	break;

	case EXC_VEC:
	KASSERT((td->td_pcb->pcb_flags & PCB_VEC) != PCB_VEC,
	("Altivec already enabled for thread"));
	enable_vec(td);
	break;

	case EXC_VSX:
	KASSERT((td->td_pcb->pcb_flags & PCB_VSX) != PCB_VSX,
	("VSX already enabled for thread"));
	if (!(td->td_pcb->pcb_flags & PCB_VEC))
	enable_vec(td);
	if (td->td_pcb->pcb_flags & PCB_FPU)
	save_fpu(td);
	td->td_pcb->pcb_flags \|= PCB_VSX;
	enable_fpu(td);
	break;

	case EXC_FAC:
	fscr = mfspr(SPR_FSCR);
	switch (fscr & FSCR_IC_MASK) {
	case FSCR_IC_HTM:
	CTR0(KTR_TRAP,
	"Hardware Transactional Memory subsystem disabled");
	sig = SIGILL;
	ucode = ILL_ILLOPC;
	break;
	case FSCR_IC_DSCR:
	td->td_pcb->pcb_flags \|= PCB_CFSCR \| PCB_CDSCR;
	fscr \|= FSCR_DSCR;
	mtspr(SPR_DSCR, 0);
	break;
	case FSCR_IC_EBB:
	td->td_pcb->pcb_flags \|= PCB_CFSCR;
	fscr \|= FSCR_EBB;
	mtspr(SPR_EBBHR, 0);
	mtspr(SPR_EBBRR, 0);
	mtspr(SPR_BESCR, 0);
	break;
	case FSCR_IC_TAR:
	td->td_pcb->pcb_flags \|= PCB_CFSCR;
	fscr \|= FSCR_TAR;
	mtspr(SPR_TAR, 0);
	break;
	case FSCR_IC_LM:
	td->td_pcb->pcb_flags \|= PCB_CFSCR;
	fscr \|= FSCR_LM;
	mtspr(SPR_LMRR, 0);
	mtspr(SPR_LMSER, 0);
	break;
	default:
	sig = SIGILL;
	ucode = ILL_ILLOPC;
	}
	mtspr(SPR_FSCR, fscr & ~FSCR_IC_MASK);
	break;
	case EXC_HEA:
	sig = SIGILL;
	ucode = ILL_ILLOPC;
	break;

	case EXC_VECAST_E:
	case EXC_VECAST_G4:
	case EXC_VECAST_G5:
	/*
	* We get a VPU assist exception for IEEE mode
	* vector operations on denormalized floats.
	* Emulating this is a giant pain, so for now,
	* just switch off IEEE mode and treat them as
	* zero.
	*/

	save_vec(td);
	td->td_pcb->pcb_vec.vscr \|= ALTIVEC_VSCR_NJ;
	enable_vec(td);
	break;

	case EXC_ALI:
	if (fix_unaligned(td, frame) != 0) {
	sig = SIGBUS;
	ucode = BUS_ADRALN;
	+ addr = frame->dar;
	}
	else
	frame->srr0 += 4;
	break;

	case EXC_DEBUG: /* Single stepping */
	mtspr(SPR_DBSR, mfspr(SPR_DBSR));
	frame->srr1 &= ~PSL_DE;
	frame->cpu.booke.dbcr0 &= ~(DBCR0_IDM \| DBCR0_IC);
	sig = SIGTRAP;
	ucode = TRAP_TRACE;
	break;

	case EXC_PGM:
	/* Identify the trap reason */
	if (frame_is_trap_inst(frame)) {
	#ifdef KDTRACE_HOOKS
	inst = fuword32((const void *)frame->srr0);
	if (inst == 0x0FFFDDDD &&
	dtrace_pid_probe_ptr != NULL) {
	(*dtrace_pid_probe_ptr)(frame);
	break;
	}
	#endif
	sig = SIGTRAP;
	ucode = TRAP_BRKPT;
	} else {
	sig = ppc_instr_emulate(frame, td);
	if (sig == SIGILL) {
	if (frame->srr1 & EXC_PGM_PRIV)
	ucode = ILL_PRVOPC;
	else if (frame->srr1 & EXC_PGM_ILLEGAL)
	ucode = ILL_ILLOPC;
	} else if (sig == SIGFPE)
	ucode = FPE_FLTINV; /* Punt for now, invalid operation. */
	}
	break;

	case EXC_MCHK:
	sig = cpu_machine_check(td, frame, &ucode);
	printtrap(frame->exc, frame, 0, (frame->srr1 & PSL_PR));
	break;

	#if defined(__powerpc64__) && defined(AIM)
	case EXC_SOFT_PATCH:
	/*
	* Point to the instruction that generated the exception to execute it again,
	* and normalize the register values.
	*/
	frame->srr0 -= 4;
	normalize_inputs();
	break;
	#endif

	default:
	trap_fatal(frame);
	}
	} else {
	/* Kernel Mode Traps */

	KASSERT(cold \|\| td->td_ucred != NULL,
	("kernel trap doesn't have ucred"));
	switch (type) {
	case EXC_PGM:
	#ifdef KDTRACE_HOOKS
	if (frame_is_trap_inst(frame)) {
	if ((uint32_t )frame->srr0 == EXC_DTRACE) {
	if (dtrace_invop_jump_addr != NULL) {
	dtrace_invop_jump_addr(frame);
	return;
	}
	}
	}
	#endif
	#ifdef KDB
	if (db_trap_glue(frame))
	return;
	#endif
	break;
	#if defined(__powerpc64__) && defined(AIM)
	case EXC_DSE:
	/* DSE on radix mmu is automatically fatal. */
	if (radix_mmu)
	break;
	if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0 &&
	(frame->dar & SEGMENT_MASK) == USER_ADDR) {
	__asm __volatile ("slbmte %0, %1" ::
	"r"(td->td_pcb->pcb_cpu.aim.usr_vsid),
	"r"(USER_SLB_SLBE));
	return;
	}
	break;
	#endif
	case EXC_DSI:
	if (trap_pfault(frame, false, NULL, NULL))
	return;
	break;
	case EXC_MCHK:
	if (handle_onfault(frame))
	return;
	break;
	default:
	break;
	}
	trap_fatal(frame);
	}

	if (sig != 0) {
	if (p->p_sysent->sv_transtrap != NULL)
	sig = (p->p_sysent->sv_transtrap)(sig, type);
	ksiginfo_init_trap(&ksi);
	ksi.ksi_signo = sig;
	ksi.ksi_code = (int) ucode; /* XXX, not POSIX */
	- ksi.ksi_addr = (void *)frame->srr0;
	+ ksi.ksi_addr = (void *)addr;
	ksi.ksi_trapno = type;
	trapsignal(td, &ksi);
	}

	userret(td, frame);
	}

	static void
	trap_fatal(struct trapframe *frame)
	{
	#ifdef KDB
	bool handled;
	#endif

	printtrap(frame->exc, frame, 1, (frame->srr1 & PSL_PR));
	#ifdef KDB
	if (debugger_on_trap) {
	kdb_why = KDB_WHY_TRAP;
	handled = kdb_trap(frame->exc, 0, frame);
	kdb_why = KDB_WHY_UNSET;
	if (handled)
	return;
	}
	#endif
	panic("%s trap", trapname(frame->exc));
	}

	static void
	cpu_printtrap(u_int vector, struct trapframe *frame, int isfatal, int user)
	{
	#ifdef AIM
	uint16_t ver;

	switch (vector) {
	case EXC_MCHK:
	ver = mfpvr() >> 16;
	if (MPC745X_P(ver))
	printf(" msssr0 = 0x%b\n",
	(int)mfspr(SPR_MSSSR0), MSSSR_BITMASK);
	case EXC_DSE:
	case EXC_DSI:
	case EXC_DTMISS:
	printf(" dsisr = 0x%lx\n",
	(u_long)frame->cpu.aim.dsisr);
	break;
	}
	#elif defined(BOOKE)
	vm_paddr_t pa;

	switch (vector) {
	case EXC_MCHK:
	pa = mfspr(SPR_MCARU);
	pa = (pa << 32) \| (u_register_t)mfspr(SPR_MCAR);
	printf(" mcsr = 0x%b\n",
	(int)mfspr(SPR_MCSR), MCSR_BITMASK);
	printf(" mcar = 0x%jx\n", (uintmax_t)pa);
	}
	printf(" esr = 0x%b\n",
	(int)frame->cpu.booke.esr, ESR_BITMASK);
	#endif
	}

	static void
	printtrap(u_int vector, struct trapframe *frame, int isfatal, int user)
	{

	printf("\n");
	printf("%s %s trap:\n", isfatal ? "fatal" : "handled",
	user ? "user" : "kernel");
	printf("\n");
	printf(" exception = 0x%x (%s)\n", vector, trapname(vector));
	switch (vector) {
	case EXC_DSE:
	case EXC_DSI:
	case EXC_DTMISS:
	case EXC_ALI:
	case EXC_MCHK:
	printf(" virtual address = 0x%" PRIxPTR "\n", frame->dar);
	break;
	case EXC_ISE:
	case EXC_ISI:
	case EXC_ITMISS:
	printf(" virtual address = 0x%" PRIxPTR "\n", frame->srr0);
	break;
	}
	cpu_printtrap(vector, frame, isfatal, user);
	printf(" srr0 = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n",
	frame->srr0, frame->srr0 - (register_t)(__startkernel - KERNBASE));
	printf(" srr1 = 0x%lx\n", (u_long)frame->srr1);
	printf(" current msr = 0x%" PRIxPTR "\n", mfmsr());
	printf(" lr = 0x%" PRIxPTR " (0x%" PRIxPTR ")\n",
	frame->lr, frame->lr - (register_t)(__startkernel - KERNBASE));
	printf(" frame = %p\n", frame);
	printf(" curthread = %p\n", curthread);
	if (curthread != NULL)
	printf(" pid = %d, comm = %s\n",
	curthread->td_proc->p_pid, curthread->td_name);
	printf("\n");
	}

	/*
	* Handles a fatal fault when we have onfault state to recover. Returns
	* non-zero if there was onfault recovery state available.
	*/
	static int
	handle_onfault(struct trapframe *frame)
	{
	struct thread *td;
	jmp_buf *fb;

	td = curthread;
	#if defined(__powerpc64__) \|\| defined(BOOKE)
	uintptr_t dispatch = (uintptr_t)td->td_pcb->pcb_onfault;

	if (dispatch == 0)
	return (0);
	/* Short-circuit radix and Book-E paths. */
	switch (dispatch) {
	case COPYFAULT:
	frame->srr0 = (uintptr_t)copy_fault;
	return (1);
	case FUSUFAULT:
	frame->srr0 = (uintptr_t)fusufault;
	return (1);
	default:
	break;
	}
	#endif
	fb = td->td_pcb->pcb_onfault;
	if (fb != NULL) {
	frame->srr0 = (*fb)->_jb[FAULTBUF_LR];
	frame->fixreg[1] = (*fb)->_jb[FAULTBUF_R1];
	frame->fixreg[2] = (*fb)->_jb[FAULTBUF_R2];
	frame->fixreg[3] = 1;
	frame->cr = (*fb)->_jb[FAULTBUF_CR];
	bcopy(&(*fb)->_jb[FAULTBUF_R14], &frame->fixreg[14],
	18 * sizeof(register_t));
	td->td_pcb->pcb_onfault = NULL; /* Returns twice, not thrice */
	return (1);
	}
	return (0);
	}

	int
	cpu_fetch_syscall_args(struct thread *td)
	{
	struct proc *p;
	struct trapframe *frame;
	struct syscall_args *sa;
	caddr_t params;
	size_t argsz;
	int error, n, i;

	p = td->td_proc;
	frame = td->td_frame;
	sa = &td->td_sa;

	sa->code = frame->fixreg[0];
	params = (caddr_t)(frame->fixreg + FIRSTARG);
	n = NARGREG;

	if (sa->code == SYS_syscall) {
	/*
	* code is first argument,
	* followed by actual args.
	*/
	sa->code = (register_t ) params;
	params += sizeof(register_t);
	n -= 1;
	} else if (sa->code == SYS___syscall) {
	/*
	* Like syscall, but code is a quad,
	* so as to maintain quad alignment
	* for the rest of the args.
	*/
	if (SV_PROC_FLAG(p, SV_ILP32)) {
	params += sizeof(register_t);
	sa->code = (register_t ) params;
	params += sizeof(register_t);
	n -= 2;
	} else {
	sa->code = (register_t ) params;
	params += sizeof(register_t);
	n -= 1;
	}
	}

	if (sa->code >= p->p_sysent->sv_size)
	sa->callp = &p->p_sysent->sv_table[0];
	else
	sa->callp = &p->p_sysent->sv_table[sa->code];

	sa->narg = sa->callp->sy_narg;

	if (SV_PROC_FLAG(p, SV_ILP32)) {
	argsz = sizeof(uint32_t);

	for (i = 0; i < n; i++)
	sa->args[i] = ((u_register_t *)(params))[i] &
	0xffffffff;
	} else {
	argsz = sizeof(uint64_t);

	for (i = 0; i < n; i++)
	sa->args[i] = ((u_register_t *)(params))[i];
	}

	if (sa->narg > n)
	error = copyin(MOREARGS(frame->fixreg[1]), sa->args + n,
	(sa->narg - n) * argsz);
	else
	error = 0;

	#ifdef __powerpc64__
	if (SV_PROC_FLAG(p, SV_ILP32) && sa->narg > n) {
	/* Expand the size of arguments copied from the stack */

	for (i = sa->narg; i >= n; i--)
	sa->args[i] = ((uint32_t *)(&sa->args[n]))[i-n];
	}
	#endif

	if (error == 0) {
	td->td_retval[0] = 0;
	td->td_retval[1] = frame->fixreg[FIRSTARG + 1];
	}
	return (error);
	}

	#include "../../kern/subr_syscall.c"

	void
	syscall(struct trapframe *frame)
	{
	struct thread *td;

	td = curthread;
	td->td_frame = frame;

	#if defined(__powerpc64__) && defined(AIM)
	/*
	* Speculatively restore last user SLB segment, which we know is
	* invalid already, since we are likely to do copyin()/copyout().
	*/
	if (td->td_pcb->pcb_cpu.aim.usr_vsid != 0)
	__asm __volatile ("slbmte %0, %1; isync" ::
	"r"(td->td_pcb->pcb_cpu.aim.usr_vsid), "r"(USER_SLB_SLBE));
	#endif

	syscallenter(td);
	syscallret(td);
	}

	static bool
	trap_pfault(struct trapframe frame, bool user, int signo, int *ucode)
	{
	vm_offset_t eva;
	struct thread *td;
	struct proc *p;
	vm_map_t map;
	vm_prot_t ftype;
	int rv, is_user;

	td = curthread;
	p = td->td_proc;
	if (frame->exc == EXC_ISI) {
	eva = frame->srr0;
	ftype = VM_PROT_EXECUTE;
	if (frame->srr1 & SRR1_ISI_PFAULT)
	ftype \|= VM_PROT_READ;
	} else {
	eva = frame->dar;
	#ifdef BOOKE
	if (frame->cpu.booke.esr & ESR_ST)
	#else
	if (frame->cpu.aim.dsisr & DSISR_STORE)
	#endif
	ftype = VM_PROT_WRITE;
	else
	ftype = VM_PROT_READ;
	}
	#if defined(__powerpc64__) && defined(AIM)
	if (radix_mmu && pmap_nofault(&p->p_vmspace->vm_pmap, eva, ftype) == 0)
	return (true);
	#endif

	if (__predict_false((td->td_pflags & TDP_NOFAULTING) == 0)) {
	/*
	* If we get a page fault while in a critical section, then
	* it is most likely a fatal kernel page fault. The kernel
	* is already going to panic trying to get a sleep lock to
	* do the VM lookup, so just consider it a fatal trap so the
	* kernel can print out a useful trap message and even get
	* to the debugger.
	*
	* If we get a page fault while holding a non-sleepable
	* lock, then it is most likely a fatal kernel page fault.
	* If WITNESS is enabled, then it's going to whine about
	* bogus LORs with various VM locks, so just skip to the
	* fatal trap handling directly.
	*/
	if (td->td_critnest != 0 \|\|
	WITNESS_CHECK(WARN_SLEEPOK \| WARN_GIANTOK, NULL,
	"Kernel page fault") != 0) {
	trap_fatal(frame);
	return (false);
	}
	}
	if (user) {
	KASSERT(p->p_vmspace != NULL, ("trap_pfault: vmspace NULL"));
	map = &p->p_vmspace->vm_map;
	} else {
	rv = pmap_decode_kernel_ptr(eva, &is_user, &eva);
	if (rv != 0)
	return (false);

	if (is_user)
	map = &p->p_vmspace->vm_map;
	else
	map = kernel_map;
	}

	/* Fault in the page. */
	rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
	/*
	* XXXDTRACE: add dtrace_doubletrap_func here?
	*/

	if (rv == KERN_SUCCESS)
	return (true);

	if (!user && handle_onfault(frame))
	return (true);

	return (false);
	}

	/*
	* For now, this only deals with the particular unaligned access case
	* that gcc tends to generate. Eventually it should handle all of the
	* possibilities that can happen on a 32-bit PowerPC in big-endian mode.
	*/

	static int
	fix_unaligned(struct thread td, struct trapframe frame)
	{
	struct thread *fputhread;
	#ifdef BOOKE
	uint32_t inst;
	#endif
	int indicator, reg;
	double *fpr;

	#ifdef __SPE__
	indicator = (frame->cpu.booke.esr & (ESR_ST\|ESR_SPE));
	if (indicator & ESR_SPE) {
	if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0)
	return (-1);
	reg = EXC_ALI_INST_RST(inst);
	fpr = (double *)td->td_pcb->pcb_vec.vr[reg];
	fputhread = PCPU_GET(vecthread);

	/* Juggle the SPE to ensure that we've initialized
	* the registers, and that their current state is in
	* the PCB.
	*/
	if (fputhread != td) {
	if (fputhread)
	save_vec(fputhread);
	enable_vec(td);
	}
	save_vec(td);

	if (!(indicator & ESR_ST)) {
	if (copyin((void *)frame->dar, fpr,
	sizeof(double)) != 0)
	return (-1);
	frame->fixreg[reg] = td->td_pcb->pcb_vec.vr[reg][1];
	enable_vec(td);
	} else {
	td->td_pcb->pcb_vec.vr[reg][1] = frame->fixreg[reg];
	if (copyout(fpr, (void *)frame->dar,
	sizeof(double)) != 0)
	return (-1);
	}
	return (0);
	}
	#else
	#ifdef BOOKE
	indicator = (frame->cpu.booke.esr & ESR_ST) ? EXC_ALI_STFD : EXC_ALI_LFD;
	#else
	indicator = EXC_ALI_OPCODE_INDICATOR(frame->cpu.aim.dsisr);
	#endif

	switch (indicator) {
	case EXC_ALI_LFD:
	case EXC_ALI_STFD:
	#ifdef BOOKE
	if (copyin((void *)frame->srr0, &inst, sizeof(inst)) != 0)
	return (-1);
	reg = EXC_ALI_INST_RST(inst);
	#else
	reg = EXC_ALI_RST(frame->cpu.aim.dsisr);
	#endif
	fpr = &td->td_pcb->pcb_fpu.fpr[reg].fpr;
	fputhread = PCPU_GET(fputhread);

	/* Juggle the FPU to ensure that we've initialized
	* the FPRs, and that their current state is in
	* the PCB.
	*/
	if (fputhread != td) {
	if (fputhread)
	save_fpu(fputhread);
	enable_fpu(td);
	}
	save_fpu(td);

	if (indicator == EXC_ALI_LFD) {
	if (copyin((void *)frame->dar, fpr,
	sizeof(double)) != 0)
	return (-1);
	enable_fpu(td);
	} else {
	if (copyout(fpr, (void *)frame->dar,
	sizeof(double)) != 0)
	return (-1);
	}
	return (0);
	break;
	}
	#endif

	return (-1);
	}

	#if defined(__powerpc64__) && defined(AIM)
	#define MSKNSHL(x, m, n) "(((" #x ") & " #m ") << " #n ")"
	#define MSKNSHR(x, m, n) "(((" #x ") & " #m ") >> " #n ")"

	/* xvcpsgndp instruction, built in opcode format.
	* This can be changed to use mnemonic after a toolchain update.
	*/
	#define XVCPSGNDP(xt, xa, xb) \
	__asm __volatile(".long (" \
	MSKNSHL(60, 0x3f, 26) " \| " \
	MSKNSHL(xt, 0x1f, 21) " \| " \
	MSKNSHL(xa, 0x1f, 16) " \| " \
	MSKNSHL(xb, 0x1f, 11) " \| " \
	MSKNSHL(240, 0xff, 3) " \| " \
	MSKNSHR(xa, 0x20, 3) " \| " \
	MSKNSHR(xa, 0x20, 4) " \| " \
	MSKNSHR(xa, 0x20, 5) ")")

	/* Macros to normalize 1 or 10 VSX registers */
	#define NORM(x) XVCPSGNDP(x, x, x)
	#define NORM10(x) \
	NORM(x ## 0); NORM(x ## 1); NORM(x ## 2); NORM(x ## 3); NORM(x ## 4); \
	NORM(x ## 5); NORM(x ## 6); NORM(x ## 7); NORM(x ## 8); NORM(x ## 9)

	static void
	normalize_inputs(void)
	{
	unsigned long msr;

	/* enable VSX */
	msr = mfmsr();
	mtmsr(msr \| PSL_VSX);

	NORM(0); NORM(1); NORM(2); NORM(3); NORM(4);
	NORM(5); NORM(6); NORM(7); NORM(8); NORM(9);
	NORM10(1); NORM10(2); NORM10(3); NORM10(4); NORM10(5);
	NORM(60); NORM(61); NORM(62); NORM(63);

	/* restore MSR */
	mtmsr(msr);
	}
	#endif

	#ifdef KDB
	int
	db_trap_glue(struct trapframe *frame)
	{

	if (!(frame->srr1 & PSL_PR)
	&& (frame->exc == EXC_TRC \|\| frame->exc == EXC_RUNMODETRC
	\|\| frame_is_trap_inst(frame)
	\|\| frame->exc == EXC_BPT
	\|\| frame->exc == EXC_DEBUG
	\|\| frame->exc == EXC_DSI)) {
	int type = frame->exc;

	/* Ignore DTrace traps. */
	if ((uint32_t )frame->srr0 == EXC_DTRACE)
	return (0);
	if (frame_is_trap_inst(frame)) {
	type = T_BREAKPOINT;
	}
	return (kdb_trap(type, 0, frame));
	}

	return (0);
	}
	#endif
	diff --git a/sys/riscv/riscv/intr_machdep.c b/sys/riscv/riscv/intr_machdep.c
	index 2a45e0905261..3e4afe11989e 100644
	--- a/sys/riscv/riscv/intr_machdep.c
	+++ b/sys/riscv/riscv/intr_machdep.c
	@@ -1,278 +1,278 @@
	/*-
	* Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com>
	* All rights reserved.
	*
	* Portions of this software were developed by SRI International and the
	* University of Cambridge Computer Laboratory under DARPA/AFRL contract
	* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
	*
	* Portions of this software were developed by the University of Cambridge
	* Computer Laboratory as part of the CTSRD Project, with support from the
	* UK Higher Education Innovation Fund (HEIF).
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/module.h>
	#include <sys/cpuset.h>
	#include <sys/interrupt.h>
	#include <sys/smp.h>

	#include <machine/bus.h>
	#include <machine/clock.h>
	#include <machine/cpu.h>
	#include <machine/cpufunc.h>
	#include <machine/frame.h>
	#include <machine/intr.h>
	#include <machine/sbi.h>

	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	#include <dev/ofw/ofw_bus_subr.h>

	#ifdef SMP
	#include <machine/smp.h>
	#endif

	void intr_irq_handler(struct trapframe *tf);

	struct intc_irqsrc {
	struct intr_irqsrc isrc;
	u_int irq;
	};

	struct intc_irqsrc isrcs[INTC_NIRQS];

	static void
	riscv_mask_irq(void *source)
	{
	uintptr_t irq;

	irq = (uintptr_t)source;

	switch (irq) {
	case IRQ_TIMER_SUPERVISOR:
	csr_clear(sie, SIE_STIE);
	break;
	case IRQ_SOFTWARE_USER:
	csr_clear(sie, SIE_USIE);
	break;
	case IRQ_SOFTWARE_SUPERVISOR:
	csr_clear(sie, SIE_SSIE);
	break;
	default:
	panic("Unknown irq %d\n", irq);
	}
	}

	static void
	riscv_unmask_irq(void *source)
	{
	uintptr_t irq;

	irq = (uintptr_t)source;

	switch (irq) {
	case IRQ_TIMER_SUPERVISOR:
	csr_set(sie, SIE_STIE);
	break;
	case IRQ_SOFTWARE_USER:
	csr_set(sie, SIE_USIE);
	break;
	case IRQ_SOFTWARE_SUPERVISOR:
	csr_set(sie, SIE_SSIE);
	break;
	default:
	panic("Unknown irq %d\n", irq);
	}
	}

	int
	riscv_setup_intr(const char name, driver_filter_t filt,
	void (handler)(void), void arg, int irq, int flags, void *cookiep)
	{
	struct intr_irqsrc *isrc;
	int error;

	if (irq < 0 \|\| irq >= INTC_NIRQS)
	panic("%s: unknown intr %d", __func__, irq);

	isrc = &isrcs[irq].isrc;
	if (isrc->isrc_event == NULL) {
	error = intr_event_create(&isrc->isrc_event, isrc, 0, irq,
	riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq);
	if (error)
	return (error);
	riscv_unmask_irq((void*)(uintptr_t)irq);
	}

	error = intr_event_add_handler(isrc->isrc_event, name,
	filt, handler, arg, intr_priority(flags), flags, cookiep);
	if (error) {
	printf("Failed to setup intr: %d\n", irq);
	return (error);
	}

	return (0);
	}

	int
	riscv_teardown_intr(void *ih)
	{

	/* TODO */

	return (0);
	}

	void
	riscv_cpu_intr(struct trapframe *frame)
	{
	struct intr_irqsrc *isrc;
	int active_irq;

	critical_enter();

	KASSERT(frame->tf_scause & EXCP_INTR,
	("riscv_cpu_intr: wrong frame passed"));

	- active_irq = (frame->tf_scause & EXCP_MASK);
	+ active_irq = frame->tf_scause & EXCP_MASK;

	switch (active_irq) {
	case IRQ_SOFTWARE_USER:
	case IRQ_SOFTWARE_SUPERVISOR:
	case IRQ_TIMER_SUPERVISOR:
	isrc = &isrcs[active_irq].isrc;
	if (intr_isrc_dispatch(isrc, frame) != 0)
	printf("stray interrupt %d\n", active_irq);
	break;
	case IRQ_EXTERNAL_SUPERVISOR:
	intr_irq_handler(frame);
	break;
	default:
	break;
	}

	critical_exit();
	}

	#ifdef SMP
	void
	riscv_setup_ipihandler(driver_filter_t *filt)
	{

	riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR,
	INTR_TYPE_MISC, NULL);
	}

	void
	riscv_unmask_ipi(void)
	{

	csr_set(sie, SIE_SSIE);
	}

	/* Sending IPI */
	static void
	ipi_send(struct pcpu *pc, int ipi)
	{
	u_long mask;

	CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi);

	atomic_set_32(&pc->pc_pending_ipis, ipi);
	mask = (1 << pc->pc_hart);

	sbi_send_ipi(&mask);

	CTR1(KTR_SMP, "%s: sent", __func__);
	}

	void
	ipi_all_but_self(u_int ipi)
	{
	cpuset_t other_cpus;

	other_cpus = all_cpus;
	CPU_CLR(PCPU_GET(cpuid), &other_cpus);

	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
	ipi_selected(other_cpus, ipi);
	}

	void
	ipi_cpu(int cpu, u_int ipi)
	{
	cpuset_t cpus;

	CPU_ZERO(&cpus);
	CPU_SET(cpu, &cpus);

	ipi_send(cpuid_to_pcpu[cpu], ipi);
	}

	void
	ipi_selected(cpuset_t cpus, u_int ipi)
	{
	struct pcpu *pc;
	u_long mask;

	CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);

	mask = 0;
	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
	if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
	CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc,
	ipi);
	atomic_set_32(&pc->pc_pending_ipis, ipi);
	mask \|= (1 << pc->pc_hart);
	}
	}
	sbi_send_ipi(&mask);
	}
	#endif

	/* Interrupt machdep initialization routine. */
	static void
	intc_init(void *dummy __unused)
	{
	int error;
	int i;

	for (i = 0; i < INTC_NIRQS; i++) {
	isrcs[i].irq = i;
	error = intr_isrc_register(&isrcs[i].isrc, NULL,
	0, "intc,%u", i);
	if (error != 0)
	printf("Can't register interrupt %d\n", i);
	}
	}

	SYSINIT(intc_init, SI_SUB_INTR, SI_ORDER_MIDDLE, intc_init, NULL);
	diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c
	index c9b6a1ea6f47..aa9ec534669d 100644
	--- a/sys/riscv/riscv/trap.c
	+++ b/sys/riscv/riscv/trap.c
	@@ -1,368 +1,372 @@
	/*-
	* Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com>
	* All rights reserved.
	*
	* Portions of this software were developed by SRI International and the
	* University of Cambridge Computer Laboratory under DARPA/AFRL contract
	* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
	*
	* Portions of this software were developed by the University of Cambridge
	* Computer Laboratory as part of the CTSRD Project, with support from the
	* UK Higher Education Innovation Fund (HEIF).
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/bus.h>
	#include <sys/proc.h>
	#include <sys/ptrace.h>
	#include <sys/syscall.h>
	#include <sys/sysent.h>
	#ifdef KDB
	#include <sys/kdb.h>
	#endif

	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_map.h>
	#include <vm/vm_param.h>
	#include <vm/vm_extern.h>

	#ifdef FPE
	#include <machine/fpe.h>
	#endif
	#include <machine/frame.h>
	#include <machine/pcb.h>
	#include <machine/pcpu.h>

	#include <machine/resource.h>
	#include <machine/intr.h>

	#ifdef KDTRACE_HOOKS
	#include <sys/dtrace_bsd.h>
	#endif

	int (dtrace_invop_jump_addr)(struct trapframe );

	extern register_t fsu_intr_fault;

	/* Called from exception.S */
	void do_trap_supervisor(struct trapframe *);
	void do_trap_user(struct trapframe *);

	static __inline void
	-call_trapsignal(struct thread td, int sig, int code, void addr)
	+call_trapsignal(struct thread td, int sig, int code, void addr, int trapno)
	{
	ksiginfo_t ksi;

	ksiginfo_init_trap(&ksi);
	ksi.ksi_signo = sig;
	ksi.ksi_code = code;
	ksi.ksi_addr = addr;
	+ ksi.ksi_trapno = trapno;
	trapsignal(td, &ksi);
	}

	int
	cpu_fetch_syscall_args(struct thread *td)
	{
	struct proc *p;
	register_t *ap;
	struct syscall_args *sa;
	int nap;

	nap = NARGREG;
	p = td->td_proc;
	sa = &td->td_sa;
	ap = &td->td_frame->tf_a[0];

	sa->code = td->td_frame->tf_t[0];

	if (sa->code == SYS_syscall \|\| sa->code == SYS___syscall) {
	sa->code = *ap++;
	nap--;
	}

	if (sa->code >= p->p_sysent->sv_size)
	sa->callp = &p->p_sysent->sv_table[0];
	else
	sa->callp = &p->p_sysent->sv_table[sa->code];

	sa->narg = sa->callp->sy_narg;
	memcpy(sa->args, ap, nap * sizeof(register_t));
	if (sa->narg > nap)
	panic("TODO: Could we have more then %d args?", NARGREG);

	td->td_retval[0] = 0;
	td->td_retval[1] = 0;

	return (0);
	}

	#include "../../kern/subr_syscall.c"

	static void
	dump_regs(struct trapframe *frame)
	{
	int n;
	int i;

	n = (sizeof(frame->tf_t) / sizeof(frame->tf_t[0]));
	for (i = 0; i < n; i++)
	printf("t[%d] == 0x%016lx\n", i, frame->tf_t[i]);

	n = (sizeof(frame->tf_s) / sizeof(frame->tf_s[0]));
	for (i = 0; i < n; i++)
	printf("s[%d] == 0x%016lx\n", i, frame->tf_s[i]);

	n = (sizeof(frame->tf_a) / sizeof(frame->tf_a[0]));
	for (i = 0; i < n; i++)
	printf("a[%d] == 0x%016lx\n", i, frame->tf_a[i]);

	printf("ra == 0x%016lx\n", frame->tf_ra);
	printf("sp == 0x%016lx\n", frame->tf_sp);
	printf("gp == 0x%016lx\n", frame->tf_gp);
	printf("tp == 0x%016lx\n", frame->tf_tp);

	printf("sepc == 0x%016lx\n", frame->tf_sepc);
	printf("sstatus == 0x%016lx\n", frame->tf_sstatus);
	}

	static void
	svc_handler(struct trapframe *frame)
	{
	struct thread *td;

	td = curthread;
	td->td_frame = frame;

	syscallenter(td);
	syscallret(td);
	}

	static void
	data_abort(struct trapframe *frame, int usermode)
	{
	struct vm_map *map;
	uint64_t stval;
	struct thread *td;
	struct pcb *pcb;
	vm_prot_t ftype;
	vm_offset_t va;
	struct proc *p;
	int error, sig, ucode;

	#ifdef KDB
	if (kdb_active) {
	kdb_reenter();
	return;
	}
	#endif

	td = curthread;
	p = td->td_proc;
	pcb = td->td_pcb;
	stval = frame->tf_stval;

	if (td->td_critnest != 0 \|\| td->td_intr_nesting_level != 0 \|\|
	WITNESS_CHECK(WARN_SLEEPOK \| WARN_GIANTOK, NULL,
	"Kernel page fault") != 0)
	goto fatal;

	if (usermode)
	map = &td->td_proc->p_vmspace->vm_map;
	else if (stval >= VM_MAX_USER_ADDRESS)
	map = kernel_map;
	else {
	if (pcb->pcb_onfault == 0)
	goto fatal;
	map = &td->td_proc->p_vmspace->vm_map;
	}

	va = trunc_page(stval);

	if ((frame->tf_scause == EXCP_FAULT_STORE) \|\|
	(frame->tf_scause == EXCP_STORE_PAGE_FAULT)) {
	ftype = VM_PROT_WRITE;
	} else if (frame->tf_scause == EXCP_INST_PAGE_FAULT) {
	ftype = VM_PROT_EXECUTE;
	} else {
	ftype = VM_PROT_READ;
	}

	if (pmap_fault_fixup(map->pmap, va, ftype))
	goto done;

	error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode);
	if (error != KERN_SUCCESS) {
	if (usermode) {
	- call_trapsignal(td, sig, ucode, (void *)stval);
	+ call_trapsignal(td, sig, ucode, (void *)stval,
	+ frame->tf_scause & EXCP_MASK);
	} else {
	if (pcb->pcb_onfault != 0) {
	frame->tf_a[0] = error;
	frame->tf_sepc = pcb->pcb_onfault;
	return;
	}
	goto fatal;
	}
	}

	done:
	if (usermode)
	userret(td, frame);
	return;

	fatal:
	dump_regs(frame);
	panic("Fatal page fault at %#lx: %#016lx", frame->tf_sepc, stval);
	}

	void
	do_trap_supervisor(struct trapframe *frame)
	{
	uint64_t exception;

	/* Ensure we came from supervisor mode, interrupts disabled */
	KASSERT((csr_read(sstatus) & (SSTATUS_SPP \| SSTATUS_SIE)) ==
	SSTATUS_SPP, ("Came from S mode with interrupts enabled"));

	- exception = (frame->tf_scause & EXCP_MASK);
	+ exception = frame->tf_scause & EXCP_MASK;
	if (frame->tf_scause & EXCP_INTR) {
	/* Interrupt */
	riscv_cpu_intr(frame);
	return;
	}

	#ifdef KDTRACE_HOOKS
	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception))
	return;
	#endif

	CTR3(KTR_TRAP, "do_trap_supervisor: curthread: %p, sepc: %lx, frame: %p",
	curthread, frame->tf_sepc, frame);

	switch (exception) {
	case EXCP_FAULT_LOAD:
	case EXCP_FAULT_STORE:
	case EXCP_FAULT_FETCH:
	case EXCP_STORE_PAGE_FAULT:
	case EXCP_LOAD_PAGE_FAULT:
	data_abort(frame, 0);
	break;
	case EXCP_BREAKPOINT:
	#ifdef KDTRACE_HOOKS
	if (dtrace_invop_jump_addr != NULL &&
	dtrace_invop_jump_addr(frame) == 0)
	break;
	#endif
	#ifdef KDB
	kdb_trap(exception, 0, frame);
	#else
	dump_regs(frame);
	panic("No debugger in kernel.\n");
	#endif
	break;
	case EXCP_ILLEGAL_INSTRUCTION:
	dump_regs(frame);
	panic("Illegal instruction at 0x%016lx\n", frame->tf_sepc);
	break;
	default:
	dump_regs(frame);
	panic("Unknown kernel exception %x trap value %lx\n",
	exception, frame->tf_stval);
	}
	}

	void
	do_trap_user(struct trapframe *frame)
	{
	uint64_t exception;
	struct thread *td;
	struct pcb *pcb;

	td = curthread;
	td->td_frame = frame;
	pcb = td->td_pcb;

	/* Ensure we came from usermode, interrupts disabled */
	KASSERT((csr_read(sstatus) & (SSTATUS_SPP \| SSTATUS_SIE)) == 0,
	("Came from U mode with interrupts enabled"));

	- exception = (frame->tf_scause & EXCP_MASK);
	+ exception = frame->tf_scause & EXCP_MASK;
	if (frame->tf_scause & EXCP_INTR) {
	/* Interrupt */
	riscv_cpu_intr(frame);
	return;
	}

	CTR3(KTR_TRAP, "do_trap_user: curthread: %p, sepc: %lx, frame: %p",
	curthread, frame->tf_sepc, frame);

	switch (exception) {
	case EXCP_FAULT_LOAD:
	case EXCP_FAULT_STORE:
	case EXCP_FAULT_FETCH:
	case EXCP_STORE_PAGE_FAULT:
	case EXCP_LOAD_PAGE_FAULT:
	case EXCP_INST_PAGE_FAULT:
	data_abort(frame, 1);
	break;
	case EXCP_USER_ECALL:
	frame->tf_sepc += 4; /* Next instruction */
	svc_handler(frame);
	break;
	case EXCP_ILLEGAL_INSTRUCTION:
	#ifdef FPE
	if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) {
	/*
	* May be a FPE trap. Enable FPE usage
	* for this thread and try again.
	*/
	fpe_state_clear();
	frame->tf_sstatus &= ~SSTATUS_FS_MASK;
	frame->tf_sstatus \|= SSTATUS_FS_CLEAN;
	pcb->pcb_fpflags \|= PCB_FP_STARTED;
	break;
	}
	#endif
	- call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc);
	+ call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc,
	+ exception);
	userret(td, frame);
	break;
	case EXCP_BREAKPOINT:
	- call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc);
	+ call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc,
	+ exception);
	userret(td, frame);
	break;
	default:
	dump_regs(frame);
	panic("Unknown userland exception %x, trap value %lx\n",
	exception, frame->tf_stval);
	}
	}
	diff --git a/sys/security/mac/mac_framework.c b/sys/security/mac/mac_framework.c
	index aea3789d572f..41c0779fa78e 100644
	--- a/sys/security/mac/mac_framework.c
	+++ b/sys/security/mac/mac_framework.c
	@@ -1,721 +1,724 @@
	/*-
	* Copyright (c) 1999-2002, 2006, 2009 Robert N. M. Watson
	* Copyright (c) 2001 Ilmar S. Habibulin
	* Copyright (c) 2001-2005 Networks Associates Technology, Inc.
	* Copyright (c) 2005-2006 SPARTA, Inc.
	* Copyright (c) 2008-2009 Apple Inc.
	* All rights reserved.
	*
	* This software was developed by Robert Watson and Ilmar Habibulin for the
	* TrustedBSD Project.
	*
	* This software was developed for the FreeBSD Project in part by Network
	* Associates Laboratories, the Security Research Division of Network
	* Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
	* as part of the DARPA CHATS research program.
	*
	* This software was enhanced by SPARTA ISSO under SPAWAR contract
	* N66001-04-C-6019 ("SEFOS").
	*
	* This software was developed at the University of Cambridge Computer
	* Laboratory with support from a grant from Google, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	/*-
	* Framework for extensible kernel access control. This file contains core
	* kernel infrastructure for the TrustedBSD MAC Framework, including policy
	* registration, versioning, locking, error composition operator, and system
	* calls.
	*
	* The MAC Framework implements three programming interfaces:
	*
	* - The kernel MAC interface, defined in mac_framework.h, and invoked
	* throughout the kernel to request security decisions, notify of security
	* related events, etc.
	*
	* - The MAC policy module interface, defined in mac_policy.h, which is
	* implemented by MAC policy modules and invoked by the MAC Framework to
	* forward kernel security requests and notifications to policy modules.
	*
	* - The user MAC API, defined in mac.h, which allows user programs to query
	* and set label state on objects.
	*
	* The majority of the MAC Framework implementation may be found in
	* src/sys/security/mac. Sample policy modules may be found in
	* src/sys/security/mac_*.
	*/

	#include "opt_mac.h"

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/condvar.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/mac.h>
	#include <sys/module.h>
	#include <sys/rmlock.h>
	#include <sys/sdt.h>
	#include <sys/sx.h>
	#include <sys/sysctl.h>

	#include <security/mac/mac_framework.h>
	#include <security/mac/mac_internal.h>
	#include <security/mac/mac_policy.h>

	/*
	* DTrace SDT providers for MAC.
	*/
	SDT_PROVIDER_DEFINE(mac);
	SDT_PROVIDER_DEFINE(mac_framework);

	SDT_PROBE_DEFINE2(mac, , policy, modevent, "int",
	"struct mac_policy_conf *");
	SDT_PROBE_DEFINE1(mac, , policy, register,
	"struct mac_policy_conf *");
	SDT_PROBE_DEFINE1(mac, , policy, unregister,
	"struct mac_policy_conf *");

	/*
	* Root sysctl node for all MAC and MAC policy controls.
	*/
	SYSCTL_NODE(_security, OID_AUTO, mac, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"TrustedBSD MAC policy controls");

	/*
	* Declare that the kernel provides MAC support, version 3 (FreeBSD 7.x).
	* This permits modules to refuse to be loaded if the necessary support isn't
	* present, even if it's pre-boot.
	*/
	MODULE_VERSION(kernel_mac_support, MAC_VERSION);

	static unsigned int mac_version = MAC_VERSION;
	SYSCTL_UINT(_security_mac, OID_AUTO, version, CTLFLAG_RD, &mac_version, 0,
	"");

	/*
	* Flags for inlined checks. Note this would be best hotpatched at runtime.
	* The following is a band-aid.
	*
	* Use FPFLAG for hooks running in commonly executed paths and FPFLAG_RARE
	* for the rest.
	*/
	#define FPFLAG(f) \
	bool __read_frequently mac_##f##_fp_flag

	#define FPFLAG_RARE(f) \
	bool __read_mostly mac_##f##_fp_flag

	FPFLAG(priv_check);
	FPFLAG(priv_grant);
	FPFLAG(vnode_check_lookup);
	FPFLAG(vnode_check_open);
	FPFLAG(vnode_check_stat);
	FPFLAG(vnode_check_read);
	FPFLAG(vnode_check_write);
	FPFLAG(vnode_check_mmap);
	FPFLAG_RARE(vnode_check_poll);
	+FPFLAG_RARE(vnode_check_rename_from);

	#undef FPFLAG
	#undef FPFLAG_RARE

	/*
	* Labels consist of a indexed set of "slots", which are allocated policies
	* as required. The MAC Framework maintains a bitmask of slots allocated so
	* far to prevent reuse. Slots cannot be reused, as the MAC Framework
	* guarantees that newly allocated slots in labels will be NULL unless
	* otherwise initialized, and because we do not have a mechanism to garbage
	* collect slots on policy unload. As labeled policies tend to be statically
	* loaded during boot, and not frequently unloaded and reloaded, this is not
	* generally an issue.
	*/
	#if MAC_MAX_SLOTS > 32
	#error "MAC_MAX_SLOTS too large"
	#endif

	static unsigned int mac_max_slots = MAC_MAX_SLOTS;
	static unsigned int mac_slot_offsets_free = (1 << MAC_MAX_SLOTS) - 1;
	SYSCTL_UINT(_security_mac, OID_AUTO, max_slots, CTLFLAG_RD, &mac_max_slots,
	0, "");

	/*
	* Has the kernel started generating labeled objects yet? All read/write
	* access to this variable is serialized during the boot process. Following
	* the end of serialization, we don't update this flag; no locking.
	*/
	static int mac_late = 0;

	/*
	* Each policy declares a mask of object types requiring labels to be
	* allocated for them. For convenience, we combine and cache the bitwise or
	* of the per-policy object flags to track whether we will allocate a label
	* for an object type at run-time.
	*/
	uint64_t mac_labeled;
	SYSCTL_UQUAD(_security_mac, OID_AUTO, labeled, CTLFLAG_RD, &mac_labeled, 0,
	"Mask of object types being labeled");

	MALLOC_DEFINE(M_MACTEMP, "mactemp", "MAC temporary label storage");

	/*
	* MAC policy modules are placed in one of two lists: mac_static_policy_list,
	* for policies that are loaded early and cannot be unloaded, and
	* mac_policy_list, which holds policies either loaded later in the boot
	* cycle or that may be unloaded. The static policy list does not require
	* locks to iterate over, but the dynamic list requires synchronization.
	* Support for dynamic policy loading can be compiled out using the
	* MAC_STATIC kernel option.
	*
	* The dynamic policy list is protected by two locks: modifying the list
	* requires both locks to be held exclusively. One of the locks,
	* mac_policy_rm, is acquired over policy entry points that will never sleep;
	* the other, mac_policy_rms, is acquired over policy entry points that may
	* sleep. The former category will be used when kernel locks may be held
	* over calls to the MAC Framework, during network processing in ithreads,
	* etc. The latter will tend to involve potentially blocking memory
	* allocations, extended attribute I/O, etc.
	*/
	#ifndef MAC_STATIC
	static struct rmlock mac_policy_rm; /* Non-sleeping entry points. */
	static struct rmslock mac_policy_rms; /* Sleeping entry points. */
	#endif

	struct mac_policy_list_head mac_policy_list;
	struct mac_policy_list_head mac_static_policy_list;
	u_int mac_policy_count; /* Registered policy count. */

	static void mac_policy_xlock(void);
	static void mac_policy_xlock_assert(void);
	static void mac_policy_xunlock(void);

	void
	mac_policy_slock_nosleep(struct rm_priotracker *tracker)
	{

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rm_rlock(&mac_policy_rm, tracker);
	#endif
	}

	void
	mac_policy_slock_sleep(void)
	{

	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
	"mac_policy_slock_sleep");

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rms_rlock(&mac_policy_rms);
	#endif
	}

	void
	mac_policy_sunlock_nosleep(struct rm_priotracker *tracker)
	{

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rm_runlock(&mac_policy_rm, tracker);
	#endif
	}

	void
	mac_policy_sunlock_sleep(void)
	{

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rms_runlock(&mac_policy_rms);
	#endif
	}

	static void
	mac_policy_xlock(void)
	{

	WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
	"mac_policy_xlock()");

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rms_wlock(&mac_policy_rms);
	rm_wlock(&mac_policy_rm);
	#endif
	}

	static void
	mac_policy_xunlock(void)
	{

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rm_wunlock(&mac_policy_rm);
	rms_wunlock(&mac_policy_rms);
	#endif
	}

	static void
	mac_policy_xlock_assert(void)
	{

	#ifndef MAC_STATIC
	if (!mac_late)
	return;

	rm_assert(&mac_policy_rm, RA_WLOCKED);
	#endif
	}

	/*
	* Initialize the MAC subsystem, including appropriate SMP locks.
	*/
	static void
	mac_init(void)
	{

	LIST_INIT(&mac_static_policy_list);
	LIST_INIT(&mac_policy_list);
	mac_labelzone_init();

	#ifndef MAC_STATIC
	rm_init_flags(&mac_policy_rm, "mac_policy_rm", RM_NOWITNESS \|
	RM_RECURSE);
	rms_init(&mac_policy_rms, "mac_policy_rms");
	#endif
	}

	/*
	* For the purposes of modules that want to know if they were loaded "early",
	* set the mac_late flag once we've processed modules either linked into the
	* kernel, or loaded before the kernel startup.
	*/
	static void
	mac_late_init(void)
	{

	mac_late = 1;
	}

	/*
	* Given a policy, derive from its set of non-NULL label init methods what
	* object types the policy is interested in.
	*/
	static uint64_t
	mac_policy_getlabeled(struct mac_policy_conf *mpc)
	{
	uint64_t labeled;

	#define MPC_FLAG(method, flag) \
	if (mpc->mpc_ops->mpo_ ## method != NULL) \
	labeled \|= (flag); \

	labeled = 0;
	MPC_FLAG(cred_init_label, MPC_OBJECT_CRED);
	MPC_FLAG(proc_init_label, MPC_OBJECT_PROC);
	MPC_FLAG(vnode_init_label, MPC_OBJECT_VNODE);
	MPC_FLAG(inpcb_init_label, MPC_OBJECT_INPCB);
	MPC_FLAG(socket_init_label, MPC_OBJECT_SOCKET);
	MPC_FLAG(devfs_init_label, MPC_OBJECT_DEVFS);
	MPC_FLAG(mbuf_init_label, MPC_OBJECT_MBUF);
	MPC_FLAG(ipq_init_label, MPC_OBJECT_IPQ);
	MPC_FLAG(ifnet_init_label, MPC_OBJECT_IFNET);
	MPC_FLAG(bpfdesc_init_label, MPC_OBJECT_BPFDESC);
	MPC_FLAG(pipe_init_label, MPC_OBJECT_PIPE);
	MPC_FLAG(mount_init_label, MPC_OBJECT_MOUNT);
	MPC_FLAG(posixsem_init_label, MPC_OBJECT_POSIXSEM);
	MPC_FLAG(posixshm_init_label, MPC_OBJECT_POSIXSHM);
	MPC_FLAG(sysvmsg_init_label, MPC_OBJECT_SYSVMSG);
	MPC_FLAG(sysvmsq_init_label, MPC_OBJECT_SYSVMSQ);
	MPC_FLAG(sysvsem_init_label, MPC_OBJECT_SYSVSEM);
	MPC_FLAG(sysvshm_init_label, MPC_OBJECT_SYSVSHM);
	MPC_FLAG(syncache_init_label, MPC_OBJECT_SYNCACHE);
	MPC_FLAG(ip6q_init_label, MPC_OBJECT_IP6Q);

	#undef MPC_FLAG
	return (labeled);
	}

	/*
	* When policies are loaded or unloaded, walk the list of registered policies
	* and built mac_labeled, a bitmask representing the union of all objects
	* requiring labels across all policies.
	*/
	static void
	mac_policy_update(void)
	{
	struct mac_policy_conf *mpc;

	mac_policy_xlock_assert();

	mac_labeled = 0;
	mac_policy_count = 0;
	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {
	mac_labeled \|= mac_policy_getlabeled(mpc);
	mac_policy_count++;
	}
	LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {
	mac_labeled \|= mac_policy_getlabeled(mpc);
	mac_policy_count++;
	}
	}

	/*
	* There are frequently used code paths which check for rarely installed
	* policies. Gross hack below enables doing it in a cheap manner.
	*/

	#define FPO(f) (offsetof(struct mac_policy_ops, mpo_##f) / sizeof(uintptr_t))

	struct mac_policy_fastpath_elem {
	int count;
	bool *flag;
	size_t offset;
	};

	struct mac_policy_fastpath_elem mac_policy_fastpath_array[] = {
	{ .offset = FPO(priv_check), .flag = &mac_priv_check_fp_flag },
	{ .offset = FPO(priv_grant), .flag = &mac_priv_grant_fp_flag },
	{ .offset = FPO(vnode_check_lookup),
	.flag = &mac_vnode_check_lookup_fp_flag },
	{ .offset = FPO(vnode_check_open),
	.flag = &mac_vnode_check_open_fp_flag },
	{ .offset = FPO(vnode_check_stat),
	.flag = &mac_vnode_check_stat_fp_flag },
	{ .offset = FPO(vnode_check_read),
	.flag = &mac_vnode_check_read_fp_flag },
	{ .offset = FPO(vnode_check_write),
	.flag = &mac_vnode_check_write_fp_flag },
	{ .offset = FPO(vnode_check_mmap),
	.flag = &mac_vnode_check_mmap_fp_flag },
	{ .offset = FPO(vnode_check_poll),
	.flag = &mac_vnode_check_poll_fp_flag },
	+ { .offset = FPO(vnode_check_rename_from),
	+ .flag = &mac_vnode_check_rename_from_fp_flag },
	};

	static void
	mac_policy_fastpath_enable(struct mac_policy_fastpath_elem *mpfe)
	{

	MPASS(mpfe->count >= 0);
	mpfe->count++;
	if (mpfe->count == 1) {
	MPASS(*mpfe->flag == false);
	*mpfe->flag = true;
	}
	}

	static void
	mac_policy_fastpath_disable(struct mac_policy_fastpath_elem *mpfe)
	{

	MPASS(mpfe->count >= 1);
	mpfe->count--;
	if (mpfe->count == 0) {
	MPASS(*mpfe->flag == true);
	*mpfe->flag = false;
	}
	}

	static void
	mac_policy_fastpath_register(struct mac_policy_conf *mpc)
	{
	struct mac_policy_fastpath_elem *mpfe;
	uintptr_t **ops;
	int i;

	mac_policy_xlock_assert();

	ops = (uintptr_t **)mpc->mpc_ops;
	for (i = 0; i < nitems(mac_policy_fastpath_array); i++) {
	mpfe = &mac_policy_fastpath_array[i];
	if (ops[mpfe->offset] != NULL)
	mac_policy_fastpath_enable(mpfe);
	}
	}

	static void
	mac_policy_fastpath_unregister(struct mac_policy_conf *mpc)
	{
	struct mac_policy_fastpath_elem *mpfe;
	uintptr_t **ops;
	int i;

	mac_policy_xlock_assert();

	ops = (uintptr_t **)mpc->mpc_ops;
	for (i = 0; i < nitems(mac_policy_fastpath_array); i++) {
	mpfe = &mac_policy_fastpath_array[i];
	if (ops[mpfe->offset] != NULL)
	mac_policy_fastpath_disable(mpfe);
	}
	}

	#undef FPO

	static int
	mac_policy_register(struct mac_policy_conf *mpc)
	{
	struct mac_policy_conf *tmpc;
	int error, slot, static_entry;

	error = 0;

	/*
	* We don't technically need exclusive access while !mac_late, but
	* hold it for assertion consistency.
	*/
	mac_policy_xlock();

	/*
	* If the module can potentially be unloaded, or we're loading late,
	* we have to stick it in the non-static list and pay an extra
	* performance overhead. Otherwise, we can pay a light locking cost
	* and stick it in the static list.
	*/
	static_entry = (!mac_late &&
	!(mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_UNLOADOK));

	if (static_entry) {
	LIST_FOREACH(tmpc, &mac_static_policy_list, mpc_list) {
	if (strcmp(tmpc->mpc_name, mpc->mpc_name) == 0) {
	error = EEXIST;
	goto out;
	}
	}
	} else {
	LIST_FOREACH(tmpc, &mac_policy_list, mpc_list) {
	if (strcmp(tmpc->mpc_name, mpc->mpc_name) == 0) {
	error = EEXIST;
	goto out;
	}
	}
	}
	if (mpc->mpc_field_off != NULL) {
	slot = ffs(mac_slot_offsets_free);
	if (slot == 0) {
	error = ENOMEM;
	goto out;
	}
	slot--;
	mac_slot_offsets_free &= ~(1 << slot);
	*mpc->mpc_field_off = slot;
	}
	mpc->mpc_runtime_flags \|= MPC_RUNTIME_FLAG_REGISTERED;

	/*
	* If we're loading a MAC module after the framework has initialized,
	* it has to go into the dynamic list. If we're loading it before
	* we've finished initializing, it can go into the static list with
	* weaker locker requirements.
	*/
	if (static_entry)
	LIST_INSERT_HEAD(&mac_static_policy_list, mpc, mpc_list);
	else
	LIST_INSERT_HEAD(&mac_policy_list, mpc, mpc_list);

	/*
	* Per-policy initialization. Currently, this takes place under the
	* exclusive lock, so policies must not sleep in their init method.
	* In the future, we may want to separate "init" from "start", with
	* "init" occurring without the lock held. Likewise, on tear-down,
	* breaking out "stop" from "destroy".
	*/
	if (mpc->mpc_ops->mpo_init != NULL)
	(*(mpc->mpc_ops->mpo_init))(mpc);

	mac_policy_fastpath_register(mpc);

	mac_policy_update();

	SDT_PROBE1(mac, , policy, register, mpc);
	printf("Security policy loaded: %s (%s)\n", mpc->mpc_fullname,
	mpc->mpc_name);

	out:
	mac_policy_xunlock();
	return (error);
	}

	static int
	mac_policy_unregister(struct mac_policy_conf *mpc)
	{

	/*
	* If we fail the load, we may get a request to unload. Check to see
	* if we did the run-time registration, and if not, silently succeed.
	*/
	mac_policy_xlock();
	if ((mpc->mpc_runtime_flags & MPC_RUNTIME_FLAG_REGISTERED) == 0) {
	mac_policy_xunlock();
	return (0);
	}
	#if 0
	/*
	* Don't allow unloading modules with private data.
	*/
	if (mpc->mpc_field_off != NULL) {
	mac_policy_xunlock();
	return (EBUSY);
	}
	#endif
	/*
	* Only allow the unload to proceed if the module is unloadable by
	* its own definition.
	*/
	if ((mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_UNLOADOK) == 0) {
	mac_policy_xunlock();
	return (EBUSY);
	}

	mac_policy_fastpath_unregister(mpc);

	if (mpc->mpc_ops->mpo_destroy != NULL)
	(*(mpc->mpc_ops->mpo_destroy))(mpc);

	LIST_REMOVE(mpc, mpc_list);
	mpc->mpc_runtime_flags &= ~MPC_RUNTIME_FLAG_REGISTERED;
	mac_policy_update();
	mac_policy_xunlock();

	SDT_PROBE1(mac, , policy, unregister, mpc);
	printf("Security policy unload: %s (%s)\n", mpc->mpc_fullname,
	mpc->mpc_name);

	return (0);
	}

	/*
	* Allow MAC policy modules to register during boot, etc.
	*/
	int
	mac_policy_modevent(module_t mod, int type, void *data)
	{
	struct mac_policy_conf *mpc;
	int error;

	error = 0;
	mpc = (struct mac_policy_conf *) data;

	#ifdef MAC_STATIC
	if (mac_late) {
	printf("mac_policy_modevent: MAC_STATIC and late\n");
	return (EBUSY);
	}
	#endif

	SDT_PROBE2(mac, , policy, modevent, type, mpc);
	switch (type) {
	case MOD_LOAD:
	if (mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_NOTLATE &&
	mac_late) {
	printf("mac_policy_modevent: can't load %s policy "
	"after booting\n", mpc->mpc_name);
	error = EBUSY;
	break;
	}
	error = mac_policy_register(mpc);
	break;
	case MOD_UNLOAD:
	/* Don't unregister the module if it was never registered. */
	if ((mpc->mpc_runtime_flags & MPC_RUNTIME_FLAG_REGISTERED)
	!= 0)
	error = mac_policy_unregister(mpc);
	else
	error = 0;
	break;
	default:
	error = EOPNOTSUPP;
	break;
	}

	return (error);
	}

	/*
	* Define an error value precedence, and given two arguments, selects the
	* value with the higher precedence.
	*/
	int
	mac_error_select(int error1, int error2)
	{

	/* Certain decision-making errors take top priority. */
	if (error1 == EDEADLK \|\| error2 == EDEADLK)
	return (EDEADLK);

	/* Invalid arguments should be reported where possible. */
	if (error1 == EINVAL \|\| error2 == EINVAL)
	return (EINVAL);

	/* Precedence goes to "visibility", with both process and file. */
	if (error1 == ESRCH \|\| error2 == ESRCH)
	return (ESRCH);

	if (error1 == ENOENT \|\| error2 == ENOENT)
	return (ENOENT);

	/* Precedence goes to DAC/MAC protections. */
	if (error1 == EACCES \|\| error2 == EACCES)
	return (EACCES);

	/* Precedence goes to privilege. */
	if (error1 == EPERM \|\| error2 == EPERM)
	return (EPERM);

	/* Precedence goes to error over success; otherwise, arbitrary. */
	if (error1 != 0)
	return (error1);
	return (error2);
	}

	int
	mac_check_structmac_consistent(struct mac *mac)
	{

	/* Require that labels have a non-zero length. */
	if (mac->m_buflen > MAC_MAX_LABEL_BUF_LEN \|\|
	mac->m_buflen <= sizeof(""))
	return (EINVAL);

	return (0);
	}

	SYSINIT(mac, SI_SUB_MAC, SI_ORDER_FIRST, mac_init, NULL);
	SYSINIT(mac_late, SI_SUB_MAC_LATE, SI_ORDER_FIRST, mac_late_init, NULL);
	diff --git a/sys/security/mac/mac_framework.h b/sys/security/mac/mac_framework.h
	index e917eeb3c893..6ae634bd2dfe 100644
	--- a/sys/security/mac/mac_framework.h
	+++ b/sys/security/mac/mac_framework.h
	@@ -1,567 +1,579 @@
	/*-
	* Copyright (c) 1999-2002, 2007-2011 Robert N. M. Watson
	* Copyright (c) 2001-2005 Networks Associates Technology, Inc.
	* Copyright (c) 2005-2006 SPARTA, Inc.
	* All rights reserved.
	*
	* This software was developed by Robert Watson for the TrustedBSD Project.
	*
	* This software was developed for the FreeBSD Project in part by Network
	* Associates Laboratories, the Security Research Division of Network
	* Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
	* as part of the DARPA CHATS research program.
	*
	* This software was enhanced by SPARTA ISSO under SPAWAR contract
	* N66001-04-C-6019 ("SEFOS").
	*
	* This software was developed at the University of Cambridge Computer
	* Laboratory with support from a grant from Google, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	/*
	* Kernel interface for Mandatory Access Control -- how kernel services
	* interact with the TrustedBSD MAC Framework.
	*/

	#ifndef _SECURITY_MAC_MAC_FRAMEWORK_H_
	#define _SECURITY_MAC_MAC_FRAMEWORK_H_

	#ifndef _KERNEL
	#error "no user-serviceable parts inside"
	#endif

	struct auditinfo;
	struct auditinfo_addr;
	struct bpf_d;
	struct cdev;
	struct componentname;
	struct devfs_dirent;
	struct ifnet;
	struct ifreq;
	struct image_params;
	struct inpcb;
	struct ip6q;
	struct ipq;
	struct ksem;
	struct label;
	struct m_tag;
	struct mac;
	struct mbuf;
	struct mount;
	struct msg;
	struct msqid_kernel;
	struct proc;
	struct semid_kernel;
	struct shmfd;
	struct shmid_kernel;
	struct sockaddr;
	struct socket;
	struct sysctl_oid;
	struct sysctl_req;
	struct pipepair;
	struct thread;
	struct timespec;
	struct ucred;
	struct vattr;
	struct vnode;
	struct vop_setlabel_args;

	#include <sys/acl.h> /* XXX acl_type_t */
	#include <sys/types.h> /* accmode_t */

	/*
	* Entry points to the TrustedBSD MAC Framework from the remainder of the
	* kernel: entry points are named based on a principle object type and an
	* action relating to it. They are sorted alphabetically first by object
	* type and then action. In some situations, the principle object type is
	* obvious, and in other cases, less so as multiple objects may be inolved
	* in the operation.
	*/
	int mac_bpfdesc_check_receive(struct bpf_d d, struct ifnet ifp);
	void mac_bpfdesc_create(struct ucred cred, struct bpf_d d);
	void mac_bpfdesc_create_mbuf(struct bpf_d d, struct mbuf m);
	void mac_bpfdesc_destroy(struct bpf_d *);
	void mac_bpfdesc_init(struct bpf_d *);

	void mac_cred_associate_nfsd(struct ucred *cred);
	int mac_cred_check_setaudit(struct ucred cred, struct auditinfo ai);
	int mac_cred_check_setaudit_addr(struct ucred *cred,
	struct auditinfo_addr *aia);
	int mac_cred_check_setauid(struct ucred *cred, uid_t auid);
	int mac_cred_check_setegid(struct ucred *cred, gid_t egid);
	int mac_cred_check_seteuid(struct ucred *cred, uid_t euid);
	int mac_cred_check_setgid(struct ucred *cred, gid_t gid);
	int mac_cred_check_setgroups(struct ucred *cred, int ngroups,
	gid_t *gidset);
	int mac_cred_check_setregid(struct ucred *cred, gid_t rgid, gid_t egid);
	int mac_cred_check_setresgid(struct ucred *cred, gid_t rgid, gid_t egid,
	gid_t sgid);
	int mac_cred_check_setresuid(struct ucred *cred, uid_t ruid, uid_t euid,
	uid_t suid);
	int mac_cred_check_setreuid(struct ucred *cred, uid_t ruid, uid_t euid);
	int mac_cred_check_setuid(struct ucred *cred, uid_t uid);
	int mac_cred_check_visible(struct ucred cr1, struct ucred cr2);
	void mac_cred_copy(struct ucred cr1, struct ucred cr2);
	void mac_cred_create_init(struct ucred *cred);
	void mac_cred_create_swapper(struct ucred *cred);
	void mac_cred_destroy(struct ucred *);
	void mac_cred_init(struct ucred *);

	void mac_devfs_create_device(struct ucred cred, struct mount mp,
	struct cdev dev, struct devfs_dirent de);
	void mac_devfs_create_directory(struct mount mp, char dirname,
	int dirnamelen, struct devfs_dirent *de);
	void mac_devfs_create_symlink(struct ucred cred, struct mount mp,
	struct devfs_dirent dd, struct devfs_dirent de);
	void mac_devfs_destroy(struct devfs_dirent *);
	void mac_devfs_init(struct devfs_dirent *);
	void mac_devfs_update(struct mount mp, struct devfs_dirent de,
	struct vnode *vp);
	void mac_devfs_vnode_associate(struct mount mp, struct devfs_dirent de,
	struct vnode *vp);

	int mac_ifnet_check_transmit(struct ifnet ifp, struct mbuf m);
	void mac_ifnet_create(struct ifnet *ifp);
	void mac_ifnet_create_mbuf(struct ifnet ifp, struct mbuf m);
	void mac_ifnet_destroy(struct ifnet *);
	void mac_ifnet_init(struct ifnet *);
	int mac_ifnet_ioctl_get(struct ucred cred, struct ifreq ifr,
	struct ifnet *ifp);
	int mac_ifnet_ioctl_set(struct ucred cred, struct ifreq ifr,
	struct ifnet *ifp);

	int mac_inpcb_check_deliver(struct inpcb inp, struct mbuf m);
	int mac_inpcb_check_visible(struct ucred cred, struct inpcb inp);
	void mac_inpcb_create(struct socket so, struct inpcb inp);
	void mac_inpcb_create_mbuf(struct inpcb inp, struct mbuf m);
	void mac_inpcb_destroy(struct inpcb *);
	int mac_inpcb_init(struct inpcb *, int);
	void mac_inpcb_sosetlabel(struct socket so, struct inpcb inp);

	void mac_ip6q_create(struct mbuf m, struct ip6q q6);
	void mac_ip6q_destroy(struct ip6q *q6);
	int mac_ip6q_init(struct ip6q *q6, int);
	int mac_ip6q_match(struct mbuf m, struct ip6q q6);
	void mac_ip6q_reassemble(struct ip6q q6, struct mbuf m);
	void mac_ip6q_update(struct mbuf m, struct ip6q q6);

	void mac_ipq_create(struct mbuf m, struct ipq q);
	void mac_ipq_destroy(struct ipq *q);
	int mac_ipq_init(struct ipq *q, int);
	int mac_ipq_match(struct mbuf m, struct ipq q);
	void mac_ipq_reassemble(struct ipq q, struct mbuf m);
	void mac_ipq_update(struct mbuf m, struct ipq q);

	int mac_kenv_check_dump(struct ucred *cred);
	int mac_kenv_check_get(struct ucred cred, char name);
	int mac_kenv_check_set(struct ucred cred, char name, char *value);
	int mac_kenv_check_unset(struct ucred cred, char name);

	int mac_kld_check_load(struct ucred cred, struct vnode vp);
	int mac_kld_check_stat(struct ucred *cred);

	void mac_mbuf_copy(struct mbuf , struct mbuf );
	int mac_mbuf_init(struct mbuf *, int);

	void mac_mbuf_tag_copy(struct m_tag , struct m_tag );
	void mac_mbuf_tag_destroy(struct m_tag *);
	int mac_mbuf_tag_init(struct m_tag *, int);

	int mac_mount_check_stat(struct ucred cred, struct mount mp);
	void mac_mount_create(struct ucred cred, struct mount mp);
	void mac_mount_destroy(struct mount *);
	void mac_mount_init(struct mount *);

	void mac_netinet_arp_send(struct ifnet ifp, struct mbuf m);
	void mac_netinet_firewall_reply(struct mbuf mrecv, struct mbuf msend);
	void mac_netinet_firewall_send(struct mbuf *m);
	void mac_netinet_fragment(struct mbuf m, struct mbuf frag);
	void mac_netinet_icmp_reply(struct mbuf mrecv, struct mbuf msend);
	void mac_netinet_icmp_replyinplace(struct mbuf *m);
	void mac_netinet_igmp_send(struct ifnet ifp, struct mbuf m);
	void mac_netinet_tcp_reply(struct mbuf *m);

	void mac_netinet6_nd6_send(struct ifnet ifp, struct mbuf m);

	int mac_pipe_check_ioctl(struct ucred cred, struct pipepair pp,
	unsigned long cmd, void *data);
	int mac_pipe_check_poll(struct ucred cred, struct pipepair pp);
	int mac_pipe_check_read(struct ucred cred, struct pipepair pp);
	int mac_pipe_check_stat(struct ucred cred, struct pipepair pp);
	int mac_pipe_check_write(struct ucred cred, struct pipepair pp);
	void mac_pipe_create(struct ucred cred, struct pipepair pp);
	void mac_pipe_destroy(struct pipepair *);
	void mac_pipe_init(struct pipepair *);
	int mac_pipe_label_set(struct ucred cred, struct pipepair pp,
	struct label *label);

	int mac_posixsem_check_getvalue(struct ucred *active_cred,
	struct ucred file_cred, struct ksem ks);
	int mac_posixsem_check_open(struct ucred cred, struct ksem ks);
	int mac_posixsem_check_post(struct ucred *active_cred,
	struct ucred file_cred, struct ksem ks);
	int mac_posixsem_check_setmode(struct ucred cred, struct ksem ks,
	mode_t mode);
	int mac_posixsem_check_setowner(struct ucred cred, struct ksem ks,
	uid_t uid, gid_t gid);
	int mac_posixsem_check_stat(struct ucred *active_cred,
	struct ucred file_cred, struct ksem ks);
	int mac_posixsem_check_unlink(struct ucred cred, struct ksem ks);
	int mac_posixsem_check_wait(struct ucred *active_cred,
	struct ucred file_cred, struct ksem ks);
	void mac_posixsem_create(struct ucred cred, struct ksem ks);
	void mac_posixsem_destroy(struct ksem *);
	void mac_posixsem_init(struct ksem *);

	int mac_posixshm_check_create(struct ucred cred, const char path);
	int mac_posixshm_check_mmap(struct ucred cred, struct shmfd shmfd,
	int prot, int flags);
	int mac_posixshm_check_open(struct ucred cred, struct shmfd shmfd,
	accmode_t accmode);
	int mac_posixshm_check_read(struct ucred *active_cred,
	struct ucred file_cred, struct shmfd shmfd);
	int mac_posixshm_check_setmode(struct ucred cred, struct shmfd shmfd,
	mode_t mode);
	int mac_posixshm_check_setowner(struct ucred cred, struct shmfd shmfd,
	uid_t uid, gid_t gid);
	int mac_posixshm_check_stat(struct ucred *active_cred,
	struct ucred file_cred, struct shmfd shmfd);
	int mac_posixshm_check_truncate(struct ucred *active_cred,
	struct ucred file_cred, struct shmfd shmfd);
	int mac_posixshm_check_unlink(struct ucred cred, struct shmfd shmfd);
	int mac_posixshm_check_write(struct ucred *active_cred,
	struct ucred file_cred, struct shmfd shmfd);
	void mac_posixshm_create(struct ucred cred, struct shmfd shmfd);
	void mac_posixshm_destroy(struct shmfd *);
	void mac_posixshm_init(struct shmfd *);

	int mac_priv_check_impl(struct ucred *cred, int priv);
	+#ifdef MAC
	extern bool mac_priv_check_fp_flag;
	+#else
	+#define mac_priv_check_fp_flag 0
	+#endif
	static inline int
	mac_priv_check(struct ucred *cred, int priv)
	{

	if (__predict_false(mac_priv_check_fp_flag))
	return (mac_priv_check_impl(cred, priv));
	return (0);
	}

	int mac_priv_grant_impl(struct ucred *cred, int priv);
	+#ifdef MAC
	extern bool mac_priv_grant_fp_flag;
	+#else
	+#define mac_priv_grant_fp_flag 0
	+#endif
	static inline int
	mac_priv_grant(struct ucred *cred, int priv)
	{

	if (__predict_false(mac_priv_grant_fp_flag))
	return (mac_priv_grant_impl(cred, priv));
	return (EPERM);
	}

	int mac_proc_check_debug(struct ucred cred, struct proc p);
	int mac_proc_check_sched(struct ucred cred, struct proc p);
	int mac_proc_check_signal(struct ucred cred, struct proc p,
	int signum);
	int mac_proc_check_wait(struct ucred cred, struct proc p);
	void mac_proc_destroy(struct proc *);
	void mac_proc_init(struct proc *);
	void mac_proc_vm_revoke(struct thread *td);
	int mac_execve_enter(struct image_params imgp, struct mac mac_p);
	void mac_execve_exit(struct image_params *imgp);
	void mac_execve_interpreter_enter(struct vnode *interpvp,
	struct label **interplabel);
	void mac_execve_interpreter_exit(struct label *interpvplabel);

	int mac_socket_check_accept(struct ucred cred, struct socket so);
	int mac_socket_check_bind(struct ucred cred, struct socket so,
	struct sockaddr *sa);
	int mac_socket_check_connect(struct ucred cred, struct socket so,
	struct sockaddr *sa);
	int mac_socket_check_create(struct ucred *cred, int domain, int type,
	int proto);
	int mac_socket_check_deliver(struct socket so, struct mbuf m);
	int mac_socket_check_listen(struct ucred cred, struct socket so);
	int mac_socket_check_poll(struct ucred cred, struct socket so);
	int mac_socket_check_receive(struct ucred cred, struct socket so);
	int mac_socket_check_send(struct ucred cred, struct socket so);
	int mac_socket_check_stat(struct ucred cred, struct socket so);
	int mac_socket_check_visible(struct ucred cred, struct socket so);
	void mac_socket_create_mbuf(struct socket so, struct mbuf m);
	void mac_socket_create(struct ucred cred, struct socket so);
	void mac_socket_destroy(struct socket *);
	int mac_socket_init(struct socket *, int);
	void mac_socket_newconn(struct socket oldso, struct socket newso);
	int mac_getsockopt_label(struct ucred cred, struct socket so,
	struct mac *extmac);
	int mac_getsockopt_peerlabel(struct ucred cred, struct socket so,
	struct mac *extmac);
	int mac_setsockopt_label(struct ucred cred, struct socket so,
	struct mac *extmac);

	void mac_socketpeer_set_from_mbuf(struct mbuf m, struct socket so);
	void mac_socketpeer_set_from_socket(struct socket *oldso,
	struct socket *newso);

	void mac_syncache_create(struct label l, struct inpcb inp);
	void mac_syncache_create_mbuf(struct label l, struct mbuf m);
	void mac_syncache_destroy(struct label **l);
	int mac_syncache_init(struct label **l);

	int mac_system_check_acct(struct ucred cred, struct vnode vp);
	int mac_system_check_audit(struct ucred cred, void record, int length);
	int mac_system_check_auditctl(struct ucred cred, struct vnode vp);
	int mac_system_check_auditon(struct ucred *cred, int cmd);
	int mac_system_check_reboot(struct ucred *cred, int howto);
	int mac_system_check_swapon(struct ucred cred, struct vnode vp);
	int mac_system_check_swapoff(struct ucred cred, struct vnode vp);
	int mac_system_check_sysctl(struct ucred cred, struct sysctl_oid oidp,
	void arg1, int arg2, struct sysctl_req req);

	void mac_sysvmsg_cleanup(struct msg *msgptr);
	void mac_sysvmsg_create(struct ucred cred, struct msqid_kernel msqkptr,
	struct msg *msgptr);
	void mac_sysvmsg_destroy(struct msg *);
	void mac_sysvmsg_init(struct msg *);

	int mac_sysvmsq_check_msgmsq(struct ucred cred, struct msg msgptr,
	struct msqid_kernel *msqkptr);
	int mac_sysvmsq_check_msgrcv(struct ucred cred, struct msg msgptr);
	int mac_sysvmsq_check_msgrmid(struct ucred cred, struct msg msgptr);
	int mac_sysvmsq_check_msqctl(struct ucred *cred,
	struct msqid_kernel *msqkptr, int cmd);
	int mac_sysvmsq_check_msqget(struct ucred *cred,
	struct msqid_kernel *msqkptr);
	int mac_sysvmsq_check_msqrcv(struct ucred *cred,
	struct msqid_kernel *msqkptr);
	int mac_sysvmsq_check_msqsnd(struct ucred *cred,
	struct msqid_kernel *msqkptr);
	void mac_sysvmsq_cleanup(struct msqid_kernel *msqkptr);
	void mac_sysvmsq_create(struct ucred cred, struct msqid_kernel msqkptr);
	void mac_sysvmsq_destroy(struct msqid_kernel *);
	void mac_sysvmsq_init(struct msqid_kernel *);

	int mac_sysvsem_check_semctl(struct ucred *cred,
	struct semid_kernel *semakptr, int cmd);
	int mac_sysvsem_check_semget(struct ucred *cred,
	struct semid_kernel *semakptr);
	int mac_sysvsem_check_semop(struct ucred *cred,
	struct semid_kernel *semakptr, size_t accesstype);
	void mac_sysvsem_cleanup(struct semid_kernel *semakptr);
	void mac_sysvsem_create(struct ucred *cred,
	struct semid_kernel *semakptr);
	void mac_sysvsem_destroy(struct semid_kernel *);
	void mac_sysvsem_init(struct semid_kernel *);

	int mac_sysvshm_check_shmat(struct ucred *cred,
	struct shmid_kernel *shmsegptr, int shmflg);
	int mac_sysvshm_check_shmctl(struct ucred *cred,
	struct shmid_kernel *shmsegptr, int cmd);
	int mac_sysvshm_check_shmdt(struct ucred *cred,
	struct shmid_kernel *shmsegptr);
	int mac_sysvshm_check_shmget(struct ucred *cred,
	struct shmid_kernel *shmsegptr, int shmflg);
	void mac_sysvshm_cleanup(struct shmid_kernel *shmsegptr);
	void mac_sysvshm_create(struct ucred *cred,
	struct shmid_kernel *shmsegptr);
	void mac_sysvshm_destroy(struct shmid_kernel *);
	void mac_sysvshm_init(struct shmid_kernel *);

	void mac_thread_userret(struct thread *td);

	#ifdef DEBUG_VFS_LOCKS
	void mac_vnode_assert_locked(struct vnode vp, const char func);
	#else
	#define mac_vnode_assert_locked(vp, func) do { } while (0)
	#endif

	int mac_vnode_associate_extattr(struct mount mp, struct vnode vp);
	void mac_vnode_associate_singlelabel(struct mount mp, struct vnode vp);
	int mac_vnode_check_access(struct ucred cred, struct vnode vp,
	accmode_t accmode);
	int mac_vnode_check_chdir(struct ucred cred, struct vnode dvp);
	int mac_vnode_check_chroot(struct ucred cred, struct vnode dvp);
	int mac_vnode_check_create(struct ucred cred, struct vnode dvp,
	struct componentname cnp, struct vattr vap);
	int mac_vnode_check_deleteacl(struct ucred cred, struct vnode vp,
	acl_type_t type);
	int mac_vnode_check_deleteextattr(struct ucred cred, struct vnode vp,
	int attrnamespace, const char *name);
	int mac_vnode_check_exec(struct ucred cred, struct vnode vp,
	struct image_params *imgp);
	int mac_vnode_check_getacl(struct ucred cred, struct vnode vp,
	acl_type_t type);
	int mac_vnode_check_getextattr(struct ucred cred, struct vnode vp,
	int attrnamespace, const char *name);
	int mac_vnode_check_link(struct ucred cred, struct vnode dvp,
	struct vnode vp, struct componentname cnp);
	int mac_vnode_check_listextattr(struct ucred cred, struct vnode vp,
	int attrnamespace);

	int mac_vnode_check_lookup_impl(struct ucred cred, struct vnode dvp,
	struct componentname *cnp);
	extern bool mac_vnode_check_lookup_fp_flag;
	#define mac_vnode_check_lookup_enabled() __predict_false(mac_vnode_check_lookup_fp_flag)
	static inline int
	mac_vnode_check_lookup(struct ucred cred, struct vnode dvp,
	struct componentname *cnp)
	{

	mac_vnode_assert_locked(dvp, "mac_vnode_check_lookup");
	if (mac_vnode_check_lookup_enabled())
	return (mac_vnode_check_lookup_impl(cred, dvp, cnp));
	return (0);
	}

	int mac_vnode_check_mmap_impl(struct ucred cred, struct vnode vp, int prot,
	int flags);
	extern bool mac_vnode_check_mmap_fp_flag;
	static inline int
	mac_vnode_check_mmap(struct ucred cred, struct vnode vp, int prot,
	int flags)
	{

	mac_vnode_assert_locked(vp, "mac_vnode_check_mmap");
	if (__predict_false(mac_vnode_check_mmap_fp_flag))
	return (mac_vnode_check_mmap_impl(cred, vp, prot, flags));
	return (0);
	}

	int mac_vnode_check_open_impl(struct ucred cred, struct vnode vp,
	accmode_t accmode);
	extern bool mac_vnode_check_open_fp_flag;
	static inline int
	mac_vnode_check_open(struct ucred cred, struct vnode vp,
	accmode_t accmode)
	{

	mac_vnode_assert_locked(vp, "mac_vnode_check_open");
	if (__predict_false(mac_vnode_check_open_fp_flag))
	return (mac_vnode_check_open_impl(cred, vp, accmode));
	return (0);
	}

	int mac_vnode_check_mprotect(struct ucred cred, struct vnode vp,
	int prot);

	#define mac_vnode_check_poll_enabled() __predict_false(mac_vnode_check_poll_fp_flag)
	#ifdef MAC
	extern bool mac_vnode_check_poll_fp_flag;
	int mac_vnode_check_poll(struct ucred *active_cred,
	struct ucred file_cred, struct vnode vp);
	#else
	#define mac_vnode_check_poll_fp_flag 0
	static inline int
	mac_vnode_check_poll(struct ucred active_cred, struct ucred file_cred,
	struct vnode *vp)
	{

	return (0);
	}
	#endif
	int mac_vnode_check_readdir(struct ucred cred, struct vnode vp);
	int mac_vnode_check_readlink(struct ucred cred, struct vnode vp);
	+#define mac_vnode_check_rename_from_enabled() __predict_false(mac_vnode_check_rename_from_fp_flag)
	+#ifdef MAC
	+extern bool mac_vnode_check_rename_from_fp_flag;
	+#endif
	int mac_vnode_check_rename_from(struct ucred cred, struct vnode dvp,
	struct vnode vp, struct componentname cnp);
	int mac_vnode_check_rename_to(struct ucred cred, struct vnode dvp,
	struct vnode vp, int samedir, struct componentname cnp);
	int mac_vnode_check_revoke(struct ucred cred, struct vnode vp);
	int mac_vnode_check_setacl(struct ucred cred, struct vnode vp,
	acl_type_t type, struct acl *acl);
	int mac_vnode_check_setextattr(struct ucred cred, struct vnode vp,
	int attrnamespace, const char *name);
	int mac_vnode_check_setflags(struct ucred cred, struct vnode vp,
	u_long flags);
	int mac_vnode_check_setmode(struct ucred cred, struct vnode vp,
	mode_t mode);
	int mac_vnode_check_setowner(struct ucred cred, struct vnode vp,
	uid_t uid, gid_t gid);
	int mac_vnode_check_setutimes(struct ucred cred, struct vnode vp,
	struct timespec atime, struct timespec mtime);

	int mac_vnode_check_stat_impl(struct ucred *active_cred,
	struct ucred file_cred, struct vnode vp);
	extern bool mac_vnode_check_stat_fp_flag;
	static inline int
	mac_vnode_check_stat(struct ucred active_cred, struct ucred file_cred,
	struct vnode *vp)
	{

	mac_vnode_assert_locked(vp, "mac_vnode_check_stat");
	if (__predict_false(mac_vnode_check_stat_fp_flag))
	return (mac_vnode_check_stat_impl(active_cred, file_cred, vp));
	return (0);
	}

	int mac_vnode_check_read_impl(struct ucred *active_cred,
	struct ucred file_cred, struct vnode vp);
	extern bool mac_vnode_check_read_fp_flag;
	static inline int
	mac_vnode_check_read(struct ucred active_cred, struct ucred file_cred,
	struct vnode *vp)
	{

	mac_vnode_assert_locked(vp, "mac_vnode_check_read");
	if (__predict_false(mac_vnode_check_read_fp_flag))
	return (mac_vnode_check_read_impl(active_cred, file_cred, vp));
	return (0);
	}

	int mac_vnode_check_write_impl(struct ucred *active_cred,
	struct ucred file_cred, struct vnode vp);
	extern bool mac_vnode_check_write_fp_flag;
	static inline int
	mac_vnode_check_write(struct ucred active_cred, struct ucred file_cred,
	struct vnode *vp)
	{

	mac_vnode_assert_locked(vp, "mac_vnode_check_write");
	if (__predict_false(mac_vnode_check_write_fp_flag))
	return (mac_vnode_check_write_impl(active_cred, file_cred, vp));
	return (0);
	}

	int mac_vnode_check_unlink(struct ucred cred, struct vnode dvp,
	struct vnode vp, struct componentname cnp);
	void mac_vnode_copy_label(struct label , struct label );
	void mac_vnode_init(struct vnode *);
	int mac_vnode_create_extattr(struct ucred cred, struct mount mp,
	struct vnode dvp, struct vnode vp, struct componentname *cnp);
	void mac_vnode_destroy(struct vnode *);
	void mac_vnode_execve_transition(struct ucred *oldcred,
	struct ucred newcred, struct vnode vp,
	struct label interpvplabel, struct image_params imgp);
	int mac_vnode_execve_will_transition(struct ucred *cred,
	struct vnode vp, struct label interpvplabel,
	struct image_params *imgp);
	void mac_vnode_relabel(struct ucred cred, struct vnode vp,
	struct label *newlabel);

	/*
	* Calls to help various file systems implement labeling functionality using
	* their existing EA implementation.
	*/
	int vop_stdsetlabel_ea(struct vop_setlabel_args *ap);

	#endif /* !_SECURITY_MAC_MAC_FRAMEWORK_H_ */
	diff --git a/sys/sys/ata.h b/sys/sys/ata.h
	index a08d5253578f..7ec4526be7df 100644
	--- a/sys/sys/ata.h
	+++ b/sys/sys/ata.h
	@@ -1,1058 +1,1059 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2000 - 2008 Søren Schmidt <sos@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer,
	* without modification, immediately at the beginning of the file.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef _SYS_ATA_H_
	#define _SYS_ATA_H_

	#include <sys/ioccom.h>

	/* ATA/ATAPI device parameters */
	struct ata_params {
	/000/ u_int16_t config; /* configuration info */
	#define ATA_PROTO_MASK 0x8003
	#define ATA_PROTO_ATAPI 0x8000
	#define ATA_PROTO_ATAPI_12 0x8000
	#define ATA_PROTO_ATAPI_16 0x8001
	#define ATA_PROTO_CFA 0x848a
	#define ATA_ATAPI_TYPE_MASK 0x1f00
	#define ATA_ATAPI_TYPE_DIRECT 0x0000 /* disk/floppy */
	#define ATA_ATAPI_TYPE_TAPE 0x0100 /* streaming tape */
	#define ATA_ATAPI_TYPE_CDROM 0x0500 /* CD-ROM device */
	#define ATA_ATAPI_TYPE_OPTICAL 0x0700 /* optical disk */
	+#define ATA_ATAPI_REMOVABLE 0x0080
	#define ATA_DRQ_MASK 0x0060
	#define ATA_DRQ_SLOW 0x0000 /* cpu 3 ms delay */
	#define ATA_DRQ_INTR 0x0020 /* interrupt 10 ms delay */
	#define ATA_DRQ_FAST 0x0040 /* accel 50 us delay */
	#define ATA_RESP_INCOMPLETE 0x0004

	/001/ u_int16_t cylinders; /* # of cylinders */
	/002/ u_int16_t specconf; /* specific configuration */
	/003/ u_int16_t heads; /* # heads */
	u_int16_t obsolete4;
	u_int16_t obsolete5;
	/006/ u_int16_t sectors; /* # sectors/track */
	/007/ u_int16_t vendor7[3];
	/010/ u_int8_t serial[20]; /* serial number */
	/020/ u_int16_t retired20;
	u_int16_t retired21;
	u_int16_t obsolete22;
	/023/ u_int8_t revision[8]; /* firmware revision */
	/027/ u_int8_t model[40]; /* model name */
	/047/ u_int16_t sectors_intr; /* sectors per interrupt */
	/048/ u_int16_t tcg; /* Trusted Computing Group */
	#define ATA_SUPPORT_TCG 0x0001
	/049/ u_int16_t capabilities1;
	#define ATA_SUPPORT_DMA 0x0100
	#define ATA_SUPPORT_LBA 0x0200
	#define ATA_SUPPORT_IORDYDIS 0x0400
	#define ATA_SUPPORT_IORDY 0x0800
	#define ATA_SUPPORT_OVERLAP 0x4000

	/050/ u_int16_t capabilities2;
	/051/ u_int16_t retired_piomode; /* PIO modes 0-2 */
	#define ATA_RETIRED_PIO_MASK 0x0300

	/052/ u_int16_t retired_dmamode; /* DMA modes */
	#define ATA_RETIRED_DMA_MASK 0x0003

	/053/ u_int16_t atavalid; /* fields valid */
	#define ATA_FLAG_54_58 0x0001 /* words 54-58 valid */
	#define ATA_FLAG_64_70 0x0002 /* words 64-70 valid */
	#define ATA_FLAG_88 0x0004 /* word 88 valid */

	/054/ u_int16_t current_cylinders;
	/055/ u_int16_t current_heads;
	/056/ u_int16_t current_sectors;
	/057/ u_int16_t current_size_1;
	/058/ u_int16_t current_size_2;
	/059/ u_int16_t multi;
	#define ATA_SUPPORT_BLOCK_ERASE_EXT 0x8000
	#define ATA_SUPPORT_OVERWRITE_EXT 0x4000
	#define ATA_SUPPORT_CRYPTO_SCRAMBLE_EXT 0x2000
	#define ATA_SUPPORT_SANITIZE 0x1000
	#define ATA_SUPPORT_SANITIZE_ALLOWED 0x0800
	#define ATA_SUPPORT_ANTIFREEZE_LOCK_EXT 0x0400
	#define ATA_MULTI_VALID 0x0100

	/060/ u_int16_t lba_size_1;
	u_int16_t lba_size_2;
	u_int16_t obsolete62;
	/063/ u_int16_t mwdmamodes; /* multiword DMA modes */
	/064/ u_int16_t apiomodes; /* advanced PIO modes */

	/065/ u_int16_t mwdmamin; /* min. M/W DMA time/word ns */
	/066/ u_int16_t mwdmarec; /* rec. M/W DMA time ns */
	/067/ u_int16_t pioblind; /* min. PIO cycle w/o flow */
	/068/ u_int16_t pioiordy; /* min. PIO cycle IORDY flow */
	/069/ u_int16_t support3;
	#define ATA_SUPPORT_RZAT 0x0020
	#define ATA_SUPPORT_DRAT 0x4000
	#define ATA_ENCRYPTS_ALL_USER_DATA 0x0010 /* Self-encrypting drive */
	#define ATA_SUPPORT_ZONE_MASK 0x0003
	#define ATA_SUPPORT_ZONE_NR 0x0000
	#define ATA_SUPPORT_ZONE_HOST_AWARE 0x0001
	#define ATA_SUPPORT_ZONE_DEV_MANAGED 0x0002
	u_int16_t reserved70;
	/071/ u_int16_t rlsovlap; /* rel time (us) for overlap */
	/072/ u_int16_t rlsservice; /* rel time (us) for service */
	u_int16_t reserved73;
	u_int16_t reserved74;
	/075/ u_int16_t queue;
	#define ATA_QUEUE_LEN(x) ((x) & 0x001f)

	/76/ u_int16_t satacapabilities;
	#define ATA_SATA_GEN1 0x0002
	#define ATA_SATA_GEN2 0x0004
	#define ATA_SATA_GEN3 0x0008
	#define ATA_SUPPORT_NCQ 0x0100
	#define ATA_SUPPORT_IFPWRMNGTRCV 0x0200
	#define ATA_SUPPORT_PHYEVENTCNT 0x0400
	#define ATA_SUPPORT_NCQ_UNLOAD 0x0800
	#define ATA_SUPPORT_NCQ_PRIO 0x1000
	#define ATA_SUPPORT_HAPST 0x2000
	#define ATA_SUPPORT_DAPST 0x4000
	#define ATA_SUPPORT_READLOGDMAEXT 0x8000

	/77/ u_int16_t satacapabilities2;
	#define ATA_SATA_CURR_GEN_MASK 0x0006
	#define ATA_SUPPORT_NCQ_STREAM 0x0010
	#define ATA_SUPPORT_NCQ_NON_DATA 0x0020
	#define ATA_SUPPORT_NCQ_QMANAGEMENT ATA_SUPPORT_NCQ_NON_DATA
	#define ATA_SUPPORT_RCVSND_FPDMA_QUEUED 0x0040
	/78/ u_int16_t satasupport;
	#define ATA_SUPPORT_NONZERO 0x0002
	#define ATA_SUPPORT_AUTOACTIVATE 0x0004
	#define ATA_SUPPORT_IFPWRMNGT 0x0008
	#define ATA_SUPPORT_INORDERDATA 0x0010
	#define ATA_SUPPORT_ASYNCNOTIF 0x0020
	#define ATA_SUPPORT_SOFTSETPRESERVE 0x0040
	#define ATA_SUPPORT_NCQ_AUTOSENSE 0x0080
	/79/ u_int16_t sataenabled;
	#define ATA_ENABLED_DAPST 0x0080

	/080/ u_int16_t version_major;
	/081/ u_int16_t version_minor;

	struct {
	/082/085/ u_int16_t command1;
	#define ATA_SUPPORT_SMART 0x0001
	#define ATA_SUPPORT_SECURITY 0x0002
	#define ATA_SUPPORT_REMOVABLE 0x0004
	#define ATA_SUPPORT_POWERMGT 0x0008
	#define ATA_SUPPORT_PACKET 0x0010
	#define ATA_SUPPORT_WRITECACHE 0x0020
	#define ATA_SUPPORT_LOOKAHEAD 0x0040
	#define ATA_SUPPORT_RELEASEIRQ 0x0080
	#define ATA_SUPPORT_SERVICEIRQ 0x0100
	#define ATA_SUPPORT_RESET 0x0200
	#define ATA_SUPPORT_PROTECTED 0x0400
	#define ATA_SUPPORT_WRITEBUFFER 0x1000
	#define ATA_SUPPORT_READBUFFER 0x2000
	#define ATA_SUPPORT_NOP 0x4000

	/083/086/ u_int16_t command2;
	#define ATA_SUPPORT_MICROCODE 0x0001
	#define ATA_SUPPORT_QUEUED 0x0002
	#define ATA_SUPPORT_CFA 0x0004
	#define ATA_SUPPORT_APM 0x0008
	#define ATA_SUPPORT_NOTIFY 0x0010
	#define ATA_SUPPORT_STANDBY 0x0020
	#define ATA_SUPPORT_SPINUP 0x0040
	#define ATA_SUPPORT_MAXSECURITY 0x0100
	#define ATA_SUPPORT_AUTOACOUSTIC 0x0200
	#define ATA_SUPPORT_ADDRESS48 0x0400
	#define ATA_SUPPORT_OVERLAY 0x0800
	#define ATA_SUPPORT_FLUSHCACHE 0x1000
	#define ATA_SUPPORT_FLUSHCACHE48 0x2000

	/084/087/ u_int16_t extension;
	#define ATA_SUPPORT_SMARTLOG 0x0001
	#define ATA_SUPPORT_SMARTTEST 0x0002
	#define ATA_SUPPORT_MEDIASN 0x0004
	#define ATA_SUPPORT_MEDIAPASS 0x0008
	#define ATA_SUPPORT_STREAMING 0x0010
	#define ATA_SUPPORT_GENLOG 0x0020
	#define ATA_SUPPORT_WRITEDMAFUAEXT 0x0040
	#define ATA_SUPPORT_WRITEDMAQFUAEXT 0x0080
	#define ATA_SUPPORT_64BITWWN 0x0100
	#define ATA_SUPPORT_UNLOAD 0x2000
	} __packed support, enabled;

	/088/ u_int16_t udmamodes; /* UltraDMA modes */
	/089/ u_int16_t erase_time; /* time req'd in 2min units */
	/090/ u_int16_t enhanced_erase_time; /* time req'd in 2min units */
	/091/ u_int16_t apm_value;
	/092/ u_int16_t master_passwd_revision; /* password revision code */
	/093/ u_int16_t hwres;
	#define ATA_CABLE_ID 0x2000

	/094/ u_int16_t acoustic;
	#define ATA_ACOUSTIC_CURRENT(x) ((x) & 0x00ff)
	#define ATA_ACOUSTIC_VENDOR(x) (((x) & 0xff00) >> 8)

	/095/ u_int16_t stream_min_req_size;
	/096/ u_int16_t stream_transfer_time;
	/097/ u_int16_t stream_access_latency;
	/098/ u_int32_t stream_granularity;
	/100/ u_int16_t lba_size48_1;
	u_int16_t lba_size48_2;
	u_int16_t lba_size48_3;
	u_int16_t lba_size48_4;
	u_int16_t reserved104;
	/105/ u_int16_t max_dsm_blocks;
	/106/ u_int16_t pss;
	#define ATA_PSS_LSPPS 0x000F
	#define ATA_PSS_LSSABOVE512 0x1000
	#define ATA_PSS_MULTLS 0x2000
	#define ATA_PSS_VALID_MASK 0xC000
	#define ATA_PSS_VALID_VALUE 0x4000
	/107/ u_int16_t isd;
	/108/ u_int16_t wwn[4];
	u_int16_t reserved112[5];
	/117/ u_int16_t lss_1;
	/118/ u_int16_t lss_2;
	/119/ u_int16_t support2;
	#define ATA_SUPPORT_WRITEREADVERIFY 0x0002
	#define ATA_SUPPORT_WRITEUNCORREXT 0x0004
	#define ATA_SUPPORT_RWLOGDMAEXT 0x0008
	#define ATA_SUPPORT_MICROCODE3 0x0010
	#define ATA_SUPPORT_FREEFALL 0x0020
	#define ATA_SUPPORT_SENSE_REPORT 0x0040
	#define ATA_SUPPORT_EPC 0x0080
	#define ATA_SUPPORT_AMAX_ADDR 0x0100
	#define ATA_SUPPORT_DSN 0x0200
	/120/ u_int16_t enabled2;
	#define ATA_ENABLED_WRITEREADVERIFY 0x0002
	#define ATA_ENABLED_WRITEUNCORREXT 0x0004
	#define ATA_ENABLED_FREEFALL 0x0020
	#define ATA_ENABLED_SENSE_REPORT 0x0040
	#define ATA_ENABLED_EPC 0x0080
	#define ATA_ENABLED_DSN 0x0200
	u_int16_t reserved121[6];
	/127/ u_int16_t removable_status;
	/128/ u_int16_t security_status;
	#define ATA_SECURITY_LEVEL 0x0100 /* 0: high, 1: maximum */
	#define ATA_SECURITY_ENH_SUPP 0x0020 /* enhanced erase supported */
	#define ATA_SECURITY_COUNT_EXP 0x0010 /* count expired */
	#define ATA_SECURITY_FROZEN 0x0008 /* security config is frozen */
	#define ATA_SECURITY_LOCKED 0x0004 /* drive is locked */
	#define ATA_SECURITY_ENABLED 0x0002 /* ATA Security is enabled */
	#define ATA_SECURITY_SUPPORTED 0x0001 /* ATA Security is supported */

	u_int16_t reserved129[31];
	/160/ u_int16_t cfa_powermode1;
	u_int16_t reserved161;
	/162/ u_int16_t cfa_kms_support;
	/163/ u_int16_t cfa_trueide_modes;
	/164/ u_int16_t cfa_memory_modes;
	u_int16_t reserved165[3];
	/168/ u_int16_t form_factor;
	#define ATA_FORM_FACTOR_MASK 0x000f
	#define ATA_FORM_FACTOR_NOT_REPORTED 0x0000
	#define ATA_FORM_FACTOR_5_25 0x0001
	#define ATA_FORM_FACTOR_3_5 0x0002
	#define ATA_FORM_FACTOR_2_5 0x0003
	#define ATA_FORM_FACTOR_1_8 0x0004
	#define ATA_FORM_FACTOR_SUB_1_8 0x0005
	#define ATA_FORM_FACTOR_MSATA 0x0006
	#define ATA_FORM_FACTOR_M_2 0x0007
	#define ATA_FORM_FACTOR_MICRO_SSD 0x0008
	#define ATA_FORM_FACTOR_C_FAST 0x0009
	/169/ u_int16_t support_dsm;
	#define ATA_SUPPORT_DSM_TRIM 0x0001
	/170/ u_int8_t product_id[8]; /* Additional Product Identifier */
	u_int16_t reserved174[2];
	/176/ u_int8_t media_serial[60];
	/206/ u_int16_t sct;
	u_int16_t reserved207[2];
	/209/ u_int16_t lsalign;
	/210/ u_int16_t wrv_sectors_m3_1;
	u_int16_t wrv_sectors_m3_2;
	/212/ u_int16_t wrv_sectors_m2_1;
	u_int16_t wrv_sectors_m2_2;
	/214/ u_int16_t nv_cache_caps;
	/215/ u_int16_t nv_cache_size_1;
	u_int16_t nv_cache_size_2;
	/217/ u_int16_t media_rotation_rate;
	#define ATA_RATE_NOT_REPORTED 0x0000
	#define ATA_RATE_NON_ROTATING 0x0001
	u_int16_t reserved218;
	/219/ u_int16_t nv_cache_opt;
	/220/ u_int16_t wrv_mode;
	u_int16_t reserved221;
	/222/ u_int16_t transport_major;
	/223/ u_int16_t transport_minor;
	u_int16_t reserved224[31];
	/255/ u_int16_t integrity;
	} __packed __aligned(2);

	/* ATA Dataset Management */
	#define ATA_DSM_BLK_SIZE 512
	#define ATA_DSM_BLK_RANGES 64
	#define ATA_DSM_RANGE_SIZE 8
	#define ATA_DSM_RANGE_MAX 65535

	/*
	* ATA Device Register
	*
	* bit 7 Obsolete (was 1 in early ATA specs)
	* bit 6 Sets LBA/CHS mode. 1=LBA, 0=CHS
	* bit 5 Obsolete (was 1 in early ATA specs)
	* bit 4 1 = Slave Drive, 0 = Master Drive
	* bit 3-0 In LBA mode, 27-24 of address. In CHS mode, head number
	*/

	#define ATA_DEV_MASTER 0x00
	#define ATA_DEV_SLAVE 0x10
	#define ATA_DEV_LBA 0x40

	/* ATA limits */
	#define ATA_MAX_28BIT_LBA 268435455UL

	/* ATA Status Register */
	#define ATA_STATUS_ERROR 0x01
	#define ATA_STATUS_SENSE_AVAIL 0x02
	#define ATA_STATUS_ALIGN_ERR 0x04
	#define ATA_STATUS_DATA_REQ 0x08
	#define ATA_STATUS_DEF_WRITE_ERR 0x10
	#define ATA_STATUS_DEVICE_FAULT 0x20
	#define ATA_STATUS_DEVICE_READY 0x40
	#define ATA_STATUS_BUSY 0x80

	/* ATA Error Register */
	#define ATA_ERROR_ABORT 0x04
	#define ATA_ERROR_ID_NOT_FOUND 0x10

	/* ATA HPA Features */
	#define ATA_HPA_FEAT_MAX_ADDR 0x00
	#define ATA_HPA_FEAT_SET_PWD 0x01
	#define ATA_HPA_FEAT_LOCK 0x02
	#define ATA_HPA_FEAT_UNLOCK 0x03
	#define ATA_HPA_FEAT_FREEZE 0x04

	/* ATA transfer modes */
	#define ATA_MODE_MASK 0x0f
	#define ATA_DMA_MASK 0xf0
	#define ATA_PIO 0x00
	#define ATA_PIO0 0x08
	#define ATA_PIO1 0x09
	#define ATA_PIO2 0x0a
	#define ATA_PIO3 0x0b
	#define ATA_PIO4 0x0c
	#define ATA_PIO_MAX 0x0f
	#define ATA_DMA 0x10
	#define ATA_WDMA0 0x20
	#define ATA_WDMA1 0x21
	#define ATA_WDMA2 0x22
	#define ATA_UDMA0 0x40
	#define ATA_UDMA1 0x41
	#define ATA_UDMA2 0x42
	#define ATA_UDMA3 0x43
	#define ATA_UDMA4 0x44
	#define ATA_UDMA5 0x45
	#define ATA_UDMA6 0x46
	#define ATA_SA150 0x47
	#define ATA_SA300 0x48
	#define ATA_SA600 0x49
	#define ATA_DMA_MAX 0x4f


	/* ATA commands */
	#define ATA_NOP 0x00 /* NOP */
	#define ATA_NF_FLUSHQUEUE 0x00 /* flush queued cmd's */
	#define ATA_NF_AUTOPOLL 0x01 /* start autopoll function */
	#define ATA_DATA_SET_MANAGEMENT 0x06
	#define ATA_DSM_TRIM 0x01
	#define ATA_DEVICE_RESET 0x08 /* reset device */
	#define ATA_READ 0x20 /* read */
	#define ATA_READ48 0x24 /* read 48bit LBA */
	#define ATA_READ_DMA48 0x25 /* read DMA 48bit LBA */
	#define ATA_READ_DMA_QUEUED48 0x26 /* read DMA QUEUED 48bit LBA */
	#define ATA_READ_NATIVE_MAX_ADDRESS48 0x27 /* read native max addr 48bit */
	#define ATA_READ_MUL48 0x29 /* read multi 48bit LBA */
	#define ATA_READ_STREAM_DMA48 0x2a /* read DMA stream 48bit LBA */
	#define ATA_READ_LOG_EXT 0x2f /* read log ext - PIO Data-In */
	#define ATA_READ_STREAM48 0x2b /* read stream 48bit LBA */
	#define ATA_WRITE 0x30 /* write */
	#define ATA_WRITE48 0x34 /* write 48bit LBA */
	#define ATA_WRITE_DMA48 0x35 /* write DMA 48bit LBA */
	#define ATA_WRITE_DMA_QUEUED48 0x36 /* write DMA QUEUED 48bit LBA*/
	#define ATA_SET_MAX_ADDRESS48 0x37 /* set max address 48bit */
	#define ATA_WRITE_MUL48 0x39 /* write multi 48bit LBA */
	#define ATA_WRITE_STREAM_DMA48 0x3a
	#define ATA_WRITE_STREAM48 0x3b
	#define ATA_WRITE_DMA_FUA48 0x3d
	#define ATA_WRITE_DMA_QUEUED_FUA48 0x3e
	#define ATA_WRITE_LOG_EXT 0x3f
	#define ATA_READ_VERIFY 0x40
	#define ATA_READ_VERIFY48 0x42
	#define ATA_WRITE_UNCORRECTABLE48 0x45 /* write uncorrectable 48bit LBA */
	#define ATA_WU_PSEUDO 0x55 /* pseudo-uncorrectable error */
	#define ATA_WU_FLAGGED 0xaa /* flagged-uncorrectable error */
	#define ATA_READ_LOG_DMA_EXT 0x47 /* read log DMA ext - PIO Data-In */
	#define ATA_ZAC_MANAGEMENT_IN 0x4a /* ZAC management in */
	#define ATA_ZM_REPORT_ZONES 0x00 /* report zones */
	#define ATA_WRITE_LOG_DMA_EXT 0x57 /* WRITE LOG DMA EXT */
	#define ATA_TRUSTED_NON_DATA 0x5b /* TRUSTED NON-DATA */
	#define ATA_TRUSTED_RECEIVE 0x5c /* TRUSTED RECEIVE */
	#define ATA_TRUSTED_RECEIVE_DMA 0x5d /* TRUSTED RECEIVE DMA */
	#define ATA_TRUSTED_SEND 0x5e /* TRUSTED SEND */
	#define ATA_TRUSTED_SEND_DMA 0x5f /* TRUSTED SEND DMA */
	#define ATA_READ_FPDMA_QUEUED 0x60 /* read DMA NCQ */
	#define ATA_WRITE_FPDMA_QUEUED 0x61 /* write DMA NCQ */
	#define ATA_NCQ_NON_DATA 0x63 /* NCQ non-data command */
	#define ATA_ABORT_NCQ_QUEUE 0x00 /* abort NCQ queue */
	#define ATA_DEADLINE_HANDLING 0x01 /* deadline handling */
	#define ATA_SET_FEATURES 0x05 /* set features */
	#define ATA_ZERO_EXT 0x06 /* zero ext */
	#define ATA_NCQ_ZAC_MGMT_OUT 0x07 /* NCQ ZAC mgmt out no data */
	#define ATA_SEND_FPDMA_QUEUED 0x64 /* send DMA NCQ */
	#define ATA_SFPDMA_DSM 0x00 /* Data set management */
	#define ATA_SFPDMA_DSM_TRIM 0x01 /* Set trim bit in auxiliary */
	#define ATA_SFPDMA_HYBRID_EVICT 0x01 /* Hybrid Evict */
	#define ATA_SFPDMA_WLDMA 0x02 /* Write Log DMA EXT */
	#define ATA_SFPDMA_ZAC_MGMT_OUT 0x03 /* NCQ ZAC mgmt out w/data */
	#define ATA_RECV_FPDMA_QUEUED 0x65 /* receive DMA NCQ */
	#define ATA_RFPDMA_RL_DMA_EXT 0x00 /* Read Log DMA EXT */
	#define ATA_RFPDMA_ZAC_MGMT_IN 0x02 /* NCQ ZAC mgmt in w/data */
	#define ATA_SEP_ATTN 0x67 /* SEP request */
	#define ATA_SEEK 0x70 /* seek */
	#define ATA_AMAX_ADDR 0x78 /* Accessible Max Address */
	#define ATA_AMAX_ADDR_GET 0x00 /* GET NATIVE MAX ADDRESS EXT */
	#define ATA_AMAX_ADDR_SET 0x01 /* SET ACCESSIBLE MAX ADDRESS EXT */
	#define ATA_AMAX_ADDR_FREEZE 0x02 /* FREEZE ACCESSIBLE MAX ADDRESS EXT */
	#define ATA_ZAC_MANAGEMENT_OUT 0x9f /* ZAC management out */
	#define ATA_ZM_CLOSE_ZONE 0x01 /* close zone */
	#define ATA_ZM_FINISH_ZONE 0x02 /* finish zone */
	#define ATA_ZM_OPEN_ZONE 0x03 /* open zone */
	#define ATA_ZM_RWP 0x04 /* reset write pointer */
	#define ATA_DOWNLOAD_MICROCODE 0x92 /* DOWNLOAD MICROCODE */
	#define ATA_DOWNLOAD_MICROCODE_DMA 0x93 /* DOWNLOAD MICROCODE DMA */
	#define ATA_PACKET_CMD 0xa0 /* packet command */
	#define ATA_ATAPI_IDENTIFY 0xa1 /* get ATAPI params*/
	#define ATA_SERVICE 0xa2 /* service command */
	#define ATA_SMART_CMD 0xb0 /* SMART command */
	#define ATA_SANITIZE 0xb4 /* sanitize device */
	#define ATA_CFA_ERASE 0xc0 /* CFA erase */
	#define ATA_READ_MUL 0xc4 /* read multi */
	#define ATA_WRITE_MUL 0xc5 /* write multi */
	#define ATA_SET_MULTI 0xc6 /* set multi size */
	#define ATA_READ_DMA_QUEUED 0xc7 /* read DMA QUEUED */
	#define ATA_READ_DMA 0xc8 /* read DMA */
	#define ATA_WRITE_DMA 0xca /* write DMA */
	#define ATA_WRITE_DMA_QUEUED 0xcc /* write DMA QUEUED */
	#define ATA_WRITE_MUL_FUA48 0xce
	#define ATA_STANDBY_IMMEDIATE 0xe0 /* standby immediate */
	#define ATA_IDLE_IMMEDIATE 0xe1 /* idle immediate */
	#define ATA_STANDBY_CMD 0xe2 /* standby */
	#define ATA_IDLE_CMD 0xe3 /* idle */
	#define ATA_READ_BUFFER 0xe4 /* read buffer */
	#define ATA_READ_PM 0xe4 /* read portmultiplier */
	#define ATA_CHECK_POWER_MODE 0xe5 /* device power mode */
	#define ATA_SLEEP 0xe6 /* sleep */
	#define ATA_FLUSHCACHE 0xe7 /* flush cache to disk */
	#define ATA_WRITE_BUFFER 0xe8 /* write buffer */
	#define ATA_WRITE_PM 0xe8 /* write portmultiplier */
	#define ATA_READ_BUFFER_DMA 0xe9 /* read buffer DMA */
	#define ATA_FLUSHCACHE48 0xea /* flush cache to disk */
	#define ATA_WRITE_BUFFER_DMA 0xeb /* write buffer DMA */
	#define ATA_ATA_IDENTIFY 0xec /* get ATA params */
	#define ATA_SETFEATURES 0xef /* features command */
	#define ATA_SF_ENAB_WCACHE 0x02 /* enable write cache */
	#define ATA_SF_DIS_WCACHE 0x82 /* disable write cache */
	#define ATA_SF_SETXFER 0x03 /* set transfer mode */
	#define ATA_SF_APM 0x05 /* Enable APM feature set */
	#define ATA_SF_ENAB_PUIS 0x06 /* enable PUIS */
	#define ATA_SF_DIS_PUIS 0x86 /* disable PUIS */
	#define ATA_SF_PUIS_SPINUP 0x07 /* PUIS spin-up */
	#define ATA_SF_WRV 0x0b /* Enable Write-Read-Verify */
	#define ATA_SF_DLC 0x0c /* Enable device life control */
	#define ATA_SF_SATA 0x10 /* Enable use of SATA feature */
	#define ATA_SF_FFC 0x41 /* Free-fall Control */
	#define ATA_SF_MHIST 0x43 /* Set Max Host Sect. Times */
	#define ATA_SF_RATE 0x45 /* Set Rate Basis */
	#define ATA_SF_EPC 0x4A /* Extended Power Conditions */
	#define ATA_SF_ENAB_RCACHE 0xaa /* enable readahead cache */
	#define ATA_SF_DIS_RCACHE 0x55 /* disable readahead cache */
	#define ATA_SF_ENAB_RELIRQ 0x5d /* enable release interrupt */
	#define ATA_SF_DIS_RELIRQ 0xdd /* disable release interrupt */
	#define ATA_SF_ENAB_SRVIRQ 0x5e /* enable service interrupt */
	#define ATA_SF_DIS_SRVIRQ 0xde /* disable service interrupt */
	#define ATA_SF_LPSAERC 0x62 /* Long Phys Sect Align ErrRep*/
	#define ATA_SF_DSN 0x63 /* Device Stats Notification */
	#define ATA_SECURITY_SET_PASSWORD 0xf1 /* set drive password */
	#define ATA_SECURITY_UNLOCK 0xf2 /* unlock drive using passwd */
	#define ATA_SECURITY_ERASE_PREPARE 0xf3 /* prepare to erase drive */
	#define ATA_SECURITY_ERASE_UNIT 0xf4 /* erase all blocks on drive */
	#define ATA_SECURITY_FREEZE_LOCK 0xf5 /* freeze security config */
	#define ATA_SECURITY_DISABLE_PASSWORD 0xf6 /* disable drive password */
	#define ATA_READ_NATIVE_MAX_ADDRESS 0xf8 /* read native max address */
	#define ATA_SET_MAX_ADDRESS 0xf9 /* set max address */


	/* ATAPI commands */
	#define ATAPI_TEST_UNIT_READY 0x00 /* check if device is ready */
	#define ATAPI_REZERO 0x01 /* rewind */
	#define ATAPI_REQUEST_SENSE 0x03 /* get sense data */
	#define ATAPI_FORMAT 0x04 /* format unit */
	#define ATAPI_READ 0x08 /* read data */
	#define ATAPI_WRITE 0x0a /* write data */
	#define ATAPI_WEOF 0x10 /* write filemark */
	#define ATAPI_WF_WRITE 0x01
	#define ATAPI_SPACE 0x11 /* space command */
	#define ATAPI_SP_FM 0x01
	#define ATAPI_SP_EOD 0x03
	#define ATAPI_INQUIRY 0x12 /* get inquiry data */
	#define ATAPI_MODE_SELECT 0x15 /* mode select */
	#define ATAPI_ERASE 0x19 /* erase */
	#define ATAPI_MODE_SENSE 0x1a /* mode sense */
	#define ATAPI_START_STOP 0x1b /* start/stop unit */
	#define ATAPI_SS_LOAD 0x01
	#define ATAPI_SS_RETENSION 0x02
	#define ATAPI_SS_EJECT 0x04
	#define ATAPI_PREVENT_ALLOW 0x1e /* media removal */
	#define ATAPI_READ_FORMAT_CAPACITIES 0x23 /* get format capacities */
	#define ATAPI_READ_CAPACITY 0x25 /* get volume capacity */
	#define ATAPI_READ_BIG 0x28 /* read data */
	#define ATAPI_WRITE_BIG 0x2a /* write data */
	#define ATAPI_LOCATE 0x2b /* locate to position */
	#define ATAPI_READ_POSITION 0x34 /* read position */
	#define ATAPI_SYNCHRONIZE_CACHE 0x35 /* flush buf, close channel */
	#define ATAPI_WRITE_BUFFER 0x3b /* write device buffer */
	#define ATAPI_READ_BUFFER 0x3c /* read device buffer */
	#define ATAPI_READ_SUBCHANNEL 0x42 /* get subchannel info */
	#define ATAPI_READ_TOC 0x43 /* get table of contents */
	#define ATAPI_PLAY_10 0x45 /* play by lba */
	#define ATAPI_PLAY_MSF 0x47 /* play by MSF address */
	#define ATAPI_PLAY_TRACK 0x48 /* play by track number */
	#define ATAPI_PAUSE 0x4b /* pause audio operation */
	#define ATAPI_READ_DISK_INFO 0x51 /* get disk info structure */
	#define ATAPI_READ_TRACK_INFO 0x52 /* get track info structure */
	#define ATAPI_RESERVE_TRACK 0x53 /* reserve track */
	#define ATAPI_SEND_OPC_INFO 0x54 /* send OPC structurek */
	#define ATAPI_MODE_SELECT_BIG 0x55 /* set device parameters */
	#define ATAPI_REPAIR_TRACK 0x58 /* repair track */
	#define ATAPI_READ_MASTER_CUE 0x59 /* read master CUE info */
	#define ATAPI_MODE_SENSE_BIG 0x5a /* get device parameters */
	#define ATAPI_CLOSE_TRACK 0x5b /* close track/session */
	#define ATAPI_READ_BUFFER_CAPACITY 0x5c /* get buffer capicity */
	#define ATAPI_SEND_CUE_SHEET 0x5d /* send CUE sheet */
	#define ATAPI_SERVICE_ACTION_IN 0x96 /* get service data */
	#define ATAPI_BLANK 0xa1 /* blank the media */
	#define ATAPI_SEND_KEY 0xa3 /* send DVD key structure */
	#define ATAPI_REPORT_KEY 0xa4 /* get DVD key structure */
	#define ATAPI_PLAY_12 0xa5 /* play by lba */
	#define ATAPI_LOAD_UNLOAD 0xa6 /* changer control command */
	#define ATAPI_READ_STRUCTURE 0xad /* get DVD structure */
	#define ATAPI_PLAY_CD 0xb4 /* universal play command */
	#define ATAPI_SET_SPEED 0xbb /* set drive speed */
	#define ATAPI_MECH_STATUS 0xbd /* get changer status */
	#define ATAPI_READ_CD 0xbe /* read data */
	#define ATAPI_POLL_DSC 0xff /* poll DSC status bit */


	struct ata_ioc_devices {
	int channel;
	char name[2][32];
	struct ata_params params[2];
	};

	/* pr channel ATA ioctl calls */
	#define IOCATAGMAXCHANNEL _IOR('a', 1, int)
	#define IOCATAREINIT _IOW('a', 2, int)
	#define IOCATAATTACH _IOW('a', 3, int)
	#define IOCATADETACH _IOW('a', 4, int)
	#define IOCATADEVICES _IOWR('a', 5, struct ata_ioc_devices)

	/* ATAPI request sense structure */
	struct atapi_sense {
	u_int8_t error; /* current or deferred errors */
	#define ATA_SENSE_VALID 0x80

	u_int8_t segment; /* segment number */
	u_int8_t key; /* sense key */
	#define ATA_SENSE_KEY_MASK 0x0f /* sense key mask */
	#define ATA_SENSE_NO_SENSE 0x00 /* no specific sense key info */
	#define ATA_SENSE_RECOVERED_ERROR 0x01 /* command OK, data recovered */
	#define ATA_SENSE_NOT_READY 0x02 /* no access to drive */
	#define ATA_SENSE_MEDIUM_ERROR 0x03 /* non-recovered data error */
	#define ATA_SENSE_HARDWARE_ERROR 0x04 /* non-recoverable HW failure */
	#define ATA_SENSE_ILLEGAL_REQUEST 0x05 /* invalid command param(s) */
	#define ATA_SENSE_UNIT_ATTENTION 0x06 /* media changed */
	#define ATA_SENSE_DATA_PROTECT 0x07 /* write protect */
	#define ATA_SENSE_BLANK_CHECK 0x08 /* blank check */
	#define ATA_SENSE_VENDOR_SPECIFIC 0x09 /* vendor specific skey */
	#define ATA_SENSE_COPY_ABORTED 0x0a /* copy aborted */
	#define ATA_SENSE_ABORTED_COMMAND 0x0b /* command aborted, try again */
	#define ATA_SENSE_EQUAL 0x0c /* equal */
	#define ATA_SENSE_VOLUME_OVERFLOW 0x0d /* volume overflow */
	#define ATA_SENSE_MISCOMPARE 0x0e /* data dont match the medium */
	#define ATA_SENSE_RESERVED 0x0f
	#define ATA_SENSE_ILI 0x20;
	#define ATA_SENSE_EOM 0x40;
	#define ATA_SENSE_FILEMARK 0x80;

	u_int32_t cmd_info; /* cmd information */
	u_int8_t sense_length; /* additional sense len (n-7) */
	u_int32_t cmd_specific_info; /* additional cmd spec info */
	u_int8_t asc; /* additional sense code */
	u_int8_t ascq; /* additional sense code qual */
	u_int8_t replaceable_unit_code; /* replaceable unit code */
	u_int8_t specific; /* sense key specific */
	#define ATA_SENSE_SPEC_VALID 0x80
	#define ATA_SENSE_SPEC_MASK 0x7f

	u_int8_t specific1; /* sense key specific */
	u_int8_t specific2; /* sense key specific */
	} __packed;

	/*
	* SET FEATURES subcommands
	*/

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* These values go in the LBA 3:0.
	*/
	#define ATA_SF_EPC_RESTORE 0x00 /* Restore Power Condition Settings */
	#define ATA_SF_EPC_GOTO 0x01 /* Go To Power Condition */
	#define ATA_SF_EPC_SET_TIMER 0x02 /* Set Power Condition Timer */
	#define ATA_SF_EPC_SET_STATE 0x03 /* Set Power Condition State */
	#define ATA_SF_EPC_ENABLE 0x04 /* Enable the EPC feature set */
	#define ATA_SF_EPC_DISABLE 0x05 /* Disable the EPC feature set */
	#define ATA_SF_EPC_SET_SOURCE 0x06 /* Set EPC Power Source */

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* Power Condition ID field
	* These values go in the count register.
	*/
	#define ATA_EPC_STANDBY_Z 0x00 /* Substate of PM2:Standby */
	#define ATA_EPC_STANDBY_Y 0x01 /* Substate of PM2:Standby */
	#define ATA_EPC_IDLE_A 0x81 /* Substate of PM1:Idle */
	#define ATA_EPC_IDLE_B 0x82 /* Substate of PM1:Idle */
	#define ATA_EPC_IDLE_C 0x83 /* Substate of PM1:Idle */
	#define ATA_EPC_ALL 0xff /* All supported power conditions */

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* Restore Power Conditions Settings subcommand
	* These values go in the LBA register.
	*/
	#define ATA_SF_EPC_RST_DFLT 0x40 /* 1=Rst from Default, 0= from Saved */
	#define ATA_SF_EPC_RST_SAVE 0x10 /* 1=Save on completion */

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* Got To Power Condition subcommand
	* These values go in the LBA register.
	*/
	#define ATA_SF_EPC_GOTO_DELAY 0x02000000 /* Delayed entry bit */
	#define ATA_SF_EPC_GOTO_HOLD 0x01000000 /* Hold Power Cond bit */

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* Set Power Condition Timer subcommand
	* These values go in the LBA register.
	*/
	#define ATA_SF_EPC_TIMER_MASK 0x00ffff00 /* Timer field */
	#define ATA_SF_EPC_TIMER_SHIFT 8
	#define ATA_SF_EPC_TIMER_SEC 0x00000080 /* Timer units, 1=sec, 0=.1s */
	#define ATA_SF_EPC_TIMER_EN 0x00000020 /* Enable/disable cond. */
	#define ATA_SF_EPC_TIMER_SAVE 0x00000010 /* Save settings on comp. */

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* Set Power Condition State subcommand
	* These values go in the LBA register.
	*/
	#define ATA_SF_EPC_SETCON_EN 0x00000020 /* Enable power cond. */
	#define ATA_SF_EPC_SETCON_SAVE 0x00000010 /* Save settings on comp */

	/*
	* SET FEATURES command
	* Extended Power Conditions subcommand -- ATA_SF_EPC (0x4A)
	* Set EPC Power Source subcommand
	* These values go in the count register.
	*/
	#define ATA_SF_EPC_SRC_UNKNOWN 0x0000 /* Unknown source */
	#define ATA_SF_EPC_SRC_BAT 0x0001 /* battery source */
	#define ATA_SF_EPC_SRC_NOT_BAT 0x0002 /* not battery source */

	#define ATA_LOG_DIRECTORY 0x00 /* Directory of all logs */
	#define ATA_POWER_COND_LOG 0x08 /* Power Conditions Log */
	#define ATA_PCL_IDLE 0x00 /* Idle Power Conditions Page */
	#define ATA_PCL_STANDBY 0x01 /* Standby Power Conditions Page */
	#define ATA_IDENTIFY_DATA_LOG 0x30 /* Identify Device Data Log */
	#define ATA_IDL_PAGE_LIST 0x00 /* List of supported pages */
	#define ATA_IDL_IDENTIFY_DATA 0x01 /* Copy of Identify Device data */
	#define ATA_IDL_CAPACITY 0x02 /* Capacity */
	#define ATA_IDL_SUP_CAP 0x03 /* Supported Capabilities */
	#define ATA_IDL_CUR_SETTINGS 0x04 /* Current Settings */
	#define ATA_IDL_ATA_STRINGS 0x05 /* ATA Strings */
	#define ATA_IDL_SECURITY 0x06 /* Security */
	#define ATA_IDL_PARALLEL_ATA 0x07 /* Parallel ATA */
	#define ATA_IDL_SERIAL_ATA 0x08 /* Serial ATA */
	#define ATA_IDL_ZDI 0x09 /* Zoned Device Information */

	struct ata_gp_log_dir {
	uint8_t header[2];
	#define ATA_GP_LOG_DIR_VERSION 0x0001
	uint8_t num_pages[2552]; / Number of log pages at address */
	};

	/*
	* ATA Power Conditions log descriptor
	*/
	struct ata_power_cond_log_desc {
	uint8_t reserved1;
	uint8_t flags;
	#define ATA_PCL_COND_SUPPORTED 0x80
	#define ATA_PCL_COND_SAVEABLE 0x40
	#define ATA_PCL_COND_CHANGEABLE 0x20
	#define ATA_PCL_DEFAULT_TIMER_EN 0x10
	#define ATA_PCL_SAVED_TIMER_EN 0x08
	#define ATA_PCL_CURRENT_TIMER_EN 0x04
	#define ATA_PCL_HOLD_PC_NOT_SUP 0x02
	uint8_t reserved2[2];
	uint8_t default_timer[4];
	uint8_t saved_timer[4];
	uint8_t current_timer[4];
	uint8_t nom_time_to_active[4];
	uint8_t min_timer[4];
	uint8_t max_timer[4];
	uint8_t num_transitions_to_pc[4];
	uint8_t hours_in_pc[4];
	uint8_t reserved3[28];
	};

	/*
	* ATA Power Conditions Log (0x08), Idle power conditions page (0x00)
	*/
	struct ata_power_cond_log_idle {
	struct ata_power_cond_log_desc idle_a_desc;
	struct ata_power_cond_log_desc idle_b_desc;
	struct ata_power_cond_log_desc idle_c_desc;
	uint8_t reserved[320];
	};

	/*
	* ATA Power Conditions Log (0x08), Standby power conditions page (0x01)
	*/
	struct ata_power_cond_log_standby {
	uint8_t reserved[384];
	struct ata_power_cond_log_desc standby_y_desc;
	struct ata_power_cond_log_desc standby_z_desc;
	};

	/*
	* ATA IDENTIFY DEVICE data log (0x30) page 0x00
	* List of Supported IDENTIFY DEVICE data pages.
	*/
	struct ata_identify_log_pages {
	uint8_t header[8];
	#define ATA_IDLOG_REVISION 0x0000000000000001
	uint8_t entry_count;
	uint8_t entries[503];
	};

	/*
	* ATA IDENTIFY DEVICE data log (0x30)
	* Capacity (Page 0x02).
	*/
	struct ata_identify_log_capacity {
	uint8_t header[8];
	#define ATA_CAP_HEADER_VALID 0x8000000000000000
	#define ATA_CAP_PAGE_NUM_MASK 0x0000000000ff0000
	#define ATA_CAP_PAGE_NUM_SHIFT 16
	#define ATA_CAP_REV_MASK 0x00000000000000ff
	uint8_t capacity[8];
	#define ATA_CAP_CAPACITY_VALID 0x8000000000000000
	#define ATA_CAP_ACCESSIBLE_CAP 0x0000ffffffffffff
	uint8_t phys_logical_sect_size[8];
	#define ATA_CAP_PL_VALID 0x8000000000000000
	#define ATA_CAP_LTOP_REL_SUP 0x4000000000000000
	#define ATA_CAP_LOG_SECT_SUP 0x2000000000000000
	#define ATA_CAP_ALIGN_ERR_MASK 0x0000000000300000
	#define ATA_CAP_LTOP_MASK 0x00000000000f0000
	#define ATA_CAP_LOG_SECT_OFF 0x000000000000ffff
	uint8_t logical_sect_size[8];
	#define ATA_CAP_LOG_SECT_VALID 0x8000000000000000
	#define ATA_CAP_LOG_SECT_SIZE 0x00000000ffffffff
	uint8_t nominal_buffer_size[8];
	#define ATA_CAP_NOM_BUF_VALID 0x8000000000000000
	#define ATA_CAP_NOM_BUF_SIZE 0x7fffffffffffffff
	uint8_t reserved[472];
	};

	/*
	* ATA IDENTIFY DEVICE data log (0x30)
	* Supported Capabilities (Page 0x03).
	*/

	struct ata_identify_log_sup_cap {
	uint8_t header[8];
	#define ATA_SUP_CAP_HEADER_VALID 0x8000000000000000
	#define ATA_SUP_CAP_PAGE_NUM_MASK 0x0000000000ff0000
	#define ATA_SUP_CAP_PAGE_NUM_SHIFT 16
	#define ATA_SUP_CAP_REV_MASK 0x00000000000000ff
	uint8_t sup_cap[8];
	#define ATA_SUP_CAP_VALID 0x8000000000000000
	#define ATA_SC_SET_SECT_CONFIG_SUP 0x0002000000000000 /* Set Sect Conf*/
	#define ATA_SC_ZERO_EXT_SUP 0x0001000000000000 /* Zero EXT */
	#define ATA_SC_SUCC_NCQ_SENSE_SUP 0x0000800000000000 /* Succ. NCQ Sns */
	#define ATA_SC_DLC_SUP 0x0000400000000000 /* DLC */
	#define ATA_SC_RQSN_DEV_FAULT_SUP 0x0000200000000000 /* Req Sns Dev Flt*/
	#define ATA_SC_DSN_SUP 0x0000100000000000 /* DSN */
	#define ATA_SC_LP_STANDBY_SUP 0x0000080000000000 /* LP Standby */
	#define ATA_SC_SET_EPC_PS_SUP 0x0000040000000000 /* Set EPC PS */
	#define ATA_SC_AMAX_ADDR_SUP 0x0000020000000000 /* AMAX Addr */
	#define ATA_SC_DRAT_SUP 0x0000008000000000 /* DRAT */
	#define ATA_SC_LPS_MISALGN_SUP 0x0000004000000000 /* LPS Misalign */
	#define ATA_SC_RB_DMA_SUP 0x0000001000000000 /* Read Buf DMA */
	#define ATA_SC_WB_DMA_SUP 0x0000000800000000 /* Write Buf DMA */
	#define ATA_SC_DNLD_MC_DMA_SUP 0x0000000200000000 /* DL MCode DMA */
	#define ATA_SC_28BIT_SUP 0x0000000100000000 /* 28-bit */
	#define ATA_SC_RZAT_SUP 0x0000000080000000 /* RZAT */
	#define ATA_SC_NOP_SUP 0x0000000020000000 /* NOP */
	#define ATA_SC_READ_BUFFER_SUP 0x0000000010000000 /* Read Buffer */
	#define ATA_SC_WRITE_BUFFER_SUP 0x0000000008000000 /* Write Buffer */
	#define ATA_SC_READ_LOOK_AHEAD_SUP 0x0000000002000000 /* Read Look-Ahead*/
	#define ATA_SC_VOLATILE_WC_SUP 0x0000000001000000 /* Volatile WC */
	#define ATA_SC_SMART_SUP 0x0000000000800000 /* SMART */
	#define ATA_SC_FLUSH_CACHE_EXT_SUP 0x0000000000400000 /* Flush Cache Ext */
	#define ATA_SC_48BIT_SUP 0x0000000000100000 /* 48-Bit */
	#define ATA_SC_SPINUP_SUP 0x0000000000040000 /* Spin-Up */
	#define ATA_SC_PUIS_SUP 0x0000000000020000 /* PUIS */
	#define ATA_SC_APM_SUP 0x0000000000010000 /* APM */
	#define ATA_SC_DL_MICROCODE_SUP 0x0000000000004000 /* DL Microcode */
	#define ATA_SC_UNLOAD_SUP 0x0000000000002000 /* Unload */
	#define ATA_SC_WRITE_FUA_EXT_SUP 0x0000000000001000 /* Write FUA EXT */
	#define ATA_SC_GPL_SUP 0x0000000000000800 /* GPL */
	#define ATA_SC_STREAMING_SUP 0x0000000000000400 /* Streaming */
	#define ATA_SC_SMART_SELFTEST_SUP 0x0000000000000100 /* SMART self-test */
	#define ATA_SC_SMART_ERR_LOG_SUP 0x0000000000000080 /* SMART Err Log */
	#define ATA_SC_EPC_SUP 0x0000000000000040 /* EPC */
	#define ATA_SC_SENSE_SUP 0x0000000000000020 /* Sense data */
	#define ATA_SC_FREEFALL_SUP 0x0000000000000010 /* Free-Fall */
	#define ATA_SC_DM_MODE3_SUP 0x0000000000000008 /* DM Mode 3 */
	#define ATA_SC_GPL_DMA_SUP 0x0000000000000004 /* GPL DMA */
	#define ATA_SC_WRITE_UNCOR_SUP 0x0000000000000002 /* Write uncorr. */
	#define ATA_SC_WRV_SUP 0x0000000000000001 /* WRV */
	uint8_t download_code_cap[8];
	#define ATA_DL_CODE_VALID 0x8000000000000000
	#define ATA_DLC_DM_OFFSETS_DEFER_SUP 0x0000000400000000
	#define ATA_DLC_DM_IMMED_SUP 0x0000000200000000
	#define ATA_DLC_DM_OFF_IMMED_SUP 0x0000000100000000
	#define ATA_DLC_DM_MAX_XFER_SIZE_MASK 0x00000000ffff0000
	#define ATA_DLC_DM_MAX_XFER_SIZE_SHIFT 16
	#define ATA_DLC_DM_MIN_XFER_SIZE_MASK 0x000000000000ffff
	uint8_t nom_media_rotation_rate[8];
	#define ATA_NOM_MEDIA_ROTATION_VALID 0x8000000000000000
	#define ATA_ROTATION_MASK 0x000000000000ffff
	uint8_t form_factor[8];
	#define ATA_FORM_FACTOR_VALID 0x8000000000000000
	#define ATA_FF_MASK 0x000000000000000f
	#define ATA_FF_NOT_REPORTED 0x0000000000000000 /* Not reported */
	#define ATA_FF_525_IN 0x0000000000000001 /* 5.25 inch */
	#define ATA_FF_35_IN 0x0000000000000002 /* 3.5 inch */
	#define ATA_FF_25_IN 0x0000000000000003 /* 2.5 inch */
	#define ATA_FF_18_IN 0x0000000000000004 /* 1.8 inch */
	#define ATA_FF_LT_18_IN 0x0000000000000005 /* < 1.8 inch */
	#define ATA_FF_MSATA 0x0000000000000006 /* mSATA */
	#define ATA_FF_M2 0x0000000000000007 /* M.2 */
	#define ATA_FF_MICROSSD 0x0000000000000008 /* MicroSSD */
	#define ATA_FF_CFAST 0x0000000000000009 /* CFast */
	uint8_t wrv_sec_cnt_mode3[8];
	#define ATA_WRV_MODE3_VALID 0x8000000000000000
	#define ATA_WRV_MODE3_COUNT 0x00000000ffffffff
	uint8_t wrv_sec_cnt_mode2[8];
	#define ATA_WRV_MODE2_VALID 0x8000000000000000
	#define ATA_WRV_MODE2_COUNT 0x00000000ffffffff
	uint8_t wwn[16];
	/* XXX KDM need to figure out how to handle 128-bit fields */
	uint8_t dsm[8];
	#define ATA_DSM_VALID 0x8000000000000000
	#define ATA_LB_MARKUP_SUP 0x000000000000ff00
	#define ATA_TRIM_SUP 0x0000000000000001
	uint8_t util_per_unit_time[16];
	/* XXX KDM need to figure out how to handle 128-bit fields */
	uint8_t util_usage_rate_sup[8];
	#define ATA_UTIL_USAGE_RATE_VALID 0x8000000000000000
	#define ATA_SETTING_RATE_SUP 0x0000000000800000
	#define ATA_SINCE_POWERON_SUP 0x0000000000000100
	#define ATA_POH_RATE_SUP 0x0000000000000010
	#define ATA_DATE_TIME_RATE_SUP 0x0000000000000001
	uint8_t zoned_cap[8];
	#define ATA_ZONED_VALID 0x8000000000000000
	#define ATA_ZONED_MASK 0x0000000000000003
	uint8_t sup_zac_cap[8];
	#define ATA_SUP_ZAC_CAP_VALID 0x8000000000000000
	#define ATA_ND_RWP_SUP 0x0000000000000010 /* Reset Write Ptr*/
	#define ATA_ND_FINISH_ZONE_SUP 0x0000000000000008 /* Finish Zone */
	#define ATA_ND_CLOSE_ZONE_SUP 0x0000000000000004 /* Close Zone */
	#define ATA_ND_OPEN_ZONE_SUP 0x0000000000000002 /* Open Zone */
	#define ATA_REPORT_ZONES_SUP 0x0000000000000001 /* Report Zones */
	uint8_t reserved[392];
	};

	/*
	* ATA Identify Device Data Log Zoned Device Information Page (0x09).
	* Current as of ZAC r04a, August 25, 2015.
	*/
	struct ata_zoned_info_log {
	uint8_t header[8];
	#define ATA_ZDI_HEADER_VALID 0x8000000000000000
	#define ATA_ZDI_PAGE_NUM_MASK 0x0000000000ff0000
	#define ATA_ZDI_PAGE_NUM_SHIFT 16
	#define ATA_ZDI_REV_MASK 0x00000000000000ff
	uint8_t zoned_cap[8];
	#define ATA_ZDI_CAP_VALID 0x8000000000000000
	#define ATA_ZDI_CAP_URSWRZ 0x0000000000000001
	uint8_t zoned_settings[8];
	#define ATA_ZDI_SETTINGS_VALID 0x8000000000000000
	uint8_t optimal_seq_zones[8];
	#define ATA_ZDI_OPT_SEQ_VALID 0x8000000000000000
	#define ATA_ZDI_OPT_SEQ_MASK 0x00000000ffffffff
	uint8_t optimal_nonseq_zones[8];
	#define ATA_ZDI_OPT_NS_VALID 0x8000000000000000
	#define ATA_ZDI_OPT_NS_MASK 0x00000000ffffffff
	uint8_t max_seq_req_zones[8];
	#define ATA_ZDI_MAX_SEQ_VALID 0x8000000000000000
	#define ATA_ZDI_MAX_SEQ_MASK 0x00000000ffffffff
	uint8_t version_info[8];
	#define ATA_ZDI_VER_VALID 0x8000000000000000
	#define ATA_ZDI_VER_ZAC_SUP 0x0100000000000000
	#define ATA_ZDI_VER_ZAC_MASK 0x00000000000000ff
	uint8_t reserved[456];
	};

	struct ata_ioc_request {
	union {
	struct {
	u_int8_t command;
	u_int8_t feature;
	u_int64_t lba;
	u_int16_t count;
	} ata;
	struct {
	char ccb[16];
	struct atapi_sense sense;
	} atapi;
	} u;
	caddr_t data;
	int count;
	int flags;
	#define ATA_CMD_CONTROL 0x01
	#define ATA_CMD_READ 0x02
	#define ATA_CMD_WRITE 0x04
	#define ATA_CMD_ATAPI 0x08

	int timeout;
	int error;
	};

	struct ata_security_password {
	u_int16_t ctrl;
	#define ATA_SECURITY_PASSWORD_USER 0x0000
	#define ATA_SECURITY_PASSWORD_MASTER 0x0001
	#define ATA_SECURITY_ERASE_NORMAL 0x0000
	#define ATA_SECURITY_ERASE_ENHANCED 0x0002
	#define ATA_SECURITY_LEVEL_HIGH 0x0000
	#define ATA_SECURITY_LEVEL_MAXIMUM 0x0100

	u_int8_t password[32];
	u_int16_t revision;
	u_int16_t reserved[238];
	};

	/* pr device ATA ioctl calls */
	#define IOCATAREQUEST _IOWR('a', 100, struct ata_ioc_request)
	#define IOCATAGPARM _IOR('a', 101, struct ata_params)
	#define IOCATAGMODE _IOR('a', 102, int)
	#define IOCATASMODE _IOW('a', 103, int)

	#define IOCATAGSPINDOWN _IOR('a', 104, int)
	#define IOCATASSPINDOWN _IOW('a', 105, int)


	struct ata_ioc_raid_config {
	int lun;
	int type;
	#define AR_JBOD 0x0001
	#define AR_SPAN 0x0002
	#define AR_RAID0 0x0004
	#define AR_RAID1 0x0008
	#define AR_RAID01 0x0010
	#define AR_RAID3 0x0020
	#define AR_RAID4 0x0040
	#define AR_RAID5 0x0080

	int interleave;
	int status;
	#define AR_READY 1
	#define AR_DEGRADED 2
	#define AR_REBUILDING 4

	int progress;
	int total_disks;
	int disks[16];
	};

	struct ata_ioc_raid_status {
	int lun;
	int type;
	int interleave;
	int status;
	int progress;
	int total_disks;
	struct {
	int state;
	#define AR_DISK_ONLINE 0x01
	#define AR_DISK_PRESENT 0x02
	#define AR_DISK_SPARE 0x04
	int lun;
	} disks[16];
	};

	/* ATA RAID ioctl calls */
	#define IOCATARAIDCREATE _IOWR('a', 200, struct ata_ioc_raid_config)
	#define IOCATARAIDDELETE _IOW('a', 201, int)
	#define IOCATARAIDSTATUS _IOWR('a', 202, struct ata_ioc_raid_status)
	#define IOCATARAIDADDSPARE _IOW('a', 203, struct ata_ioc_raid_config)
	#define IOCATARAIDREBUILD _IOW('a', 204, int)

	#endif /* _SYS_ATA_H_ */
	diff --git a/sys/sys/file.h b/sys/sys/file.h
	index 65b73158f8d9..10d483764467 100644
	--- a/sys/sys/file.h
	+++ b/sys/sys/file.h
	@@ -1,460 +1,461 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1982, 1986, 1989, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)file.h 8.3 (Berkeley) 1/9/95
	* $FreeBSD$
	*/

	#ifndef _SYS_FILE_H_
	#define _SYS_FILE_H_

	#ifndef _KERNEL
	#include <sys/types.h> /* XXX */
	#include <sys/fcntl.h>
	#include <sys/unistd.h>
	#else
	#include <sys/queue.h>
	#include <sys/refcount.h>
	#include <sys/_lock.h>
	#include <sys/_mutex.h>
	#include <vm/vm.h>

	struct filedesc;
	struct stat;
	struct thread;
	struct uio;
	struct knote;
	struct vnode;

	#endif /* _KERNEL */

	#define DTYPE_NONE 0 /* not yet initialized */
	#define DTYPE_VNODE 1 /* file */
	#define DTYPE_SOCKET 2 /* communications endpoint */
	#define DTYPE_PIPE 3 /* pipe */
	#define DTYPE_FIFO 4 /* fifo (named pipe) */
	#define DTYPE_KQUEUE 5 /* event queue */
	#define DTYPE_CRYPTO 6 /* crypto */
	#define DTYPE_MQUEUE 7 /* posix message queue */
	#define DTYPE_SHM 8 /* swap-backed shared memory */
	#define DTYPE_SEM 9 /* posix semaphore */
	#define DTYPE_PTS 10 /* pseudo teletype master device */
	#define DTYPE_DEV 11 /* Device specific fd type */
	#define DTYPE_PROCDESC 12 /* process descriptor */
	#define DTYPE_LINUXEFD 13 /* emulation eventfd type */
	#define DTYPE_LINUXTFD 14 /* emulation timerfd type */

	#ifdef _KERNEL

	struct file;
	struct filecaps;
	struct kaiocb;
	struct kinfo_file;
	struct ucred;

	#define FOF_OFFSET 0x01 /* Use the offset in uio argument */
	#define FOF_NOLOCK 0x02 /* Do not take FOFFSET_LOCK */
	#define FOF_NEXTOFF_R 0x04 /* Also update f_nextoff[UIO_READ] */
	#define FOF_NEXTOFF_W 0x08 /* Also update f_nextoff[UIO_WRITE] */
	#define FOF_NOUPDATE 0x10 /* Do not update f_offset */
	off_t foffset_lock(struct file *fp, int flags);
	void foffset_lock_uio(struct file fp, struct uio uio, int flags);
	void foffset_unlock(struct file *fp, off_t val, int flags);
	void foffset_unlock_uio(struct file fp, struct uio uio, int flags);

	static inline off_t
	foffset_get(struct file *fp)
	{

	return (foffset_lock(fp, FOF_NOLOCK));
	}

	typedef int fo_rdwr_t(struct file fp, struct uio uio,
	struct ucred *active_cred, int flags,
	struct thread *td);
	typedef int fo_truncate_t(struct file *fp, off_t length,
	struct ucred active_cred, struct thread td);
	typedef int fo_ioctl_t(struct file fp, u_long com, void data,
	struct ucred active_cred, struct thread td);
	typedef int fo_poll_t(struct file *fp, int events,
	struct ucred active_cred, struct thread td);
	typedef int fo_kqfilter_t(struct file fp, struct knote kn);
	typedef int fo_stat_t(struct file fp, struct stat sb,
	struct ucred active_cred, struct thread td);
	typedef int fo_close_t(struct file fp, struct thread td);
	typedef int fo_chmod_t(struct file *fp, mode_t mode,
	struct ucred active_cred, struct thread td);
	typedef int fo_chown_t(struct file *fp, uid_t uid, gid_t gid,
	struct ucred active_cred, struct thread td);
	typedef int fo_sendfile_t(struct file fp, int sockfd, struct uio hdr_uio,
	struct uio *trl_uio, off_t offset, size_t nbytes,
	off_t sent, int flags, struct thread td);
	typedef int fo_seek_t(struct file *fp, off_t offset, int whence,
	struct thread *td);
	typedef int fo_fill_kinfo_t(struct file fp, struct kinfo_file kif,
	struct filedesc *fdp);
	typedef int fo_mmap_t(struct file fp, vm_map_t map, vm_offset_t addr,
	vm_size_t size, vm_prot_t prot, vm_prot_t cap_maxprot,
	int flags, vm_ooffset_t foff, struct thread *td);
	typedef int fo_aio_queue_t(struct file fp, struct kaiocb job);
	typedef int fo_add_seals_t(struct file *fp, int flags);
	typedef int fo_get_seals_t(struct file fp, int flags);
	typedef int fo_fallocate_t(struct file *fp, off_t offset, off_t len,
	struct thread *td);
	typedef int fo_flags_t;

	struct fileops {
	fo_rdwr_t *fo_read;
	fo_rdwr_t *fo_write;
	fo_truncate_t *fo_truncate;
	fo_ioctl_t *fo_ioctl;
	fo_poll_t *fo_poll;
	fo_kqfilter_t *fo_kqfilter;
	fo_stat_t *fo_stat;
	fo_close_t *fo_close;
	fo_chmod_t *fo_chmod;
	fo_chown_t *fo_chown;
	fo_sendfile_t *fo_sendfile;
	fo_seek_t *fo_seek;
	fo_fill_kinfo_t *fo_fill_kinfo;
	fo_mmap_t *fo_mmap;
	fo_aio_queue_t *fo_aio_queue;
	fo_add_seals_t *fo_add_seals;
	fo_get_seals_t *fo_get_seals;
	fo_fallocate_t *fo_fallocate;
	fo_flags_t fo_flags; /* DFLAG_* below */
	};

	#define DFLAG_PASSABLE 0x01 /* may be passed via unix sockets. */
	#define DFLAG_SEEKABLE 0x02 /* seekable / nonsequential */
	#endif /* _KERNEL */

	#if defined(_KERNEL) \|\| defined(_WANT_FILE)
	/*
	* Kernel descriptor table.
	* One entry for each open kernel vnode and socket.
	*
	* Below is the list of locks that protects members in struct file.
	*
	* (a) f_vnode lock required (shared allows both reads and writes)
	* (f) updated with atomics and blocking on sleepq
	* (d) cdevpriv_mtx
	* none not locked
	*/

	struct fadvise_info {
	int fa_advice; /* (f) FADV_* type. */
	off_t fa_start; /* (f) Region start. */
	off_t fa_end; /* (f) Region end. */
	};

	struct file {
	void f_data; / file descriptor specific data */
	struct fileops f_ops; / File operations */
	struct ucred f_cred; / associated credentials. */
	struct vnode f_vnode; / NULL or applicable vnode */
	short f_type; /* descriptor type */
	short f_vnread_flags; /* (f) Sleep lock for f_offset */
	volatile u_int f_flag; /* see fcntl.h */
	volatile u_int f_count; /* reference count */
	/*
	* DTYPE_VNODE specific fields.
	*/
	union {
	int16_t f_seqcount[2]; /* (a) Count of seq. reads and writes. */
	int f_pipegen;
	};
	off_t f_nextoff[2]; /* next expected read/write offset. */
	union {
	struct cdev_privdata *fvn_cdevpriv;
	/* (d) Private data for the cdev. */
	struct fadvise_info *fvn_advice;
	} f_vnun;
	/*
	* DFLAG_SEEKABLE specific fields
	*/
	off_t f_offset;
	};

	#define f_cdevpriv f_vnun.fvn_cdevpriv
	#define f_advice f_vnun.fvn_advice

	#define FOFFSET_LOCKED 0x1
	#define FOFFSET_LOCK_WAITING 0x2

	#endif /* _KERNEL \|\| _WANT_FILE */

	/*
	* Userland version of struct file, for sysctl
	*/
	struct xfile {
	ksize_t xf_size; /* size of struct xfile */
	pid_t xf_pid; /* owning process */
	uid_t xf_uid; /* effective uid of owning process */
	int xf_fd; /* descriptor number */
	int _xf_int_pad1;
	kvaddr_t xf_file; /* address of struct file */
	short xf_type; /* descriptor type */
	short _xf_short_pad1;
	int xf_count; /* reference count */
	int xf_msgcount; /* references from message queue */
	int _xf_int_pad2;
	off_t xf_offset; /* file offset */
	kvaddr_t xf_data; /* file descriptor specific data */
	kvaddr_t xf_vnode; /* vnode pointer */
	u_int xf_flag; /* flags (see fcntl.h) */
	int _xf_int_pad3;
	int64_t _xf_int64_pad[6];
	};

	#ifdef _KERNEL

	extern struct fileops vnops;
	extern struct fileops badfileops;
	extern struct fileops socketops;
	extern int maxfiles; /* kernel limit on number of open files */
	extern int maxfilesperproc; /* per process limit on number of open files */

	int fget(struct thread td, int fd, cap_rights_t rightsp, struct file **fpp);
	int fget_mmap(struct thread td, int fd, cap_rights_t rightsp,
	vm_prot_t maxprotp, struct file *fpp);
	int fget_read(struct thread td, int fd, cap_rights_t rightsp,
	struct file **fpp);
	int fget_write(struct thread td, int fd, cap_rights_t rightsp,
	struct file **fpp);
	int fget_fcntl(struct thread td, int fd, cap_rights_t rightsp,
	int needfcntl, struct file **fpp);
	int _fdrop(struct file fp, struct thread td);

	fo_rdwr_t invfo_rdwr;
	fo_truncate_t invfo_truncate;
	fo_ioctl_t invfo_ioctl;
	fo_poll_t invfo_poll;
	fo_kqfilter_t invfo_kqfilter;
	fo_chmod_t invfo_chmod;
	fo_chown_t invfo_chown;
	fo_sendfile_t invfo_sendfile;

	fo_sendfile_t vn_sendfile;
	fo_seek_t vn_seek;
	fo_fill_kinfo_t vn_fill_kinfo;
	int vn_fill_kinfo_vnode(struct vnode vp, struct kinfo_file kif);

	void finit(struct file , u_int, short, void , struct fileops *);
	int fgetvp(struct thread td, int fd, cap_rights_t rightsp,
	struct vnode **vpp);
	int fgetvp_exec(struct thread td, int fd, cap_rights_t rightsp,
	struct vnode **vpp);
	int fgetvp_rights(struct thread td, int fd, cap_rights_t needrightsp,
	struct filecaps havecaps, struct vnode *vpp);
	int fgetvp_read(struct thread td, int fd, cap_rights_t rightsp,
	struct vnode **vpp);
	int fgetvp_write(struct thread td, int fd, cap_rights_t rightsp,
	struct vnode **vpp);

	-static __inline int
	-_fnoop(void)
	-{
	-
	- return (0);
	-}
	-
	static __inline __result_use_check bool
	fhold(struct file *fp)
	{
	return (refcount_acquire_checked(&fp->f_count));
	}

	-#define fdrop(fp, td) \
	- (refcount_release(&(fp)->f_count) ? _fdrop((fp), (td)) : _fnoop())
	+#define fdrop(fp, td) ({ \
	+ struct file *_fp; \
	+ int _error; \
	+ \
	+ _error = 0; \
	+ _fp = (fp); \
	+ if (__predict_false(refcount_release(&_fp->f_count))) \
	+ _error = _fdrop(_fp, td); \
	+ _error; \
	+})

	static __inline fo_rdwr_t fo_read;
	static __inline fo_rdwr_t fo_write;
	static __inline fo_truncate_t fo_truncate;
	static __inline fo_ioctl_t fo_ioctl;
	static __inline fo_poll_t fo_poll;
	static __inline fo_kqfilter_t fo_kqfilter;
	static __inline fo_stat_t fo_stat;
	static __inline fo_close_t fo_close;
	static __inline fo_chmod_t fo_chmod;
	static __inline fo_chown_t fo_chown;
	static __inline fo_sendfile_t fo_sendfile;

	static __inline int
	fo_read(struct file fp, struct uio uio, struct ucred *active_cred,
	int flags, struct thread *td)
	{

	return ((*fp->f_ops->fo_read)(fp, uio, active_cred, flags, td));
	}

	static __inline int
	fo_write(struct file fp, struct uio uio, struct ucred *active_cred,
	int flags, struct thread *td)
	{

	return ((*fp->f_ops->fo_write)(fp, uio, active_cred, flags, td));
	}

	static __inline int
	fo_truncate(struct file fp, off_t length, struct ucred active_cred,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_truncate)(fp, length, active_cred, td));
	}

	static __inline int
	fo_ioctl(struct file fp, u_long com, void data, struct ucred *active_cred,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_ioctl)(fp, com, data, active_cred, td));
	}

	static __inline int
	fo_poll(struct file fp, int events, struct ucred active_cred,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_poll)(fp, events, active_cred, td));
	}

	static __inline int
	fo_stat(struct file fp, struct stat sb, struct ucred *active_cred,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_stat)(fp, sb, active_cred, td));
	}

	static __inline int
	fo_close(struct file fp, struct thread td)
	{

	return ((*fp->f_ops->fo_close)(fp, td));
	}

	static __inline int
	fo_kqfilter(struct file fp, struct knote kn)
	{

	return ((*fp->f_ops->fo_kqfilter)(fp, kn));
	}

	static __inline int
	fo_chmod(struct file fp, mode_t mode, struct ucred active_cred,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_chmod)(fp, mode, active_cred, td));
	}

	static __inline int
	fo_chown(struct file fp, uid_t uid, gid_t gid, struct ucred active_cred,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_chown)(fp, uid, gid, active_cred, td));
	}

	static __inline int
	fo_sendfile(struct file fp, int sockfd, struct uio hdr_uio,
	struct uio trl_uio, off_t offset, size_t nbytes, off_t sent, int flags,
	struct thread *td)
	{

	return ((*fp->f_ops->fo_sendfile)(fp, sockfd, hdr_uio, trl_uio, offset,
	nbytes, sent, flags, td));
	}

	static __inline int
	fo_seek(struct file fp, off_t offset, int whence, struct thread td)
	{

	return ((*fp->f_ops->fo_seek)(fp, offset, whence, td));
	}

	static __inline int
	fo_fill_kinfo(struct file fp, struct kinfo_file kif, struct filedesc *fdp)
	{

	return ((*fp->f_ops->fo_fill_kinfo)(fp, kif, fdp));
	}

	static __inline int
	fo_mmap(struct file fp, vm_map_t map, vm_offset_t addr, vm_size_t size,
	vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff,
	struct thread *td)
	{

	if (fp->f_ops->fo_mmap == NULL)
	return (ENODEV);
	return ((*fp->f_ops->fo_mmap)(fp, map, addr, size, prot, cap_maxprot,
	flags, foff, td));
	}

	static __inline int
	fo_aio_queue(struct file fp, struct kaiocb job)
	{

	return ((*fp->f_ops->fo_aio_queue)(fp, job));
	}

	static __inline int
	fo_add_seals(struct file *fp, int seals)
	{

	if (fp->f_ops->fo_add_seals == NULL)
	return (EINVAL);
	return ((*fp->f_ops->fo_add_seals)(fp, seals));
	}

	static __inline int
	fo_get_seals(struct file fp, int seals)
	{

	if (fp->f_ops->fo_get_seals == NULL)
	return (EINVAL);
	return ((*fp->f_ops->fo_get_seals)(fp, seals));
	}

	static __inline int
	fo_fallocate(struct file fp, off_t offset, off_t len, struct thread td)
	{

	if (fp->f_ops->fo_fallocate == NULL)
	return (ENODEV);
	return ((*fp->f_ops->fo_fallocate)(fp, offset, len, td));
	}

	#endif /* _KERNEL */

	#endif /* !SYS_FILE_H */
	diff --git a/sys/sys/namei.h b/sys/sys/namei.h
	index 1fa20081a552..0950b0a8f583 100644
	--- a/sys/sys/namei.h
	+++ b/sys/sys/namei.h
	@@ -1,237 +1,245 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1985, 1989, 1991, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)namei.h 8.5 (Berkeley) 1/9/95
	* $FreeBSD$
	*/

	#ifndef _SYS_NAMEI_H_
	#define _SYS_NAMEI_H_

	#include <sys/caprights.h>
	#include <sys/filedesc.h>
	#include <sys/queue.h>
	#include <sys/_uio.h>

	struct componentname {
	/*
	* Arguments to lookup.
	*/
	u_long cn_nameiop; /* namei operation */
	u_int64_t cn_flags; /* flags to namei */
	struct thread cn_thread;/ thread requesting lookup */
	struct ucred cn_cred; / credentials */
	int cn_lkflags; /* Lock flags LK_EXCLUSIVE or LK_SHARED */
	/*
	* Shared between lookup and commit routines.
	*/
	char cn_pnbuf; / pathname buffer */
	char cn_nameptr; / pointer to looked up name */
	long cn_namelen; /* length of looked up component */
	};

	struct nameicap_tracker;
	TAILQ_HEAD(nameicap_tracker_head, nameicap_tracker);

	/*
	* Encapsulation of namei parameters.
	*/
	struct nameidata {
	/*
	* Arguments to namei/lookup.
	*/
	const char ni_dirp; / pathname pointer */
	enum uio_seg ni_segflg; /* location of pathname */
	cap_rights_t ni_rightsneeded; /* rights required to look up vnode */
	/*
	* Arguments to lookup.
	*/
	struct vnode ni_startdir; / starting directory */
	struct vnode ni_rootdir; / logical root directory */
	struct vnode ni_topdir; / logical top directory */
	int ni_dirfd; /* starting directory for at functions /
	int ni_lcf; /* local call flags */
	/*
	* Results: returned from namei
	*/
	struct filecaps ni_filecaps; /* rights the at base has /
	/*
	* Results: returned from/manipulated by lookup
	*/
	struct vnode ni_vp; / vnode of result */
	struct vnode ni_dvp; / vnode of intermediate directory */
	/*
	* Results: flags returned from namei
	*/
	u_int ni_resflags;
	/*
	* Shared between namei and lookup/commit routines.
	*/
	size_t ni_pathlen; /* remaining chars in path */
	char ni_next; / next location in pathname */
	u_int ni_loopcnt; /* count of symlinks encountered */
	/*
	* Lookup parameters: this structure describes the subset of
	* information from the nameidata structure that is passed
	* through the VOP interface.
	*/
	struct componentname ni_cnd;
	struct nameicap_tracker_head ni_cap_tracker;
	struct vnode *ni_beneath_latch;
	};

	#ifdef _KERNEL

	enum cache_fpl_status { CACHE_FPL_STATUS_ABORTED, CACHE_FPL_STATUS_PARTIAL,
	CACHE_FPL_STATUS_HANDLED, CACHE_FPL_STATUS_UNSET };
	int cache_fplookup(struct nameidata ndp, enum cache_fpl_status status,
	struct pwd **pwdp);

	/*
	* namei operations
	*/
	#define LOOKUP 0 /* perform name lookup only */
	#define CREATE 1 /* setup for file creation */
	#define DELETE 2 /* setup for file deletion */
	#define RENAME 3 /* setup for file renaming */
	#define OPMASK 3 /* mask for operation */
	/*
	* namei operational modifier flags, stored in ni_cnd.flags
	*/
	#define LOCKLEAF 0x0004 /* lock vnode on return */
	#define LOCKPARENT 0x0008 /* want parent vnode returned locked */
	#define WANTPARENT 0x0010 /* want parent vnode returned unlocked */
	#define NOCACHE 0x0020 /* name must not be left in cache */
	#define FOLLOW 0x0040 /* follow symbolic links */
	#define BENEATH 0x0080 /* No escape from the start dir */
	#define LOCKSHARED 0x0100 /* Shared lock leaf */
	#define NOFOLLOW 0x0000 /* do not follow symbolic links (pseudo) */
	#define MODMASK 0x01fc /* mask of operational modifiers */
	/*
	* Namei parameter descriptors.
	*
	* SAVENAME may be set by either the callers of namei or by VOP_LOOKUP.
	* If the caller of namei sets the flag (for example execve wants to
	* know the name of the program that is being executed), then it must
	* free the buffer. If VOP_LOOKUP sets the flag, then the buffer must
	* be freed by either the commit routine or the VOP_ABORT routine.
	* SAVESTART is set only by the callers of namei. It implies SAVENAME
	* plus the addition of saving the parent directory that contains the
	* name in ni_startdir. It allows repeated calls to lookup for the
	* name being sought. The caller is responsible for releasing the
	* buffer and for vrele'ing ni_startdir.
	*/
	#define RDONLY 0x00000200 /* lookup with read-only semantics */
	#define HASBUF 0x00000400 /* has allocated pathname buffer */
	#define SAVENAME 0x00000800 /* save pathname buffer */
	#define SAVESTART 0x00001000 /* save starting directory */
	#define ISDOTDOT 0x00002000 /* current component name is .. */
	#define MAKEENTRY 0x00004000 /* entry is to be added to name cache */
	#define ISLASTCN 0x00008000 /* this is last component of pathname */
	#define ISSYMLINK 0x00010000 /* symlink needs interpretation */
	#define ISWHITEOUT 0x00020000 /* found whiteout */
	#define DOWHITEOUT 0x00040000 /* do whiteouts */
	#define WILLBEDIR 0x00080000 /* new files will be dirs; allow trailing / */
	#define ISUNICODE 0x00100000 /* current component name is unicode*/
	#define ISOPEN 0x00200000 /* caller is opening; return a real vnode. */
	#define NOCROSSMOUNT 0x00400000 /* do not cross mount points */
	#define NOMACCHECK 0x00800000 /* do not perform MAC checks */
	#define AUDITVNODE1 0x04000000 /* audit the looked up vnode information */
	#define AUDITVNODE2 0x08000000 /* audit the looked up vnode information */
	#define TRAILINGSLASH 0x10000000 /* path ended in a slash */
	#define NOCAPCHECK 0x20000000 /* do not perform capability checks */
	#define NOEXECCHECK 0x40000000 /* do not perform exec check on dir */
	#define PARAMASK 0x7ffffe00 /* mask of parameter descriptors */

	/*
	* Namei results flags
	*/
	#define NIRES_ABS 0x00000001 /* Path was absolute */

	/*
	* Flags in ni_lcf, valid for the duration of the namei call.
	*/
	#define NI_LCF_STRICTRELATIVE 0x0001 /* relative lookup only */
	#define NI_LCF_CAP_DOTDOT 0x0002 /* ".." in strictrelative case */
	#define NI_LCF_BENEATH_ABS 0x0004 /* BENEATH with absolute path */
	#define NI_LCF_BENEATH_LATCHED 0x0008 /* BENEATH_ABS traversed starting dir */
	#define NI_LCF_LATCH 0x0010 /* ni_beneath_latch valid */

	/*
	* Initialization of a nameidata structure.
	*/
	#define NDINIT(ndp, op, flags, segflg, namep, td) \
	NDINIT_ALL(ndp, op, flags, segflg, namep, AT_FDCWD, NULL, 0, td)
	#define NDINIT_AT(ndp, op, flags, segflg, namep, dirfd, td) \
	NDINIT_ALL(ndp, op, flags, segflg, namep, dirfd, NULL, 0, td)
	#define NDINIT_ATRIGHTS(ndp, op, flags, segflg, namep, dirfd, rightsp, td) \
	NDINIT_ALL(ndp, op, flags, segflg, namep, dirfd, NULL, rightsp, td)
	#define NDINIT_ATVP(ndp, op, flags, segflg, namep, vp, td) \
	NDINIT_ALL(ndp, op, flags, segflg, namep, AT_FDCWD, vp, 0, td)

	void NDINIT_ALL(struct nameidata *ndp, u_long op, u_long flags,
	enum uio_seg segflg, const char namep, int dirfd, struct vnode startdir,
	cap_rights_t rightsp, struct thread td);

	#define NDF_NO_DVP_RELE 0x00000001
	#define NDF_NO_DVP_UNLOCK 0x00000002
	#define NDF_NO_DVP_PUT 0x00000003
	#define NDF_NO_VP_RELE 0x00000004
	#define NDF_NO_VP_UNLOCK 0x00000008
	#define NDF_NO_VP_PUT 0x0000000c
	#define NDF_NO_STARTDIR_RELE 0x00000010
	#define NDF_NO_FREE_PNBUF 0x00000020
	#define NDF_ONLY_PNBUF (~NDF_NO_FREE_PNBUF)

	+void NDFREE_PNBUF(struct nameidata *);
	void NDFREE(struct nameidata *, const u_int);
	+#define NDFREE(ndp, flags) do { \
	+ struct nameidata *_ndp = (ndp); \
	+ if (__builtin_constant_p(flags) && flags == NDF_ONLY_PNBUF) \
	+ NDFREE_PNBUF(_ndp); \
	+ else \
	+ NDFREE(_ndp, flags); \
	+} while (0)

	int namei(struct nameidata *ndp);
	int lookup(struct nameidata *ndp);
	int relookup(struct vnode dvp, struct vnode *vpp,
	struct componentname *cnp);
	#endif

	/*
	* Stats on usefulness of namei caches.
	*/
	struct nchstats {
	long ncs_goodhits; /* hits that we can really use */
	long ncs_neghits; /* negative hits that we can use */
	long ncs_badhits; /* hits we must drop */
	long ncs_falsehits; /* hits with id mismatch */
	long ncs_miss; /* misses */
	long ncs_long; /* long names that ignore cache */
	long ncs_pass2; /* names found with passes == 2 */
	long ncs_2passes; /* number of times we attempt it */
	};

	extern struct nchstats nchstats;

	#endif /* !_SYS_NAMEI_H_ */
	diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h
	index 08fa64bed407..d0390ac42f4d 100644
	--- a/sys/sys/sysctl.h
	+++ b/sys/sys/sysctl.h
	@@ -1,1189 +1,1189 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* Mike Karels at Berkeley Software Design, Inc.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)sysctl.h 8.1 (Berkeley) 6/2/93
	* $FreeBSD$
	*/

	#ifndef _SYS_SYSCTL_H_
	#define _SYS_SYSCTL_H_

	#ifdef _KERNEL
	#include <sys/queue.h>
	#endif

	struct thread;
	/*
	* Definitions for sysctl call. The sysctl call uses a hierarchical name
	* for objects that can be examined or modified. The name is expressed as
	* a sequence of integers. Like a file path name, the meaning of each
	* component depends on its place in the hierarchy. The top-level and kern
	* identifiers are defined here, and other identifiers are defined in the
	* respective subsystem header files.
	*/

	#define CTL_MAXNAME 24 /* largest number of components supported */

	/*
	* Each subsystem defined by sysctl defines a list of variables
	* for that subsystem. Each name is either a node with further
	* levels defined below it, or it is a leaf of some particular
	* type given below. Each sysctl level defines a set of name/type
	* pairs to be used by sysctl(8) in manipulating the subsystem.
	*/
	struct ctlname {
	char ctl_name; / subsystem name */
	int ctl_type; /* type of name */
	};

	#define CTLTYPE 0xf /* mask for the type */
	#define CTLTYPE_NODE 1 /* name is a node */
	#define CTLTYPE_INT 2 /* name describes an integer */
	#define CTLTYPE_STRING 3 /* name describes a string */
	#define CTLTYPE_S64 4 /* name describes a signed 64-bit number */
	#define CTLTYPE_OPAQUE 5 /* name describes a structure */
	#define CTLTYPE_STRUCT CTLTYPE_OPAQUE /* name describes a structure */
	#define CTLTYPE_UINT 6 /* name describes an unsigned integer */
	#define CTLTYPE_LONG 7 /* name describes a long */
	#define CTLTYPE_ULONG 8 /* name describes an unsigned long */
	#define CTLTYPE_U64 9 /* name describes an unsigned 64-bit number */
	#define CTLTYPE_U8 0xa /* name describes an unsigned 8-bit number */
	#define CTLTYPE_U16 0xb /* name describes an unsigned 16-bit number */
	#define CTLTYPE_S8 0xc /* name describes a signed 8-bit number */
	#define CTLTYPE_S16 0xd /* name describes a signed 16-bit number */
	#define CTLTYPE_S32 0xe /* name describes a signed 32-bit number */
	#define CTLTYPE_U32 0xf /* name describes an unsigned 32-bit number */

	#define CTLFLAG_RD 0x80000000 /* Allow reads of variable */
	#define CTLFLAG_WR 0x40000000 /* Allow writes to the variable */
	#define CTLFLAG_RW (CTLFLAG_RD\|CTLFLAG_WR)
	#define CTLFLAG_DORMANT 0x20000000 /* This sysctl is not active yet */
	#define CTLFLAG_ANYBODY 0x10000000 /* All users can set this var */
	#define CTLFLAG_SECURE 0x08000000 /* Permit set only if securelevel<=0 */
	#define CTLFLAG_PRISON 0x04000000 /* Prisoned roots can fiddle */
	#define CTLFLAG_DYN 0x02000000 /* Dynamic oid - can be freed */
	#define CTLFLAG_SKIP 0x01000000 /* Skip this sysctl when listing */
	#define CTLMASK_SECURE 0x00F00000 /* Secure level */
	#define CTLFLAG_TUN 0x00080000 /* Default value is loaded from getenv() */
	#define CTLFLAG_RDTUN (CTLFLAG_RD\|CTLFLAG_TUN)
	#define CTLFLAG_RWTUN (CTLFLAG_RW\|CTLFLAG_TUN)
	#define CTLFLAG_MPSAFE 0x00040000 /* Handler is MP safe */
	#define CTLFLAG_VNET 0x00020000 /* Prisons with vnet can fiddle */
	#define CTLFLAG_DYING 0x00010000 /* Oid is being removed */
	#define CTLFLAG_CAPRD 0x00008000 /* Can be read in capability mode */
	#define CTLFLAG_CAPWR 0x00004000 /* Can be written in capability mode */
	#define CTLFLAG_STATS 0x00002000 /* Statistics, not a tuneable */
	#define CTLFLAG_NOFETCH 0x00001000 /* Don't fetch tunable from getenv() */
	#define CTLFLAG_CAPRW (CTLFLAG_CAPRD\|CTLFLAG_CAPWR)
	/*
	* This is transient flag to be used until all sysctl handlers are converted
	* to not lock Giant.
	* One, and only one of CTLFLAG_MPSAFE or CTLFLAG_NEEDGIANT is required
	* for SYSCTL_PROC and SYSCTL_NODE.
	*/
	#define CTLFLAG_NEEDGIANT 0x00000800 /* Handler require Giant */

	/*
	* Secure level. Note that CTLFLAG_SECURE == CTLFLAG_SECURE1.
	*
	* Secure when the securelevel is raised to at least N.
	*/
	#define CTLSHIFT_SECURE 20
	#define CTLFLAG_SECURE1 (CTLFLAG_SECURE \| (0 << CTLSHIFT_SECURE))
	#define CTLFLAG_SECURE2 (CTLFLAG_SECURE \| (1 << CTLSHIFT_SECURE))
	#define CTLFLAG_SECURE3 (CTLFLAG_SECURE \| (2 << CTLSHIFT_SECURE))

	/*
	* USE THIS instead of a hardwired number from the categories below
	* to get dynamically assigned sysctl entries using the linker-set
	* technology. This is the way nearly all new sysctl variables should
	* be implemented.
	* e.g. SYSCTL_INT(_parent, OID_AUTO, name, CTLFLAG_RW, &variable, 0, "");
	*/
	#define OID_AUTO (-1)

	/*
	* The starting number for dynamically-assigned entries. WARNING!
	* ALL static sysctl entries should have numbers LESS than this!
	*/
	#define CTL_AUTO_START 0x100

	#ifdef _KERNEL
	#include <sys/linker_set.h>

	#ifdef KLD_MODULE
	/* XXX allow overspecification of type in external kernel modules */
	#define SYSCTL_CT_ASSERT_MASK CTLTYPE
	#else
	#define SYSCTL_CT_ASSERT_MASK 0
	#endif

	#define SYSCTL_HANDLER_ARGS struct sysctl_oid oidp, void arg1, \
	intmax_t arg2, struct sysctl_req *req

	/* definitions for sysctl_req 'lock' member */
	#define REQ_UNWIRED 1
	#define REQ_WIRED 2

	/* definitions for sysctl_req 'flags' member */
	#ifdef COMPAT_FREEBSD32
	#define SCTL_MASK32 1 /* 32 bit emulation */
	#endif

	/*
	* This describes the access space for a sysctl request. This is needed
	* so that we can use the interface from the kernel or from user-space.
	*/
	struct sysctl_req {
	struct thread td; / used for access checking */
	int lock; /* wiring state */
	void *oldptr;
	size_t oldlen;
	size_t oldidx;
	int (oldfunc)(struct sysctl_req , const void *, size_t);
	const void *newptr;
	size_t newlen;
	size_t newidx;
	int (newfunc)(struct sysctl_req , void *, size_t);
	size_t validlen;
	int flags;
	};

	SLIST_HEAD(sysctl_oid_list, sysctl_oid);

	/*
	* This describes one "oid" in the MIB tree. Potentially more nodes can
	* be hidden behind it, expanded by the handler.
	*/
	struct sysctl_oid {
	struct sysctl_oid_list oid_children;
	struct sysctl_oid_list *oid_parent;
	SLIST_ENTRY(sysctl_oid) oid_link;
	int oid_number;
	u_int oid_kind;
	void *oid_arg1;
	intmax_t oid_arg2;
	const char *oid_name;
	int (*oid_handler)(SYSCTL_HANDLER_ARGS);
	const char *oid_fmt;
	int oid_refcnt;
	u_int oid_running;
	const char *oid_descr;
	const char *oid_label;
	};

	#define SYSCTL_IN(r, p, l) (r->newfunc)(r, p, l)
	#define SYSCTL_OUT(r, p, l) (r->oldfunc)(r, p, l)
	#define SYSCTL_OUT_STR(r, p) (r->oldfunc)(r, p, strlen(p) + 1)

	int sysctl_handle_bool(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_8(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_16(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_32(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_64(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_int(SYSCTL_HANDLER_ARGS);
	int sysctl_msec_to_ticks(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_long(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_string(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_opaque(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_counter_u64_array(SYSCTL_HANDLER_ARGS);

	int sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS);
	int sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS);

	int sysctl_msec_to_sbintime(SYSCTL_HANDLER_ARGS);
	int sysctl_usec_to_sbintime(SYSCTL_HANDLER_ARGS);
	int sysctl_sec_to_timeval(SYSCTL_HANDLER_ARGS);

	int sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS);
	int sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS);
	int sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS);

	/*
	* These functions are used to add/remove an oid from the mib.
	*/
	void sysctl_register_oid(struct sysctl_oid *oidp);
	void sysctl_register_disabled_oid(struct sysctl_oid *oidp);
	void sysctl_enable_oid(struct sysctl_oid *oidp);
	void sysctl_unregister_oid(struct sysctl_oid *oidp);

	/* Declare a static oid to allow child oids to be added to it. */
	#define SYSCTL_DECL(name) \
	extern struct sysctl_oid sysctl__##name

	/* Hide these in macros. */
	#define SYSCTL_CHILDREN(oid_ptr) (&(oid_ptr)->oid_children)
	#define SYSCTL_PARENT(oid_ptr) \
	(((oid_ptr)->oid_parent != &sysctl__children) ? \
	__containerof((oid_ptr)->oid_parent, struct sysctl_oid, \
	oid_children) : (struct sysctl_oid *)NULL)
	#define SYSCTL_STATIC_CHILDREN(oid_name) (&sysctl__##oid_name.oid_children)

	/* === Structs and macros related to context handling. === */

	/* All dynamically created sysctls can be tracked in a context list. */
	struct sysctl_ctx_entry {
	struct sysctl_oid *entry;
	TAILQ_ENTRY(sysctl_ctx_entry) link;
	};

	TAILQ_HEAD(sysctl_ctx_list, sysctl_ctx_entry);

	#define SYSCTL_NODE_CHILDREN(parent, name) \
	sysctl__##parent##_##name.oid_children

	#ifndef NO_SYSCTL_DESCR
	#define __DESCR(d) d
	#else
	#define __DESCR(d) ""
	#endif

	#ifdef notyet
	#define SYSCTL_ENFORCE_FLAGS(x) \
	_Static_assert((((x) & CTLFLAG_MPSAFE) != 0) ^ (((x) & CTLFLAG_NEEDGIANT) != 0), \
	"Has to be either CTLFLAG_MPSAFE or CTLFLAG_NEEDGIANT")
	#else
	#define SYSCTL_ENFORCE_FLAGS(x)
	#endif

	/* This macro is only for internal use */
	#define SYSCTL_OID_RAW(id, parent_child_head, nbr, name, kind, a1, a2, handler, fmt, descr, label) \
	struct sysctl_oid id = { \
	.oid_parent = (parent_child_head), \
	.oid_children = SLIST_HEAD_INITIALIZER(&id.oid_children), \
	.oid_number = (nbr), \
	.oid_kind = (kind), \
	.oid_arg1 = (a1), \
	.oid_arg2 = (a2), \
	.oid_name = (name), \
	.oid_handler = (handler), \
	.oid_fmt = (fmt), \
	.oid_descr = __DESCR(descr), \
	.oid_label = (label), \
	}; \
	DATA_SET(sysctl_set, id); \
	SYSCTL_ENFORCE_FLAGS(kind)

	/* This constructs a static "raw" MIB oid. */
	#define SYSCTL_OID(parent, nbr, name, kind, a1, a2, handler, fmt, descr) \
	SYSCTL_OID_WITH_LABEL(parent, nbr, name, kind, a1, a2, \
	handler, fmt, descr, NULL)

	#define SYSCTL_OID_WITH_LABEL(parent, nbr, name, kind, a1, a2, handler, fmt, descr, label) \
	static SYSCTL_OID_RAW(sysctl__##parent##_##name, \
	SYSCTL_CHILDREN(&sysctl__##parent), \
	nbr, #name, kind, a1, a2, handler, fmt, descr, label)

	/* This constructs a global "raw" MIB oid. */
	#define SYSCTL_OID_GLOBAL(parent, nbr, name, kind, a1, a2, handler, fmt, descr, label) \
	SYSCTL_OID_RAW(sysctl__##parent##_##name, \
	SYSCTL_CHILDREN(&sysctl__##parent), \
	nbr, #name, kind, a1, a2, handler, fmt, descr, label)

	#define SYSCTL_ADD_OID(ctx, parent, nbr, name, kind, a1, a2, handler, fmt, descr) \
	({ \
	SYSCTL_ENFORCE_FLAGS(kind); \
	sysctl_add_oid(ctx, parent, nbr, name, kind, a1, a2,handler, \
	fmt, __DESCR(descr), NULL); \
	})

	/* This constructs a root node from which other nodes can hang. */
	#define SYSCTL_ROOT_NODE(nbr, name, access, handler, descr) \
	SYSCTL_OID_RAW(sysctl___##name, &sysctl__children, \
	nbr, #name, CTLTYPE_NODE\|(access), NULL, 0, \
	handler, "N", descr, NULL); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_NODE)

	/* This constructs a node from which other oids can hang. */
	#define SYSCTL_NODE(parent, nbr, name, access, handler, descr) \
	SYSCTL_NODE_WITH_LABEL(parent, nbr, name, access, handler, descr, NULL)

	#define SYSCTL_NODE_WITH_LABEL(parent, nbr, name, access, handler, descr, label) \
	SYSCTL_OID_GLOBAL(parent, nbr, name, CTLTYPE_NODE\|(access), \
	NULL, 0, handler, "N", descr, label); \
	SYSCTL_ENFORCE_FLAGS(access); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_NODE)

	#define SYSCTL_ADD_NODE(ctx, parent, nbr, name, access, handler, descr) \
	SYSCTL_ADD_NODE_WITH_LABEL(ctx, parent, nbr, name, access, \
	handler, descr, NULL)

	#define SYSCTL_ADD_NODE_WITH_LABEL(ctx, parent, nbr, name, access, handler, descr, label) \
	({ \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_NODE); \
	SYSCTL_ENFORCE_FLAGS(access); \
	sysctl_add_oid(ctx, parent, nbr, name, CTLTYPE_NODE\|(access), \
	NULL, 0, handler, "N", __DESCR(descr), label); \
	})

	#define SYSCTL_ADD_ROOT_NODE(ctx, nbr, name, access, handler, descr) \
	({ \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_NODE); \
	SYSCTL_ENFORCE_FLAGS(access); \
	sysctl_add_oid(ctx, &sysctl__children, nbr, name, \
	CTLTYPE_NODE\|(access), \
	NULL, 0, handler, "N", __DESCR(descr), NULL); \
	})

	/* Oid for a string. len can be 0 to indicate '\0' termination. */
	#define SYSCTL_STRING(parent, nbr, name, access, arg, len, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_STRING \| CTLFLAG_MPSAFE \| (access), \
	arg, len, sysctl_handle_string, "A", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_STRING)

	#define SYSCTL_ADD_STRING(ctx, parent, nbr, name, access, arg, len, descr) \
	({ \
	char *__arg = (arg); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_STRING); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_STRING \| CTLFLAG_MPSAFE \| (access), \
	__arg, len, sysctl_handle_string, "A", __DESCR(descr), \
	NULL); \
	})

	/* Oid for a constant '\0' terminated string. */
	#define SYSCTL_CONST_STRING(parent, nbr, name, access, arg, descr) \
	SYSCTL_OID(parent, nbr, name, CTLTYPE_STRING \| CTLFLAG_MPSAFE \| (access),\
	__DECONST(char *, arg), 0, sysctl_handle_string, "A", descr); \
	CTASSERT(!(access & CTLFLAG_WR)); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_STRING)

	#define SYSCTL_ADD_CONST_STRING(ctx, parent, nbr, name, access, arg, descr) \
	({ \
	char __arg = __DECONST(char , arg); \
	CTASSERT(!(access & CTLFLAG_WR)); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_STRING); \
	sysctl_add_oid(ctx, parent, nbr, name, CTLTYPE_STRING \| \
	CTLFLAG_MPSAFE \| (access), __arg, 0, sysctl_handle_string, "A",\
	__DESCR(descr), NULL); \
	})

	/* Oid for a bool. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_BOOL_PTR ((bool *)NULL)
	#define SYSCTL_BOOL(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U8 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_bool, "CU", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 && \
	sizeof(bool) == sizeof(*(ptr)))

	#define SYSCTL_ADD_BOOL(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	bool *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U8 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_bool, "CU", __DESCR(descr), \
	NULL); \
	})

	/* Oid for a signed 8-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_S8_PTR ((int8_t *)NULL)
	#define SYSCTL_S8(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_S8 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_8, "C", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S8) && \
	sizeof(int8_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_S8(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	int8_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S8); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_S8 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_8, "C", __DESCR(descr), NULL); \
	})

	/* Oid for an unsigned 8-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_U8_PTR ((uint8_t *)NULL)
	#define SYSCTL_U8(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U8 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_8, "CU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U8) && \
	sizeof(uint8_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_U8(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	uint8_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U8); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U8 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_8, "CU", __DESCR(descr), NULL); \
	})

	/* Oid for a signed 16-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_S16_PTR ((int16_t *)NULL)
	#define SYSCTL_S16(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_S16 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_16, "S", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S16) && \
	sizeof(int16_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_S16(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	int16_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S16); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_S16 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_16, "S", __DESCR(descr), NULL); \
	})

	/* Oid for an unsigned 16-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_U16_PTR ((uint16_t *)NULL)
	#define SYSCTL_U16(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U16 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_16, "SU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U16) && \
	sizeof(uint16_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_U16(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	uint16_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U16); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U16 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_16, "SU", __DESCR(descr), NULL); \
	})

	/* Oid for a signed 32-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_S32_PTR ((int32_t *)NULL)
	#define SYSCTL_S32(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_S32 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_32, "I", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S32) && \
	sizeof(int32_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_S32(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	int32_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S32); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_S32 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_32, "I", __DESCR(descr), NULL); \
	})

	/* Oid for an unsigned 32-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_U32_PTR ((uint32_t *)NULL)
	#define SYSCTL_U32(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U32 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_32, "IU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U32) && \
	sizeof(uint32_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_U32(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	uint32_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U32); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U32 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_32, "IU", __DESCR(descr), NULL); \
	})

	/* Oid for a signed 64-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_S64_PTR ((int64_t *)NULL)
	#define SYSCTL_S64(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_S64 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_64, "Q", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64) && \
	sizeof(int64_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_S64(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	int64_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_S64 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_64, "Q", __DESCR(descr), NULL); \
	})

	/* Oid for an unsigned 64-bit int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_U64_PTR ((uint64_t *)NULL)
	#define SYSCTL_U64(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_64, "QU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U64) && \
	sizeof(uint64_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_U64(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	uint64_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_64, "QU", __DESCR(descr), NULL); \
	})

	/* Oid for an int. If ptr is SYSCTL_NULL_INT_PTR, val is returned. */
	#define SYSCTL_NULL_INT_PTR ((int *)NULL)
	#define SYSCTL_INT(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_INT_WITH_LABEL(parent, nbr, name, access, ptr, val, descr, NULL)

	#define SYSCTL_INT_WITH_LABEL(parent, nbr, name, access, ptr, val, descr, label) \
	SYSCTL_OID_WITH_LABEL(parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_int, "I", descr, label); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT) && \
	sizeof(int) == sizeof(*(ptr)))

	#define SYSCTL_ADD_INT(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	int *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_int, "I", __DESCR(descr), NULL); \
	})

	/* Oid for an unsigned int. If ptr is NULL, val is returned. */
	#define SYSCTL_NULL_UINT_PTR ((unsigned *)NULL)
	#define SYSCTL_UINT(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_UINT \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_int, "IU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_UINT) && \
	sizeof(unsigned) == sizeof(*(ptr)))

	#define SYSCTL_ADD_UINT(ctx, parent, nbr, name, access, ptr, val, descr) \
	({ \
	unsigned *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_UINT); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_UINT \| CTLFLAG_MPSAFE \| (access), \
	__ptr, val, sysctl_handle_int, "IU", __DESCR(descr), NULL); \
	})

	/* Oid for a long. The pointer must be non NULL. */
	#define SYSCTL_NULL_LONG_PTR ((long *)NULL)
	#define SYSCTL_LONG(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_LONG \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_long, "L", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_LONG) && \
	sizeof(long) == sizeof(*(ptr)))

	#define SYSCTL_ADD_LONG(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	long *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_LONG); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_LONG \| CTLFLAG_MPSAFE \| (access), \
	__ptr, 0, sysctl_handle_long, "L", __DESCR(descr), NULL); \
	})

	/* Oid for an unsigned long. The pointer must be non NULL. */
	#define SYSCTL_NULL_ULONG_PTR ((unsigned long *)NULL)
	#define SYSCTL_ULONG(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_ULONG \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_long, "LU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_ULONG) && \
	sizeof(unsigned long) == sizeof(*(ptr)))

	#define SYSCTL_ADD_ULONG(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	unsigned long *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_ULONG); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_ULONG \| CTLFLAG_MPSAFE \| (access), \
	__ptr, 0, sysctl_handle_long, "LU", __DESCR(descr), NULL); \
	})

	/* Oid for a quad. The pointer must be non NULL. */
	#define SYSCTL_NULL_QUAD_PTR ((int64_t *)NULL)
	#define SYSCTL_QUAD(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_S64 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_64, "Q", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64) && \
	sizeof(int64_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_QUAD(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	int64_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_S64 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, 0, sysctl_handle_64, "Q", __DESCR(descr), NULL); \
	})

	#define SYSCTL_NULL_UQUAD_PTR ((uint64_t *)NULL)
	#define SYSCTL_UQUAD(parent, nbr, name, access, ptr, val, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| (access), \
	ptr, val, sysctl_handle_64, "QU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U64) && \
	sizeof(uint64_t) == sizeof(*(ptr)))

	#define SYSCTL_ADD_UQUAD(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	uint64_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| (access), \
	__ptr, 0, sysctl_handle_64, "QU", __DESCR(descr), NULL); \
	})

	/* Oid for a CPU dependent variable */
	#define SYSCTL_ADD_UAUTO(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	struct sysctl_oid *__ret; \
	CTASSERT((sizeof(uint64_t) == sizeof(*(ptr)) \|\| \
	sizeof(unsigned) == sizeof(*(ptr))) && \
	((access) & CTLTYPE) == 0); \
	if (sizeof(uint64_t) == sizeof(*(ptr))) { \
	__ret = sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| (access), \
	(ptr), 0, sysctl_handle_64, "QU", \
	__DESCR(descr), NULL); \
	} else { \
	__ret = sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_UINT \| CTLFLAG_MPSAFE \| (access), \
	(ptr), 0, sysctl_handle_int, "IU", \
	__DESCR(descr), NULL); \
	} \
	__ret; \
	})

	/* Oid for a 64-bit unsigned counter(9). The pointer must be non NULL. */
	#define SYSCTL_COUNTER_U64(parent, nbr, name, access, ptr, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| CTLFLAG_STATS \| (access), \
	(ptr), 0, sysctl_handle_counter_u64, "QU", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U64) && \
	sizeof(counter_u64_t) == sizeof(*(ptr)) && \
	sizeof(uint64_t) == sizeof(**(ptr)))

	#define SYSCTL_ADD_COUNTER_U64(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	counter_u64_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_U64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_U64 \| CTLFLAG_MPSAFE \| CTLFLAG_STATS \| (access), \
	__ptr, 0, sysctl_handle_counter_u64, "QU", __DESCR(descr), \
	NULL); \
	})

	/* Oid for an array of counter(9)s. The pointer and length must be non zero. */
	#define SYSCTL_COUNTER_U64_ARRAY(parent, nbr, name, access, ptr, len, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| CTLFLAG_STATS \| (access), \
	(ptr), (len), sysctl_handle_counter_u64_array, "S", descr); \
	CTASSERT((((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_OPAQUE) && \
	sizeof(counter_u64_t) == sizeof(*(ptr)) && \
	sizeof(uint64_t) == sizeof(**(ptr)))

	#define SYSCTL_ADD_COUNTER_U64_ARRAY(ctx, parent, nbr, name, access, \
	ptr, len, descr) \
	({ \
	counter_u64_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_OPAQUE); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| CTLFLAG_STATS \| (access), \
	__ptr, len, sysctl_handle_counter_u64_array, "S", \
	__DESCR(descr), NULL); \
	})

	/* Oid for an opaque object. Specified by a pointer and a length. */
	#define SYSCTL_OPAQUE(parent, nbr, name, access, ptr, len, fmt, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| (access), \
	ptr, len, sysctl_handle_opaque, fmt, descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_OPAQUE)

	#define SYSCTL_ADD_OPAQUE(ctx, parent, nbr, name, access, ptr, len, fmt, descr) \
	({ \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_OPAQUE); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| (access), \
	ptr, len, sysctl_handle_opaque, fmt, __DESCR(descr), NULL); \
	})

	/* Oid for a struct. Specified by a pointer and a type. */
	#define SYSCTL_STRUCT(parent, nbr, name, access, ptr, type, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| (access), \
	ptr, sizeof(struct type), sysctl_handle_opaque, \
	"S," #type, descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_OPAQUE)

	#define SYSCTL_ADD_STRUCT(ctx, parent, nbr, name, access, ptr, type, descr) \
	({ \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_OPAQUE); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_OPAQUE \| CTLFLAG_MPSAFE \| (access), \
	(ptr), sizeof(struct type), \
	sysctl_handle_opaque, "S," #type, __DESCR(descr), NULL); \
	})

	/* Oid for a procedure. Specified by a pointer and an arg. */
	#define SYSCTL_PROC(parent, nbr, name, access, ptr, arg, handler, fmt, descr) \
	SYSCTL_OID(parent, nbr, name, (access), \
	ptr, arg, handler, fmt, descr); \
	CTASSERT(((access) & CTLTYPE) != 0)

	#define SYSCTL_ADD_PROC(ctx, parent, nbr, name, access, ptr, arg, handler, fmt, descr) \
	({ \
	CTASSERT(((access) & CTLTYPE) != 0); \
	SYSCTL_ENFORCE_FLAGS(access); \
	sysctl_add_oid(ctx, parent, nbr, name, (access), \
	(ptr), (arg), (handler), (fmt), __DESCR(descr), NULL); \
	})

	/* Oid to handle limits on uma(9) zone specified by pointer. */
	#define SYSCTL_UMA_MAX(parent, nbr, name, access, ptr, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| (access), \
	(ptr), 0, sysctl_handle_uma_zone_max, "I", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT)

	#define SYSCTL_ADD_UMA_MAX(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	uma_zone_t __ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| (access), \
	__ptr, 0, sysctl_handle_uma_zone_max, "I", __DESCR(descr), \
	NULL); \
	})

	/* Oid to obtain current use of uma(9) zone specified by pointer. */
	#define SYSCTL_UMA_CUR(parent, nbr, name, access, ptr, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	(ptr), 0, sysctl_handle_uma_zone_cur, "I", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT)

	#define SYSCTL_ADD_UMA_CUR(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	uma_zone_t __ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	__ptr, 0, sysctl_handle_uma_zone_cur, "I", __DESCR(descr), \
	NULL); \
	})

	/* OID expressing a sbintime_t as microseconds */
	#define SYSCTL_SBINTIME_USEC(parent, nbr, name, access, ptr, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	(ptr), 0, sysctl_usec_to_sbintime, "Q", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64)
	#define SYSCTL_ADD_SBINTIME_USEC(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	sbintime_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	__ptr, 0, sysctl_usec_to_sbintime, "Q", __DESCR(descr), \
	NULL); \
	})

	/* OID expressing a sbintime_t as milliseconds */
	#define SYSCTL_SBINTIME_MSEC(parent, nbr, name, access, ptr, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	(ptr), 0, sysctl_msec_to_sbintime, "Q", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64)
	#define SYSCTL_ADD_SBINTIME_MSEC(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	sbintime_t *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_S64); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	__ptr, 0, sysctl_msec_to_sbintime, "Q", __DESCR(descr), \
	NULL); \
	})

	/* OID expressing a struct timeval as seconds */
	#define SYSCTL_TIMEVAL_SEC(parent, nbr, name, access, ptr, descr) \
	SYSCTL_OID(parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	(ptr), 0, sysctl_sec_to_timeval, "I", descr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT)
	#define SYSCTL_ADD_TIMEVAL_SEC(ctx, parent, nbr, name, access, ptr, descr) \
	({ \
	struct timeval *__ptr = (ptr); \
	CTASSERT(((access) & CTLTYPE) == 0 \|\| \
	((access) & SYSCTL_CT_ASSERT_MASK) == CTLTYPE_INT); \
	sysctl_add_oid(ctx, parent, nbr, name, \
	CTLTYPE_INT \| CTLFLAG_MPSAFE \| CTLFLAG_RD \| (access), \
	__ptr, 0, sysctl_sec_to_timeval, "I", __DESCR(descr), \
	NULL); \
	})

	/*
	* A macro to generate a read-only sysctl to indicate the presence of optional
	* kernel features.
	*/
	#define FEATURE(name, desc) \
	SYSCTL_INT_WITH_LABEL(_kern_features, OID_AUTO, name, \
	CTLFLAG_RD \| CTLFLAG_CAPRD, SYSCTL_NULL_INT_PTR, 1, desc, "feature")

	#endif /* _KERNEL */

	/*
	* Top-level identifiers
	*/
	#define CTL_SYSCTL 0 /* "magic" numbers */
	#define CTL_KERN 1 /* "high kernel": proc, limits */
	#define CTL_VM 2 /* virtual memory */
	#define CTL_VFS 3 /* filesystem, mount type is next */
	#define CTL_NET 4 /* network, see socket.h */
	#define CTL_DEBUG 5 /* debugging parameters */
	#define CTL_HW 6 /* generic cpu/io */
	#define CTL_MACHDEP 7 /* machine dependent */
	#define CTL_USER 8 /* user-level */
	#define CTL_P1003_1B 9 /* POSIX 1003.1B */

	/*
	* CTL_SYSCTL identifiers
	*/
	#define CTL_SYSCTL_DEBUG 0 /* printf all nodes */
	#define CTL_SYSCTL_NAME 1 /* string name of OID */
	#define CTL_SYSCTL_NEXT 2 /* next OID */
	#define CTL_SYSCTL_NAME2OID 3 /* int array of name */
	#define CTL_SYSCTL_OIDFMT 4 /* OID's kind and format */
	#define CTL_SYSCTL_OIDDESCR 5 /* OID's description */
	#define CTL_SYSCTL_OIDLABEL 6 /* aggregation label */

	/*
	* CTL_KERN identifiers
	*/
	#define KERN_OSTYPE 1 /* string: system version */
	#define KERN_OSRELEASE 2 /* string: system release */
	#define KERN_OSREV 3 /* int: system revision */
	#define KERN_VERSION 4 /* string: compile time info */
	#define KERN_MAXVNODES 5 /* int: max vnodes */
	#define KERN_MAXPROC 6 /* int: max processes */
	#define KERN_MAXFILES 7 /* int: max open files */
	#define KERN_ARGMAX 8 /* int: max arguments to exec */
	#define KERN_SECURELVL 9 /* int: system security level */
	#define KERN_HOSTNAME 10 /* string: hostname */
	#define KERN_HOSTID 11 /* int: host identifier */
	#define KERN_CLOCKRATE 12 /* struct: struct clockrate */
	#define KERN_VNODE 13 /* struct: vnode structures */
	#define KERN_PROC 14 /* struct: process entries */
	#define KERN_FILE 15 /* struct: file entries */
	#define KERN_PROF 16 /* node: kernel profiling info */
	#define KERN_POSIX1 17 /* int: POSIX.1 version */
	#define KERN_NGROUPS 18 /* int: # of supplemental group ids */
	#define KERN_JOB_CONTROL 19 /* int: is job control available */
	#define KERN_SAVED_IDS 20 /* int: saved set-user/group-ID */
	#define KERN_BOOTTIME 21 /* struct: time kernel was booted */
	#define KERN_NISDOMAINNAME 22 /* string: YP domain name */
	#define KERN_UPDATEINTERVAL 23 /* int: update process sleep time */
	#define KERN_OSRELDATE 24 /* int: kernel release date */
	#define KERN_NTP_PLL 25 /* node: NTP PLL control */
	#define KERN_BOOTFILE 26 /* string: name of booted kernel */
	#define KERN_MAXFILESPERPROC 27 /* int: max open files per proc */
	#define KERN_MAXPROCPERUID 28 /* int: max processes per uid */
	#define KERN_DUMPDEV 29 /* struct cdev : device to dump on /
	#define KERN_IPC 30 /* node: anything related to IPC */
	#define KERN_DUMMY 31 /* unused */
	#define KERN_PS_STRINGS 32 /* int: address of PS_STRINGS */
	#define KERN_USRSTACK 33 /* int: address of USRSTACK */
	#define KERN_LOGSIGEXIT 34 /* int: do we log sigexit procs? */
	#define KERN_IOV_MAX 35 /* int: value of UIO_MAXIOV */
	#define KERN_HOSTUUID 36 /* string: host UUID identifier */
	#define KERN_ARND 37 /* int: from arc4rand() */
	#define KERN_MAXPHYS 38 /* int: MAXPHYS value */
	/*
	* KERN_PROC subtypes
	*/
	#define KERN_PROC_ALL 0 /* everything */
	#define KERN_PROC_PID 1 /* by process id */
	#define KERN_PROC_PGRP 2 /* by process group id */
	#define KERN_PROC_SESSION 3 /* by session of pid */
	#define KERN_PROC_TTY 4 /* by controlling tty */
	#define KERN_PROC_UID 5 /* by effective uid */
	#define KERN_PROC_RUID 6 /* by real uid */
	#define KERN_PROC_ARGS 7 /* get/set arguments/proctitle */
	#define KERN_PROC_PROC 8 /* only return procs */
	#define KERN_PROC_SV_NAME 9 /* get syscall vector name */
	#define KERN_PROC_RGID 10 /* by real group id */
	#define KERN_PROC_GID 11 /* by effective group id */
	#define KERN_PROC_PATHNAME 12 /* path to executable */
	#define KERN_PROC_OVMMAP 13 /* Old VM map entries for process */
	#define KERN_PROC_OFILEDESC 14 /* Old file descriptors for process */
	#define KERN_PROC_KSTACK 15 /* Kernel stacks for process */
	#define KERN_PROC_INC_THREAD 0x10 /*
	* modifier for pid, pgrp, tty,
	* uid, ruid, gid, rgid and proc
	* This effectively uses 16-31
	*/
	#define KERN_PROC_VMMAP 32 /* VM map entries for process */
	#define KERN_PROC_FILEDESC 33 /* File descriptors for process */
	#define KERN_PROC_GROUPS 34 /* process groups */
	#define KERN_PROC_ENV 35 /* get environment */
	#define KERN_PROC_AUXV 36 /* get ELF auxiliary vector */
	#define KERN_PROC_RLIMIT 37 /* process resource limits */
	#define KERN_PROC_PS_STRINGS 38 /* get ps_strings location */
	#define KERN_PROC_UMASK 39 /* process umask */
	#define KERN_PROC_OSREL 40 /* osreldate for process binary */
	#define KERN_PROC_SIGTRAMP 41 /* signal trampoline location */
	#define KERN_PROC_CWD 42 /* process current working directory */
	#define KERN_PROC_NFDS 43 /* number of open file descriptors */
	#define KERN_PROC_SIGFASTBLK 44 /* address of fastsigblk magic word */

	/*
	* KERN_IPC identifiers
	*/
	#define KIPC_MAXSOCKBUF 1 /* int: max size of a socket buffer */
	#define KIPC_SOCKBUF_WASTE 2 /* int: wastage factor in sockbuf */
	#define KIPC_SOMAXCONN 3 /* int: max length of connection q */
	#define KIPC_MAX_LINKHDR 4 /* int: max length of link header */
	#define KIPC_MAX_PROTOHDR 5 /* int: max length of network header */
	#define KIPC_MAX_HDR 6 /* int: max total length of headers */
	#define KIPC_MAX_DATALEN 7 /* int: max length of data? */

	/*
	* CTL_HW identifiers
	*/
	#define HW_MACHINE 1 /* string: machine class */
	#define HW_MODEL 2 /* string: specific machine model */
	#define HW_NCPU 3 /* int: number of cpus */
	#define HW_BYTEORDER 4 /* int: machine byte order */
	#define HW_PHYSMEM 5 /* int: total memory */
	#define HW_USERMEM 6 /* int: non-kernel memory */
	#define HW_PAGESIZE 7 /* int: software page size */
	#define HW_DISKNAMES 8 /* strings: disk drive names */
	#define HW_DISKSTATS 9 /* struct: diskstats[] */
	#define HW_FLOATINGPT 10 /* int: has HW floating point? */
	#define HW_MACHINE_ARCH 11 /* string: machine architecture */
	#define HW_REALMEM 12 /* int: 'real' memory */

	/*
	* CTL_USER definitions
	*/
	#define USER_CS_PATH 1 /* string: _CS_PATH */
	#define USER_BC_BASE_MAX 2 /* int: BC_BASE_MAX */
	#define USER_BC_DIM_MAX 3 /* int: BC_DIM_MAX */
	#define USER_BC_SCALE_MAX 4 /* int: BC_SCALE_MAX */
	#define USER_BC_STRING_MAX 5 /* int: BC_STRING_MAX */
	#define USER_COLL_WEIGHTS_MAX 6 /* int: COLL_WEIGHTS_MAX */
	#define USER_EXPR_NEST_MAX 7 /* int: EXPR_NEST_MAX */
	#define USER_LINE_MAX 8 /* int: LINE_MAX */
	#define USER_RE_DUP_MAX 9 /* int: RE_DUP_MAX */
	#define USER_POSIX2_VERSION 10 /* int: POSIX2_VERSION */
	#define USER_POSIX2_C_BIND 11 /* int: POSIX2_C_BIND */
	#define USER_POSIX2_C_DEV 12 /* int: POSIX2_C_DEV */
	#define USER_POSIX2_CHAR_TERM 13 /* int: POSIX2_CHAR_TERM */
	#define USER_POSIX2_FORT_DEV 14 /* int: POSIX2_FORT_DEV */
	#define USER_POSIX2_FORT_RUN 15 /* int: POSIX2_FORT_RUN */
	#define USER_POSIX2_LOCALEDEF 16 /* int: POSIX2_LOCALEDEF */
	#define USER_POSIX2_SW_DEV 17 /* int: POSIX2_SW_DEV */
	#define USER_POSIX2_UPE 18 /* int: POSIX2_UPE */
	#define USER_STREAM_MAX 19 /* int: POSIX2_STREAM_MAX */
	#define USER_TZNAME_MAX 20 /* int: POSIX2_TZNAME_MAX */

	#define CTL_P1003_1B_ASYNCHRONOUS_IO 1 /* boolean */
	#define CTL_P1003_1B_MAPPED_FILES 2 /* boolean */
	#define CTL_P1003_1B_MEMLOCK 3 /* boolean */
	#define CTL_P1003_1B_MEMLOCK_RANGE 4 /* boolean */
	#define CTL_P1003_1B_MEMORY_PROTECTION 5 /* boolean */
	#define CTL_P1003_1B_MESSAGE_PASSING 6 /* boolean */
	#define CTL_P1003_1B_PRIORITIZED_IO 7 /* boolean */
	#define CTL_P1003_1B_PRIORITY_SCHEDULING 8 /* boolean */
	#define CTL_P1003_1B_REALTIME_SIGNALS 9 /* boolean */
	#define CTL_P1003_1B_SEMAPHORES 10 /* boolean */
	#define CTL_P1003_1B_FSYNC 11 /* boolean */
	#define CTL_P1003_1B_SHARED_MEMORY_OBJECTS 12 /* boolean */
	#define CTL_P1003_1B_SYNCHRONIZED_IO 13 /* boolean */
	#define CTL_P1003_1B_TIMERS 14 /* boolean */
	#define CTL_P1003_1B_AIO_LISTIO_MAX 15 /* int */
	#define CTL_P1003_1B_AIO_MAX 16 /* int */
	#define CTL_P1003_1B_AIO_PRIO_DELTA_MAX 17 /* int */
	#define CTL_P1003_1B_DELAYTIMER_MAX 18 /* int */
	#define CTL_P1003_1B_MQ_OPEN_MAX 19 /* int */
	#define CTL_P1003_1B_PAGESIZE 20 /* int */
	#define CTL_P1003_1B_RTSIG_MAX 21 /* int */
	#define CTL_P1003_1B_SEM_NSEMS_MAX 22 /* int */
	#define CTL_P1003_1B_SEM_VALUE_MAX 23 /* int */
	#define CTL_P1003_1B_SIGQUEUE_MAX 24 /* int */
	#define CTL_P1003_1B_TIMER_MAX 25 /* int */

	-#define CTL_P1003_1B_MAXID 26
	-
	#ifdef _KERNEL

	+#define CTL_P1003_1B_MAXID 26
	+
	/*
	* Declare some common oids.
	*/
	extern struct sysctl_oid_list sysctl__children;
	SYSCTL_DECL(_kern);
	SYSCTL_DECL(_kern_features);
	SYSCTL_DECL(_kern_ipc);
	SYSCTL_DECL(_kern_proc);
	SYSCTL_DECL(_kern_sched);
	SYSCTL_DECL(_kern_sched_stats);
	SYSCTL_DECL(_sysctl);
	SYSCTL_DECL(_vm);
	SYSCTL_DECL(_vm_stats);
	SYSCTL_DECL(_vm_stats_misc);
	SYSCTL_DECL(_vfs);
	SYSCTL_DECL(_net);
	SYSCTL_DECL(_debug);
	SYSCTL_DECL(_debug_sizeof);
	SYSCTL_DECL(_dev);
	SYSCTL_DECL(_hw);
	SYSCTL_DECL(_hw_bus);
	SYSCTL_DECL(_hw_bus_devices);
	SYSCTL_DECL(_machdep);
	SYSCTL_DECL(_machdep_mitigations);
	SYSCTL_DECL(_user);
	SYSCTL_DECL(_compat);
	SYSCTL_DECL(_regression);
	SYSCTL_DECL(_security);
	SYSCTL_DECL(_security_bsd);

	extern char machine[];
	extern char osrelease[];
	extern char ostype[];
	extern char kern_ident[];

	/* Dynamic oid handling */
	struct sysctl_oid sysctl_add_oid(struct sysctl_ctx_list clist,
	struct sysctl_oid_list parent, int nbr, const char name, int kind,
	void arg1, intmax_t arg2, int (handler)(SYSCTL_HANDLER_ARGS),
	const char fmt, const char descr, const char *label);
	int sysctl_remove_name(struct sysctl_oid parent, const char name, int del,
	int recurse);
	void sysctl_rename_oid(struct sysctl_oid oidp, const char name);
	int sysctl_move_oid(struct sysctl_oid *oidp,
	struct sysctl_oid_list *parent);
	int sysctl_remove_oid(struct sysctl_oid *oidp, int del, int recurse);
	int sysctl_ctx_init(struct sysctl_ctx_list *clist);
	int sysctl_ctx_free(struct sysctl_ctx_list *clist);
	struct sysctl_ctx_entry sysctl_ctx_entry_add(struct sysctl_ctx_list clist,
	struct sysctl_oid *oidp);
	struct sysctl_ctx_entry sysctl_ctx_entry_find(struct sysctl_ctx_list clist,
	struct sysctl_oid *oidp);
	int sysctl_ctx_entry_del(struct sysctl_ctx_list *clist,
	struct sysctl_oid *oidp);

	int kernel_sysctl(struct thread td, int name, u_int namelen, void *old,
	size_t oldlenp, void new, size_t newlen, size_t *retval,
	int flags);
	int kernel_sysctlbyname(struct thread td, char name, void *old,
	size_t oldlenp, void new, size_t newlen, size_t *retval,
	int flags);
	int userland_sysctl(struct thread td, int name, u_int namelen, void *old,
	size_t oldlenp, int inkernel, const void new, size_t newlen,
	size_t *retval, int flags);
	int sysctl_find_oid(int name, u_int namelen, struct sysctl_oid *noid,
	int nindx, struct sysctl_req req);
	void sysctl_wlock(void);
	void sysctl_wunlock(void);
	int sysctl_wire_old_buffer(struct sysctl_req *req, size_t len);
	int kern___sysctlbyname(struct thread td, const char name,
	size_t namelen, void old, size_t oldlenp, void *new,
	size_t newlen, size_t *retval, int flags, bool inkernel);

	struct sbuf;
	struct sbuf sbuf_new_for_sysctl(struct sbuf , char *, int,
	struct sysctl_req *);
	#else /* !_KERNEL */
	#include <sys/cdefs.h>

	__BEGIN_DECLS
	int sysctl(const int , u_int, void , size_t , const void , size_t);
	int sysctlbyname(const char , void , size_t , const void , size_t);
	int sysctlnametomib(const char , int , size_t *);
	__END_DECLS
	#endif /* _KERNEL */

	#endif /* !_SYS_SYSCTL_H_ */
	diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
	index 8273842a91f5..3a83ea5af6e1 100644
	--- a/sys/sys/vnode.h
	+++ b/sys/sys/vnode.h
	@@ -1,1050 +1,1050 @@
	/*-
	* SPDX-License-Identifier: BSD-3-Clause
	*
	* Copyright (c) 1989, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* @(#)vnode.h 8.7 (Berkeley) 2/4/94
	* $FreeBSD$
	*/

	#ifndef _SYS_VNODE_H_
	#define _SYS_VNODE_H_

	#include <sys/bufobj.h>
	#include <sys/queue.h>
	#include <sys/lock.h>
	#include <sys/lockmgr.h>
	#include <sys/mutex.h>
	#include <sys/rangelock.h>
	#include <sys/selinfo.h>
	#include <sys/uio.h>
	#include <sys/acl.h>
	#include <sys/ktr.h>
	#include <sys/_seqc.h>

	/*
	* The vnode is the focus of all file activity in UNIX. There is a
	* unique vnode allocated for each active file, each current directory,
	* each mounted-on file, text file, and the root.
	*/

	/*
	* Vnode types. VNON means no type.
	*/
	enum vtype { VNON, VREG, VDIR, VBLK, VCHR, VLNK, VSOCK, VFIFO, VBAD,
	VMARKER };

	enum vgetstate { VGET_NONE, VGET_HOLDCNT, VGET_USECOUNT };
	/*
	* Each underlying filesystem allocates its own private area and hangs
	* it from v_data. If non-null, this area is freed in getnewvnode().
	*/

	struct namecache;

	struct vpollinfo {
	struct mtx vpi_lock; /* lock to protect below */
	struct selinfo vpi_selinfo; /* identity of poller(s) */
	short vpi_events; /* what they are looking for */
	short vpi_revents; /* what has happened */
	};

	/*
	* Reading or writing any of these items requires holding the appropriate lock.
	*
	* Lock reference:
	* c - namecache mutex
	* i - interlock
	* l - mp mnt_listmtx or freelist mutex
	* I - updated with atomics, 0->1 and 1->0 transitions with interlock held
	* m - mount point interlock
	* p - pollinfo lock
	* u - Only a reference to the vnode is needed to read.
	* v - vnode lock
	*
	* Vnodes may be found on many lists. The general way to deal with operating
	* on a vnode that is on a list is:
	* 1) Lock the list and find the vnode.
	* 2) Lock interlock so that the vnode does not go away.
	* 3) Unlock the list to avoid lock order reversals.
	* 4) vget with LK_INTERLOCK and check for ENOENT, or
	* 5) Check for DOOMED if the vnode lock is not required.
	* 6) Perform your operation, then vput().
	*/

	#if defined(_KERNEL) \|\| defined(_KVM_VNODE)

	struct vnode {
	/*
	* Fields which define the identity of the vnode. These fields are
	* owned by the filesystem (XXX: and vgone() ?)
	*/
	enum vtype v_type:8; /* u vnode type */
	short v_irflag; /* i frequently read flags */
	seqc_t v_seqc; /* i modification count */
	struct vop_vector v_op; / u vnode operations vector */
	void v_data; / u private data for fs */

	/*
	* Filesystem instance stuff
	*/
	struct mount v_mount; / u ptr to vfs we are in */
	TAILQ_ENTRY(vnode) v_nmntvnodes; /* m vnodes for mount point */

	/*
	* Type specific fields, only one applies to any given vnode.
	*/
	union {
	struct mount v_mountedhere; / v ptr to mountpoint (VDIR) */
	struct unpcb v_unpcb; / v unix domain net (VSOCK) */
	struct cdev v_rdev; / v device (VCHR, VBLK) */
	struct fifoinfo v_fifoinfo; / v fifo (VFIFO) */
	};

	/*
	* vfs_hash: (mount + inode) -> vnode hash. The hash value
	* itself is grouped with other int fields, to avoid padding.
	*/
	LIST_ENTRY(vnode) v_hashlist;

	/*
	* VFS_namecache stuff
	*/
	LIST_HEAD(, namecache) v_cache_src; /* c Cache entries from us */
	TAILQ_HEAD(, namecache) v_cache_dst; /* c Cache entries to us */
	struct namecache v_cache_dd; / c Cache entry for .. vnode */

	/*
	* Locking
	*/
	struct lock v_lock; /* u (if fs don't have one) */
	struct mtx v_interlock; /* lock for "i" things */
	struct lock v_vnlock; / u pointer to vnode lock */

	/*
	* The machinery of being a vnode
	*/
	TAILQ_ENTRY(vnode) v_vnodelist; /* l vnode lists */
	TAILQ_ENTRY(vnode) v_lazylist; /* l vnode lazy list */
	struct bufobj v_bufobj; /* * Buffer cache object */

	/*
	* Hooks for various subsystems and features.
	*/
	struct vpollinfo v_pollinfo; / i Poll events, p for v_pi /
	struct label v_label; / MAC label for vnode */
	struct lockf v_lockf; / Byte-level advisory lock list */
	struct rangelock v_rl; /* Byte-range lock */

	/*
	* clustering stuff
	*/
	daddr_t v_cstart; /* v start block of cluster */
	daddr_t v_lasta; /* v last allocation */
	daddr_t v_lastw; /* v last write */
	int v_clen; /* v length of cur. cluster */

	u_int v_holdcnt; /* I prevents recycling. */
	u_int v_usecount; /* I ref count of users */
	u_int v_iflag; /* i vnode flags (see below) */
	u_int v_vflag; /* v vnode flags */
	u_short v_mflag; /* l mnt-specific vnode flags */
	short v_dbatchcpu; /* i LRU requeue deferral batch */
	int v_writecount; /* I ref count of writers or
	(negative) text users */
	int v_seqc_users; /* i modifications pending */
	u_int v_hash;
	};

	#endif /* defined(_KERNEL) \|\| defined(_KVM_VNODE) */

	#define bo2vnode(bo) __containerof((bo), struct vnode, v_bufobj)

	/* XXX: These are temporary to avoid a source sweep at this time */
	#define v_object v_bufobj.bo_object

	/*
	* Userland version of struct vnode, for sysctl.
	*/
	struct xvnode {
	size_t xv_size; /* sizeof(struct xvnode) */
	void xv_vnode; / address of real vnode */
	u_long xv_flag; /* vnode vflags */
	int xv_usecount; /* reference count of users */
	int xv_writecount; /* reference count of writers */
	int xv_holdcnt; /* page & buffer references */
	u_long xv_id; /* capability identifier */
	void xv_mount; / address of parent mount */
	long xv_numoutput; /* num of writes in progress */
	enum vtype xv_type; /* vnode type */
	union {
	void xvu_socket; / unpcb, if VSOCK */
	void xvu_fifo; / fifo, if VFIFO */
	dev_t xvu_rdev; /* maj/min, if VBLK/VCHR */
	struct {
	dev_t xvu_dev; /* device, if VDIR/VREG/VLNK */
	ino_t xvu_ino; /* id, if VDIR/VREG/VLNK */
	} xv_uns;
	} xv_un;
	};
	#define xv_socket xv_un.xvu_socket
	#define xv_fifo xv_un.xvu_fifo
	#define xv_rdev xv_un.xvu_rdev
	#define xv_dev xv_un.xv_uns.xvu_dev
	#define xv_ino xv_un.xv_uns.xvu_ino

	/* We don't need to lock the knlist */
	#define VN_KNLIST_EMPTY(vp) ((vp)->v_pollinfo == NULL \|\| \
	KNLIST_EMPTY(&(vp)->v_pollinfo->vpi_selinfo.si_note))

	#define VN_KNOTE(vp, b, a) \
	do { \
	if (!VN_KNLIST_EMPTY(vp)) \
	KNOTE(&vp->v_pollinfo->vpi_selinfo.si_note, (b), \
	(a) \| KNF_NOKQLOCK); \
	} while (0)
	#define VN_KNOTE_LOCKED(vp, b) VN_KNOTE(vp, b, KNF_LISTLOCKED)
	#define VN_KNOTE_UNLOCKED(vp, b) VN_KNOTE(vp, b, 0)

	/*
	* Vnode flags.
	* VI flags are protected by interlock and live in v_iflag
	* VV flags are protected by the vnode lock and live in v_vflag
	*
	* VIRF_DOOMED is doubly protected by the interlock and vnode lock. Both
	* are required for writing but the status may be checked with either.
	*/
	#define VHOLD_NO_SMR (1<<29) /* Disable vhold_smr */
	#define VHOLD_ALL_FLAGS (VHOLD_NO_SMR)

	#define VIRF_DOOMED 0x0001 /* This vnode is being recycled */

	#define VI_TEXT_REF 0x0001 /* Text ref grabbed use ref */
	#define VI_MOUNT 0x0020 /* Mount in progress */
	#define VI_DOINGINACT 0x0800 /* VOP_INACTIVE is in progress */
	#define VI_OWEINACT 0x1000 /* Need to call inactive */
	#define VI_DEFINACT 0x2000 /* deferred inactive */

	#define VV_ROOT 0x0001 /* root of its filesystem */
	#define VV_ISTTY 0x0002 /* vnode represents a tty */
	#define VV_NOSYNC 0x0004 /* unlinked, stop syncing */
	#define VV_ETERNALDEV 0x0008 /* device that is never destroyed */
	#define VV_CACHEDLABEL 0x0010 /* Vnode has valid cached MAC label */
	#define VV_VMSIZEVNLOCK 0x0020 /* object size check requires vnode lock */
	#define VV_COPYONWRITE 0x0040 /* vnode is doing copy-on-write */
	#define VV_SYSTEM 0x0080 /* vnode being used by kernel */
	#define VV_PROCDEP 0x0100 /* vnode is process dependent */
	#define VV_NOKNOTE 0x0200 /* don't activate knotes on this vnode */
	#define VV_DELETED 0x0400 /* should be removed */
	#define VV_MD 0x0800 /* vnode backs the md device */
	#define VV_FORCEINSMQ 0x1000 /* force the insmntque to succeed */
	#define VV_READLINK 0x2000 /* fdescfs linux vnode */

	#define VMP_LAZYLIST 0x0001 /* Vnode is on mnt's lazy list */

	/*
	* Vnode attributes. A field value of VNOVAL represents a field whose value
	* is unavailable (getattr) or which is not to be changed (setattr).
	*/
	struct vattr {
	enum vtype va_type; /* vnode type (for create) */
	u_short va_mode; /* files access mode and type */
	u_short va_padding0;
	uid_t va_uid; /* owner user id */
	gid_t va_gid; /* owner group id */
	nlink_t va_nlink; /* number of references to file */
	dev_t va_fsid; /* filesystem id */
	ino_t va_fileid; /* file id */
	u_quad_t va_size; /* file size in bytes */
	long va_blocksize; /* blocksize preferred for i/o */
	struct timespec va_atime; /* time of last access */
	struct timespec va_mtime; /* time of last modification */
	struct timespec va_ctime; /* time file changed */
	struct timespec va_birthtime; /* time file created */
	u_long va_gen; /* generation number of file */
	u_long va_flags; /* flags defined for file */
	dev_t va_rdev; /* device the special file represents */
	u_quad_t va_bytes; /* bytes of disk space held by file */
	u_quad_t va_filerev; /* file modification number */
	u_int va_vaflags; /* operations flags, see below */
	long va_spare; /* remain quad aligned */
	};

	/*
	* Flags for va_vaflags.
	*/
	#define VA_UTIMES_NULL 0x01 /* utimes argument was NULL */
	#define VA_EXCLUSIVE 0x02 /* exclusive create request */
	#define VA_SYNC 0x04 /* O_SYNC truncation */

	/*
	* Flags for ioflag. (high 16 bits used to ask for read-ahead and
	* help with write clustering)
	* NB: IO_NDELAY and IO_DIRECT are linked to fcntl.h
	*/
	#define IO_UNIT 0x0001 /* do I/O as atomic unit */
	#define IO_APPEND 0x0002 /* append write to end */
	#define IO_NDELAY 0x0004 /* FNDELAY flag set in file table */
	#define IO_NODELOCKED 0x0008 /* underlying node already locked */
	#define IO_ASYNC 0x0010 /* bawrite rather then bdwrite */
	#define IO_VMIO 0x0020 /* data already in VMIO space */
	#define IO_INVAL 0x0040 /* invalidate after I/O */
	#define IO_SYNC 0x0080 /* do I/O synchronously */
	#define IO_DIRECT 0x0100 /* attempt to bypass buffer cache */
	#define IO_NOREUSE 0x0200 /* VMIO data won't be reused */
	#define IO_EXT 0x0400 /* operate on external attributes */
	#define IO_NORMAL 0x0800 /* operate on regular data */
	#define IO_NOMACCHECK 0x1000 /* MAC checks unnecessary */
	#define IO_BUFLOCKED 0x2000 /* ffs flag; indir buf is locked */
	#define IO_RANGELOCKED 0x4000 /* range locked */

	#define IO_SEQMAX 0x7F /* seq heuristic max value */
	#define IO_SEQSHIFT 16 /* seq heuristic in upper 16 bits */

	/*
	* Flags for accmode_t.
	*/
	#define VEXEC 000000000100 /* execute/search permission */
	#define VWRITE 000000000200 /* write permission */
	#define VREAD 000000000400 /* read permission */
	#define VADMIN 000000010000 /* being the file owner */
	#define VAPPEND 000000040000 /* permission to write/append */
	/*
	* VEXPLICIT_DENY makes VOP_ACCESSX(9) return EPERM or EACCES only
	* if permission was denied explicitly, by a "deny" rule in NFSv4 ACL,
	* and 0 otherwise. This never happens with ordinary unix access rights
	* or POSIX.1e ACLs. Obviously, VEXPLICIT_DENY must be OR-ed with
	* some other V* constant.
	*/
	#define VEXPLICIT_DENY 000000100000
	#define VREAD_NAMED_ATTRS 000000200000 /* not used */
	#define VWRITE_NAMED_ATTRS 000000400000 /* not used */
	#define VDELETE_CHILD 000001000000
	#define VREAD_ATTRIBUTES 000002000000 /* permission to stat(2) */
	#define VWRITE_ATTRIBUTES 000004000000 /* change {m,c,a}time */
	#define VDELETE 000010000000
	#define VREAD_ACL 000020000000 /* read ACL and file mode */
	#define VWRITE_ACL 000040000000 /* change ACL and/or file mode */
	#define VWRITE_OWNER 000100000000 /* change file owner */
	#define VSYNCHRONIZE 000200000000 /* not used */
	#define VCREAT 000400000000 /* creating new file */
	#define VVERIFY 001000000000 /* verification required */

	/*
	* Permissions that were traditionally granted only to the file owner.
	*/
	#define VADMIN_PERMS (VADMIN \| VWRITE_ATTRIBUTES \| VWRITE_ACL \| \
	VWRITE_OWNER)

	/*
	* Permissions that were traditionally granted to everyone.
	*/
	#define VSTAT_PERMS (VREAD_ATTRIBUTES \| VREAD_ACL)

	/*
	* Permissions that allow to change the state of the file in any way.
	*/
	#define VMODIFY_PERMS (VWRITE \| VAPPEND \| VADMIN_PERMS \| VDELETE_CHILD \| \
	VDELETE)

	/*
	* Token indicating no attribute value yet assigned.
	*/
	#define VNOVAL (-1)

	/*
	* LK_TIMELOCK timeout for vnode locks (used mainly by the pageout daemon)
	*/
	#define VLKTIMEOUT (hz / 20 + 1)

	#ifdef _KERNEL

	#ifdef MALLOC_DECLARE
	MALLOC_DECLARE(M_VNODE);
	#endif

	extern u_int ncsizefactor;

	/*
	* Convert between vnode types and inode formats (since POSIX.1
	* defines mode word of stat structure in terms of inode formats).
	*/
	extern enum vtype iftovt_tab[];
	extern int vttoif_tab[];
	#define IFTOVT(mode) (iftovt_tab[((mode) & S_IFMT) >> 12])
	#define VTTOIF(indx) (vttoif_tab[(int)(indx)])
	#define MAKEIMODE(indx, mode) (int)(VTTOIF(indx) \| (mode))

	/*
	* Flags to various vnode functions.
	*/
	#define SKIPSYSTEM 0x0001 /* vflush: skip vnodes marked VSYSTEM */
	#define FORCECLOSE 0x0002 /* vflush: force file closure */
	#define WRITECLOSE 0x0004 /* vflush: only close writable files */
	#define EARLYFLUSH 0x0008 /* vflush: early call for ffs_flushfiles */
	#define V_SAVE 0x0001 /* vinvalbuf: sync file first */
	#define V_ALT 0x0002 /* vinvalbuf: invalidate only alternate bufs */
	#define V_NORMAL 0x0004 /* vinvalbuf: invalidate only regular bufs */
	#define V_CLEANONLY 0x0008 /* vinvalbuf: invalidate only clean bufs */
	#define V_VMIO 0x0010 /* vinvalbuf: called during pageout */
	#define V_ALLOWCLEAN 0x0020 /* vinvalbuf: allow clean buffers after flush */
	#define REVOKEALL 0x0001 /* vop_revoke: revoke all aliases */
	#define V_WAIT 0x0001 /* vn_start_write: sleep for suspend */
	#define V_NOWAIT 0x0002 /* vn_start_write: don't sleep for suspend */
	#define V_XSLEEP 0x0004 /* vn_start_write: just return after sleep */
	#define V_MNTREF 0x0010 /* vn_start_write: mp is already ref-ed */

	#define VR_START_WRITE 0x0001 /* vfs_write_resume: start write atomically */
	#define VR_NO_SUSPCLR 0x0002 /* vfs_write_resume: do not clear suspension */

	#define VS_SKIP_UNMOUNT 0x0001 /* vfs_write_suspend: fail if the
	filesystem is being unmounted */

	#define VREF(vp) vref(vp)

	#ifdef DIAGNOSTIC
	#define VATTR_NULL(vap) vattr_null(vap)
	#else
	#define VATTR_NULL(vap) ((vap) = va_null) / initialize a vattr */
	#endif /* DIAGNOSTIC */

	#define NULLVP ((struct vnode *)NULL)

	/*
	* Global vnode data.
	*/
	extern struct vnode rootvnode; / root (i.e. "/") vnode */
	extern struct mount rootdevmp; / "/dev" mount */
	extern u_long desiredvnodes; /* number of vnodes desired */
	extern struct uma_zone *namei_zone;
	extern struct vattr va_null; /* predefined null vattr structure */

	#define VI_LOCK(vp) mtx_lock(&(vp)->v_interlock)
	#define VI_LOCK_FLAGS(vp, flags) mtx_lock_flags(&(vp)->v_interlock, (flags))
	#define VI_TRYLOCK(vp) mtx_trylock(&(vp)->v_interlock)
	#define VI_UNLOCK(vp) mtx_unlock(&(vp)->v_interlock)
	#define VI_MTX(vp) (&(vp)->v_interlock)

	#define VN_LOCK_AREC(vp) lockallowrecurse((vp)->v_vnlock)
	#define VN_LOCK_ASHARE(vp) lockallowshare((vp)->v_vnlock)
	#define VN_LOCK_DSHARE(vp) lockdisableshare((vp)->v_vnlock)

	#endif /* _KERNEL */

	/*
	* Mods for extensibility.
	*/

	/*
	* Flags for vdesc_flags:
	*/
	#define VDESC_MAX_VPS 16
	/* Low order 16 flag bits are reserved for willrele flags for vp arguments. */
	#define VDESC_VP0_WILLRELE 0x0001
	#define VDESC_VP1_WILLRELE 0x0002
	#define VDESC_VP2_WILLRELE 0x0004
	#define VDESC_VP3_WILLRELE 0x0008

	/*
	* A generic structure.
	* This can be used by bypass routines to identify generic arguments.
	*/
	struct vop_generic_args {
	struct vnodeop_desc *a_desc;
	/* other random data follows, presumably */
	};

	typedef int vop_bypass_t(struct vop_generic_args *);

	/*
	* VDESC_NO_OFFSET is used to identify the end of the offset list
	* and in places where no such field exists.
	*/
	#define VDESC_NO_OFFSET -1

	/*
	* This structure describes the vnode operation taking place.
	*/
	struct vnodeop_desc {
	char vdesc_name; / a readable name for debugging */
	int vdesc_flags; /* VDESC_* flags */
	int vdesc_vop_offset;
	vop_bypass_t vdesc_call; / Function to call */

	/*
	* These ops are used by bypass routines to map and locate arguments.
	* Creds and procs are not needed in bypass routines, but sometimes
	* they are useful to (for example) transport layers.
	* Nameidata is useful because it has a cred in it.
	*/
	int vdesc_vp_offsets; / list ended by VDESC_NO_OFFSET */
	int vdesc_vpp_offset; /* return vpp location */
	int vdesc_cred_offset; /* cred location, if any */
	int vdesc_thread_offset; /* thread location, if any */
	int vdesc_componentname_offset; /* if any */
	};

	#ifdef _KERNEL
	/*
	* A list of all the operation descs.
	*/
	extern struct vnodeop_desc *vnodeop_descs[];

	#define VOPARG_OFFSETOF(s_type, field) __offsetof(s_type, field)
	#define VOPARG_OFFSETTO(s_type, s_offset, struct_p) \
	((s_type)(((char*)(struct_p)) + (s_offset)))


	#ifdef DEBUG_VFS_LOCKS
	/*
	* Support code to aid in debugging VFS locking problems. Not totally
	* reliable since if the thread sleeps between changing the lock
	* state and checking it with the assert, some other thread could
	* change the state. They are good enough for debugging a single
	* filesystem using a single-threaded test. Note that the unreliability is
	* limited to false negatives; efforts were made to ensure that false
	* positives cannot occur.
	*/
	void assert_vi_locked(struct vnode vp, const char str);
	void assert_vi_unlocked(struct vnode vp, const char str);
	void assert_vop_elocked(struct vnode vp, const char str);
	void assert_vop_locked(struct vnode vp, const char str);
	void assert_vop_unlocked(struct vnode vp, const char str);

	#define ASSERT_VI_LOCKED(vp, str) assert_vi_locked((vp), (str))
	#define ASSERT_VI_UNLOCKED(vp, str) assert_vi_unlocked((vp), (str))
	#define ASSERT_VOP_ELOCKED(vp, str) assert_vop_elocked((vp), (str))
	#define ASSERT_VOP_LOCKED(vp, str) assert_vop_locked((vp), (str))
	#define ASSERT_VOP_UNLOCKED(vp, str) assert_vop_unlocked((vp), (str))

	#define ASSERT_VOP_IN_SEQC(vp) do { \
	struct vnode *_vp = (vp); \
	\
	VNPASS(seqc_in_modify(_vp->v_seqc), _vp); \
	} while (0)

	#define ASSERT_VOP_NOT_IN_SEQC(vp) do { \
	struct vnode *_vp = (vp); \
	\
	VNPASS(!seqc_in_modify(_vp->v_seqc), _vp); \
	} while (0)

	#else /* !DEBUG_VFS_LOCKS */

	#define ASSERT_VI_LOCKED(vp, str) ((void)0)
	#define ASSERT_VI_UNLOCKED(vp, str) ((void)0)
	#define ASSERT_VOP_ELOCKED(vp, str) ((void)0)
	#define ASSERT_VOP_LOCKED(vp, str) ((void)0)
	#define ASSERT_VOP_UNLOCKED(vp, str) ((void)0)

	#define ASSERT_VOP_IN_SEQC(vp) ((void)0)
	#define ASSERT_VOP_NOT_IN_SEQC(vp) ((void)0)

	#endif /* DEBUG_VFS_LOCKS */


	/*
	* This call works for vnodes in the kernel.
	*/
	#define VCALL(c) ((c)->a_desc->vdesc_call(c))

	#define DOINGASYNC(vp) \
	(((vp)->v_mount->mnt_kern_flag & MNTK_ASYNC) != 0 && \
	((curthread->td_pflags & TDP_SYNCIO) == 0))

	/*
	* VMIO support inline
	*/

	extern int vmiodirenable;

	static __inline int
	vn_canvmio(struct vnode *vp)
	{
	if (vp && (vp->v_type == VREG \|\| (vmiodirenable && vp->v_type == VDIR)))
	return(TRUE);
	return(FALSE);
	}

	/*
	* Finally, include the default set of vnode operations.
	*/
	typedef void vop_getpages_iodone_t(void , vm_page_t , int, int);
	#include "vnode_if.h"

	/* vn_open_flags */
	#define VN_OPEN_NOAUDIT 0x00000001
	#define VN_OPEN_NOCAPCHECK 0x00000002
	#define VN_OPEN_NAMECACHE 0x00000004
	#define VN_OPEN_INVFS 0x00000008

	/*
	* Public vnode manipulation functions.
	*/
	struct componentname;
	struct file;
	struct mount;
	struct nameidata;
	struct ostat;
	struct freebsd11_stat;
	struct thread;
	struct proc;
	struct stat;
	struct nstat;
	struct ucred;
	struct uio;
	struct vattr;
	struct vfsops;
	struct vnode;

	typedef int (vn_get_ino_t)(struct mount , void , int, struct vnode *);

	int bnoreuselist(struct bufv bufv, struct bufobj bo, daddr_t startn,
	daddr_t endn);
	/* cache_* may belong in namei.h. */
	void cache_changesize(u_long newhashsize);
	#define cache_enter(dvp, vp, cnp) \
	cache_enter_time(dvp, vp, cnp, NULL, NULL)
	void cache_enter_time(struct vnode dvp, struct vnode vp,
	struct componentname cnp, struct timespec tsp,
	struct timespec *dtsp);
	int cache_lookup(struct vnode dvp, struct vnode *vpp,
	struct componentname cnp, struct timespec tsp, int *ticksp);
	void cache_purge(struct vnode *vp);
	void cache_purge_negative(struct vnode *vp);
	void cache_purgevfs(struct mount *mp, bool force);
	int change_dir(struct vnode vp, struct thread td);
	void cvtstat(struct stat st, struct ostat ost);
	void freebsd11_cvtnstat(struct stat sb, struct nstat nsb);
	int freebsd11_cvtstat(struct stat st, struct freebsd11_stat ost);
	int getnewvnode(const char tag, struct mount mp, struct vop_vector *vops,
	struct vnode **vpp);
	void getnewvnode_reserve(void);
	void getnewvnode_drop_reserve(void);
	int insmntque1(struct vnode vp, struct mount mp,
	void (dtr)(struct vnode , void ), void dtr_arg);
	int insmntque(struct vnode vp, struct mount mp);
	u_quad_t init_va_filerev(void);
	int speedup_syncer(void);
	int vn_vptocnp(struct vnode *vp, struct ucred cred, char *buf,
	size_t *buflen);
	int vn_getcwd(struct thread td, char buf, char *retbuf, size_t buflen);
	int vn_fullpath(struct thread td, struct vnode vn,
	char retbuf, char freebuf);
	int vn_fullpath_global(struct thread td, struct vnode vn,
	char retbuf, char freebuf);
	struct vnode *
	vn_dir_dd_ino(struct vnode *vp);
	int vn_commname(struct vnode vn, char buf, u_int buflen);
	int vn_path_to_global_path(struct thread td, struct vnode vp,
	char *path, u_int pathlen);
	int vaccess(enum vtype type, mode_t file_mode, uid_t file_uid,
	gid_t file_gid, accmode_t accmode, struct ucred *cred,
	int *privused);
	int vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid,
	struct ucred *cred);
	int vaccess_acl_nfs4(enum vtype type, uid_t file_uid, gid_t file_gid,
	struct acl aclp, accmode_t accmode, struct ucred cred,
	int *privused);
	int vaccess_acl_posix1e(enum vtype type, uid_t file_uid,
	gid_t file_gid, struct acl *acl, accmode_t accmode,
	struct ucred cred, int privused);
	void vattr_null(struct vattr *vap);
	int vcount(struct vnode *vp);
	void vlazy(struct vnode *);
	void vdrop(struct vnode *);
	void vdropl(struct vnode *);
	int vflush(struct mount mp, int rootrefs, int flags, struct thread td);
	int vget(struct vnode vp, int flags, struct thread td);
	enum vgetstate vget_prep_smr(struct vnode *vp);
	enum vgetstate vget_prep(struct vnode *vp);
	int vget_finish(struct vnode *vp, int flags, enum vgetstate vs);
	void vget_finish_ref(struct vnode *vp, enum vgetstate vs);
	void vget_abort(struct vnode *vp, enum vgetstate vs);
	void vgone(struct vnode *vp);
	void vhold(struct vnode *);
	void vholdl(struct vnode *);
	void vholdnz(struct vnode *);
	bool vhold_smr(struct vnode *);
	void vinactive(struct vnode *vp);
	int vinvalbuf(struct vnode *vp, int save, int slpflag, int slptimeo);
	int vtruncbuf(struct vnode *vp, off_t length, int blksize);
	void v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn,
	int blksize);
	void vunref(struct vnode *);
	void vn_printf(struct vnode vp, const char fmt, ...) __printflike(2,3);
	int vrecycle(struct vnode *vp);
	int vrecyclel(struct vnode *vp);
	int vn_bmap_seekhole(struct vnode vp, u_long cmd, off_t off,
	struct ucred *cred);
	int vn_close(struct vnode *vp,
	int flags, struct ucred file_cred, struct thread td);
	int vn_copy_file_range(struct vnode invp, off_t inoffp,
	struct vnode outvp, off_t outoffp, size_t *lenp,
	unsigned int flags, struct ucred incred, struct ucred outcred,
	struct thread *fsize_td);
	void vn_finished_write(struct mount *mp);
	void vn_finished_secondary_write(struct mount *mp);
	int vn_fsync_buf(struct vnode *vp, int waitfor);
	int vn_generic_copy_file_range(struct vnode invp, off_t inoffp,
	struct vnode outvp, off_t outoffp, size_t *lenp,
	unsigned int flags, struct ucred incred, struct ucred outcred,
	struct thread *fsize_td);
	int vn_need_pageq_flush(struct vnode *vp);
	int vn_isdisk(struct vnode vp, int errp);
	int _vn_lock(struct vnode vp, int flags, const char file, int line);
	#define vn_lock(vp, flags) _vn_lock(vp, flags, __FILE__, __LINE__)
	int vn_open(struct nameidata ndp, int flagp, int cmode, struct file *fp);
	int vn_open_cred(struct nameidata ndp, int flagp, int cmode,
	u_int vn_open_flags, struct ucred cred, struct file fp);
	int vn_open_vnode(struct vnode vp, int fmode, struct ucred cred,
	struct thread td, struct file fp);
	void vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end);
	int vn_pollrecord(struct vnode vp, struct thread p, int events);
	int vn_rdwr(enum uio_rw rw, struct vnode vp, void base,
	int len, off_t offset, enum uio_seg segflg, int ioflg,
	struct ucred active_cred, struct ucred file_cred, ssize_t *aresid,
	struct thread *td);
	int vn_rdwr_inchunks(enum uio_rw rw, struct vnode vp, void base,
	size_t len, off_t offset, enum uio_seg segflg, int ioflg,
	struct ucred active_cred, struct ucred file_cred, size_t *aresid,
	struct thread *td);
	int vn_rlimit_fsize(const struct vnode vn, const struct uio uio,
	struct thread *td);
	int vn_stat(struct vnode vp, struct stat sb, struct ucred *active_cred,
	struct ucred file_cred, struct thread td);
	int vn_start_write(struct vnode vp, struct mount *mpp, int flags);
	int vn_start_secondary_write(struct vnode vp, struct mount *mpp,
	int flags);
	int vn_truncate_locked(struct vnode *vp, off_t length, bool sync,
	struct ucred *cred);
	int vn_writechk(struct vnode *vp);
	int vn_extattr_get(struct vnode *vp, int ioflg, int attrnamespace,
	const char attrname, int buflen, char buf, struct thread td);
	int vn_extattr_set(struct vnode *vp, int ioflg, int attrnamespace,
	const char attrname, int buflen, char buf, struct thread *td);
	int vn_extattr_rm(struct vnode *vp, int ioflg, int attrnamespace,
	const char attrname, struct thread td);
	int vn_vget_ino(struct vnode *vp, ino_t ino, int lkflags,
	struct vnode **rvp);
	int vn_vget_ino_gen(struct vnode *vp, vn_get_ino_t alloc,
	void alloc_arg, int lkflags, struct vnode *rvp);
	int vn_utimes_perm(struct vnode vp, struct vattr vap,
	struct ucred cred, struct thread td);

	int vn_io_fault_uiomove(char data, int xfersize, struct uio uio);
	int vn_io_fault_pgmove(vm_page_t ma[], vm_offset_t offset, int xfersize,
	struct uio *uio);

	void vn_seqc_write_begin_locked(struct vnode *vp);
	void vn_seqc_write_begin(struct vnode *vp);
	void vn_seqc_write_end_locked(struct vnode *vp);
	void vn_seqc_write_end(struct vnode *vp);
	#define vn_seqc_read_any(vp) seqc_read_any(&(vp)->v_seqc)
	#define vn_seqc_consistent(vp, seq) seqc_consistent(&(vp)->v_seqc, seq)

	#define vn_rangelock_unlock(vp, cookie) \
	rangelock_unlock(&(vp)->v_rl, (cookie), VI_MTX(vp))
	#define vn_rangelock_unlock_range(vp, cookie, start, end) \
	rangelock_unlock_range(&(vp)->v_rl, (cookie), (start), (end), \
	VI_MTX(vp))
	#define vn_rangelock_rlock(vp, start, end) \
	rangelock_rlock(&(vp)->v_rl, (start), (end), VI_MTX(vp))
	#define vn_rangelock_tryrlock(vp, start, end) \
	rangelock_tryrlock(&(vp)->v_rl, (start), (end), VI_MTX(vp))
	#define vn_rangelock_wlock(vp, start, end) \
	rangelock_wlock(&(vp)->v_rl, (start), (end), VI_MTX(vp))
	#define vn_rangelock_trywlock(vp, start, end) \
	rangelock_trywlock(&(vp)->v_rl, (start), (end), VI_MTX(vp))

	int vfs_cache_lookup(struct vop_lookup_args *ap);
	int vfs_cache_root(struct mount mp, int flags, struct vnode *vpp);
	void vfs_timestamp(struct timespec *);
	void vfs_write_resume(struct mount *mp, int flags);
	int vfs_write_suspend(struct mount *mp, int flags);
	int vfs_write_suspend_umnt(struct mount *mp);
	void vnlru_free(int, struct vfsops *);
	int vop_stdbmap(struct vop_bmap_args *);
	int vop_stdfdatasync_buf(struct vop_fdatasync_args *);
	int vop_stdfsync(struct vop_fsync_args *);
	int vop_stdgetwritemount(struct vop_getwritemount_args *);
	int vop_stdgetpages(struct vop_getpages_args *);
	int vop_stdinactive(struct vop_inactive_args *);
	int vop_stdioctl(struct vop_ioctl_args *);
	int vop_stdneed_inactive(struct vop_need_inactive_args *);
	int vop_stdkqfilter(struct vop_kqfilter_args *);
	int vop_stdlock(struct vop_lock1_args *);
	int vop_stdunlock(struct vop_unlock_args *);
	int vop_stdislocked(struct vop_islocked_args *);
	int vop_lock(struct vop_lock1_args *);
	int vop_unlock(struct vop_unlock_args *);
	int vop_islocked(struct vop_islocked_args *);
	int vop_stdputpages(struct vop_putpages_args *);
	int vop_nopoll(struct vop_poll_args *);
	int vop_stdaccess(struct vop_access_args *ap);
	int vop_stdaccessx(struct vop_accessx_args *ap);
	int vop_stdadvise(struct vop_advise_args *ap);
	int vop_stdadvlock(struct vop_advlock_args *ap);
	int vop_stdadvlockasync(struct vop_advlockasync_args *ap);
	int vop_stdadvlockpurge(struct vop_advlockpurge_args *ap);
	int vop_stdallocate(struct vop_allocate_args *ap);
	int vop_stdset_text(struct vop_set_text_args *ap);
	int vop_stdpathconf(struct vop_pathconf_args *);
	int vop_stdpoll(struct vop_poll_args *);
	int vop_stdvptocnp(struct vop_vptocnp_args *ap);
	int vop_stdvptofh(struct vop_vptofh_args *ap);
	int vop_stdunp_bind(struct vop_unp_bind_args *ap);
	int vop_stdunp_connect(struct vop_unp_connect_args *ap);
	int vop_stdunp_detach(struct vop_unp_detach_args *ap);
	int vop_eopnotsupp(struct vop_generic_args *ap);
	int vop_ebadf(struct vop_generic_args *ap);
	int vop_einval(struct vop_generic_args *ap);
	int vop_enoent(struct vop_generic_args *ap);
	int vop_enotty(struct vop_generic_args *ap);
	int vop_null(struct vop_generic_args *ap);
	int vop_panic(struct vop_generic_args *ap);
	int dead_poll(struct vop_poll_args *ap);
	int dead_read(struct vop_read_args *ap);
	int dead_write(struct vop_write_args *ap);

	/* These are called from within the actual VOPS. */
	void vop_close_post(void *a, int rc);
	void vop_create_pre(void *a);
	void vop_create_post(void *a, int rc);
	void vop_whiteout_pre(void *a);
	void vop_whiteout_post(void *a, int rc);
	void vop_deleteextattr_pre(void *a);
	void vop_deleteextattr_post(void *a, int rc);
	void vop_link_pre(void *a);
	void vop_link_post(void *a, int rc);
	void vop_lookup_post(void *a, int rc);
	void vop_lookup_pre(void *a);
	void vop_mkdir_pre(void *a);
	void vop_mkdir_post(void *a, int rc);
	void vop_mknod_pre(void *a);
	void vop_mknod_post(void *a, int rc);
	void vop_open_post(void *a, int rc);
	void vop_read_post(void *a, int rc);
	void vop_readdir_post(void *a, int rc);
	void vop_reclaim_post(void *a, int rc);
	void vop_remove_pre(void *a);
	void vop_remove_post(void *a, int rc);
	void vop_rename_post(void *a, int rc);
	void vop_rename_pre(void *a);
	void vop_rmdir_pre(void *a);
	void vop_rmdir_post(void *a, int rc);
	void vop_setattr_pre(void *a);
	void vop_setattr_post(void *a, int rc);
	void vop_setacl_pre(void *a);
	void vop_setacl_post(void *a, int rc);
	void vop_setextattr_pre(void *a);
	void vop_setextattr_post(void *a, int rc);
	void vop_symlink_pre(void *a);
	void vop_symlink_post(void *a, int rc);
	int vop_sigdefer(struct vop_vector vop, struct vop_generic_args a);

	#ifdef DEBUG_VFS_LOCKS
	-void vop_fplookup_vexec_pre(void *a);
	-void vop_fplookup_vexec_post(void *a, int rc);
	-void vop_strategy_pre(void *a);
	-void vop_lock_pre(void *a);
	-void vop_lock_post(void *a, int rc);
	-void vop_unlock_pre(void *a);
	-void vop_need_inactive_pre(void *a);
	-void vop_need_inactive_post(void *a, int rc);
	+void vop_fplookup_vexec_debugpre(void *a);
	+void vop_fplookup_vexec_debugpost(void *a, int rc);
	+void vop_strategy_debugpre(void *a);
	+void vop_lock_debugpre(void *a);
	+void vop_lock_debugpost(void *a, int rc);
	+void vop_unlock_debugpre(void *a);
	+void vop_need_inactive_debugpre(void *a);
	+void vop_need_inactive_debugpost(void *a, int rc);
	#else
	-#define vop_fplookup_vexec_pre(x) do { } while (0)
	-#define vop_fplookup_vexec_post(x, y) do { } while (0)
	-#define vop_strategy_pre(x) do { } while (0)
	-#define vop_lock_pre(x) do { } while (0)
	-#define vop_lock_post(x, y) do { } while (0)
	-#define vop_unlock_pre(x) do { } while (0)
	-#define vop_need_inactive_pre(x) do { } while (0)
	-#define vop_need_inactive_post(x, y) do { } while (0)
	+#define vop_fplookup_vexec_debugpre(x) do { } while (0)
	+#define vop_fplookup_vexec_debugpost(x, y) do { } while (0)
	+#define vop_strategy_debugpre(x) do { } while (0)
	+#define vop_lock_debugpre(x) do { } while (0)
	+#define vop_lock_debugpost(x, y) do { } while (0)
	+#define vop_unlock_debugpre(x) do { } while (0)
	+#define vop_need_inactive_debugpre(x) do { } while (0)
	+#define vop_need_inactive_debugpost(x, y) do { } while (0)
	#endif

	void vop_rename_fail(struct vop_rename_args *ap);

	#define VOP_WRITE_PRE(ap) \
	struct vattr va; \
	int error; \
	off_t osize, ooffset, noffset; \
	\
	osize = ooffset = noffset = 0; \
	if (!VN_KNLIST_EMPTY((ap)->a_vp)) { \
	error = VOP_GETATTR((ap)->a_vp, &va, (ap)->a_cred); \
	if (error) \
	return (error); \
	ooffset = (ap)->a_uio->uio_offset; \
	osize = (off_t)va.va_size; \
	}

	#define VOP_WRITE_POST(ap, ret) \
	noffset = (ap)->a_uio->uio_offset; \
	if (noffset > ooffset && !VN_KNLIST_EMPTY((ap)->a_vp)) { \
	VFS_KNOTE_LOCKED((ap)->a_vp, NOTE_WRITE \
	\| (noffset > osize ? NOTE_EXTEND : 0)); \
	}

	#define VOP_LOCK(vp, flags) VOP_LOCK1(vp, flags, __FILE__, __LINE__)

	#ifdef INVARIANTS
	#define VOP_ADD_WRITECOUNT_CHECKED(vp, cnt) \
	do { \
	int error_; \
	\
	error_ = VOP_ADD_WRITECOUNT((vp), (cnt)); \
	VNASSERT(error_ == 0, (vp), ("VOP_ADD_WRITECOUNT returned %d", \
	error_)); \
	} while (0)
	#define VOP_SET_TEXT_CHECKED(vp) \
	do { \
	int error_; \
	\
	error_ = VOP_SET_TEXT((vp)); \
	VNASSERT(error_ == 0, (vp), ("VOP_SET_TEXT returned %d", \
	error_)); \
	} while (0)
	#define VOP_UNSET_TEXT_CHECKED(vp) \
	do { \
	int error_; \
	\
	error_ = VOP_UNSET_TEXT((vp)); \
	VNASSERT(error_ == 0, (vp), ("VOP_UNSET_TEXT returned %d", \
	error_)); \
	} while (0)
	#else
	#define VOP_ADD_WRITECOUNT_CHECKED(vp, cnt) VOP_ADD_WRITECOUNT((vp), (cnt))
	#define VOP_SET_TEXT_CHECKED(vp) VOP_SET_TEXT((vp))
	#define VOP_UNSET_TEXT_CHECKED(vp) VOP_UNSET_TEXT((vp))
	#endif

	#define VN_IS_DOOMED(vp) __predict_false((vp)->v_irflag & VIRF_DOOMED)

	void vput(struct vnode *vp);
	void vrele(struct vnode *vp);
	void vref(struct vnode *vp);
	void vrefl(struct vnode *vp);
	void vrefact(struct vnode *vp);
	void vrefactn(struct vnode *vp, u_int n);
	int vrefcnt(struct vnode *vp);
	void v_addpollinfo(struct vnode *vp);

	int vnode_create_vobject(struct vnode vp, off_t size, struct thread td);
	void vnode_destroy_vobject(struct vnode *vp);

	extern struct vop_vector fifo_specops;
	extern struct vop_vector dead_vnodeops;
	extern struct vop_vector default_vnodeops;

	#define VOP_PANIC ((void*)(uintptr_t)vop_panic)
	#define VOP_NULL ((void*)(uintptr_t)vop_null)
	#define VOP_EBADF ((void*)(uintptr_t)vop_ebadf)
	#define VOP_ENOTTY ((void*)(uintptr_t)vop_enotty)
	#define VOP_EINVAL ((void*)(uintptr_t)vop_einval)
	#define VOP_ENOENT ((void*)(uintptr_t)vop_enoent)
	#define VOP_EOPNOTSUPP ((void*)(uintptr_t)vop_eopnotsupp)

	/* fifo_vnops.c */
	int fifo_printinfo(struct vnode *);

	/* vfs_hash.c */
	typedef int vfs_hash_cmp_t(struct vnode vp, void arg);

	void vfs_hash_changesize(u_long newhashsize);
	int vfs_hash_get(const struct mount *mp, u_int hash, int flags,
	struct thread td, struct vnode vpp, vfs_hash_cmp_t fn, void *arg);
	u_int vfs_hash_index(struct vnode *vp);
	int vfs_hash_insert(struct vnode vp, u_int hash, int flags, struct thread td,
	struct vnode *vpp, vfs_hash_cmp_t fn, void *arg);
	void vfs_hash_ref(const struct mount mp, u_int hash, struct thread td,
	struct vnode *vpp, vfs_hash_cmp_t fn, void *arg);
	void vfs_hash_rehash(struct vnode *vp, u_int hash);
	void vfs_hash_remove(struct vnode *vp);

	int vfs_kqfilter(struct vop_kqfilter_args *);
	struct dirent;
	int vfs_read_dirent(struct vop_readdir_args ap, struct dirent dp, off_t off);
	int vfs_emptydir(struct vnode *vp);

	int vfs_unixify_accmode(accmode_t *accmode);

	void vfs_unp_reclaim(struct vnode *vp);

	int setfmode(struct thread td, struct ucred cred, struct vnode *vp, int mode);
	int setfown(struct thread td, struct ucred cred, struct vnode *vp, uid_t uid,
	gid_t gid);
	int vn_chmod(struct file fp, mode_t mode, struct ucred active_cred,
	struct thread *td);
	int vn_chown(struct file fp, uid_t uid, gid_t gid, struct ucred active_cred,
	struct thread *td);

	void vn_fsid(struct vnode vp, struct vattr va);

	int vn_dir_check_exec(struct vnode vp, struct componentname cnp);

	#define VOP_UNLOCK_FLAGS(vp, flags) ({ \
	struct vnode *_vp = (vp); \
	int _flags = (flags); \
	int _error; \
	\
	if ((_flags & ~(LK_INTERLOCK \| LK_RELEASE)) != 0) \
	panic("%s: unsupported flags %x\n", __func__, flags); \
	_error = VOP_UNLOCK(_vp); \
	if (_flags & LK_INTERLOCK) \
	VI_UNLOCK(_vp); \
	_error; \
	})

	#include <sys/kernel.h>

	#define VFS_VOP_VECTOR_REGISTER(vnodeops) \
	SYSINIT(vfs_vector_##vnodeops##_f, SI_SUB_VFS, SI_ORDER_ANY, \
	vfs_vector_op_register, &vnodeops)

	#define VFS_SMR_DECLARE \
	extern smr_t vfs_smr

	#define VFS_SMR() vfs_smr
	#define vfs_smr_enter() smr_enter(VFS_SMR())
	#define vfs_smr_exit() smr_exit(VFS_SMR())
	#define vfs_smr_entered_load(ptr) smr_entered_load((ptr), VFS_SMR())
	#define VFS_SMR_ASSERT_ENTERED() SMR_ASSERT_ENTERED(VFS_SMR())
	#define VFS_SMR_ASSERT_NOT_ENTERED() SMR_ASSERT_NOT_ENTERED(VFS_SMR())
	#define VFS_SMR_ZONE_SET(zone) uma_zone_set_smr((zone), VFS_SMR())

	#define vn_load_v_data_smr(vp) ({ \
	struct vnode *_vp = (vp); \
	\
	VFS_SMR_ASSERT_ENTERED(); \
	atomic_load_ptr(&(_vp)->v_data); \
	})

	#endif /* _KERNEL */

	#endif /* !_SYS_VNODE_H_ */
	diff --git a/sys/tools/vnode_if.awk b/sys/tools/vnode_if.awk
	index cd138bef75da..486f0e6b2ce1 100644
	--- a/sys/tools/vnode_if.awk
	+++ b/sys/tools/vnode_if.awk
	@@ -1,478 +1,519 @@
	#!/usr/bin/awk -f

	#-
	# SPDX-License-Identifier: BSD-3-Clause
	#
	# Copyright (c) 1992, 1993
	# The Regents of the University of California. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	# 3. Neither the name of the University nor the names of its contributors
	# may be used to endorse or promote products derived from this software
	# without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	# ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	# SUCH DAMAGE.

	#
	# @(#)vnode_if.sh 8.1 (Berkeley) 6/10/93
	# $FreeBSD$
	#
	# Script to produce VFS front-end sugar.
	#
	# usage: vnode_if.awk <srcfile> [-c \| -h \| -p \| -q]
	# (where <srcfile> is currently /sys/kern/vnode_if.src)
	# The source file must have a .src extension
	#

	function usage()
	{
	print "usage: vnode_if.awk <srcfile> [-c\|-h\|-p\|-q]";
	exit 1;
	}

	function die(msg, what)
	{
	printf srcfile "(" fnr "): " > "/dev/stderr";
	printf msg "\n", what > "/dev/stderr";
	exit 1;
	}

	function t_spc(type)
	{
	# Append a space if the type is not a pointer
	return (type ~ /\*$/) ? type : type " ";
	}

	# These are just for convenience ...
	function printc(s) {print s > cfile;}
	function printh(s) {print s > hfile;}
	function printp(s) {print s > pfile;}
	function printq(s) {print s > qfile;}

	function add_debug_code(name, arg, pos, ind)
	{
	if (arg == "vpp")
	star = "*";
	else
	star = "";
	if (lockdata[name, arg, pos] && (lockdata[name, arg, pos] != "-")) {
	printc(ind"ASSERT_VI_UNLOCKED("star"a->a_"arg", \""uname"\");");
	# Add assertions for locking
	if (lockdata[name, arg, pos] == "L")
	printc(ind"ASSERT_VOP_LOCKED(" star "a->a_"arg", \""uname"\");");
	else if (lockdata[name, arg, pos] == "U")
	printc(ind"ASSERT_VOP_UNLOCKED(" star "a->a_"arg", \""uname"\");");
	else if (lockdata[name, arg, pos] == "E")
	printc(ind"ASSERT_VOP_ELOCKED(" star "a->a_"arg", \""uname"\");");
	else if (0) {
	# XXX More checks!
	}
	}
	}

	+function add_debugpre(name)
	+{
	+ if (lockdata[name, "debugpre"]) {
	+ printc("#ifdef DEBUG_VFS_LOCKS");
	+ printc("\t"lockdata[name, "debugpre"]"(a);");
	+ printc("#endif");
	+ }
	+}
	+
	+function add_debugpost(name)
	+{
	+ if (lockdata[name, "debugpost"]) {
	+ printc("#ifdef DEBUG_VFS_LOCKS");
	+ printc("\t"lockdata[name, "debugpost"]"(a, rc);");
	+ printc("#endif");
	+ }
	+}
	+
	function add_pre(name)
	{
	if (lockdata[name, "pre"]) {
	printc("\t"lockdata[name, "pre"]"(a);");
	}
	}

	function add_post(name)
	{
	if (lockdata[name, "post"]) {
	printc("\t"lockdata[name, "post"]"(a, rc);");
	}
	}

	+function can_inline(name)
	+{
	+ if (lockdata[name, "pre"])
	+ return 0;
	+ if (lockdata[name, "post"])
	+ return 0;
	+ return 1;
	+}
	+
	function find_arg_with_type (type)
	{
	for (jj = 0; jj < numargs; jj++) {
	if (types[jj] == type) {
	return "VOPARG_OFFSETOF(struct " \
	name "_args,a_" args[jj] ")";
	}
	}

	return "VDESC_NO_OFFSET";
	}

	BEGIN{

	# Process the command line
	for (i = 1; i < ARGC; i++) {
	arg = ARGV[i];
	if (arg !~ /^-[chpq]+$/ && arg !~ /\.src$/)
	usage();
	if (arg ~ /^-.*c/)
	cfile = "vnode_if.c";
	if (arg ~ /^-.*h/)
	hfile = "vnode_if.h";
	if (arg ~ /^-.*p/)
	pfile = "vnode_if_newproto.h";
	if (arg ~ /^-.*q/)
	qfile = "vnode_if_typedef.h";
	if (arg ~ /\.src$/)
	srcfile = arg;
	}
	ARGC = 1;

	if (!cfile && !hfile && !pfile && !qfile)
	exit 0;

	if (!srcfile)
	usage();

	# Avoid a literal generated file tag here.
	generated = "@" "generated";

	common_head = \
	"/*\n" \
	" * This file is " generated " automatically.\n" \
	" * Do not modify anything in here by hand.\n" \
	" *\n" \
	" * Created from $FreeBSD$\n" \
	" */\n" \
	"\n";

	if (pfile) {
	printp(common_head)
	printp("struct vop_vector {")
	printp("\tstruct vop_vector\t*vop_default;")
	printp("\tvop_bypass_t\t*vop_bypass;")
	}

	if (qfile) {
	printq(common_head)
	}

	if (hfile) {
	printh(common_head "extern struct vnodeop_desc vop_default_desc;");
	printh("#include \"vnode_if_typedef.h\"")
	printh("#include \"vnode_if_newproto.h\"")
	}

	if (cfile) {
	printc(common_head \
	"#include <sys/param.h>\n" \
	"#include <sys/event.h>\n" \
	"#include <sys/kernel.h>\n" \
	"#include <sys/mount.h>\n" \
	"#include <sys/sdt.h>\n" \
	"#include <sys/signalvar.h>\n" \
	"#include <sys/systm.h>\n" \
	"#include <sys/vnode.h>\n" \
	"\n" \
	"SDT_PROVIDER_DECLARE(vfs);\n" \
	"\n" \
	"struct vnodeop_desc vop_default_desc = {\n" \
	" \"default\",\n" \
	" 0,\n" \
	" 0,\n" \
	" (vop_bypass_t *)vop_panic,\n" \
	" NULL,\n" \
	" VDESC_NO_OFFSET,\n" \
	" VDESC_NO_OFFSET,\n" \
	" VDESC_NO_OFFSET,\n" \
	" VDESC_NO_OFFSET,\n" \
	"};\n");
	}

	while ((getline < srcfile) > 0) {
	fnr++;
	if (NF == 0)
	continue;
	if ($1 ~ /^%%/) {
	if (NF != 6 \|\|
	$2 !~ /^[a-z_]+$/ \|\| $3 !~ /^[a-z]+$/ \|\|
	$4 !~ /^.$/ \|\| $5 !~ /^.$/ \|\| $6 !~ /^.$/) {
	die("Invalid %s construction", "%%");
	continue;
	}
	lockdata["vop_" $2, $3, "Entry"] = $4;
	lockdata["vop_" $2, $3, "OK"] = $5;
	lockdata["vop_" $2, $3, "Error"] = $6;
	continue;
	}

	if ($1 ~ /^%!/) {
	if (NF != 4 \|\|
	- ($3 != "pre" && $3 != "post")) {
	+ ($3 != "pre" && $3 != "post" &&
	+ $3 != "debugpre" && $3 != "debugpost")) {
	die("Invalid %s construction", "%!");
	continue;
	}
	lockdata["vop_" $2, $3] = $4;
	continue;
	}
	if ($1 ~ /^#/)
	continue;

	# Get the function name.
	name = $1;
	uname = toupper(name);

	# Get the function arguments.
	for (numargs = 0; ; ++numargs) {
	if ((getline < srcfile) <= 0) {
	die("Unable to read through the arguments for \"%s\"",
	name);
	}
	fnr++;
	if ($1 ~ /^\};/)
	break;

	# Delete comments, if any.
	gsub (/\/\.\*\//, "");

	# Condense whitespace and delete leading/trailing space.
	gsub(/[[:space:]]+/, " ");
	sub(/^ /, "");
	sub(/ $/, "");

	# Pick off direction.
	if ($1 != "INOUT" && $1 != "IN" && $1 != "OUT")
	die("No IN/OUT direction for \"%s\".", $0);
	dirs[numargs] = $1;
	sub(/^[A-Z]* /, "");

	if ((reles[numargs] = $1) == "WILLRELE")
	sub(/^[A-Z]* /, "");
	else
	reles[numargs] = "WONTRELE";

	# kill trailing ;
	if (sub(/;$/, "") < 1)
	die("Missing end-of-line ; in \"%s\".", $0);

	# pick off variable name
	if ((argp = match($0, /[A-Za-z0-9_]+$/)) < 1)
	die("Missing var name \"a_foo\" in \"%s\".", $0);
	args[numargs] = substr($0, argp);
	$0 = substr($0, 1, argp - 1);

	# what is left must be type
	# remove trailing space (if any)
	sub(/ $/, "");
	types[numargs] = $0;
	}
	if (numargs > 4)
	ctrargs = 4;
	else
	ctrargs = numargs;
	ctrstr = ctrargs "(KTR_VOP, \"VOP\", \"" uname "\", (uintptr_t)a,\n\t ";
	ctrstr = ctrstr "\"" args[0] ":0x%jX\", (uintptr_t)a->a_" args[0];
	for (i = 1; i < ctrargs; ++i)
	ctrstr = ctrstr ", \"" args[i] ":0x%jX\", a->a_" args[i];
	ctrstr = ctrstr ");";

	if (pfile) {
	printp("\t"name"_t\t*"name";")
	}
	if (qfile) {
	printq("struct "name"_args;")
	printq("typedef int "name"_t(struct "name"_args *);\n")
	}

	if (hfile) {
	# Print out the vop_F_args structure.
	printh("struct "name"_args {\n\tstruct vop_generic_args a_gen;");
	for (i = 0; i < numargs; ++i)
	printh("\t" t_spc(types[i]) "a_" args[i] ";");
	printh("};");
	printh("");

	# Print out extern declaration.
	printh("extern struct vnodeop_desc " name "_desc;");
	printh("");

	# Print out function prototypes.
	printh("int " uname "_AP(struct " name "_args *);");
	printh("int " uname "_APV(struct vop_vector vop, struct " name "_args );");
	printh("");
	printh("static __inline int " uname "(");
	for (i = 0; i < numargs; ++i) {
	printh("\t" t_spc(types[i]) args[i] \
	(i < numargs - 1 ? "," : ")"));
	}
	printh("{");
	printh("\tstruct " name "_args a;");
	printh("");
	printh("\ta.a_gen.a_desc = &" name "_desc;");
	for (i = 0; i < numargs; ++i)
	printh("\ta.a_" args[i] " = " args[i] ";");
	+ if (can_inline(name)) {
	+ printh("\n#if !defined(DEBUG_VFS_LOCKS) && !defined(INVARIANTS) && !defined(KTR)");
	+ printh("\tif (!SDT_PROBES_ENABLED())");
	+ printh("\t\treturn (" args[0]"->v_op->"name"(&a));");
	+ printh("\telse");
	+ printh("\t\treturn (" uname "_APV("args[0]"->v_op, &a));");
	+ printh("#else");
	+ }
	printh("\treturn (" uname "_APV("args[0]"->v_op, &a));");
	+ if (can_inline(name))
	+ printh("#endif");
	+
	printh("}");

	printh("");
	}

	if (cfile) {
	funcarr[name] = 1;
	# Print out the vop_F_vp_offsets structure. This all depends
	# on naming conventions and nothing else.
	printc("static int " name "_vp_offsets[] = {");
	# as a side effect, figure out the releflags
	releflags = "";
	vpnum = 0;
	for (i = 0; i < numargs; i++) {
	if (types[i] == "struct vnode *") {
	printc("\tVOPARG_OFFSETOF(struct " name \
	"_args,a_" args[i] "),");
	if (reles[i] == "WILLRELE") {
	releflags = releflags \
	"\|VDESC_VP" vpnum "_WILLRELE";
	}
	vpnum++;
	}
	}

	sub(/^\\|/, "", releflags);
	printc("\tVDESC_NO_OFFSET");
	printc("};");

	printc("\n");
	printc("SDT_PROBE_DEFINE2(vfs, vop, " name ", entry, \"struct vnode \", \"struct " name "_args \");\n");
	printc("SDT_PROBE_DEFINE3(vfs, vop, " name ", return, \"struct vnode \", \"struct " name "_args \", \"int\");\n");

	# Print out function.
	printc("\nint\n" uname "_AP(struct " name "_args *a)");
	printc("{");
	printc("");
	printc("\treturn(" uname "_APV(a->a_" args[0] "->v_op, a));");
	printc("}");
	printc("\nint\n" uname "_APV(struct vop_vector vop, struct " name "_args a)");
	printc("{");
	printc("\tint rc;");
	printc("");
	printc("\tVNASSERT(a->a_gen.a_desc == &" name "_desc, a->a_" args[0]",");
	printc("\t (\"Wrong a_desc in " name "(%p, %p)\", a->a_" args[0]", a));");
	printc("\tVNASSERT(vop != NULL, a->a_" args[0]", (\"No "name"(%p, %p)\", a->a_" args[0]", a));")
	printc("\tKTR_START" ctrstr);
	+ add_debugpre(name);
	add_pre(name);
	for (i = 0; i < numargs; ++i)
	add_debug_code(name, args[i], "Entry", "\t");
	printc("\tif (!SDT_PROBES_ENABLED()) {");
	printc("\t\trc = vop->"name"(a);")
	printc("\t} else {")
	printc("\t\tSDT_PROBE2(vfs, vop, " name ", entry, a->a_" args[0] ", a);");
	printc("\t\trc = vop->"name"(a);")
	printc("\t\tSDT_PROBE3(vfs, vop, " name ", return, a->a_" args[0] ", a, rc);");
	printc("\t}")
	printc("\tif (rc == 0) {");
	for (i = 0; i < numargs; ++i)
	add_debug_code(name, args[i], "OK", "\t\t");
	printc("\t} else {");
	for (i = 0; i < numargs; ++i)
	add_debug_code(name, args[i], "Error", "\t\t");
	printc("\t}");
	add_post(name);
	+ add_debugpost(name);
	printc("\tKTR_STOP" ctrstr);
	printc("\treturn (rc);");
	printc("}\n");

	# Print out the vnodeop_desc structure.
	printc("struct vnodeop_desc " name "_desc = {");
	# printable name
	printc("\t\"" name "\",");
	# flags
	vppwillrele = "";
	for (i = 0; i < numargs; i++) {
	if (types[i] == "struct vnode **" && \
	reles[i] == "WILLRELE") {
	vppwillrele = "\|VDESC_VPP_WILLRELE";
	}
	}

	if (!releflags)
	releflags = "0";
	printc("\t" releflags vppwillrele ",");

	# index in struct vop_vector
	printc("\t__offsetof(struct vop_vector, " name "),");
	# function to call
	printc("\t(vop_bypass_t *)" uname "_AP,");
	# vp offsets
	printc("\t" name "_vp_offsets,");
	# vpp (if any)
	printc("\t" find_arg_with_type("struct vnode **") ",");
	# cred (if any)
	printc("\t" find_arg_with_type("struct ucred *") ",");
	# thread (if any)
	printc("\t" find_arg_with_type("struct thread *") ",");
	# componentname
	printc("\t" find_arg_with_type("struct componentname *") ",");
	# transport layer information
	printc("};\n");
	}
	}

	if (cfile) {
	printc("void");
	printc("vfs_vector_op_register(struct vop_vector *orig_vop)");
	printc("{");
	printc("\tstruct vop_vector *vop;");
	printc("");
	printc("\tif (orig_vop->registered)");
	printc("\t\tpanic(\"%s: vop_vector %p already registered\",")
	printc("\t\t __func__, orig_vop);");
	printc("");
	for (name in funcarr) {
	printc("\tvop = orig_vop;");
	printc("\twhile (vop != NULL && \\");
	printc("\t vop->"name" == NULL && vop->vop_bypass == NULL)")
	printc("\t\tvop = vop->vop_default;")
	printc("\tif (vop != NULL)");
	printc("\t\torig_vop->"name" = vop->"name";");
	printc("");
	}
	printc("\tvop = orig_vop;");
	printc("\twhile (vop != NULL && vop->vop_bypass == NULL)")
	printc("\t\tvop = vop->vop_default;")
	printc("\tif (vop != NULL)");
	printc("\t\torig_vop->vop_bypass = vop->vop_bypass;");
	printc("");
	for (name in funcarr) {
	printc("\tif (orig_vop->"name" == NULL)");
	printc("\t\torig_vop->"name" = (void *)orig_vop->vop_bypass;");
	}
	printc("");
	printc("\torig_vop->registered = true;");
	printc("}")
	}

	if (hfile) {
	printh("void vfs_vector_op_register(struct vop_vector *orig_vop);");
	}

	if (pfile) {
	printp("\tbool\tregistered;")
	printp("};")
	}

	if (hfile)
	close(hfile);
	if (cfile)
	close(cfile);
	if (pfile)
	close(pfile);
	close(srcfile);

	exit 0;

	}
	diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
	index a7d6f3dd9d64..ce107645d6a6 100644
	--- a/sys/vm/vm_page.c
	+++ b/sys/vm/vm_page.c
	@@ -1,5493 +1,5498 @@
	/*-
	* SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
	*
	* Copyright (c) 1991 Regents of the University of California.
	* All rights reserved.
	* Copyright (c) 1998 Matthew Dillon. All Rights Reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* The Mach Operating System project at Carnegie-Mellon University.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91
	*/

	/*-
	* Copyright (c) 1987, 1990 Carnegie-Mellon University.
	* All rights reserved.
	*
	* Authors: Avadis Tevanian, Jr., Michael Wayne Young
	*
	* Permission to use, copy, modify and distribute this software and
	* its documentation is hereby granted, provided that both the copyright
	* notice and this permission notice appear in all copies of the
	* software, derivative works or modified versions, and any portions
	* thereof, and that both notices appear in supporting documentation.
	*
	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
	* FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	*
	* Carnegie Mellon requests users of this software to return to
	*
	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	* School of Computer Science
	* Carnegie Mellon University
	* Pittsburgh PA 15213-3890
	*
	* any improvements or extensions that they make and grant Carnegie the
	* rights to redistribute these changes.
	*/

	/*
	* Resident memory management module.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_vm.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/counter.h>
	#include <sys/domainset.h>
	#include <sys/kernel.h>
	#include <sys/limits.h>
	#include <sys/linker.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/mman.h>
	#include <sys/msgbuf.h>
	#include <sys/mutex.h>
	#include <sys/proc.h>
	#include <sys/rwlock.h>
	#include <sys/sleepqueue.h>
	#include <sys/sbuf.h>
	#include <sys/sched.h>
	#include <sys/smp.h>
	#include <sys/sysctl.h>
	#include <sys/vmmeter.h>
	#include <sys/vnode.h>

	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <vm/vm_param.h>
	#include <vm/vm_domainset.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_map.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pageout.h>
	#include <vm/vm_phys.h>
	#include <vm/vm_pagequeue.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_radix.h>
	#include <vm/vm_reserv.h>
	#include <vm/vm_extern.h>
	#include <vm/uma.h>
	#include <vm/uma_int.h>

	#include <machine/md_var.h>

	struct vm_domain vm_dom[MAXMEMDOM];

	DPCPU_DEFINE_STATIC(struct vm_batchqueue, pqbatch[MAXMEMDOM][PQ_COUNT]);

	struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT];

	struct mtx_padalign __exclusive_cache_line vm_domainset_lock;
	/* The following fields are protected by the domainset lock. */
	domainset_t __exclusive_cache_line vm_min_domains;
	domainset_t __exclusive_cache_line vm_severe_domains;
	static int vm_min_waiters;
	static int vm_severe_waiters;
	static int vm_pageproc_waiters;

	static SYSCTL_NODE(_vm_stats, OID_AUTO, page, CTLFLAG_RD \| CTLFLAG_MPSAFE, 0,
	"VM page statistics");

	static COUNTER_U64_DEFINE_EARLY(pqstate_commit_retries);
	SYSCTL_COUNTER_U64(_vm_stats_page, OID_AUTO, pqstate_commit_retries,
	CTLFLAG_RD, &pqstate_commit_retries,
	"Number of failed per-page atomic queue state updates");

	static COUNTER_U64_DEFINE_EARLY(queue_ops);
	SYSCTL_COUNTER_U64(_vm_stats_page, OID_AUTO, queue_ops,
	CTLFLAG_RD, &queue_ops,
	"Number of batched queue operations");

	static COUNTER_U64_DEFINE_EARLY(queue_nops);
	SYSCTL_COUNTER_U64(_vm_stats_page, OID_AUTO, queue_nops,
	CTLFLAG_RD, &queue_nops,
	"Number of batched queue operations with no effects");

	/*
	* bogus page -- for I/O to/from partially complete buffers,
	* or for paging into sparsely invalid regions.
	*/
	vm_page_t bogus_page;

	vm_page_t vm_page_array;
	long vm_page_array_size;
	long first_page;

	static TAILQ_HEAD(, vm_page) blacklist_head;
	static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS);
	SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING \| CTLFLAG_RD \|
	CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_page_blacklist, "A", "Blacklist pages");

	static uma_zone_t fakepg_zone;

	static void vm_page_alloc_check(vm_page_t m);
	static bool _vm_page_busy_sleep(vm_object_t obj, vm_page_t m,
	vm_pindex_t pindex, const char *wmesg, int allocflags, bool locked);
	static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
	static void vm_page_enqueue(vm_page_t m, uint8_t queue);
	static bool vm_page_free_prep(vm_page_t m);
	static void vm_page_free_toq(vm_page_t m);
	static void vm_page_init(void *dummy);
	static int vm_page_insert_after(vm_page_t m, vm_object_t object,
	vm_pindex_t pindex, vm_page_t mpred);
	static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
	vm_page_t mpred);
	static void vm_page_mvqueue(vm_page_t m, const uint8_t queue,
	const uint16_t nflag);
	static int vm_page_reclaim_run(int req_class, int domain, u_long npages,
	vm_page_t m_run, vm_paddr_t high);
	static void vm_page_release_toq(vm_page_t m, uint8_t nqueue, bool noreuse);
	static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
	int req);
	static int vm_page_zone_import(void arg, void *store, int cnt, int domain,
	int flags);
	static void vm_page_zone_release(void arg, void *store, int cnt);

	SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init, NULL);

	static void
	vm_page_init(void *dummy)
	{

	fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL,
	NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
	bogus_page = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ \|
	VM_ALLOC_NORMAL \| VM_ALLOC_WIRED);
	}

	/*
	* The cache page zone is initialized later since we need to be able to allocate
	* pages before UMA is fully initialized.
	*/
	static void
	vm_page_init_cache_zones(void *dummy __unused)
	{
	struct vm_domain *vmd;
	struct vm_pgcache *pgcache;
	int cache, domain, maxcache, pool;

	maxcache = 0;
	TUNABLE_INT_FETCH("vm.pgcache_zone_max_pcpu", &maxcache);
	maxcache *= mp_ncpus;
	for (domain = 0; domain < vm_ndomains; domain++) {
	vmd = VM_DOMAIN(domain);
	for (pool = 0; pool < VM_NFREEPOOL; pool++) {
	pgcache = &vmd->vmd_pgcache[pool];
	pgcache->domain = domain;
	pgcache->pool = pool;
	pgcache->zone = uma_zcache_create("vm pgcache",
	PAGE_SIZE, NULL, NULL, NULL, NULL,
	vm_page_zone_import, vm_page_zone_release, pgcache,
	UMA_ZONE_VM);

	/*
	* Limit each pool's zone to 0.1% of the pages in the
	* domain.
	*/
	cache = maxcache != 0 ? maxcache :
	vmd->vmd_page_count / 1000;
	uma_zone_set_maxcache(pgcache->zone, cache);
	}
	}
	}
	SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL);

	/* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */
	#if PAGE_SIZE == 32768
	#ifdef CTASSERT
	CTASSERT(sizeof(u_long) >= 8);
	#endif
	#endif

	/*
	* vm_set_page_size:
	*
	* Sets the page size, perhaps based upon the memory
	* size. Must be called before any use of page-size
	* dependent functions.
	*/
	void
	vm_set_page_size(void)
	{
	if (vm_cnt.v_page_size == 0)
	vm_cnt.v_page_size = PAGE_SIZE;
	if (((vm_cnt.v_page_size - 1) & vm_cnt.v_page_size) != 0)
	panic("vm_set_page_size: page size not a power of two");
	}

	/*
	* vm_page_blacklist_next:
	*
	* Find the next entry in the provided string of blacklist
	* addresses. Entries are separated by space, comma, or newline.
	* If an invalid integer is encountered then the rest of the
	* string is skipped. Updates the list pointer to the next
	* character, or NULL if the string is exhausted or invalid.
	*/
	static vm_paddr_t
	vm_page_blacklist_next(char *list, char end)
	{
	vm_paddr_t bad;
	char cp, pos;

	if (list == NULL \|\| *list == NULL)
	return (0);
	if (**list =='\0') {
	*list = NULL;
	return (0);
	}

	/*
	* If there's no end pointer then the buffer is coming from
	* the kenv and we know it's null-terminated.
	*/
	if (end == NULL)
	end = list + strlen(list);

	/* Ensure that strtoq() won't walk off the end */
	if (*end != '\0') {
	if (end == '\n' \|\| end == ' ' \|\| *end == ',')
	*end = '\0';
	else {
	printf("Blacklist not terminated, skipping\n");
	*list = NULL;
	return (0);
	}
	}

	for (pos = list; pos != '\0'; pos = cp) {
	bad = strtoq(pos, &cp, 0);
	if (cp == '\0' \|\| cp == ' ' \|\| cp == ',' \|\| cp == '\n') {
	if (bad == 0) {
	if (++cp < end)
	continue;
	else
	break;
	}
	} else
	break;
	if (*cp == '\0' \|\| ++cp >= end)
	*list = NULL;
	else
	*list = cp;
	return (trunc_page(bad));
	}
	printf("Garbage in RAM blacklist, skipping\n");
	*list = NULL;
	return (0);
	}

	bool
	vm_page_blacklist_add(vm_paddr_t pa, bool verbose)
	{
	struct vm_domain *vmd;
	vm_page_t m;
	int ret;

	m = vm_phys_paddr_to_vm_page(pa);
	if (m == NULL)
	return (true); /* page does not exist, no failure */

	vmd = vm_pagequeue_domain(m);
	vm_domain_free_lock(vmd);
	ret = vm_phys_unfree_page(m);
	vm_domain_free_unlock(vmd);
	if (ret != 0) {
	vm_domain_freecnt_inc(vmd, -1);
	TAILQ_INSERT_TAIL(&blacklist_head, m, listq);
	if (verbose)
	printf("Skipping page with pa 0x%jx\n", (uintmax_t)pa);
	}
	return (ret);
	}

	/*
	* vm_page_blacklist_check:
	*
	* Iterate through the provided string of blacklist addresses, pulling
	* each entry out of the physical allocator free list and putting it
	* onto a list for reporting via the vm.page_blacklist sysctl.
	*/
	static void
	vm_page_blacklist_check(char list, char end)
	{
	vm_paddr_t pa;
	char *next;

	next = list;
	while (next != NULL) {
	if ((pa = vm_page_blacklist_next(&next, end)) == 0)
	continue;
	vm_page_blacklist_add(pa, bootverbose);
	}
	}

	/*
	* vm_page_blacklist_load:
	*
	* Search for a special module named "ram_blacklist". It'll be a
	* plain text file provided by the user via the loader directive
	* of the same name.
	*/
	static void
	vm_page_blacklist_load(char list, char end)
	{
	void *mod;
	u_char *ptr;
	u_int len;

	mod = NULL;
	ptr = NULL;

	mod = preload_search_by_type("ram_blacklist");
	if (mod != NULL) {
	ptr = preload_fetch_addr(mod);
	len = preload_fetch_size(mod);
	}
	*list = ptr;
	if (ptr != NULL)
	*end = ptr + len;
	else
	*end = NULL;
	return;
	}

	static int
	sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS)
	{
	vm_page_t m;
	struct sbuf sbuf;
	int error, first;

	first = 1;
	error = sysctl_wire_old_buffer(req, 0);
	if (error != 0)
	return (error);
	sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
	TAILQ_FOREACH(m, &blacklist_head, listq) {
	sbuf_printf(&sbuf, "%s%#jx", first ? "" : ",",
	(uintmax_t)m->phys_addr);
	first = 0;
	}
	error = sbuf_finish(&sbuf);
	sbuf_delete(&sbuf);
	return (error);
	}

	/*
	* Initialize a dummy page for use in scans of the specified paging queue.
	* In principle, this function only needs to set the flag PG_MARKER.
	* Nonetheless, it write busies the page as a safety precaution.
	*/
	static void
	vm_page_init_marker(vm_page_t marker, int queue, uint16_t aflags)
	{

	bzero(marker, sizeof(*marker));
	marker->flags = PG_MARKER;
	marker->a.flags = aflags;
	marker->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
	marker->a.queue = queue;
	}

	static void
	vm_page_domain_init(int domain)
	{
	struct vm_domain *vmd;
	struct vm_pagequeue *pq;
	int i;

	vmd = VM_DOMAIN(domain);
	bzero(vmd, sizeof(*vmd));
	__DECONST(const char *, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) =
	"vm inactive pagequeue";
	__DECONST(const char *, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_name) =
	"vm active pagequeue";
	__DECONST(const char *, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) =
	"vm laundry pagequeue";
	__DECONST(const char *,
	&vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_name) =
	"vm unswappable pagequeue";
	vmd->vmd_domain = domain;
	vmd->vmd_page_count = 0;
	vmd->vmd_free_count = 0;
	vmd->vmd_segs = 0;
	vmd->vmd_oom = FALSE;
	for (i = 0; i < PQ_COUNT; i++) {
	pq = &vmd->vmd_pagequeues[i];
	TAILQ_INIT(&pq->pq_pl);
	mtx_init(&pq->pq_mutex, pq->pq_name, "vm pagequeue",
	MTX_DEF \| MTX_DUPOK);
	pq->pq_pdpages = 0;
	vm_page_init_marker(&vmd->vmd_markers[i], i, 0);
	}
	mtx_init(&vmd->vmd_free_mtx, "vm page free queue", NULL, MTX_DEF);
	mtx_init(&vmd->vmd_pageout_mtx, "vm pageout lock", NULL, MTX_DEF);
	snprintf(vmd->vmd_name, sizeof(vmd->vmd_name), "%d", domain);

	/*
	* inacthead is used to provide FIFO ordering for LRU-bypassing
	* insertions.
	*/
	vm_page_init_marker(&vmd->vmd_inacthead, PQ_INACTIVE, PGA_ENQUEUED);
	TAILQ_INSERT_HEAD(&vmd->vmd_pagequeues[PQ_INACTIVE].pq_pl,
	&vmd->vmd_inacthead, plinks.q);

	/*
	* The clock pages are used to implement active queue scanning without
	* requeues. Scans start at clock[0], which is advanced after the scan
	* ends. When the two clock hands meet, they are reset and scanning
	* resumes from the head of the queue.
	*/
	vm_page_init_marker(&vmd->vmd_clock[0], PQ_ACTIVE, PGA_ENQUEUED);
	vm_page_init_marker(&vmd->vmd_clock[1], PQ_ACTIVE, PGA_ENQUEUED);
	TAILQ_INSERT_HEAD(&vmd->vmd_pagequeues[PQ_ACTIVE].pq_pl,
	&vmd->vmd_clock[0], plinks.q);
	TAILQ_INSERT_TAIL(&vmd->vmd_pagequeues[PQ_ACTIVE].pq_pl,
	&vmd->vmd_clock[1], plinks.q);
	}

	/*
	* Initialize a physical page in preparation for adding it to the free
	* lists.
	*/
	static void
	vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segind)
	{

	m->object = NULL;
	m->ref_count = 0;
	m->busy_lock = VPB_FREED;
	m->flags = m->a.flags = 0;
	m->phys_addr = pa;
	m->a.queue = PQ_NONE;
	m->psind = 0;
	m->segind = segind;
	m->order = VM_NFREEORDER;
	m->pool = VM_FREEPOOL_DEFAULT;
	m->valid = m->dirty = 0;
	pmap_page_init(m);
	}

	#ifndef PMAP_HAS_PAGE_ARRAY
	static vm_paddr_t
	vm_page_array_alloc(vm_offset_t *vaddr, vm_paddr_t end, vm_paddr_t page_range)
	{
	vm_paddr_t new_end;

	/*
	* Reserve an unmapped guard page to trap access to vm_page_array[-1].
	* However, because this page is allocated from KVM, out-of-bounds
	* accesses using the direct map will not be trapped.
	*/
	*vaddr += PAGE_SIZE;

	/*
	* Allocate physical memory for the page structures, and map it.
	*/
	new_end = trunc_page(end - page_range * sizeof(struct vm_page));
	vm_page_array = (vm_page_t)pmap_map(vaddr, new_end, end,
	VM_PROT_READ \| VM_PROT_WRITE);
	vm_page_array_size = page_range;

	return (new_end);
	}
	#endif

	/*
	* vm_page_startup:
	*
	* Initializes the resident memory module. Allocates physical memory for
	* bootstrapping UMA and some data structures that are used to manage
	* physical pages. Initializes these structures, and populates the free
	* page queues.
	*/
	vm_offset_t
	vm_page_startup(vm_offset_t vaddr)
	{
	struct vm_phys_seg *seg;
	vm_page_t m;
	char list, listend;
	vm_paddr_t end, high_avail, low_avail, new_end, size;
	vm_paddr_t page_range __unused;
	vm_paddr_t last_pa, pa;
	u_long pagecount;
	int biggestone, i, segind;
	#ifdef WITNESS
	vm_offset_t mapped;
	int witness_size;
	#endif
	#if defined(__i386__) && defined(VM_PHYSSEG_DENSE)
	long ii;
	#endif

	vaddr = round_page(vaddr);

	vm_phys_early_startup();
	biggestone = vm_phys_avail_largest();
	end = phys_avail[biggestone+1];

	/*
	* Initialize the page and queue locks.
	*/
	mtx_init(&vm_domainset_lock, "vm domainset lock", NULL, MTX_DEF);
	for (i = 0; i < PA_LOCK_COUNT; i++)
	mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF);
	for (i = 0; i < vm_ndomains; i++)
	vm_page_domain_init(i);

	new_end = end;
	#ifdef WITNESS
	witness_size = round_page(witness_startup_count());
	new_end -= witness_size;
	mapped = pmap_map(&vaddr, new_end, new_end + witness_size,
	VM_PROT_READ \| VM_PROT_WRITE);
	bzero((void *)mapped, witness_size);
	witness_startup((void *)mapped);
	#endif

	#if defined(__aarch64__) \|\| defined(__amd64__) \|\| defined(__arm__) \|\| \
	defined(__i386__) \|\| defined(__mips__) \|\| defined(__riscv) \|\| \
	defined(__powerpc64__)
	/*
	* Allocate a bitmap to indicate that a random physical page
	* needs to be included in a minidump.
	*
	* The amd64 port needs this to indicate which direct map pages
	* need to be dumped, via calls to dump_add_page()/dump_drop_page().
	*
	* However, i386 still needs this workspace internally within the
	* minidump code. In theory, they are not needed on i386, but are
	* included should the sf_buf code decide to use them.
	*/
	last_pa = 0;
	for (i = 0; dump_avail[i + 1] != 0; i += 2)
	if (dump_avail[i + 1] > last_pa)
	last_pa = dump_avail[i + 1];
	page_range = last_pa / PAGE_SIZE;
	vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
	new_end -= vm_page_dump_size;
	vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
	new_end + vm_page_dump_size, VM_PROT_READ \| VM_PROT_WRITE);
	bzero((void *)vm_page_dump, vm_page_dump_size);
	#else
	(void)last_pa;
	#endif
	#if defined(__aarch64__) \|\| defined(__amd64__) \|\| defined(__mips__) \|\| \
	defined(__riscv) \|\| defined(__powerpc64__)
	/*
	* Include the UMA bootstrap pages, witness pages and vm_page_dump
	* in a crash dump. When pmap_map() uses the direct map, they are
	* not automatically included.
	*/
	for (pa = new_end; pa < end; pa += PAGE_SIZE)
	dump_add_page(pa);
	#endif
	phys_avail[biggestone + 1] = new_end;
	#ifdef __amd64__
	/*
	* Request that the physical pages underlying the message buffer be
	* included in a crash dump. Since the message buffer is accessed
	* through the direct map, they are not automatically included.
	*/
	pa = DMAP_TO_PHYS((vm_offset_t)msgbufp->msg_ptr);
	last_pa = pa + round_page(msgbufsize);
	while (pa < last_pa) {
	dump_add_page(pa);
	pa += PAGE_SIZE;
	}
	#endif
	/*
	* Compute the number of pages of memory that will be available for
	* use, taking into account the overhead of a page structure per page.
	* In other words, solve
	* "available physical memory" - round_page(page_range *
	* sizeof(struct vm_page)) = page_range * PAGE_SIZE
	* for page_range.
	*/
	low_avail = phys_avail[0];
	high_avail = phys_avail[1];
	for (i = 0; i < vm_phys_nsegs; i++) {
	if (vm_phys_segs[i].start < low_avail)
	low_avail = vm_phys_segs[i].start;
	if (vm_phys_segs[i].end > high_avail)
	high_avail = vm_phys_segs[i].end;
	}
	/* Skip the first chunk. It is already accounted for. */
	for (i = 2; phys_avail[i + 1] != 0; i += 2) {
	if (phys_avail[i] < low_avail)
	low_avail = phys_avail[i];
	if (phys_avail[i + 1] > high_avail)
	high_avail = phys_avail[i + 1];
	}
	first_page = low_avail / PAGE_SIZE;
	#ifdef VM_PHYSSEG_SPARSE
	size = 0;
	for (i = 0; i < vm_phys_nsegs; i++)
	size += vm_phys_segs[i].end - vm_phys_segs[i].start;
	for (i = 0; phys_avail[i + 1] != 0; i += 2)
	size += phys_avail[i + 1] - phys_avail[i];
	#elif defined(VM_PHYSSEG_DENSE)
	size = high_avail - low_avail;
	#else
	#error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
	#endif

	#ifdef PMAP_HAS_PAGE_ARRAY
	pmap_page_array_startup(size / PAGE_SIZE);
	biggestone = vm_phys_avail_largest();
	end = new_end = phys_avail[biggestone + 1];
	#else
	#ifdef VM_PHYSSEG_DENSE
	/*
	* In the VM_PHYSSEG_DENSE case, the number of pages can account for
	* the overhead of a page structure per page only if vm_page_array is
	* allocated from the last physical memory chunk. Otherwise, we must
	* allocate page structures representing the physical memory
	* underlying vm_page_array, even though they will not be used.
	*/
	if (new_end != high_avail)
	page_range = size / PAGE_SIZE;
	else
	#endif
	{
	page_range = size / (PAGE_SIZE + sizeof(struct vm_page));

	/*
	* If the partial bytes remaining are large enough for
	* a page (PAGE_SIZE) without a corresponding
	* 'struct vm_page', then new_end will contain an
	* extra page after subtracting the length of the VM
	* page array. Compensate by subtracting an extra
	* page from new_end.
	*/
	if (size % (PAGE_SIZE + sizeof(struct vm_page)) >= PAGE_SIZE) {
	if (new_end == high_avail)
	high_avail -= PAGE_SIZE;
	new_end -= PAGE_SIZE;
	}
	}
	end = new_end;
	new_end = vm_page_array_alloc(&vaddr, end, page_range);
	#endif

	#if VM_NRESERVLEVEL > 0
	/*
	* Allocate physical memory for the reservation management system's
	* data structures, and map it.
	*/
	new_end = vm_reserv_startup(&vaddr, new_end);
	#endif
	#if defined(__aarch64__) \|\| defined(__amd64__) \|\| defined(__mips__) \|\| \
	defined(__riscv) \|\| defined(__powerpc64__)
	/*
	* Include vm_page_array and vm_reserv_array in a crash dump.
	*/
	for (pa = new_end; pa < end; pa += PAGE_SIZE)
	dump_add_page(pa);
	#endif
	phys_avail[biggestone + 1] = new_end;

	/*
	* Add physical memory segments corresponding to the available
	* physical pages.
	*/
	for (i = 0; phys_avail[i + 1] != 0; i += 2)
	if (vm_phys_avail_size(i) != 0)
	vm_phys_add_seg(phys_avail[i], phys_avail[i + 1]);

	/*
	* Initialize the physical memory allocator.
	*/
	vm_phys_init();

	/*
	* Initialize the page structures and add every available page to the
	* physical memory allocator's free lists.
	*/
	#if defined(__i386__) && defined(VM_PHYSSEG_DENSE)
	for (ii = 0; ii < vm_page_array_size; ii++) {
	m = &vm_page_array[ii];
	vm_page_init_page(m, (first_page + ii) << PAGE_SHIFT, 0);
	m->flags = PG_FICTITIOUS;
	}
	#endif
	vm_cnt.v_page_count = 0;
	for (segind = 0; segind < vm_phys_nsegs; segind++) {
	seg = &vm_phys_segs[segind];
	for (m = seg->first_page, pa = seg->start; pa < seg->end;
	m++, pa += PAGE_SIZE)
	vm_page_init_page(m, pa, segind);

	/*
	* Add the segment to the free lists only if it is covered by
	* one of the ranges in phys_avail. Because we've added the
	* ranges to the vm_phys_segs array, we can assume that each
	* segment is either entirely contained in one of the ranges,
	* or doesn't overlap any of them.
	*/
	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
	struct vm_domain *vmd;

	if (seg->start < phys_avail[i] \|\|
	seg->end > phys_avail[i + 1])
	continue;

	m = seg->first_page;
	pagecount = (u_long)atop(seg->end - seg->start);

	vmd = VM_DOMAIN(seg->domain);
	vm_domain_free_lock(vmd);
	vm_phys_enqueue_contig(m, pagecount);
	vm_domain_free_unlock(vmd);
	vm_domain_freecnt_inc(vmd, pagecount);
	vm_cnt.v_page_count += (u_int)pagecount;

	vmd = VM_DOMAIN(seg->domain);
	vmd->vmd_page_count += (u_int)pagecount;
	vmd->vmd_segs \|= 1UL << m->segind;
	break;
	}
	}

	/*
	* Remove blacklisted pages from the physical memory allocator.
	*/
	TAILQ_INIT(&blacklist_head);
	vm_page_blacklist_load(&list, &listend);
	vm_page_blacklist_check(list, listend);

	list = kern_getenv("vm.blacklist");
	vm_page_blacklist_check(list, NULL);

	freeenv(list);
	#if VM_NRESERVLEVEL > 0
	/*
	* Initialize the reservation management system.
	*/
	vm_reserv_init();
	#endif

	return (vaddr);
	}

	void
	vm_page_reference(vm_page_t m)
	{

	vm_page_aflag_set(m, PGA_REFERENCED);
	}

	/*
	* vm_page_trybusy
	*
	* Helper routine for grab functions to trylock busy.
	*
	* Returns true on success and false on failure.
	*/
	static bool
	vm_page_trybusy(vm_page_t m, int allocflags)
	{

	if ((allocflags & (VM_ALLOC_SBUSY \| VM_ALLOC_IGN_SBUSY)) != 0)
	return (vm_page_trysbusy(m));
	else
	return (vm_page_tryxbusy(m));
	}

	/*
	* vm_page_tryacquire
	*
	* Helper routine for grab functions to trylock busy and wire.
	*
	* Returns true on success and false on failure.
	*/
	static inline bool
	vm_page_tryacquire(vm_page_t m, int allocflags)
	{
	bool locked;

	locked = vm_page_trybusy(m, allocflags);
	if (locked && (allocflags & VM_ALLOC_WIRED) != 0)
	vm_page_wire(m);
	return (locked);
	}

	/*
	* vm_page_busy_acquire:
	*
	* Acquire the busy lock as described by VM_ALLOC_* flags. Will loop
	* and drop the object lock if necessary.
	*/
	bool
	vm_page_busy_acquire(vm_page_t m, int allocflags)
	{
	vm_object_t obj;
	bool locked;

	/*
	* The page-specific object must be cached because page
	* identity can change during the sleep, causing the
	* re-lock of a different object.
	* It is assumed that a reference to the object is already
	* held by the callers.
	*/
	obj = m->object;
	for (;;) {
	if (vm_page_tryacquire(m, allocflags))
	return (true);
	if ((allocflags & VM_ALLOC_NOWAIT) != 0)
	return (false);
	if (obj != NULL)
	locked = VM_OBJECT_WOWNED(obj);
	else
	locked = false;
	MPASS(locked \|\| vm_page_wired(m));
	if (_vm_page_busy_sleep(obj, m, m->pindex, "vmpba", allocflags,
	locked) && locked)
	VM_OBJECT_WLOCK(obj);
	if ((allocflags & VM_ALLOC_WAITFAIL) != 0)
	return (false);
	KASSERT(m->object == obj \|\| m->object == NULL,
	("vm_page_busy_acquire: page %p does not belong to %p",
	m, obj));
	}
	}

	/*
	* vm_page_busy_downgrade:
	*
	* Downgrade an exclusive busy page into a single shared busy page.
	*/
	void
	vm_page_busy_downgrade(vm_page_t m)
	{
	u_int x;

	vm_page_assert_xbusied(m);

	- x = m->busy_lock;
	+ x = vm_page_busy_fetch(m);
	for (;;) {
	if (atomic_fcmpset_rel_int(&m->busy_lock,
	&x, VPB_SHARERS_WORD(1)))
	break;
	}
	if ((x & VPB_BIT_WAITERS) != 0)
	wakeup(m);
	}

	/*
	*
	* vm_page_busy_tryupgrade:
	*
	* Attempt to upgrade a single shared busy into an exclusive busy.
	*/
	int
	vm_page_busy_tryupgrade(vm_page_t m)
	{
	u_int ce, x;

	vm_page_assert_sbusied(m);

	- x = m->busy_lock;
	+ x = vm_page_busy_fetch(m);
	ce = VPB_CURTHREAD_EXCLUSIVE;
	for (;;) {
	if (VPB_SHARERS(x) > 1)
	return (0);
	KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1),
	("vm_page_busy_tryupgrade: invalid lock state"));
	if (!atomic_fcmpset_acq_int(&m->busy_lock, &x,
	ce \| (x & VPB_BIT_WAITERS)))
	continue;
	return (1);
	}
	}

	/*
	* vm_page_sbusied:
	*
	* Return a positive value if the page is shared busied, 0 otherwise.
	*/
	int
	vm_page_sbusied(vm_page_t m)
	{
	u_int x;

	- x = m->busy_lock;
	+ x = vm_page_busy_fetch(m);
	return ((x & VPB_BIT_SHARED) != 0 && x != VPB_UNBUSIED);
	}

	/*
	* vm_page_sunbusy:
	*
	* Shared unbusy a page.
	*/
	void
	vm_page_sunbusy(vm_page_t m)
	{
	u_int x;

	vm_page_assert_sbusied(m);

	- x = m->busy_lock;
	+ x = vm_page_busy_fetch(m);
	for (;;) {
	KASSERT(x != VPB_FREED,
	("vm_page_sunbusy: Unlocking freed page."));
	if (VPB_SHARERS(x) > 1) {
	if (atomic_fcmpset_int(&m->busy_lock, &x,
	x - VPB_ONE_SHARER))
	break;
	continue;
	}
	KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1),
	("vm_page_sunbusy: invalid lock state"));
	if (!atomic_fcmpset_rel_int(&m->busy_lock, &x, VPB_UNBUSIED))
	continue;
	if ((x & VPB_BIT_WAITERS) == 0)
	break;
	wakeup(m);
	break;
	}
	}

	/*
	* vm_page_busy_sleep:
	*
	* Sleep if the page is busy, using the page pointer as wchan.
	* This is used to implement the hard-path of busying mechanism.
	*
	* If nonshared is true, sleep only if the page is xbusy.
	*
	* The object lock must be held on entry and will be released on exit.
	*/
	void
	vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
	{
	vm_object_t obj;

	obj = m->object;
	VM_OBJECT_ASSERT_LOCKED(obj);
	vm_page_lock_assert(m, MA_NOTOWNED);

	if (!_vm_page_busy_sleep(obj, m, m->pindex, wmesg,
	nonshared ? VM_ALLOC_SBUSY : 0 , true))
	VM_OBJECT_DROP(obj);
	}

	/*
	* vm_page_busy_sleep_unlocked:
	*
	* Sleep if the page is busy, using the page pointer as wchan.
	* This is used to implement the hard-path of busying mechanism.
	*
	* If nonshared is true, sleep only if the page is xbusy.
	*
	* The object lock must not be held on entry. The operation will
	* return if the page changes identity.
	*/
	void
	vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
	const char *wmesg, bool nonshared)
	{

	VM_OBJECT_ASSERT_UNLOCKED(obj);
	vm_page_lock_assert(m, MA_NOTOWNED);

	_vm_page_busy_sleep(obj, m, pindex, wmesg,
	nonshared ? VM_ALLOC_SBUSY : 0, false);
	}

	/*
	* _vm_page_busy_sleep:
	*
	* Internal busy sleep function. Verifies the page identity and
	* lockstate against parameters. Returns true if it sleeps and
	* false otherwise.
	*
	* If locked is true the lock will be dropped for any true returns
	* and held for any false returns.
	*/
	static bool
	_vm_page_busy_sleep(vm_object_t obj, vm_page_t m, vm_pindex_t pindex,
	const char *wmesg, int allocflags, bool locked)
	{
	bool xsleep;
	u_int x;

	/*
	* If the object is busy we must wait for that to drain to zero
	* before trying the page again.
	*/
	if (obj != NULL && vm_object_busied(obj)) {
	if (locked)
	VM_OBJECT_DROP(obj);
	vm_object_busy_wait(obj, wmesg);
	return (true);
	}

	if (!vm_page_busied(m))
	return (false);

	xsleep = (allocflags & (VM_ALLOC_SBUSY \| VM_ALLOC_IGN_SBUSY)) != 0;
	sleepq_lock(m);
	- x = atomic_load_int(&m->busy_lock);
	+ x = vm_page_busy_fetch(m);
	do {
	/*
	* If the page changes objects or becomes unlocked we can
	* simply return.
	*/
	if (x == VPB_UNBUSIED \|\|
	(xsleep && (x & VPB_BIT_SHARED) != 0) \|\|
	m->object != obj \|\| m->pindex != pindex) {
	sleepq_release(m);
	return (false);
	}
	if ((x & VPB_BIT_WAITERS) != 0)
	break;
	} while (!atomic_fcmpset_int(&m->busy_lock, &x, x \| VPB_BIT_WAITERS));
	if (locked)
	VM_OBJECT_DROP(obj);
	DROP_GIANT();
	sleepq_add(m, NULL, wmesg, 0, 0);
	sleepq_wait(m, PVM);
	PICKUP_GIANT();
	return (true);
	}

	/*
	* vm_page_trysbusy:
	*
	* Try to shared busy a page.
	* If the operation succeeds 1 is returned otherwise 0.
	* The operation never sleeps.
	*/
	int
	vm_page_trysbusy(vm_page_t m)
	{
	vm_object_t obj;
	u_int x;

	obj = m->object;
	- x = m->busy_lock;
	+ x = vm_page_busy_fetch(m);
	for (;;) {
	if ((x & VPB_BIT_SHARED) == 0)
	return (0);
	/*
	* Reduce the window for transient busies that will trigger
	* false negatives in vm_page_ps_test().
	*/
	if (obj != NULL && vm_object_busied(obj))
	return (0);
	if (atomic_fcmpset_acq_int(&m->busy_lock, &x,
	x + VPB_ONE_SHARER))
	break;
	}

	/* Refetch the object now that we're guaranteed that it is stable. */
	obj = m->object;
	if (obj != NULL && vm_object_busied(obj)) {
	vm_page_sunbusy(m);
	return (0);
	}
	return (1);
	}

	/*
	* vm_page_tryxbusy:
	*
	* Try to exclusive busy a page.
	* If the operation succeeds 1 is returned otherwise 0.
	* The operation never sleeps.
	*/
	int
	vm_page_tryxbusy(vm_page_t m)
	{
	vm_object_t obj;

	- if (atomic_cmpset_acq_int(&(m)->busy_lock, VPB_UNBUSIED,
	+ if (atomic_cmpset_acq_int(&m->busy_lock, VPB_UNBUSIED,
	VPB_CURTHREAD_EXCLUSIVE) == 0)
	return (0);

	obj = m->object;
	if (obj != NULL && vm_object_busied(obj)) {
	vm_page_xunbusy(m);
	return (0);
	}
	return (1);
	}

	static void
	vm_page_xunbusy_hard_tail(vm_page_t m)
	{
	atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
	/* Wake the waiter. */
	wakeup(m);
	}

	/*
	* vm_page_xunbusy_hard:
	*
	* Called when unbusy has failed because there is a waiter.
	*/
	void
	vm_page_xunbusy_hard(vm_page_t m)
	{
	vm_page_assert_xbusied(m);
	vm_page_xunbusy_hard_tail(m);
	}

	void
	vm_page_xunbusy_hard_unchecked(vm_page_t m)
	{
	vm_page_assert_xbusied_unchecked(m);
	vm_page_xunbusy_hard_tail(m);
	}

	static void
	vm_page_busy_free(vm_page_t m)
	{
	u_int x;

	atomic_thread_fence_rel();
	x = atomic_swap_int(&m->busy_lock, VPB_FREED);
	if ((x & VPB_BIT_WAITERS) != 0)
	wakeup(m);
	}

	/*
	* vm_page_unhold_pages:
	*
	* Unhold each of the pages that is referenced by the given array.
	*/
	void
	vm_page_unhold_pages(vm_page_t *ma, int count)
	{

	for (; count != 0; count--) {
	vm_page_unwire(*ma, PQ_ACTIVE);
	ma++;
	}
	}

	vm_page_t
	PHYS_TO_VM_PAGE(vm_paddr_t pa)
	{
	vm_page_t m;

	#ifdef VM_PHYSSEG_SPARSE
	m = vm_phys_paddr_to_vm_page(pa);
	if (m == NULL)
	m = vm_phys_fictitious_to_vm_page(pa);
	return (m);
	#elif defined(VM_PHYSSEG_DENSE)
	long pi;

	pi = atop(pa);
	if (pi >= first_page && (pi - first_page) < vm_page_array_size) {
	m = &vm_page_array[pi - first_page];
	return (m);
	}
	return (vm_phys_fictitious_to_vm_page(pa));
	#else
	#error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
	#endif
	}

	/*
	* vm_page_getfake:
	*
	* Create a fictitious page with the specified physical address and
	* memory attribute. The memory attribute is the only the machine-
	* dependent aspect of a fictitious page that must be initialized.
	*/
	vm_page_t
	vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr)
	{
	vm_page_t m;

	m = uma_zalloc(fakepg_zone, M_WAITOK \| M_ZERO);
	vm_page_initfake(m, paddr, memattr);
	return (m);
	}

	void
	vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
	{

	if ((m->flags & PG_FICTITIOUS) != 0) {
	/*
	* The page's memattr might have changed since the
	* previous initialization. Update the pmap to the
	* new memattr.
	*/
	goto memattr;
	}
	m->phys_addr = paddr;
	m->a.queue = PQ_NONE;
	/* Fictitious pages don't use "segind". */
	m->flags = PG_FICTITIOUS;
	/* Fictitious pages don't use "order" or "pool". */
	m->oflags = VPO_UNMANAGED;
	m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
	/* Fictitious pages are unevictable. */
	m->ref_count = 1;
	pmap_page_init(m);
	memattr:
	pmap_page_set_memattr(m, memattr);
	}

	/*
	* vm_page_putfake:
	*
	* Release a fictitious page.
	*/
	void
	vm_page_putfake(vm_page_t m)
	{

	KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed %p", m));
	KASSERT((m->flags & PG_FICTITIOUS) != 0,
	("vm_page_putfake: bad page %p", m));
	vm_page_assert_xbusied(m);
	vm_page_busy_free(m);
	uma_zfree(fakepg_zone, m);
	}

	/*
	* vm_page_updatefake:
	*
	* Update the given fictitious page to the specified physical address and
	* memory attribute.
	*/
	void
	vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
	{

	KASSERT((m->flags & PG_FICTITIOUS) != 0,
	("vm_page_updatefake: bad page %p", m));
	m->phys_addr = paddr;
	pmap_page_set_memattr(m, memattr);
	}

	/*
	* vm_page_free:
	*
	* Free a page.
	*/
	void
	vm_page_free(vm_page_t m)
	{

	m->flags &= ~PG_ZERO;
	vm_page_free_toq(m);
	}

	/*
	* vm_page_free_zero:
	*
	* Free a page to the zerod-pages queue
	*/
	void
	vm_page_free_zero(vm_page_t m)
	{

	m->flags \|= PG_ZERO;
	vm_page_free_toq(m);
	}

	/*
	* Unbusy and handle the page queueing for a page from a getpages request that
	* was optionally read ahead or behind.
	*/
	void
	vm_page_readahead_finish(vm_page_t m)
	{

	/* We shouldn't put invalid pages on queues. */
	KASSERT(!vm_page_none_valid(m), ("%s: %p is invalid", __func__, m));

	/*
	* Since the page is not the actually needed one, whether it should
	* be activated or deactivated is not obvious. Empirical results
	* have shown that deactivating the page is usually the best choice,
	* unless the page is wanted by another thread.
	*/
	- if ((m->busy_lock & VPB_BIT_WAITERS) != 0)
	+ if ((vm_page_busy_fetch(m) & VPB_BIT_WAITERS) != 0)
	vm_page_activate(m);
	else
	vm_page_deactivate(m);
	vm_page_xunbusy_unchecked(m);
	}

	/*
	* Destroy the identity of an invalid page and free it if possible.
	* This is intended to be used when reading a page from backing store fails.
	*/
	void
	vm_page_free_invalid(vm_page_t m)
	{

	KASSERT(vm_page_none_valid(m), ("page %p is valid", m));
	KASSERT(!pmap_page_is_mapped(m), ("page %p is mapped", m));
	- vm_page_assert_xbusied(m);
	KASSERT(m->object != NULL, ("page %p has no object", m));
	VM_OBJECT_ASSERT_WLOCKED(m->object);

	+ /*
	+ * We may be attempting to free the page as part of the handling for an
	+ * I/O error, in which case the page was xbusied by a different thread.
	+ */
	+ vm_page_xbusy_claim(m);
	+
	/*
	* If someone has wired this page while the object lock
	* was not held, then the thread that unwires is responsible
	* for freeing the page. Otherwise just free the page now.
	* The wire count of this unmapped page cannot change while
	* we have the page xbusy and the page's object wlocked.
	*/
	if (vm_page_remove(m))
	vm_page_free(m);
	}

	/*
	* vm_page_sleep_if_busy:
	*
	* Sleep and release the object lock if the page is busied.
	* Returns TRUE if the thread slept.
	*
	* The given page must be unlocked and object containing it must
	* be locked.
	*/
	int
	vm_page_sleep_if_busy(vm_page_t m, const char *wmesg)
	{
	vm_object_t obj;

	vm_page_lock_assert(m, MA_NOTOWNED);
	VM_OBJECT_ASSERT_WLOCKED(m->object);

	/*
	* The page-specific object must be cached because page
	* identity can change during the sleep, causing the
	* re-lock of a different object.
	* It is assumed that a reference to the object is already
	* held by the callers.
	*/
	obj = m->object;
	if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, 0, true)) {
	VM_OBJECT_WLOCK(obj);
	return (TRUE);
	}
	return (FALSE);
	}

	/*
	* vm_page_sleep_if_xbusy:
	*
	* Sleep and release the object lock if the page is xbusied.
	* Returns TRUE if the thread slept.
	*
	* The given page must be unlocked and object containing it must
	* be locked.
	*/
	int
	vm_page_sleep_if_xbusy(vm_page_t m, const char *wmesg)
	{
	vm_object_t obj;

	vm_page_lock_assert(m, MA_NOTOWNED);
	VM_OBJECT_ASSERT_WLOCKED(m->object);

	/*
	* The page-specific object must be cached because page
	* identity can change during the sleep, causing the
	* re-lock of a different object.
	* It is assumed that a reference to the object is already
	* held by the callers.
	*/
	obj = m->object;
	if (_vm_page_busy_sleep(obj, m, m->pindex, wmesg, VM_ALLOC_SBUSY,
	true)) {
	VM_OBJECT_WLOCK(obj);
	return (TRUE);
	}
	return (FALSE);
	}

	/*
	* vm_page_dirty_KBI: [ internal use only ]
	*
	* Set all bits in the page's dirty field.
	*
	* The object containing the specified page must be locked if the
	* call is made from the machine-independent layer.
	*
	* See vm_page_clear_dirty_mask().
	*
	* This function should only be called by vm_page_dirty().
	*/
	void
	vm_page_dirty_KBI(vm_page_t m)
	{

	/* Refer to this operation by its public name. */
	KASSERT(vm_page_all_valid(m), ("vm_page_dirty: page is invalid!"));
	m->dirty = VM_PAGE_BITS_ALL;
	}

	/*
	* vm_page_insert: [ internal use only ]
	*
	* Inserts the given mem entry into the object and object list.
	*
	* The object must be locked.
	*/
	int
	vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
	{
	vm_page_t mpred;

	VM_OBJECT_ASSERT_WLOCKED(object);
	mpred = vm_radix_lookup_le(&object->rtree, pindex);
	return (vm_page_insert_after(m, object, pindex, mpred));
	}

	/*
	* vm_page_insert_after:
	*
	* Inserts the page "m" into the specified object at offset "pindex".
	*
	* The page "mpred" must immediately precede the offset "pindex" within
	* the specified object.
	*
	* The object must be locked.
	*/
	static int
	vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
	vm_page_t mpred)
	{
	vm_page_t msucc;

	VM_OBJECT_ASSERT_WLOCKED(object);
	KASSERT(m->object == NULL,
	("vm_page_insert_after: page already inserted"));
	if (mpred != NULL) {
	KASSERT(mpred->object == object,
	("vm_page_insert_after: object doesn't contain mpred"));
	KASSERT(mpred->pindex < pindex,
	("vm_page_insert_after: mpred doesn't precede pindex"));
	msucc = TAILQ_NEXT(mpred, listq);
	} else
	msucc = TAILQ_FIRST(&object->memq);
	if (msucc != NULL)
	KASSERT(msucc->pindex > pindex,
	("vm_page_insert_after: msucc doesn't succeed pindex"));

	/*
	* Record the object/offset pair in this page.
	*/
	m->object = object;
	m->pindex = pindex;
	m->ref_count \|= VPRC_OBJREF;

	/*
	* Now link into the object's ordered list of backed pages.
	*/
	if (vm_radix_insert(&object->rtree, m)) {
	m->object = NULL;
	m->pindex = 0;
	m->ref_count &= ~VPRC_OBJREF;
	return (1);
	}
	vm_page_insert_radixdone(m, object, mpred);
	return (0);
	}

	/*
	* vm_page_insert_radixdone:
	*
	* Complete page "m" insertion into the specified object after the
	* radix trie hooking.
	*
	* The page "mpred" must precede the offset "m->pindex" within the
	* specified object.
	*
	* The object must be locked.
	*/
	static void
	vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred)
	{

	VM_OBJECT_ASSERT_WLOCKED(object);
	KASSERT(object != NULL && m->object == object,
	("vm_page_insert_radixdone: page %p has inconsistent object", m));
	KASSERT((m->ref_count & VPRC_OBJREF) != 0,
	("vm_page_insert_radixdone: page %p is missing object ref", m));
	if (mpred != NULL) {
	KASSERT(mpred->object == object,
	("vm_page_insert_radixdone: object doesn't contain mpred"));
	KASSERT(mpred->pindex < m->pindex,
	("vm_page_insert_radixdone: mpred doesn't precede pindex"));
	}

	if (mpred != NULL)
	TAILQ_INSERT_AFTER(&object->memq, mpred, m, listq);
	else
	TAILQ_INSERT_HEAD(&object->memq, m, listq);

	/*
	* Show that the object has one more resident page.
	*/
	object->resident_page_count++;

	/*
	* Hold the vnode until the last page is released.
	*/
	if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
	vhold(object->handle);

	/*
	* Since we are inserting a new and possibly dirty page,
	* update the object's generation count.
	*/
	if (pmap_page_is_write_mapped(m))
	vm_object_set_writeable_dirty(object);
	}

	/*
	* Do the work to remove a page from its object. The caller is responsible for
	* updating the page's fields to reflect this removal.
	*/
	static void
	vm_page_object_remove(vm_page_t m)
	{
	vm_object_t object;
	vm_page_t mrem;

	vm_page_assert_xbusied(m);
	object = m->object;
	VM_OBJECT_ASSERT_WLOCKED(object);
	KASSERT((m->ref_count & VPRC_OBJREF) != 0,
	("page %p is missing its object ref", m));

	/* Deferred free of swap space. */
	if ((m->a.flags & PGA_SWAP_FREE) != 0)
	vm_pager_page_unswapped(m);

	m->object = NULL;
	mrem = vm_radix_remove(&object->rtree, m->pindex);
	KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m));

	/*
	* Now remove from the object's list of backed pages.
	*/
	TAILQ_REMOVE(&object->memq, m, listq);

	/*
	* And show that the object has one fewer resident page.
	*/
	object->resident_page_count--;

	/*
	* The vnode may now be recycled.
	*/
	if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
	vdrop(object->handle);
	}

	/*
	* vm_page_remove:
	*
	* Removes the specified page from its containing object, but does not
	* invalidate any backing storage. Returns true if the object's reference
	* was the last reference to the page, and false otherwise.
	*
	* The object must be locked and the page must be exclusively busied.
	* The exclusive busy will be released on return. If this is not the
	* final ref and the caller does not hold a wire reference it may not
	* continue to access the page.
	*/
	bool
	vm_page_remove(vm_page_t m)
	{
	bool dropped;

	dropped = vm_page_remove_xbusy(m);
	vm_page_xunbusy(m);

	return (dropped);
	}

	/*
	* vm_page_remove_xbusy
	*
	* Removes the page but leaves the xbusy held. Returns true if this
	* removed the final ref and false otherwise.
	*/
	bool
	vm_page_remove_xbusy(vm_page_t m)
	{

	vm_page_object_remove(m);
	return (vm_page_drop(m, VPRC_OBJREF) == VPRC_OBJREF);
	}

	/*
	* vm_page_lookup:
	*
	* Returns the page associated with the object/offset
	* pair specified; if none is found, NULL is returned.
	*
	* The object must be locked.
	*/
	vm_page_t
	vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
	{

	VM_OBJECT_ASSERT_LOCKED(object);
	return (vm_radix_lookup(&object->rtree, pindex));
	}

	/*
	* vm_page_relookup:
	*
	* Returns a page that must already have been busied by
	* the caller. Used for bogus page replacement.
	*/
	vm_page_t
	vm_page_relookup(vm_object_t object, vm_pindex_t pindex)
	{
	vm_page_t m;

	m = vm_radix_lookup_unlocked(&object->rtree, pindex);
	KASSERT(m != NULL && (vm_page_busied(m) \|\| vm_page_wired(m)) &&
	m->object == object && m->pindex == pindex,
	("vm_page_relookup: Invalid page %p", m));
	return (m);
	}

	/*
	* This should only be used by lockless functions for releasing transient
	* incorrect acquires. The page may have been freed after we acquired a
	* busy lock. In this case busy_lock == VPB_FREED and we have nothing
	* further to do.
	*/
	static void
	vm_page_busy_release(vm_page_t m)
	{
	u_int x;

	- x = atomic_load_int(&m->busy_lock);
	+ x = vm_page_busy_fetch(m);
	for (;;) {
	if (x == VPB_FREED)
	break;
	if ((x & VPB_BIT_SHARED) != 0 && VPB_SHARERS(x) > 1) {
	if (atomic_fcmpset_int(&m->busy_lock, &x,
	x - VPB_ONE_SHARER))
	break;
	continue;
	}
	KASSERT((x & VPB_BIT_SHARED) != 0 \|\|
	(x & ~VPB_BIT_WAITERS) == VPB_CURTHREAD_EXCLUSIVE,
	("vm_page_busy_release: %p xbusy not owned.", m));
	if (!atomic_fcmpset_rel_int(&m->busy_lock, &x, VPB_UNBUSIED))
	continue;
	if ((x & VPB_BIT_WAITERS) != 0)
	wakeup(m);
	break;
	}
	}

	/*
	* vm_page_find_least:
	*
	* Returns the page associated with the object with least pindex
	* greater than or equal to the parameter pindex, or NULL.
	*
	* The object must be locked.
	*/
	vm_page_t
	vm_page_find_least(vm_object_t object, vm_pindex_t pindex)
	{
	vm_page_t m;

	VM_OBJECT_ASSERT_LOCKED(object);
	if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex)
	m = vm_radix_lookup_ge(&object->rtree, pindex);
	return (m);
	}

	/*
	* Returns the given page's successor (by pindex) within the object if it is
	* resident; if none is found, NULL is returned.
	*
	* The object must be locked.
	*/
	vm_page_t
	vm_page_next(vm_page_t m)
	{
	vm_page_t next;

	VM_OBJECT_ASSERT_LOCKED(m->object);
	if ((next = TAILQ_NEXT(m, listq)) != NULL) {
	MPASS(next->object == m->object);
	if (next->pindex != m->pindex + 1)
	next = NULL;
	}
	return (next);
	}

	/*
	* Returns the given page's predecessor (by pindex) within the object if it is
	* resident; if none is found, NULL is returned.
	*
	* The object must be locked.
	*/
	vm_page_t
	vm_page_prev(vm_page_t m)
	{
	vm_page_t prev;

	VM_OBJECT_ASSERT_LOCKED(m->object);
	if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL) {
	MPASS(prev->object == m->object);
	if (prev->pindex != m->pindex - 1)
	prev = NULL;
	}
	return (prev);
	}

	/*
	* Uses the page mnew as a replacement for an existing page at index
	* pindex which must be already present in the object.
	*
	* Both pages must be exclusively busied on enter. The old page is
	* unbusied on exit.
	*
	* A return value of true means mold is now free. If this is not the
	* final ref and the caller does not hold a wire reference it may not
	* continue to access the page.
	*/
	static bool
	vm_page_replace_hold(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex,
	vm_page_t mold)
	{
	vm_page_t mret;
	bool dropped;

	VM_OBJECT_ASSERT_WLOCKED(object);
	vm_page_assert_xbusied(mold);
	KASSERT(mnew->object == NULL && (mnew->ref_count & VPRC_OBJREF) == 0,
	("vm_page_replace: page %p already in object", mnew));

	/*
	* This function mostly follows vm_page_insert() and
	* vm_page_remove() without the radix, object count and vnode
	* dance. Double check such functions for more comments.
	*/

	mnew->object = object;
	mnew->pindex = pindex;
	atomic_set_int(&mnew->ref_count, VPRC_OBJREF);
	mret = vm_radix_replace(&object->rtree, mnew);
	KASSERT(mret == mold,
	("invalid page replacement, mold=%p, mret=%p", mold, mret));
	KASSERT((mold->oflags & VPO_UNMANAGED) ==
	(mnew->oflags & VPO_UNMANAGED),
	("vm_page_replace: mismatched VPO_UNMANAGED"));

	/* Keep the resident page list in sorted order. */
	TAILQ_INSERT_AFTER(&object->memq, mold, mnew, listq);
	TAILQ_REMOVE(&object->memq, mold, listq);
	mold->object = NULL;

	/*
	* The object's resident_page_count does not change because we have
	* swapped one page for another, but the generation count should
	* change if the page is dirty.
	*/
	if (pmap_page_is_write_mapped(mnew))
	vm_object_set_writeable_dirty(object);
	dropped = vm_page_drop(mold, VPRC_OBJREF) == VPRC_OBJREF;
	vm_page_xunbusy(mold);

	return (dropped);
	}

	void
	vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex,
	vm_page_t mold)
	{

	vm_page_assert_xbusied(mnew);

	if (vm_page_replace_hold(mnew, object, pindex, mold))
	vm_page_free(mold);
	}

	/*
	* vm_page_rename:
	*
	* Move the given memory entry from its
	* current object to the specified target object/offset.
	*
	* Note: swap associated with the page must be invalidated by the move. We
	* have to do this for several reasons: (1) we aren't freeing the
	* page, (2) we are dirtying the page, (3) the VM system is probably
	* moving the page from object A to B, and will then later move
	* the backing store from A to B and we can't have a conflict.
	*
	* Note: we always dirty the page. It is necessary both for the
	* fact that we moved it, and because we may be invalidating
	* swap.
	*
	* The objects must be locked.
	*/
	int
	vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
	{
	vm_page_t mpred;
	vm_pindex_t opidx;

	VM_OBJECT_ASSERT_WLOCKED(new_object);

	KASSERT(m->ref_count != 0, ("vm_page_rename: page %p has no refs", m));
	mpred = vm_radix_lookup_le(&new_object->rtree, new_pindex);
	KASSERT(mpred == NULL \|\| mpred->pindex != new_pindex,
	("vm_page_rename: pindex already renamed"));

	/*
	* Create a custom version of vm_page_insert() which does not depend
	* by m_prev and can cheat on the implementation aspects of the
	* function.
	*/
	opidx = m->pindex;
	m->pindex = new_pindex;
	if (vm_radix_insert(&new_object->rtree, m)) {
	m->pindex = opidx;
	return (1);
	}

	/*
	* The operation cannot fail anymore. The removal must happen before
	* the listq iterator is tainted.
	*/
	m->pindex = opidx;
	vm_page_object_remove(m);

	/* Return back to the new pindex to complete vm_page_insert(). */
	m->pindex = new_pindex;
	m->object = new_object;

	vm_page_insert_radixdone(m, new_object, mpred);
	vm_page_dirty(m);
	return (0);
	}

	/*
	* vm_page_alloc:
	*
	* Allocate and return a page that is associated with the specified
	* object and offset pair. By default, this page is exclusive busied.
	*
	* The caller must always specify an allocation class.
	*
	* allocation classes:
	* VM_ALLOC_NORMAL normal process request
	* VM_ALLOC_SYSTEM system really needs a page
	* VM_ALLOC_INTERRUPT interrupt time request
	*
	* optional allocation flags:
	* VM_ALLOC_COUNT(number) the number of additional pages that the caller
	* intends to allocate
	* VM_ALLOC_NOBUSY do not exclusive busy the page
	* VM_ALLOC_NODUMP do not include the page in a kernel core dump
	* VM_ALLOC_NOOBJ page is not associated with an object and
	* should not be exclusive busy
	* VM_ALLOC_SBUSY shared busy the allocated page
	* VM_ALLOC_WIRED wire the allocated page
	* VM_ALLOC_ZERO prefer a zeroed page
	*/
	vm_page_t
	vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
	{

	return (vm_page_alloc_after(object, pindex, req, object != NULL ?
	vm_radix_lookup_le(&object->rtree, pindex) : NULL));
	}

	vm_page_t
	vm_page_alloc_domain(vm_object_t object, vm_pindex_t pindex, int domain,
	int req)
	{

	return (vm_page_alloc_domain_after(object, pindex, domain, req,
	object != NULL ? vm_radix_lookup_le(&object->rtree, pindex) :
	NULL));
	}

	/*
	* Allocate a page in the specified object with the given page index. To
	* optimize insertion of the page into the object, the caller must also specifiy
	* the resident page in the object with largest index smaller than the given
	* page index, or NULL if no such page exists.
	*/
	vm_page_t
	vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,
	int req, vm_page_t mpred)
	{
	struct vm_domainset_iter di;
	vm_page_t m;
	int domain;

	vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
	do {
	m = vm_page_alloc_domain_after(object, pindex, domain, req,
	mpred);
	if (m != NULL)
	break;
	} while (vm_domainset_iter_page(&di, object, &domain) == 0);

	return (m);
	}

	/*
	* Returns true if the number of free pages exceeds the minimum
	* for the request class and false otherwise.
	*/
	static int
	_vm_domain_allocate(struct vm_domain *vmd, int req_class, int npages)
	{
	u_int limit, old, new;

	if (req_class == VM_ALLOC_INTERRUPT)
	limit = 0;
	else if (req_class == VM_ALLOC_SYSTEM)
	limit = vmd->vmd_interrupt_free_min;
	else
	limit = vmd->vmd_free_reserved;

	/*
	* Attempt to reserve the pages. Fail if we're below the limit.
	*/
	limit += npages;
	old = vmd->vmd_free_count;
	do {
	if (old < limit)
	return (0);
	new = old - npages;
	} while (atomic_fcmpset_int(&vmd->vmd_free_count, &old, new) == 0);

	/* Wake the page daemon if we've crossed the threshold. */
	if (vm_paging_needed(vmd, new) && !vm_paging_needed(vmd, old))
	pagedaemon_wakeup(vmd->vmd_domain);

	/* Only update bitsets on transitions. */
	if ((old >= vmd->vmd_free_min && new < vmd->vmd_free_min) \|\|
	(old >= vmd->vmd_free_severe && new < vmd->vmd_free_severe))
	vm_domain_set(vmd);

	return (1);
	}

	int
	vm_domain_allocate(struct vm_domain *vmd, int req, int npages)
	{
	int req_class;

	/*
	* The page daemon is allowed to dig deeper into the free page list.
	*/
	req_class = req & VM_ALLOC_CLASS_MASK;
	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
	req_class = VM_ALLOC_SYSTEM;
	return (_vm_domain_allocate(vmd, req_class, npages));
	}

	vm_page_t
	vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,
	int req, vm_page_t mpred)
	{
	struct vm_domain *vmd;
	vm_page_t m;
	int flags, pool;

	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
	(object != NULL \|\| (req & VM_ALLOC_SBUSY) == 0) &&
	((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)) !=
	(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)),
	("inconsistent object(%p)/req(%x)", object, req));
	KASSERT(object == NULL \|\| (req & VM_ALLOC_WAITOK) == 0,
	("Can't sleep and retry object insertion."));
	KASSERT(mpred == NULL \|\| mpred->pindex < pindex,
	("mpred %p doesn't precede pindex 0x%jx", mpred,
	(uintmax_t)pindex));
	if (object != NULL)
	VM_OBJECT_ASSERT_WLOCKED(object);

	flags = 0;
	m = NULL;
	pool = object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT;
	again:
	#if VM_NRESERVLEVEL > 0
	/*
	* Can we allocate the page from a reservation?
	*/
	if (vm_object_reserv(object) &&
	(m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) !=
	NULL) {
	goto found;
	}
	#endif
	vmd = VM_DOMAIN(domain);
	if (vmd->vmd_pgcache[pool].zone != NULL) {
	m = uma_zalloc(vmd->vmd_pgcache[pool].zone, M_NOWAIT \| M_NOVM);
	if (m != NULL) {
	flags \|= PG_PCPU_CACHE;
	goto found;
	}
	}
	if (vm_domain_allocate(vmd, req, 1)) {
	/*
	* If not, allocate it from the free page queues.
	*/
	vm_domain_free_lock(vmd);
	m = vm_phys_alloc_pages(domain, pool, 0);
	vm_domain_free_unlock(vmd);
	if (m == NULL) {
	vm_domain_freecnt_inc(vmd, 1);
	#if VM_NRESERVLEVEL > 0
	if (vm_reserv_reclaim_inactive(domain))
	goto again;
	#endif
	}
	}
	if (m == NULL) {
	/*
	* Not allocatable, give up.
	*/
	if (vm_domain_alloc_fail(vmd, object, req))
	goto again;
	return (NULL);
	}

	/*
	* At this point we had better have found a good page.
	*/
	found:
	vm_page_dequeue(m);
	vm_page_alloc_check(m);

	/*
	* Initialize the page. Only the PG_ZERO flag is inherited.
	*/
	if ((req & VM_ALLOC_ZERO) != 0)
	flags \|= (m->flags & PG_ZERO);
	if ((req & VM_ALLOC_NODUMP) != 0)
	flags \|= PG_NODUMP;
	m->flags = flags;
	m->a.flags = 0;
	m->oflags = object == NULL \|\| (object->flags & OBJ_UNMANAGED) != 0 ?
	VPO_UNMANAGED : 0;
	if ((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_NOOBJ \| VM_ALLOC_SBUSY)) == 0)
	m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
	else if ((req & VM_ALLOC_SBUSY) != 0)
	m->busy_lock = VPB_SHARERS_WORD(1);
	else
	m->busy_lock = VPB_UNBUSIED;
	if (req & VM_ALLOC_WIRED) {
	vm_wire_add(1);
	m->ref_count = 1;
	}
	m->a.act_count = 0;

	if (object != NULL) {
	if (vm_page_insert_after(m, object, pindex, mpred)) {
	if (req & VM_ALLOC_WIRED) {
	vm_wire_sub(1);
	m->ref_count = 0;
	}
	KASSERT(m->object == NULL, ("page %p has object", m));
	m->oflags = VPO_UNMANAGED;
	m->busy_lock = VPB_UNBUSIED;
	/* Don't change PG_ZERO. */
	vm_page_free_toq(m);
	if (req & VM_ALLOC_WAITFAIL) {
	VM_OBJECT_WUNLOCK(object);
	vm_radix_wait();
	VM_OBJECT_WLOCK(object);
	}
	return (NULL);
	}

	/* Ignore device objects; the pager sets "memattr" for them. */
	if (object->memattr != VM_MEMATTR_DEFAULT &&
	(object->flags & OBJ_FICTITIOUS) == 0)
	pmap_page_set_memattr(m, object->memattr);
	} else
	m->pindex = pindex;

	return (m);
	}

	/*
	* vm_page_alloc_contig:
	*
	* Allocate a contiguous set of physical pages of the given size "npages"
	* from the free lists. All of the physical pages must be at or above
	* the given physical address "low" and below the given physical address
	* "high". The given value "alignment" determines the alignment of the
	* first physical page in the set. If the given value "boundary" is
	* non-zero, then the set of physical pages cannot cross any physical
	* address boundary that is a multiple of that value. Both "alignment"
	* and "boundary" must be a power of two.
	*
	* If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT,
	* then the memory attribute setting for the physical pages is configured
	* to the object's memory attribute setting. Otherwise, the memory
	* attribute setting for the physical pages is configured to "memattr",
	* overriding the object's memory attribute setting. However, if the
	* object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the
	* memory attribute setting for the physical pages cannot be configured
	* to VM_MEMATTR_DEFAULT.
	*
	* The specified object may not contain fictitious pages.
	*
	* The caller must always specify an allocation class.
	*
	* allocation classes:
	* VM_ALLOC_NORMAL normal process request
	* VM_ALLOC_SYSTEM system really needs a page
	* VM_ALLOC_INTERRUPT interrupt time request
	*
	* optional allocation flags:
	* VM_ALLOC_NOBUSY do not exclusive busy the page
	* VM_ALLOC_NODUMP do not include the page in a kernel core dump
	* VM_ALLOC_NOOBJ page is not associated with an object and
	* should not be exclusive busy
	* VM_ALLOC_SBUSY shared busy the allocated page
	* VM_ALLOC_WIRED wire the allocated page
	* VM_ALLOC_ZERO prefer a zeroed page
	*/
	vm_page_t
	vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
	u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
	vm_paddr_t boundary, vm_memattr_t memattr)
	{
	struct vm_domainset_iter di;
	vm_page_t m;
	int domain;

	vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);
	do {
	m = vm_page_alloc_contig_domain(object, pindex, domain, req,
	npages, low, high, alignment, boundary, memattr);
	if (m != NULL)
	break;
	} while (vm_domainset_iter_page(&di, object, &domain) == 0);

	return (m);
	}

	vm_page_t
	vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain,
	int req, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
	vm_paddr_t boundary, vm_memattr_t memattr)
	{
	struct vm_domain *vmd;
	vm_page_t m, m_ret, mpred;
	u_int busy_lock, flags, oflags;

	mpred = NULL; /* XXX: pacify gcc */
	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
	(object != NULL \|\| (req & VM_ALLOC_SBUSY) == 0) &&
	((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)) !=
	(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY)),
	("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object,
	req));
	KASSERT(object == NULL \|\| (req & VM_ALLOC_WAITOK) == 0,
	("Can't sleep and retry object insertion."));
	if (object != NULL) {
	VM_OBJECT_ASSERT_WLOCKED(object);
	KASSERT((object->flags & OBJ_FICTITIOUS) == 0,
	("vm_page_alloc_contig: object %p has fictitious pages",
	object));
	}
	KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));

	if (object != NULL) {
	mpred = vm_radix_lookup_le(&object->rtree, pindex);
	KASSERT(mpred == NULL \|\| mpred->pindex != pindex,
	("vm_page_alloc_contig: pindex already allocated"));
	}

	/*
	* Can we allocate the pages without the number of free pages falling
	* below the lower bound for the allocation class?
	*/
	m_ret = NULL;
	again:
	#if VM_NRESERVLEVEL > 0
	/*
	* Can we allocate the pages from a reservation?
	*/
	if (vm_object_reserv(object) &&
	(m_ret = vm_reserv_alloc_contig(object, pindex, domain, req,
	mpred, npages, low, high, alignment, boundary)) != NULL) {
	goto found;
	}
	#endif
	vmd = VM_DOMAIN(domain);
	if (vm_domain_allocate(vmd, req, npages)) {
	/*
	* allocate them from the free page queues.
	*/
	vm_domain_free_lock(vmd);
	m_ret = vm_phys_alloc_contig(domain, npages, low, high,
	alignment, boundary);
	vm_domain_free_unlock(vmd);
	if (m_ret == NULL) {
	vm_domain_freecnt_inc(vmd, npages);
	#if VM_NRESERVLEVEL > 0
	if (vm_reserv_reclaim_contig(domain, npages, low,
	high, alignment, boundary))
	goto again;
	#endif
	}
	}
	if (m_ret == NULL) {
	if (vm_domain_alloc_fail(vmd, object, req))
	goto again;
	return (NULL);
	}
	#if VM_NRESERVLEVEL > 0
	found:
	#endif
	for (m = m_ret; m < &m_ret[npages]; m++) {
	vm_page_dequeue(m);
	vm_page_alloc_check(m);
	}

	/*
	* Initialize the pages. Only the PG_ZERO flag is inherited.
	*/
	flags = 0;
	if ((req & VM_ALLOC_ZERO) != 0)
	flags = PG_ZERO;
	if ((req & VM_ALLOC_NODUMP) != 0)
	flags \|= PG_NODUMP;
	oflags = object == NULL \|\| (object->flags & OBJ_UNMANAGED) != 0 ?
	VPO_UNMANAGED : 0;
	if ((req & (VM_ALLOC_NOBUSY \| VM_ALLOC_NOOBJ \| VM_ALLOC_SBUSY)) == 0)
	busy_lock = VPB_CURTHREAD_EXCLUSIVE;
	else if ((req & VM_ALLOC_SBUSY) != 0)
	busy_lock = VPB_SHARERS_WORD(1);
	else
	busy_lock = VPB_UNBUSIED;
	if ((req & VM_ALLOC_WIRED) != 0)
	vm_wire_add(npages);
	if (object != NULL) {
	if (object->memattr != VM_MEMATTR_DEFAULT &&
	memattr == VM_MEMATTR_DEFAULT)
	memattr = object->memattr;
	}
	for (m = m_ret; m < &m_ret[npages]; m++) {
	m->a.flags = 0;
	m->flags = (m->flags \| PG_NODUMP) & flags;
	m->busy_lock = busy_lock;
	if ((req & VM_ALLOC_WIRED) != 0)
	m->ref_count = 1;
	m->a.act_count = 0;
	m->oflags = oflags;
	if (object != NULL) {
	if (vm_page_insert_after(m, object, pindex, mpred)) {
	if ((req & VM_ALLOC_WIRED) != 0)
	vm_wire_sub(npages);
	KASSERT(m->object == NULL,
	("page %p has object", m));
	mpred = m;
	for (m = m_ret; m < &m_ret[npages]; m++) {
	if (m <= mpred &&
	(req & VM_ALLOC_WIRED) != 0)
	m->ref_count = 0;
	m->oflags = VPO_UNMANAGED;
	m->busy_lock = VPB_UNBUSIED;
	/* Don't change PG_ZERO. */
	vm_page_free_toq(m);
	}
	if (req & VM_ALLOC_WAITFAIL) {
	VM_OBJECT_WUNLOCK(object);
	vm_radix_wait();
	VM_OBJECT_WLOCK(object);
	}
	return (NULL);
	}
	mpred = m;
	} else
	m->pindex = pindex;
	if (memattr != VM_MEMATTR_DEFAULT)
	pmap_page_set_memattr(m, memattr);
	pindex++;
	}
	return (m_ret);
	}

	/*
	* Check a page that has been freshly dequeued from a freelist.
	*/
	static void
	vm_page_alloc_check(vm_page_t m)
	{

	KASSERT(m->object == NULL, ("page %p has object", m));
	KASSERT(m->a.queue == PQ_NONE &&
	(m->a.flags & PGA_QUEUE_STATE_MASK) == 0,
	("page %p has unexpected queue %d, flags %#x",
	m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK)));
	KASSERT(m->ref_count == 0, ("page %p has references", m));
	KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m));
	KASSERT(m->dirty == 0, ("page %p is dirty", m));
	KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
	("page %p has unexpected memattr %d",
	m, pmap_page_get_memattr(m)));
	KASSERT(m->valid == 0, ("free page %p is valid", m));
	}

	/*
	* vm_page_alloc_freelist:
	*
	* Allocate a physical page from the specified free page list.
	*
	* The caller must always specify an allocation class.
	*
	* allocation classes:
	* VM_ALLOC_NORMAL normal process request
	* VM_ALLOC_SYSTEM system really needs a page
	* VM_ALLOC_INTERRUPT interrupt time request
	*
	* optional allocation flags:
	* VM_ALLOC_COUNT(number) the number of additional pages that the caller
	* intends to allocate
	* VM_ALLOC_WIRED wire the allocated page
	* VM_ALLOC_ZERO prefer a zeroed page
	*/
	vm_page_t
	vm_page_alloc_freelist(int freelist, int req)
	{
	struct vm_domainset_iter di;
	vm_page_t m;
	int domain;

	vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
	do {
	m = vm_page_alloc_freelist_domain(domain, freelist, req);
	if (m != NULL)
	break;
	} while (vm_domainset_iter_page(&di, NULL, &domain) == 0);

	return (m);
	}

	vm_page_t
	vm_page_alloc_freelist_domain(int domain, int freelist, int req)
	{
	struct vm_domain *vmd;
	vm_page_t m;
	u_int flags;

	m = NULL;
	vmd = VM_DOMAIN(domain);
	again:
	if (vm_domain_allocate(vmd, req, 1)) {
	vm_domain_free_lock(vmd);
	m = vm_phys_alloc_freelist_pages(domain, freelist,
	VM_FREEPOOL_DIRECT, 0);
	vm_domain_free_unlock(vmd);
	if (m == NULL)
	vm_domain_freecnt_inc(vmd, 1);
	}
	if (m == NULL) {
	if (vm_domain_alloc_fail(vmd, NULL, req))
	goto again;
	return (NULL);
	}
	vm_page_dequeue(m);
	vm_page_alloc_check(m);

	/*
	* Initialize the page. Only the PG_ZERO flag is inherited.
	*/
	m->a.flags = 0;
	flags = 0;
	if ((req & VM_ALLOC_ZERO) != 0)
	flags = PG_ZERO;
	m->flags &= flags;
	if ((req & VM_ALLOC_WIRED) != 0) {
	vm_wire_add(1);
	m->ref_count = 1;
	}
	/* Unmanaged pages don't use "act_count". */
	m->oflags = VPO_UNMANAGED;
	return (m);
	}

	static int
	vm_page_zone_import(void arg, void *store, int cnt, int domain, int flags)
	{
	struct vm_domain *vmd;
	struct vm_pgcache *pgcache;
	int i;

	pgcache = arg;
	vmd = VM_DOMAIN(pgcache->domain);

	/*
	* The page daemon should avoid creating extra memory pressure since its
	* main purpose is to replenish the store of free pages.
	*/
	if (vmd->vmd_severeset \|\| curproc == pageproc \|\|
	!_vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt))
	return (0);
	domain = vmd->vmd_domain;
	vm_domain_free_lock(vmd);
	i = vm_phys_alloc_npages(domain, pgcache->pool, cnt,
	(vm_page_t *)store);
	vm_domain_free_unlock(vmd);
	if (cnt != i)
	vm_domain_freecnt_inc(vmd, cnt - i);

	return (i);
	}

	static void
	vm_page_zone_release(void arg, void *store, int cnt)
	{
	struct vm_domain *vmd;
	struct vm_pgcache *pgcache;
	vm_page_t m;
	int i;

	pgcache = arg;
	vmd = VM_DOMAIN(pgcache->domain);
	vm_domain_free_lock(vmd);
	for (i = 0; i < cnt; i++) {
	m = (vm_page_t)store[i];
	vm_phys_free_pages(m, 0);
	}
	vm_domain_free_unlock(vmd);
	vm_domain_freecnt_inc(vmd, cnt);
	}

	#define VPSC_ANY 0 /* No restrictions. */
	#define VPSC_NORESERV 1 /* Skip reservations; implies VPSC_NOSUPER. */
	#define VPSC_NOSUPER 2 /* Skip superpages. */

	/*
	* vm_page_scan_contig:
	*
	* Scan vm_page_array[] between the specified entries "m_start" and
	* "m_end" for a run of contiguous physical pages that satisfy the
	* specified conditions, and return the lowest page in the run. The
	* specified "alignment" determines the alignment of the lowest physical
	* page in the run. If the specified "boundary" is non-zero, then the
	* run of physical pages cannot span a physical address that is a
	* multiple of "boundary".
	*
	* "m_end" is never dereferenced, so it need not point to a vm_page
	* structure within vm_page_array[].
	*
	* "npages" must be greater than zero. "m_start" and "m_end" must not
	* span a hole (or discontiguity) in the physical address space. Both
	* "alignment" and "boundary" must be a power of two.
	*/
	vm_page_t
	vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end,
	u_long alignment, vm_paddr_t boundary, int options)
	{
	vm_object_t object;
	vm_paddr_t pa;
	vm_page_t m, m_run;
	#if VM_NRESERVLEVEL > 0
	int level;
	#endif
	int m_inc, order, run_ext, run_len;

	KASSERT(npages > 0, ("npages is 0"));
	KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
	KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
	m_run = NULL;
	run_len = 0;
	for (m = m_start; m < m_end && run_len < npages; m += m_inc) {
	KASSERT((m->flags & PG_MARKER) == 0,
	("page %p is PG_MARKER", m));
	KASSERT((m->flags & PG_FICTITIOUS) == 0 \|\| m->ref_count >= 1,
	("fictitious page %p has invalid ref count", m));

	/*
	* If the current page would be the start of a run, check its
	* physical address against the end, alignment, and boundary
	* conditions. If it doesn't satisfy these conditions, either
	* terminate the scan or advance to the next page that
	* satisfies the failed condition.
	*/
	if (run_len == 0) {
	KASSERT(m_run == NULL, ("m_run != NULL"));
	if (m + npages > m_end)
	break;
	pa = VM_PAGE_TO_PHYS(m);
	if ((pa & (alignment - 1)) != 0) {
	m_inc = atop(roundup2(pa, alignment) - pa);
	continue;
	}
	if (rounddown2(pa ^ (pa + ptoa(npages) - 1),
	boundary) != 0) {
	m_inc = atop(roundup2(pa, boundary) - pa);
	continue;
	}
	} else
	KASSERT(m_run != NULL, ("m_run == NULL"));

	retry:
	m_inc = 1;
	if (vm_page_wired(m))
	run_ext = 0;
	#if VM_NRESERVLEVEL > 0
	else if ((level = vm_reserv_level(m)) >= 0 &&
	(options & VPSC_NORESERV) != 0) {
	run_ext = 0;
	/* Advance to the end of the reservation. */
	pa = VM_PAGE_TO_PHYS(m);
	m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) -
	pa);
	}
	#endif
	else if ((object = atomic_load_ptr(&m->object)) != NULL) {
	/*
	* The page is considered eligible for relocation if
	* and only if it could be laundered or reclaimed by
	* the page daemon.
	*/
	VM_OBJECT_RLOCK(object);
	if (object != m->object) {
	VM_OBJECT_RUNLOCK(object);
	goto retry;
	}
	/* Don't care: PG_NODUMP, PG_ZERO. */
	if (object->type != OBJT_DEFAULT &&
	object->type != OBJT_SWAP &&
	object->type != OBJT_VNODE) {
	run_ext = 0;
	#if VM_NRESERVLEVEL > 0
	} else if ((options & VPSC_NOSUPER) != 0 &&
	(level = vm_reserv_level_iffullpop(m)) >= 0) {
	run_ext = 0;
	/* Advance to the end of the superpage. */
	pa = VM_PAGE_TO_PHYS(m);
	m_inc = atop(roundup2(pa + 1,
	vm_reserv_size(level)) - pa);
	#endif
	} else if (object->memattr == VM_MEMATTR_DEFAULT &&
	vm_page_queue(m) != PQ_NONE && !vm_page_busied(m)) {
	/*
	* The page is allocated but eligible for
	* relocation. Extend the current run by one
	* page.
	*/
	KASSERT(pmap_page_get_memattr(m) ==
	VM_MEMATTR_DEFAULT,
	("page %p has an unexpected memattr", m));
	KASSERT((m->oflags & (VPO_SWAPINPROG \|
	VPO_SWAPSLEEP \| VPO_UNMANAGED)) == 0,
	("page %p has unexpected oflags", m));
	/* Don't care: PGA_NOSYNC. */
	run_ext = 1;
	} else
	run_ext = 0;
	VM_OBJECT_RUNLOCK(object);
	#if VM_NRESERVLEVEL > 0
	} else if (level >= 0) {
	/*
	* The page is reserved but not yet allocated. In
	* other words, it is still free. Extend the current
	* run by one page.
	*/
	run_ext = 1;
	#endif
	} else if ((order = m->order) < VM_NFREEORDER) {
	/*
	* The page is enqueued in the physical memory
	* allocator's free page queues. Moreover, it is the
	* first page in a power-of-two-sized run of
	* contiguous free pages. Add these pages to the end
	* of the current run, and jump ahead.
	*/
	run_ext = 1 << order;
	m_inc = 1 << order;
	} else {
	/*
	* Skip the page for one of the following reasons: (1)
	* It is enqueued in the physical memory allocator's
	* free page queues. However, it is not the first
	* page in a run of contiguous free pages. (This case
	* rarely occurs because the scan is performed in
	* ascending order.) (2) It is not reserved, and it is
	* transitioning from free to allocated. (Conversely,
	* the transition from allocated to free for managed
	* pages is blocked by the page lock.) (3) It is
	* allocated but not contained by an object and not
	* wired, e.g., allocated by Xen's balloon driver.
	*/
	run_ext = 0;
	}

	/*
	* Extend or reset the current run of pages.
	*/
	if (run_ext > 0) {
	if (run_len == 0)
	m_run = m;
	run_len += run_ext;
	} else {
	if (run_len > 0) {
	m_run = NULL;
	run_len = 0;
	}
	}
	}
	if (run_len >= npages)
	return (m_run);
	return (NULL);
	}

	/*
	* vm_page_reclaim_run:
	*
	* Try to relocate each of the allocated virtual pages within the
	* specified run of physical pages to a new physical address. Free the
	* physical pages underlying the relocated virtual pages. A virtual page
	* is relocatable if and only if it could be laundered or reclaimed by
	* the page daemon. Whenever possible, a virtual page is relocated to a
	* physical address above "high".
	*
	* Returns 0 if every physical page within the run was already free or
	* just freed by a successful relocation. Otherwise, returns a non-zero
	* value indicating why the last attempt to relocate a virtual page was
	* unsuccessful.
	*
	* "req_class" must be an allocation class.
	*/
	static int
	vm_page_reclaim_run(int req_class, int domain, u_long npages, vm_page_t m_run,
	vm_paddr_t high)
	{
	struct vm_domain *vmd;
	struct spglist free;
	vm_object_t object;
	vm_paddr_t pa;
	vm_page_t m, m_end, m_new;
	int error, order, req;

	KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class,
	("req_class is not an allocation class"));
	SLIST_INIT(&free);
	error = 0;
	m = m_run;
	m_end = m_run + npages;
	for (; error == 0 && m < m_end; m++) {
	KASSERT((m->flags & (PG_FICTITIOUS \| PG_MARKER)) == 0,
	("page %p is PG_FICTITIOUS or PG_MARKER", m));

	/*
	* Racily check for wirings. Races are handled once the object
	* lock is held and the page is unmapped.
	*/
	if (vm_page_wired(m))
	error = EBUSY;
	else if ((object = atomic_load_ptr(&m->object)) != NULL) {
	/*
	* The page is relocated if and only if it could be
	* laundered or reclaimed by the page daemon.
	*/
	VM_OBJECT_WLOCK(object);
	/* Don't care: PG_NODUMP, PG_ZERO. */
	if (m->object != object \|\|
	(object->type != OBJT_DEFAULT &&
	object->type != OBJT_SWAP &&
	object->type != OBJT_VNODE))
	error = EINVAL;
	else if (object->memattr != VM_MEMATTR_DEFAULT)
	error = EINVAL;
	else if (vm_page_queue(m) != PQ_NONE &&
	vm_page_tryxbusy(m) != 0) {
	if (vm_page_wired(m)) {
	vm_page_xunbusy(m);
	error = EBUSY;
	goto unlock;
	}
	KASSERT(pmap_page_get_memattr(m) ==
	VM_MEMATTR_DEFAULT,
	("page %p has an unexpected memattr", m));
	KASSERT(m->oflags == 0,
	("page %p has unexpected oflags", m));
	/* Don't care: PGA_NOSYNC. */
	if (!vm_page_none_valid(m)) {
	/*
	* First, try to allocate a new page
	* that is above "high". Failing
	* that, try to allocate a new page
	* that is below "m_run". Allocate
	* the new page between the end of
	* "m_run" and "high" only as a last
	* resort.
	*/
	req = req_class \| VM_ALLOC_NOOBJ;
	if ((m->flags & PG_NODUMP) != 0)
	req \|= VM_ALLOC_NODUMP;
	if (trunc_page(high) !=
	~(vm_paddr_t)PAGE_MASK) {
	m_new = vm_page_alloc_contig(
	NULL, 0, req, 1,
	round_page(high),
	~(vm_paddr_t)0,
	PAGE_SIZE, 0,
	VM_MEMATTR_DEFAULT);
	} else
	m_new = NULL;
	if (m_new == NULL) {
	pa = VM_PAGE_TO_PHYS(m_run);
	m_new = vm_page_alloc_contig(
	NULL, 0, req, 1,
	0, pa - 1, PAGE_SIZE, 0,
	VM_MEMATTR_DEFAULT);
	}
	if (m_new == NULL) {
	pa += ptoa(npages);
	m_new = vm_page_alloc_contig(
	NULL, 0, req, 1,
	pa, high, PAGE_SIZE, 0,
	VM_MEMATTR_DEFAULT);
	}
	if (m_new == NULL) {
	vm_page_xunbusy(m);
	error = ENOMEM;
	goto unlock;
	}

	/*
	* Unmap the page and check for new
	* wirings that may have been acquired
	* through a pmap lookup.
	*/
	if (object->ref_count != 0 &&
	!vm_page_try_remove_all(m)) {
	vm_page_xunbusy(m);
	vm_page_free(m_new);
	error = EBUSY;
	goto unlock;
	}

	/*
	* Replace "m" with the new page. For
	* vm_page_replace(), "m" must be busy
	* and dequeued. Finally, change "m"
	* as if vm_page_free() was called.
	*/
	m_new->a.flags = m->a.flags &
	~PGA_QUEUE_STATE_MASK;
	KASSERT(m_new->oflags == VPO_UNMANAGED,
	("page %p is managed", m_new));
	m_new->oflags = 0;
	pmap_copy_page(m, m_new);
	m_new->valid = m->valid;
	m_new->dirty = m->dirty;
	m->flags &= ~PG_ZERO;
	vm_page_dequeue(m);
	if (vm_page_replace_hold(m_new, object,
	m->pindex, m) &&
	vm_page_free_prep(m))
	SLIST_INSERT_HEAD(&free, m,
	plinks.s.ss);

	/*
	* The new page must be deactivated
	* before the object is unlocked.
	*/
	vm_page_deactivate(m_new);
	} else {
	m->flags &= ~PG_ZERO;
	vm_page_dequeue(m);
	if (vm_page_free_prep(m))
	SLIST_INSERT_HEAD(&free, m,
	plinks.s.ss);
	KASSERT(m->dirty == 0,
	("page %p is dirty", m));
	}
	} else
	error = EBUSY;
	unlock:
	VM_OBJECT_WUNLOCK(object);
	} else {
	MPASS(vm_phys_domain(m) == domain);
	vmd = VM_DOMAIN(domain);
	vm_domain_free_lock(vmd);
	order = m->order;
	if (order < VM_NFREEORDER) {
	/*
	* The page is enqueued in the physical memory
	* allocator's free page queues. Moreover, it
	* is the first page in a power-of-two-sized
	* run of contiguous free pages. Jump ahead
	* to the last page within that run, and
	* continue from there.
	*/
	m += (1 << order) - 1;
	}
	#if VM_NRESERVLEVEL > 0
	else if (vm_reserv_is_page_free(m))
	order = 0;
	#endif
	vm_domain_free_unlock(vmd);
	if (order == VM_NFREEORDER)
	error = EINVAL;
	}
	}
	if ((m = SLIST_FIRST(&free)) != NULL) {
	int cnt;

	vmd = VM_DOMAIN(domain);
	cnt = 0;
	vm_domain_free_lock(vmd);
	do {
	MPASS(vm_phys_domain(m) == domain);
	SLIST_REMOVE_HEAD(&free, plinks.s.ss);
	vm_phys_free_pages(m, 0);
	cnt++;
	} while ((m = SLIST_FIRST(&free)) != NULL);
	vm_domain_free_unlock(vmd);
	vm_domain_freecnt_inc(vmd, cnt);
	}
	return (error);
	}

	#define NRUNS 16

	CTASSERT(powerof2(NRUNS));

	#define RUN_INDEX(count) ((count) & (NRUNS - 1))

	#define MIN_RECLAIM 8

	/*
	* vm_page_reclaim_contig:
	*
	* Reclaim allocated, contiguous physical memory satisfying the specified
	* conditions by relocating the virtual pages using that physical memory.
	* Returns true if reclamation is successful and false otherwise. Since
	* relocation requires the allocation of physical pages, reclamation may
	* fail due to a shortage of free pages. When reclamation fails, callers
	* are expected to perform vm_wait() before retrying a failed allocation
	* operation, e.g., vm_page_alloc_contig().
	*
	* The caller must always specify an allocation class through "req".
	*
	* allocation classes:
	* VM_ALLOC_NORMAL normal process request
	* VM_ALLOC_SYSTEM system really needs a page
	* VM_ALLOC_INTERRUPT interrupt time request
	*
	* The optional allocation flags are ignored.
	*
	* "npages" must be greater than zero. Both "alignment" and "boundary"
	* must be a power of two.
	*/
	bool
	vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
	vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary)
	{
	struct vm_domain *vmd;
	vm_paddr_t curr_low;
	vm_page_t m_run, m_runs[NRUNS];
	u_long count, reclaimed;
	int error, i, options, req_class;

	KASSERT(npages > 0, ("npages is 0"));
	KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
	KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
	req_class = req & VM_ALLOC_CLASS_MASK;

	/*
	* The page daemon is allowed to dig deeper into the free page list.
	*/
	if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
	req_class = VM_ALLOC_SYSTEM;

	/*
	* Return if the number of free pages cannot satisfy the requested
	* allocation.
	*/
	vmd = VM_DOMAIN(domain);
	count = vmd->vmd_free_count;
	if (count < npages + vmd->vmd_free_reserved \|\| (count < npages +
	vmd->vmd_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) \|\|
	(count < npages && req_class == VM_ALLOC_INTERRUPT))
	return (false);

	/*
	* Scan up to three times, relaxing the restrictions ("options") on
	* the reclamation of reservations and superpages each time.
	*/
	for (options = VPSC_NORESERV;;) {
	/*
	* Find the highest runs that satisfy the given constraints
	* and restrictions, and record them in "m_runs".
	*/
	curr_low = low;
	count = 0;
	for (;;) {
	m_run = vm_phys_scan_contig(domain, npages, curr_low,
	high, alignment, boundary, options);
	if (m_run == NULL)
	break;
	curr_low = VM_PAGE_TO_PHYS(m_run) + ptoa(npages);
	m_runs[RUN_INDEX(count)] = m_run;
	count++;
	}

	/*
	* Reclaim the highest runs in LIFO (descending) order until
	* the number of reclaimed pages, "reclaimed", is at least
	* MIN_RECLAIM. Reset "reclaimed" each time because each
	* reclamation is idempotent, and runs will (likely) recur
	* from one scan to the next as restrictions are relaxed.
	*/
	reclaimed = 0;
	for (i = 0; count > 0 && i < NRUNS; i++) {
	count--;
	m_run = m_runs[RUN_INDEX(count)];
	error = vm_page_reclaim_run(req_class, domain, npages,
	m_run, high);
	if (error == 0) {
	reclaimed += npages;
	if (reclaimed >= MIN_RECLAIM)
	return (true);
	}
	}

	/*
	* Either relax the restrictions on the next scan or return if
	* the last scan had no restrictions.
	*/
	if (options == VPSC_NORESERV)
	options = VPSC_NOSUPER;
	else if (options == VPSC_NOSUPER)
	options = VPSC_ANY;
	else if (options == VPSC_ANY)
	return (reclaimed != 0);
	}
	}

	bool
	vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
	u_long alignment, vm_paddr_t boundary)
	{
	struct vm_domainset_iter di;
	int domain;
	bool ret;

	vm_domainset_iter_page_init(&di, NULL, 0, &domain, &req);
	do {
	ret = vm_page_reclaim_contig_domain(domain, req, npages, low,
	high, alignment, boundary);
	if (ret)
	break;
	} while (vm_domainset_iter_page(&di, NULL, &domain) == 0);

	return (ret);
	}

	/*
	* Set the domain in the appropriate page level domainset.
	*/
	void
	vm_domain_set(struct vm_domain *vmd)
	{

	mtx_lock(&vm_domainset_lock);
	if (!vmd->vmd_minset && vm_paging_min(vmd)) {
	vmd->vmd_minset = 1;
	DOMAINSET_SET(vmd->vmd_domain, &vm_min_domains);
	}
	if (!vmd->vmd_severeset && vm_paging_severe(vmd)) {
	vmd->vmd_severeset = 1;
	DOMAINSET_SET(vmd->vmd_domain, &vm_severe_domains);
	}
	mtx_unlock(&vm_domainset_lock);
	}

	/*
	* Clear the domain from the appropriate page level domainset.
	*/
	void
	vm_domain_clear(struct vm_domain *vmd)
	{

	mtx_lock(&vm_domainset_lock);
	if (vmd->vmd_minset && !vm_paging_min(vmd)) {
	vmd->vmd_minset = 0;
	DOMAINSET_CLR(vmd->vmd_domain, &vm_min_domains);
	if (vm_min_waiters != 0) {
	vm_min_waiters = 0;
	wakeup(&vm_min_domains);
	}
	}
	if (vmd->vmd_severeset && !vm_paging_severe(vmd)) {
	vmd->vmd_severeset = 0;
	DOMAINSET_CLR(vmd->vmd_domain, &vm_severe_domains);
	if (vm_severe_waiters != 0) {
	vm_severe_waiters = 0;
	wakeup(&vm_severe_domains);
	}
	}

	/*
	* If pageout daemon needs pages, then tell it that there are
	* some free.
	*/
	if (vmd->vmd_pageout_pages_needed &&
	vmd->vmd_free_count >= vmd->vmd_pageout_free_min) {
	wakeup(&vmd->vmd_pageout_pages_needed);
	vmd->vmd_pageout_pages_needed = 0;
	}

	/* See comments in vm_wait_doms(). */
	if (vm_pageproc_waiters) {
	vm_pageproc_waiters = 0;
	wakeup(&vm_pageproc_waiters);
	}
	mtx_unlock(&vm_domainset_lock);
	}

	/*
	* Wait for free pages to exceed the min threshold globally.
	*/
	void
	vm_wait_min(void)
	{

	mtx_lock(&vm_domainset_lock);
	while (vm_page_count_min()) {
	vm_min_waiters++;
	msleep(&vm_min_domains, &vm_domainset_lock, PVM, "vmwait", 0);
	}
	mtx_unlock(&vm_domainset_lock);
	}

	/*
	* Wait for free pages to exceed the severe threshold globally.
	*/
	void
	vm_wait_severe(void)
	{

	mtx_lock(&vm_domainset_lock);
	while (vm_page_count_severe()) {
	vm_severe_waiters++;
	msleep(&vm_severe_domains, &vm_domainset_lock, PVM,
	"vmwait", 0);
	}
	mtx_unlock(&vm_domainset_lock);
	}

	u_int
	vm_wait_count(void)
	{

	return (vm_severe_waiters + vm_min_waiters + vm_pageproc_waiters);
	}

	void
	vm_wait_doms(const domainset_t *wdoms)
	{

	/*
	* We use racey wakeup synchronization to avoid expensive global
	* locking for the pageproc when sleeping with a non-specific vm_wait.
	* To handle this, we only sleep for one tick in this instance. It
	* is expected that most allocations for the pageproc will come from
	* kmem or vm_page_grab* which will use the more specific and
	* race-free vm_wait_domain().
	*/
	if (curproc == pageproc) {
	mtx_lock(&vm_domainset_lock);
	vm_pageproc_waiters++;
	msleep(&vm_pageproc_waiters, &vm_domainset_lock, PVM \| PDROP,
	"pageprocwait", 1);
	} else {
	/*
	* XXX Ideally we would wait only until the allocation could
	* be satisfied. This condition can cause new allocators to
	* consume all freed pages while old allocators wait.
	*/
	mtx_lock(&vm_domainset_lock);
	if (vm_page_count_min_set(wdoms)) {
	vm_min_waiters++;
	msleep(&vm_min_domains, &vm_domainset_lock,
	PVM \| PDROP, "vmwait", 0);
	} else
	mtx_unlock(&vm_domainset_lock);
	}
	}

	/*
	* vm_wait_domain:
	*
	* Sleep until free pages are available for allocation.
	* - Called in various places after failed memory allocations.
	*/
	void
	vm_wait_domain(int domain)
	{
	struct vm_domain *vmd;
	domainset_t wdom;

	vmd = VM_DOMAIN(domain);
	vm_domain_free_assert_unlocked(vmd);

	if (curproc == pageproc) {
	mtx_lock(&vm_domainset_lock);
	if (vmd->vmd_free_count < vmd->vmd_pageout_free_min) {
	vmd->vmd_pageout_pages_needed = 1;
	msleep(&vmd->vmd_pageout_pages_needed,
	&vm_domainset_lock, PDROP \| PSWP, "VMWait", 0);
	} else
	mtx_unlock(&vm_domainset_lock);
	} else {
	if (pageproc == NULL)
	panic("vm_wait in early boot");
	DOMAINSET_ZERO(&wdom);
	DOMAINSET_SET(vmd->vmd_domain, &wdom);
	vm_wait_doms(&wdom);
	}
	}

	/*
	* vm_wait:
	*
	* Sleep until free pages are available for allocation in the
	* affinity domains of the obj. If obj is NULL, the domain set
	* for the calling thread is used.
	* Called in various places after failed memory allocations.
	*/
	void
	vm_wait(vm_object_t obj)
	{
	struct domainset *d;

	d = NULL;

	/*
	* Carefully fetch pointers only once: the struct domainset
	* itself is ummutable but the pointer might change.
	*/
	if (obj != NULL)
	d = obj->domain.dr_policy;
	if (d == NULL)
	d = curthread->td_domain.dr_policy;

	vm_wait_doms(&d->ds_mask);
	}

	/*
	* vm_domain_alloc_fail:
	*
	* Called when a page allocation function fails. Informs the
	* pagedaemon and performs the requested wait. Requires the
	* domain_free and object lock on entry. Returns with the
	* object lock held and free lock released. Returns an error when
	* retry is necessary.
	*
	*/
	static int
	vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, int req)
	{

	vm_domain_free_assert_unlocked(vmd);

	atomic_add_int(&vmd->vmd_pageout_deficit,
	max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
	if (req & (VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL)) {
	if (object != NULL)
	VM_OBJECT_WUNLOCK(object);
	vm_wait_domain(vmd->vmd_domain);
	if (object != NULL)
	VM_OBJECT_WLOCK(object);
	if (req & VM_ALLOC_WAITOK)
	return (EAGAIN);
	}

	return (0);
	}

	/*
	* vm_waitpfault:
	*
	* Sleep until free pages are available for allocation.
	* - Called only in vm_fault so that processes page faulting
	* can be easily tracked.
	* - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
	* processes will be able to grab memory first. Do not change
	* this balance without careful testing first.
	*/
	void
	vm_waitpfault(struct domainset *dset, int timo)
	{

	/*
	* XXX Ideally we would wait only until the allocation could
	* be satisfied. This condition can cause new allocators to
	* consume all freed pages while old allocators wait.
	*/
	mtx_lock(&vm_domainset_lock);
	if (vm_page_count_min_set(&dset->ds_mask)) {
	vm_min_waiters++;
	msleep(&vm_min_domains, &vm_domainset_lock, PUSER \| PDROP,
	"pfault", timo);
	} else
	mtx_unlock(&vm_domainset_lock);
	}

	static struct vm_pagequeue *
	_vm_page_pagequeue(vm_page_t m, uint8_t queue)
	{

	return (&vm_pagequeue_domain(m)->vmd_pagequeues[queue]);
	}

	#ifdef INVARIANTS
	static struct vm_pagequeue *
	vm_page_pagequeue(vm_page_t m)
	{

	return (_vm_page_pagequeue(m, vm_page_astate_load(m).queue));
	}
	#endif

	static __always_inline bool
	vm_page_pqstate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new)
	{
	vm_page_astate_t tmp;

	tmp = *old;
	do {
	if (__predict_true(vm_page_astate_fcmpset(m, old, new)))
	return (true);
	counter_u64_add(pqstate_commit_retries, 1);
	} while (old->_bits == tmp._bits);

	return (false);
	}

	/*
	* Do the work of committing a queue state update that moves the page out of
	* its current queue.
	*/
	static bool
	_vm_page_pqstate_commit_dequeue(struct vm_pagequeue *pq, vm_page_t m,
	vm_page_astate_t *old, vm_page_astate_t new)
	{
	vm_page_t next;

	vm_pagequeue_assert_locked(pq);
	KASSERT(vm_page_pagequeue(m) == pq,
	("%s: queue %p does not match page %p", __func__, pq, m));
	KASSERT(old->queue != PQ_NONE && new.queue != old->queue,
	("%s: invalid queue indices %d %d",
	__func__, old->queue, new.queue));

	/*
	* Once the queue index of the page changes there is nothing
	* synchronizing with further updates to the page's physical
	* queue state. Therefore we must speculatively remove the page
	* from the queue now and be prepared to roll back if the queue
	* state update fails. If the page is not physically enqueued then
	* we just update its queue index.
	*/
	if ((old->flags & PGA_ENQUEUED) != 0) {
	new.flags &= ~PGA_ENQUEUED;
	next = TAILQ_NEXT(m, plinks.q);
	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
	vm_pagequeue_cnt_dec(pq);
	if (!vm_page_pqstate_fcmpset(m, old, new)) {
	if (next == NULL)
	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
	else
	TAILQ_INSERT_BEFORE(next, m, plinks.q);
	vm_pagequeue_cnt_inc(pq);
	return (false);
	} else {
	return (true);
	}
	} else {
	return (vm_page_pqstate_fcmpset(m, old, new));
	}
	}

	static bool
	vm_page_pqstate_commit_dequeue(vm_page_t m, vm_page_astate_t *old,
	vm_page_astate_t new)
	{
	struct vm_pagequeue *pq;
	vm_page_astate_t as;
	bool ret;

	pq = _vm_page_pagequeue(m, old->queue);

	/*
	* The queue field and PGA_ENQUEUED flag are stable only so long as the
	* corresponding page queue lock is held.
	*/
	vm_pagequeue_lock(pq);
	as = vm_page_astate_load(m);
	if (__predict_false(as._bits != old->_bits)) {
	*old = as;
	ret = false;
	} else {
	ret = _vm_page_pqstate_commit_dequeue(pq, m, old, new);
	}
	vm_pagequeue_unlock(pq);
	return (ret);
	}

	/*
	* Commit a queue state update that enqueues or requeues a page.
	*/
	static bool
	_vm_page_pqstate_commit_requeue(struct vm_pagequeue *pq, vm_page_t m,
	vm_page_astate_t *old, vm_page_astate_t new)
	{
	struct vm_domain *vmd;

	vm_pagequeue_assert_locked(pq);
	KASSERT(old->queue != PQ_NONE && new.queue == old->queue,
	("%s: invalid queue indices %d %d",
	__func__, old->queue, new.queue));

	new.flags \|= PGA_ENQUEUED;
	if (!vm_page_pqstate_fcmpset(m, old, new))
	return (false);

	if ((old->flags & PGA_ENQUEUED) != 0)
	TAILQ_REMOVE(&pq->pq_pl, m, plinks.q);
	else
	vm_pagequeue_cnt_inc(pq);

	/*
	* Give PGA_REQUEUE_HEAD precedence over PGA_REQUEUE. In particular, if
	* both flags are set in close succession, only PGA_REQUEUE_HEAD will be
	* applied, even if it was set first.
	*/
	if ((old->flags & PGA_REQUEUE_HEAD) != 0) {
	vmd = vm_pagequeue_domain(m);
	KASSERT(pq == &vmd->vmd_pagequeues[PQ_INACTIVE],
	("%s: invalid page queue for page %p", __func__, m));
	TAILQ_INSERT_BEFORE(&vmd->vmd_inacthead, m, plinks.q);
	} else {
	TAILQ_INSERT_TAIL(&pq->pq_pl, m, plinks.q);
	}
	return (true);
	}

	/*
	* Commit a queue state update that encodes a request for a deferred queue
	* operation.
	*/
	static bool
	vm_page_pqstate_commit_request(vm_page_t m, vm_page_astate_t *old,
	vm_page_astate_t new)
	{

	KASSERT(old->queue == new.queue \|\| new.queue != PQ_NONE,
	("%s: invalid state, queue %d flags %x",
	__func__, new.queue, new.flags));

	if (old->_bits != new._bits &&
	!vm_page_pqstate_fcmpset(m, old, new))
	return (false);
	vm_page_pqbatch_submit(m, new.queue);
	return (true);
	}

	/*
	* A generic queue state update function. This handles more cases than the
	* specialized functions above.
	*/
	bool
	vm_page_pqstate_commit(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new)
	{

	if (old->_bits == new._bits)
	return (true);

	if (old->queue != PQ_NONE && new.queue != old->queue) {
	if (!vm_page_pqstate_commit_dequeue(m, old, new))
	return (false);
	if (new.queue != PQ_NONE)
	vm_page_pqbatch_submit(m, new.queue);
	} else {
	if (!vm_page_pqstate_fcmpset(m, old, new))
	return (false);
	if (new.queue != PQ_NONE &&
	((new.flags & ~old->flags) & PGA_QUEUE_OP_MASK) != 0)
	vm_page_pqbatch_submit(m, new.queue);
	}
	return (true);
	}

	/*
	* Apply deferred queue state updates to a page.
	*/
	static inline void
	vm_pqbatch_process_page(struct vm_pagequeue *pq, vm_page_t m, uint8_t queue)
	{
	vm_page_astate_t new, old;

	CRITICAL_ASSERT(curthread);
	vm_pagequeue_assert_locked(pq);
	KASSERT(queue < PQ_COUNT,
	("%s: invalid queue index %d", __func__, queue));
	KASSERT(pq == _vm_page_pagequeue(m, queue),
	("%s: page %p does not belong to queue %p", __func__, m, pq));

	for (old = vm_page_astate_load(m);;) {
	if (__predict_false(old.queue != queue \|\|
	(old.flags & PGA_QUEUE_OP_MASK) == 0)) {
	counter_u64_add(queue_nops, 1);
	break;
	}
	KASSERT(old.queue != PQ_NONE \|\| (old.flags & PGA_QUEUE_STATE_MASK) == 0,
	("%s: page %p has unexpected queue state", __func__, m));

	new = old;
	if ((old.flags & PGA_DEQUEUE) != 0) {
	new.flags &= ~PGA_QUEUE_OP_MASK;
	new.queue = PQ_NONE;
	if (__predict_true(_vm_page_pqstate_commit_dequeue(pq,
	m, &old, new))) {
	counter_u64_add(queue_ops, 1);
	break;
	}
	} else {
	new.flags &= ~(PGA_REQUEUE \| PGA_REQUEUE_HEAD);
	if (__predict_true(_vm_page_pqstate_commit_requeue(pq,
	m, &old, new))) {
	counter_u64_add(queue_ops, 1);
	break;
	}
	}
	}
	}

	static void
	vm_pqbatch_process(struct vm_pagequeue pq, struct vm_batchqueue bq,
	uint8_t queue)
	{
	int i;

	for (i = 0; i < bq->bq_cnt; i++)
	vm_pqbatch_process_page(pq, bq->bq_pa[i], queue);
	vm_batchqueue_init(bq);
	}

	/*
	* vm_page_pqbatch_submit: [ internal use only ]
	*
	* Enqueue a page in the specified page queue's batched work queue.
	* The caller must have encoded the requested operation in the page
	* structure's a.flags field.
	*/
	void
	vm_page_pqbatch_submit(vm_page_t m, uint8_t queue)
	{
	struct vm_batchqueue *bq;
	struct vm_pagequeue *pq;
	int domain;

	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
	("page %p is unmanaged", m));
	KASSERT(queue < PQ_COUNT, ("invalid queue %d", queue));

	domain = vm_phys_domain(m);
	pq = &vm_pagequeue_domain(m)->vmd_pagequeues[queue];

	critical_enter();
	bq = DPCPU_PTR(pqbatch[domain][queue]);
	if (vm_batchqueue_insert(bq, m)) {
	critical_exit();
	return;
	}
	critical_exit();
	vm_pagequeue_lock(pq);
	critical_enter();
	bq = DPCPU_PTR(pqbatch[domain][queue]);
	vm_pqbatch_process(pq, bq, queue);
	vm_pqbatch_process_page(pq, m, queue);
	vm_pagequeue_unlock(pq);
	critical_exit();
	}

	/*
	* vm_page_pqbatch_drain: [ internal use only ]
	*
	* Force all per-CPU page queue batch queues to be drained. This is
	* intended for use in severe memory shortages, to ensure that pages
	* do not remain stuck in the batch queues.
	*/
	void
	vm_page_pqbatch_drain(void)
	{
	struct thread *td;
	struct vm_domain *vmd;
	struct vm_pagequeue *pq;
	int cpu, domain, queue;

	td = curthread;
	CPU_FOREACH(cpu) {
	thread_lock(td);
	sched_bind(td, cpu);
	thread_unlock(td);

	for (domain = 0; domain < vm_ndomains; domain++) {
	vmd = VM_DOMAIN(domain);
	for (queue = 0; queue < PQ_COUNT; queue++) {
	pq = &vmd->vmd_pagequeues[queue];
	vm_pagequeue_lock(pq);
	critical_enter();
	vm_pqbatch_process(pq,
	DPCPU_PTR(pqbatch[domain][queue]), queue);
	critical_exit();
	vm_pagequeue_unlock(pq);
	}
	}
	}
	thread_lock(td);
	sched_unbind(td);
	thread_unlock(td);
	}

	/*
	* vm_page_dequeue_deferred: [ internal use only ]
	*
	* Request removal of the given page from its current page
	* queue. Physical removal from the queue may be deferred
	* indefinitely.
	*
	* The page must be locked.
	*/
	void
	vm_page_dequeue_deferred(vm_page_t m)
	{
	vm_page_astate_t new, old;

	old = vm_page_astate_load(m);
	do {
	if (old.queue == PQ_NONE) {
	KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0,
	("%s: page %p has unexpected queue state",
	__func__, m));
	break;
	}
	new = old;
	new.flags \|= PGA_DEQUEUE;
	} while (!vm_page_pqstate_commit_request(m, &old, new));
	}

	/*
	* vm_page_dequeue:
	*
	* Remove the page from whichever page queue it's in, if any, before
	* returning.
	*/
	void
	vm_page_dequeue(vm_page_t m)
	{
	vm_page_astate_t new, old;

	old = vm_page_astate_load(m);
	do {
	if (old.queue == PQ_NONE) {
	KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0,
	("%s: page %p has unexpected queue state",
	__func__, m));
	break;
	}
	new = old;
	new.flags &= ~PGA_QUEUE_OP_MASK;
	new.queue = PQ_NONE;
	} while (!vm_page_pqstate_commit_dequeue(m, &old, new));

	}

	/*
	* Schedule the given page for insertion into the specified page queue.
	* Physical insertion of the page may be deferred indefinitely.
	*/
	static void
	vm_page_enqueue(vm_page_t m, uint8_t queue)
	{

	KASSERT(m->a.queue == PQ_NONE &&
	(m->a.flags & PGA_QUEUE_STATE_MASK) == 0,
	("%s: page %p is already enqueued", __func__, m));
	KASSERT(m->ref_count > 0,
	("%s: page %p does not carry any references", __func__, m));

	m->a.queue = queue;
	if ((m->a.flags & PGA_REQUEUE) == 0)
	vm_page_aflag_set(m, PGA_REQUEUE);
	vm_page_pqbatch_submit(m, queue);
	}

	/*
	* vm_page_free_prep:
	*
	* Prepares the given page to be put on the free list,
	* disassociating it from any VM object. The caller may return
	* the page to the free list only if this function returns true.
	*
	* The object, if it exists, must be locked, and then the page must
	* be xbusy. Otherwise the page must be not busied. A managed
	* page must be unmapped.
	*/
	static bool
	vm_page_free_prep(vm_page_t m)
	{

	/*
	* Synchronize with threads that have dropped a reference to this
	* page.
	*/
	atomic_thread_fence_acq();

	#if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP)
	if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) {
	uint64_t *p;
	int i;
	p = (uint64_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
	for (i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++, p++)
	KASSERT(*p == 0, ("vm_page_free_prep %p PG_ZERO %d %jx",
	m, i, (uintmax_t)*p));
	}
	#endif
	if ((m->oflags & VPO_UNMANAGED) == 0) {
	KASSERT(!pmap_page_is_mapped(m),
	("vm_page_free_prep: freeing mapped page %p", m));
	KASSERT((m->a.flags & (PGA_EXECUTABLE \| PGA_WRITEABLE)) == 0,
	("vm_page_free_prep: mapping flags set in page %p", m));
	} else {
	KASSERT(m->a.queue == PQ_NONE,
	("vm_page_free_prep: unmanaged page %p is queued", m));
	}
	VM_CNT_INC(v_tfree);

	if (m->object != NULL) {
	KASSERT(((m->oflags & VPO_UNMANAGED) != 0) ==
	((m->object->flags & OBJ_UNMANAGED) != 0),
	("vm_page_free_prep: managed flag mismatch for page %p",
	m));
	vm_page_assert_xbusied(m);

	/*
	* The object reference can be released without an atomic
	* operation.
	*/
	KASSERT((m->flags & PG_FICTITIOUS) != 0 \|\|
	m->ref_count == VPRC_OBJREF,
	("vm_page_free_prep: page %p has unexpected ref_count %u",
	m, m->ref_count));
	vm_page_object_remove(m);
	m->ref_count -= VPRC_OBJREF;
	} else
	vm_page_assert_unbusied(m);

	vm_page_busy_free(m);

	/*
	* If fictitious remove object association and
	* return.
	*/
	if ((m->flags & PG_FICTITIOUS) != 0) {
	KASSERT(m->ref_count == 1,
	("fictitious page %p is referenced", m));
	KASSERT(m->a.queue == PQ_NONE,
	("fictitious page %p is queued", m));
	return (false);
	}

	/*
	* Pages need not be dequeued before they are returned to the physical
	* memory allocator, but they must at least be marked for a deferred
	* dequeue.
	*/
	if ((m->oflags & VPO_UNMANAGED) == 0)
	vm_page_dequeue_deferred(m);

	m->valid = 0;
	vm_page_undirty(m);

	if (m->ref_count != 0)
	panic("vm_page_free_prep: page %p has references", m);

	/*
	* Restore the default memory attribute to the page.
	*/
	if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
	pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);

	#if VM_NRESERVLEVEL > 0
	/*
	* Determine whether the page belongs to a reservation. If the page was
	* allocated from a per-CPU cache, it cannot belong to a reservation, so
	* as an optimization, we avoid the check in that case.
	*/
	if ((m->flags & PG_PCPU_CACHE) == 0 && vm_reserv_free_page(m))
	return (false);
	#endif

	return (true);
	}

	/*
	* vm_page_free_toq:
	*
	* Returns the given page to the free list, disassociating it
	* from any VM object.
	*
	* The object must be locked. The page must be locked if it is
	* managed.
	*/
	static void
	vm_page_free_toq(vm_page_t m)
	{
	struct vm_domain *vmd;
	uma_zone_t zone;

	if (!vm_page_free_prep(m))
	return;

	vmd = vm_pagequeue_domain(m);
	zone = vmd->vmd_pgcache[m->pool].zone;
	if ((m->flags & PG_PCPU_CACHE) != 0 && zone != NULL) {
	uma_zfree(zone, m);
	return;
	}
	vm_domain_free_lock(vmd);
	vm_phys_free_pages(m, 0);
	vm_domain_free_unlock(vmd);
	vm_domain_freecnt_inc(vmd, 1);
	}

	/*
	* vm_page_free_pages_toq:
	*
	* Returns a list of pages to the free list, disassociating it
	* from any VM object. In other words, this is equivalent to
	* calling vm_page_free_toq() for each page of a list of VM objects.
	*
	* The objects must be locked. The pages must be locked if it is
	* managed.
	*/
	void
	vm_page_free_pages_toq(struct spglist *free, bool update_wire_count)
	{
	vm_page_t m;
	int count;

	if (SLIST_EMPTY(free))
	return;

	count = 0;
	while ((m = SLIST_FIRST(free)) != NULL) {
	count++;
	SLIST_REMOVE_HEAD(free, plinks.s.ss);
	vm_page_free_toq(m);
	}

	if (update_wire_count)
	vm_wire_sub(count);
	}

	/*
	* Mark this page as wired down, preventing reclamation by the page daemon
	* or when the containing object is destroyed.
	*/
	void
	vm_page_wire(vm_page_t m)
	{
	u_int old;

	KASSERT(m->object != NULL,
	("vm_page_wire: page %p does not belong to an object", m));
	if (!vm_page_busied(m) && !vm_object_busied(m->object))
	VM_OBJECT_ASSERT_LOCKED(m->object);
	KASSERT((m->flags & PG_FICTITIOUS) == 0 \|\|
	VPRC_WIRE_COUNT(m->ref_count) >= 1,
	("vm_page_wire: fictitious page %p has zero wirings", m));

	old = atomic_fetchadd_int(&m->ref_count, 1);
	KASSERT(VPRC_WIRE_COUNT(old) != VPRC_WIRE_COUNT_MAX,
	("vm_page_wire: counter overflow for page %p", m));
	if (VPRC_WIRE_COUNT(old) == 0) {
	if ((m->oflags & VPO_UNMANAGED) == 0)
	vm_page_aflag_set(m, PGA_DEQUEUE);
	vm_wire_add(1);
	}
	}

	/*
	* Attempt to wire a mapped page following a pmap lookup of that page.
	* This may fail if a thread is concurrently tearing down mappings of the page.
	* The transient failure is acceptable because it translates to the
	* failure of the caller pmap_extract_and_hold(), which should be then
	* followed by the vm_fault() fallback, see e.g. vm_fault_quick_hold_pages().
	*/
	bool
	vm_page_wire_mapped(vm_page_t m)
	{
	u_int old;

	old = m->ref_count;
	do {
	KASSERT(old > 0,
	("vm_page_wire_mapped: wiring unreferenced page %p", m));
	if ((old & VPRC_BLOCKED) != 0)
	return (false);
	} while (!atomic_fcmpset_int(&m->ref_count, &old, old + 1));

	if (VPRC_WIRE_COUNT(old) == 0) {
	if ((m->oflags & VPO_UNMANAGED) == 0)
	vm_page_aflag_set(m, PGA_DEQUEUE);
	vm_wire_add(1);
	}
	return (true);
	}

	/*
	* Release a wiring reference to a managed page. If the page still belongs to
	* an object, update its position in the page queues to reflect the reference.
	* If the wiring was the last reference to the page, free the page.
	*/
	static void
	vm_page_unwire_managed(vm_page_t m, uint8_t nqueue, bool noreuse)
	{
	u_int old;

	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
	("%s: page %p is unmanaged", __func__, m));

	/*
	* Update LRU state before releasing the wiring reference.
	* Use a release store when updating the reference count to
	* synchronize with vm_page_free_prep().
	*/
	old = m->ref_count;
	do {
	KASSERT(VPRC_WIRE_COUNT(old) > 0,
	("vm_page_unwire: wire count underflow for page %p", m));

	if (old > VPRC_OBJREF + 1) {
	/*
	* The page has at least one other wiring reference. An
	* earlier iteration of this loop may have called
	* vm_page_release_toq() and cleared PGA_DEQUEUE, so
	* re-set it if necessary.
	*/
	if ((vm_page_astate_load(m).flags & PGA_DEQUEUE) == 0)
	vm_page_aflag_set(m, PGA_DEQUEUE);
	} else if (old == VPRC_OBJREF + 1) {
	/*
	* This is the last wiring. Clear PGA_DEQUEUE and
	* update the page's queue state to reflect the
	* reference. If the page does not belong to an object
	* (i.e., the VPRC_OBJREF bit is clear), we only need to
	* clear leftover queue state.
	*/
	vm_page_release_toq(m, nqueue, false);
	} else if (old == 1) {
	vm_page_aflag_clear(m, PGA_DEQUEUE);
	}
	} while (!atomic_fcmpset_rel_int(&m->ref_count, &old, old - 1));

	if (VPRC_WIRE_COUNT(old) == 1) {
	vm_wire_sub(1);
	if (old == 1)
	vm_page_free(m);
	}
	}

	/*
	* Release one wiring of the specified page, potentially allowing it to be
	* paged out.
	*
	* Only managed pages belonging to an object can be paged out. If the number
	* of wirings transitions to zero and the page is eligible for page out, then
	* the page is added to the specified paging queue. If the released wiring
	* represented the last reference to the page, the page is freed.
	*
	* A managed page must be locked.
	*/
	void
	vm_page_unwire(vm_page_t m, uint8_t nqueue)
	{

	KASSERT(nqueue < PQ_COUNT,
	("vm_page_unwire: invalid queue %u request for page %p",
	nqueue, m));

	if ((m->oflags & VPO_UNMANAGED) != 0) {
	if (vm_page_unwire_noq(m) && m->ref_count == 0)
	vm_page_free(m);
	return;
	}
	vm_page_unwire_managed(m, nqueue, false);
	}

	/*
	* Unwire a page without (re-)inserting it into a page queue. It is up
	* to the caller to enqueue, requeue, or free the page as appropriate.
	* In most cases involving managed pages, vm_page_unwire() should be used
	* instead.
	*/
	bool
	vm_page_unwire_noq(vm_page_t m)
	{
	u_int old;

	old = vm_page_drop(m, 1);
	KASSERT(VPRC_WIRE_COUNT(old) != 0,
	("vm_page_unref: counter underflow for page %p", m));
	KASSERT((m->flags & PG_FICTITIOUS) == 0 \|\| VPRC_WIRE_COUNT(old) > 1,
	("vm_page_unref: missing ref on fictitious page %p", m));

	if (VPRC_WIRE_COUNT(old) > 1)
	return (false);
	if ((m->oflags & VPO_UNMANAGED) == 0)
	vm_page_aflag_clear(m, PGA_DEQUEUE);
	vm_wire_sub(1);
	return (true);
	}

	/*
	* Ensure that the page ends up in the specified page queue. If the page is
	* active or being moved to the active queue, ensure that its act_count is
	* at least ACT_INIT but do not otherwise mess with it.
	*
	* A managed page must be locked.
	*/
	static __always_inline void
	vm_page_mvqueue(vm_page_t m, const uint8_t nqueue, const uint16_t nflag)
	{
	vm_page_astate_t old, new;

	KASSERT(m->ref_count > 0,
	("%s: page %p does not carry any references", __func__, m));
	KASSERT(nflag == PGA_REQUEUE \|\| nflag == PGA_REQUEUE_HEAD,
	("%s: invalid flags %x", __func__, nflag));

	if ((m->oflags & VPO_UNMANAGED) != 0 \|\| vm_page_wired(m))
	return;

	old = vm_page_astate_load(m);
	do {
	if ((old.flags & PGA_DEQUEUE) != 0)
	break;
	new = old;
	new.flags &= ~PGA_QUEUE_OP_MASK;
	if (nqueue == PQ_ACTIVE)
	new.act_count = max(old.act_count, ACT_INIT);
	if (old.queue == nqueue) {
	if (nqueue != PQ_ACTIVE)
	new.flags \|= nflag;
	} else {
	new.flags \|= nflag;
	new.queue = nqueue;
	}
	} while (!vm_page_pqstate_commit(m, &old, new));
	}

	/*
	* Put the specified page on the active list (if appropriate).
	*/
	void
	vm_page_activate(vm_page_t m)
	{

	vm_page_mvqueue(m, PQ_ACTIVE, PGA_REQUEUE);
	}

	/*
	* Move the specified page to the tail of the inactive queue, or requeue
	* the page if it is already in the inactive queue.
	*/
	void
	vm_page_deactivate(vm_page_t m)
	{

	vm_page_mvqueue(m, PQ_INACTIVE, PGA_REQUEUE);
	}

	void
	vm_page_deactivate_noreuse(vm_page_t m)
	{

	vm_page_mvqueue(m, PQ_INACTIVE, PGA_REQUEUE_HEAD);
	}

	/*
	* Put a page in the laundry, or requeue it if it is already there.
	*/
	void
	vm_page_launder(vm_page_t m)
	{

	vm_page_mvqueue(m, PQ_LAUNDRY, PGA_REQUEUE);
	}

	/*
	* Put a page in the PQ_UNSWAPPABLE holding queue.
	*/
	void
	vm_page_unswappable(vm_page_t m)
	{

	KASSERT(!vm_page_wired(m) && (m->oflags & VPO_UNMANAGED) == 0,
	("page %p already unswappable", m));

	vm_page_dequeue(m);
	vm_page_enqueue(m, PQ_UNSWAPPABLE);
	}

	/*
	* Release a page back to the page queues in preparation for unwiring.
	*/
	static void
	vm_page_release_toq(vm_page_t m, uint8_t nqueue, const bool noreuse)
	{
	vm_page_astate_t old, new;
	uint16_t nflag;

	/*
	* Use a check of the valid bits to determine whether we should
	* accelerate reclamation of the page. The object lock might not be
	* held here, in which case the check is racy. At worst we will either
	* accelerate reclamation of a valid page and violate LRU, or
	* unnecessarily defer reclamation of an invalid page.
	*
	* If we were asked to not cache the page, place it near the head of the
	* inactive queue so that is reclaimed sooner.
	*/
	if (noreuse \|\| m->valid == 0) {
	nqueue = PQ_INACTIVE;
	nflag = PGA_REQUEUE_HEAD;
	} else {
	nflag = PGA_REQUEUE;
	}

	old = vm_page_astate_load(m);
	do {
	new = old;

	/*
	* If the page is already in the active queue and we are not
	* trying to accelerate reclamation, simply mark it as
	* referenced and avoid any queue operations.
	*/
	new.flags &= ~PGA_QUEUE_OP_MASK;
	if (nflag != PGA_REQUEUE_HEAD && old.queue == PQ_ACTIVE)
	new.flags \|= PGA_REFERENCED;
	else {
	new.flags \|= nflag;
	new.queue = nqueue;
	}
	} while (!vm_page_pqstate_commit(m, &old, new));
	}

	/*
	* Unwire a page and either attempt to free it or re-add it to the page queues.
	*/
	void
	vm_page_release(vm_page_t m, int flags)
	{
	vm_object_t object;

	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
	("vm_page_release: page %p is unmanaged", m));

	if ((flags & VPR_TRYFREE) != 0) {
	for (;;) {
	object = atomic_load_ptr(&m->object);
	if (object == NULL)
	break;
	/* Depends on type-stability. */
	if (vm_page_busied(m) \|\| !VM_OBJECT_TRYWLOCK(object))
	break;
	if (object == m->object) {
	vm_page_release_locked(m, flags);
	VM_OBJECT_WUNLOCK(object);
	return;
	}
	VM_OBJECT_WUNLOCK(object);
	}
	}
	vm_page_unwire_managed(m, PQ_INACTIVE, flags != 0);
	}

	/* See vm_page_release(). */
	void
	vm_page_release_locked(vm_page_t m, int flags)
	{

	VM_OBJECT_ASSERT_WLOCKED(m->object);
	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
	("vm_page_release_locked: page %p is unmanaged", m));

	if (vm_page_unwire_noq(m)) {
	if ((flags & VPR_TRYFREE) != 0 &&
	(m->object->ref_count == 0 \|\| !pmap_page_is_mapped(m)) &&
	m->dirty == 0 && vm_page_tryxbusy(m)) {
	/*
	* An unlocked lookup may have wired the page before the
	* busy lock was acquired, in which case the page must
	* not be freed.
	*/
	if (__predict_true(!vm_page_wired(m))) {
	vm_page_free(m);
	return;
	}
	vm_page_xunbusy(m);
	} else {
	vm_page_release_toq(m, PQ_INACTIVE, flags != 0);
	}
	}
	}

	static bool
	vm_page_try_blocked_op(vm_page_t m, void (*op)(vm_page_t))
	{
	u_int old;

	KASSERT(m->object != NULL && (m->oflags & VPO_UNMANAGED) == 0,
	("vm_page_try_blocked_op: page %p has no object", m));
	KASSERT(vm_page_busied(m),
	("vm_page_try_blocked_op: page %p is not busy", m));
	VM_OBJECT_ASSERT_LOCKED(m->object);

	old = m->ref_count;
	do {
	KASSERT(old != 0,
	("vm_page_try_blocked_op: page %p has no references", m));
	if (VPRC_WIRE_COUNT(old) != 0)
	return (false);
	} while (!atomic_fcmpset_int(&m->ref_count, &old, old \| VPRC_BLOCKED));

	(op)(m);

	/*
	* If the object is read-locked, new wirings may be created via an
	* object lookup.
	*/
	old = vm_page_drop(m, VPRC_BLOCKED);
	KASSERT(!VM_OBJECT_WOWNED(m->object) \|\|
	old == (VPRC_BLOCKED \| VPRC_OBJREF),
	("vm_page_try_blocked_op: unexpected refcount value %u for %p",
	old, m));
	return (true);
	}

	/*
	* Atomically check for wirings and remove all mappings of the page.
	*/
	bool
	vm_page_try_remove_all(vm_page_t m)
	{

	return (vm_page_try_blocked_op(m, pmap_remove_all));
	}

	/*
	* Atomically check for wirings and remove all writeable mappings of the page.
	*/
	bool
	vm_page_try_remove_write(vm_page_t m)
	{

	return (vm_page_try_blocked_op(m, pmap_remove_write));
	}

	/*
	* vm_page_advise
	*
	* Apply the specified advice to the given page.
	*
	* The object and page must be locked.
	*/
	void
	vm_page_advise(vm_page_t m, int advice)
	{

	VM_OBJECT_ASSERT_WLOCKED(m->object);
	if (advice == MADV_FREE)
	/*
	* Mark the page clean. This will allow the page to be freed
	* without first paging it out. MADV_FREE pages are often
	* quickly reused by malloc(3), so we do not do anything that
	* would result in a page fault on a later access.
	*/
	vm_page_undirty(m);
	else if (advice != MADV_DONTNEED) {
	if (advice == MADV_WILLNEED)
	vm_page_activate(m);
	return;
	}

	if (advice != MADV_FREE && m->dirty == 0 && pmap_is_modified(m))
	vm_page_dirty(m);

	/*
	* Clear any references to the page. Otherwise, the page daemon will
	* immediately reactivate the page.
	*/
	vm_page_aflag_clear(m, PGA_REFERENCED);

	/*
	* Place clean pages near the head of the inactive queue rather than
	* the tail, thus defeating the queue's LRU operation and ensuring that
	* the page will be reused quickly. Dirty pages not already in the
	* laundry are moved there.
	*/
	if (m->dirty == 0)
	vm_page_deactivate_noreuse(m);
	else if (!vm_page_in_laundry(m))
	vm_page_launder(m);
	}

	/*
	* vm_page_grab_release
	*
	* Helper routine for grab functions to release busy on return.
	*/
	static inline void
	vm_page_grab_release(vm_page_t m, int allocflags)
	{

	if ((allocflags & VM_ALLOC_NOBUSY) != 0) {
	if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
	vm_page_sunbusy(m);
	else
	vm_page_xunbusy(m);
	}
	}

	/*
	* vm_page_grab_sleep
	*
	* Sleep for busy according to VM_ALLOC_ parameters. Returns true
	* if the caller should retry and false otherwise.
	*
	* If the object is locked on entry the object will be unlocked with
	* false returns and still locked but possibly having been dropped
	* with true returns.
	*/
	static bool
	vm_page_grab_sleep(vm_object_t object, vm_page_t m, vm_pindex_t pindex,
	const char *wmesg, int allocflags, bool locked)
	{

	if ((allocflags & VM_ALLOC_NOWAIT) != 0)
	return (false);

	/*
	* Reference the page before unlocking and sleeping so that
	* the page daemon is less likely to reclaim it.
	*/
	if (locked && (allocflags & VM_ALLOC_NOCREAT) == 0)
	vm_page_reference(m);

	if (_vm_page_busy_sleep(object, m, m->pindex, wmesg, allocflags,
	locked) && locked)
	VM_OBJECT_WLOCK(object);
	if ((allocflags & VM_ALLOC_WAITFAIL) != 0)
	return (false);

	return (true);
	}

	/*
	* Assert that the grab flags are valid.
	*/
	static inline void
	vm_page_grab_check(int allocflags)
	{

	KASSERT((allocflags & VM_ALLOC_NOBUSY) == 0 \|\|
	(allocflags & VM_ALLOC_WIRED) != 0,
	("vm_page_grab*: the pages must be busied or wired"));

	KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
	(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
	("vm_page_grab*: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
	}

	/*
	* Calculate the page allocation flags for grab.
	*/
	static inline int
	vm_page_grab_pflags(int allocflags)
	{
	int pflags;

	pflags = allocflags &
	~(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITOK \| VM_ALLOC_WAITFAIL \|
	VM_ALLOC_NOBUSY);
	if ((allocflags & VM_ALLOC_NOWAIT) == 0)
	pflags \|= VM_ALLOC_WAITFAIL;
	if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0)
	pflags \|= VM_ALLOC_SBUSY;

	return (pflags);
	}

	/*
	* Grab a page, waiting until we are waken up due to the page
	* changing state. We keep on waiting, if the page continues
	* to be in the object. If the page doesn't exist, first allocate it
	* and then conditionally zero it.
	*
	* This routine may sleep.
	*
	* The object must be locked on entry. The lock will, however, be released
	* and reacquired if the routine sleeps.
	*/
	vm_page_t
	vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
	{
	vm_page_t m;

	VM_OBJECT_ASSERT_WLOCKED(object);
	vm_page_grab_check(allocflags);

	retrylookup:
	if ((m = vm_page_lookup(object, pindex)) != NULL) {
	if (!vm_page_tryacquire(m, allocflags)) {
	if (vm_page_grab_sleep(object, m, pindex, "pgrbwt",
	allocflags, true))
	goto retrylookup;
	return (NULL);
	}
	goto out;
	}
	if ((allocflags & VM_ALLOC_NOCREAT) != 0)
	return (NULL);
	m = vm_page_alloc(object, pindex, vm_page_grab_pflags(allocflags));
	if (m == NULL) {
	if ((allocflags & (VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL)) != 0)
	return (NULL);
	goto retrylookup;
	}
	if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);

	out:
	vm_page_grab_release(m, allocflags);

	return (m);
	}

	/*
	* Locklessly attempt to acquire a page given a (object, pindex) tuple
	* and an optional previous page to avoid the radix lookup. The resulting
	* page will be validated against the identity tuple and busied or wired
	* as requested. A NULL *mp return guarantees that the page was not in
	* radix at the time of the call but callers must perform higher level
	* synchronization or retry the operation under a lock if they require
	* an atomic answer. This is the only lock free validation routine,
	* other routines can depend on the resulting page state.
	*
	* The return value indicates whether the operation failed due to caller
	* flags. The return is tri-state with mp:
	*
	* (true, *mp != NULL) - The operation was successful.
	* (true, *mp == NULL) - The page was not found in tree.
	* (false, *mp == NULL) - WAITFAIL or NOWAIT prevented acquisition.
	*/
	static bool
	vm_page_acquire_unlocked(vm_object_t object, vm_pindex_t pindex,
	vm_page_t prev, vm_page_t *mp, int allocflags)
	{
	vm_page_t m;

	vm_page_grab_check(allocflags);
	MPASS(prev == NULL \|\| vm_page_busied(prev) \|\| vm_page_wired(prev));

	*mp = NULL;
	for (;;) {
	/*
	* We may see a false NULL here because the previous page
	* has been removed or just inserted and the list is loaded
	* without barriers. Switch to radix to verify.
	*/
	if (prev == NULL \|\| (m = TAILQ_NEXT(prev, listq)) == NULL \|\|
	QMD_IS_TRASHED(m) \|\| m->pindex != pindex \|\|
	atomic_load_ptr(&m->object) != object) {
	prev = NULL;
	/*
	* This guarantees the result is instantaneously
	* correct.
	*/
	m = vm_radix_lookup_unlocked(&object->rtree, pindex);
	}
	if (m == NULL)
	return (true);
	if (vm_page_trybusy(m, allocflags)) {
	if (m->object == object && m->pindex == pindex)
	break;
	/* relookup. */
	vm_page_busy_release(m);
	cpu_spinwait();
	continue;
	}
	if (!vm_page_grab_sleep(object, m, pindex, "pgnslp",
	allocflags, false))
	return (false);
	}
	if ((allocflags & VM_ALLOC_WIRED) != 0)
	vm_page_wire(m);
	vm_page_grab_release(m, allocflags);
	*mp = m;
	return (true);
	}

	/*
	* Try to locklessly grab a page and fall back to the object lock if NOCREAT
	* is not set.
	*/
	vm_page_t
	vm_page_grab_unlocked(vm_object_t object, vm_pindex_t pindex, int allocflags)
	{
	vm_page_t m;

	vm_page_grab_check(allocflags);

	if (!vm_page_acquire_unlocked(object, pindex, NULL, &m, allocflags))
	return (NULL);
	if (m != NULL)
	return (m);

	/*
	* The radix lockless lookup should never return a false negative
	* errors. If the user specifies NOCREAT they are guaranteed there
	* was no page present at the instant of the call. A NOCREAT caller
	* must handle create races gracefully.
	*/
	if ((allocflags & VM_ALLOC_NOCREAT) != 0)
	return (NULL);

	VM_OBJECT_WLOCK(object);
	m = vm_page_grab(object, pindex, allocflags);
	VM_OBJECT_WUNLOCK(object);

	return (m);
	}

	/*
	* Grab a page and make it valid, paging in if necessary. Pages missing from
	* their pager are zero filled and validated. If a VM_ALLOC_COUNT is supplied
	* and the page is not valid as many as VM_INITIAL_PAGEIN pages can be brought
	* in simultaneously. Additional pages will be left on a paging queue but
	* will neither be wired nor busy regardless of allocflags.
	*/
	int
	vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)
	{
	vm_page_t m;
	vm_page_t ma[VM_INITIAL_PAGEIN];
	int after, i, pflags, rv;

	KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
	(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
	("vm_page_grab_valid: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
	KASSERT((allocflags &
	(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,
	("vm_page_grab_valid: Invalid flags 0x%X", allocflags));
	VM_OBJECT_ASSERT_WLOCKED(object);
	pflags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_SBUSY \|
	VM_ALLOC_WIRED);
	pflags \|= VM_ALLOC_WAITFAIL;

	retrylookup:
	if ((m = vm_page_lookup(object, pindex)) != NULL) {
	/*
	* If the page is fully valid it can only become invalid
	* with the object lock held. If it is not valid it can
	* become valid with the busy lock held. Therefore, we
	* may unnecessarily lock the exclusive busy here if we
	* race with I/O completion not using the object lock.
	* However, we will not end up with an invalid page and a
	* shared lock.
	*/
	if (!vm_page_trybusy(m,
	vm_page_all_valid(m) ? allocflags : 0)) {
	(void)vm_page_grab_sleep(object, m, pindex, "pgrbwt",
	allocflags, true);
	goto retrylookup;
	}
	if (vm_page_all_valid(m))
	goto out;
	if ((allocflags & VM_ALLOC_NOCREAT) != 0) {
	vm_page_busy_release(m);
	*mp = NULL;
	return (VM_PAGER_FAIL);
	}
	} else if ((allocflags & VM_ALLOC_NOCREAT) != 0) {
	*mp = NULL;
	return (VM_PAGER_FAIL);
	} else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) {
	goto retrylookup;
	}

	vm_page_assert_xbusied(m);
	if (vm_pager_has_page(object, pindex, NULL, &after)) {
	after = MIN(after, VM_INITIAL_PAGEIN);
	after = MIN(after, allocflags >> VM_ALLOC_COUNT_SHIFT);
	after = MAX(after, 1);
	ma[0] = m;
	for (i = 1; i < after; i++) {
	if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) {
	if (ma[i]->valid \|\| !vm_page_tryxbusy(ma[i]))
	break;
	} else {
	ma[i] = vm_page_alloc(object, m->pindex + i,
	VM_ALLOC_NORMAL);
	if (ma[i] == NULL)
	break;
	}
	}
	after = i;
	vm_object_pip_add(object, after);
	VM_OBJECT_WUNLOCK(object);
	rv = vm_pager_get_pages(object, ma, after, NULL, NULL);
	VM_OBJECT_WLOCK(object);
	vm_object_pip_wakeupn(object, after);
	/* Pager may have replaced a page. */
	m = ma[0];
	if (rv != VM_PAGER_OK) {
	for (i = 0; i < after; i++) {
	if (!vm_page_wired(ma[i]))
	vm_page_free(ma[i]);
	else
	vm_page_xunbusy(ma[i]);
	}
	*mp = NULL;
	return (rv);
	}
	for (i = 1; i < after; i++)
	vm_page_readahead_finish(ma[i]);
	MPASS(vm_page_all_valid(m));
	} else {
	vm_page_zero_invalid(m, TRUE);
	}
	out:
	if ((allocflags & VM_ALLOC_WIRED) != 0)
	vm_page_wire(m);
	if ((allocflags & VM_ALLOC_SBUSY) != 0 && vm_page_xbusied(m))
	vm_page_busy_downgrade(m);
	else if ((allocflags & VM_ALLOC_NOBUSY) != 0)
	vm_page_busy_release(m);
	*mp = m;
	return (VM_PAGER_OK);
	}

	/*
	* Locklessly grab a valid page. If the page is not valid or not yet
	* allocated this will fall back to the object lock method.
	*/
	int
	vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object,
	vm_pindex_t pindex, int allocflags)
	{
	vm_page_t m;
	int flags;
	int error;

	KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 \|\|
	(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
	("vm_page_grab_valid_unlocked: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY "
	"mismatch"));
	KASSERT((allocflags &
	(VM_ALLOC_NOWAIT \| VM_ALLOC_WAITFAIL \| VM_ALLOC_ZERO)) == 0,
	("vm_page_grab_valid_unlocked: Invalid flags 0x%X", allocflags));

	/*
	* Attempt a lockless lookup and busy. We need at least an sbusy
	* before we can inspect the valid field and return a wired page.
	*/
	flags = allocflags & ~(VM_ALLOC_NOBUSY \| VM_ALLOC_WIRED);
	if (!vm_page_acquire_unlocked(object, pindex, NULL, mp, flags))
	return (VM_PAGER_FAIL);
	if ((m = *mp) != NULL) {
	if (vm_page_all_valid(m)) {
	if ((allocflags & VM_ALLOC_WIRED) != 0)
	vm_page_wire(m);
	vm_page_grab_release(m, allocflags);
	return (VM_PAGER_OK);
	}
	vm_page_busy_release(m);
	}
	if ((allocflags & VM_ALLOC_NOCREAT) != 0) {
	*mp = NULL;
	return (VM_PAGER_FAIL);
	}
	VM_OBJECT_WLOCK(object);
	error = vm_page_grab_valid(mp, object, pindex, allocflags);
	VM_OBJECT_WUNLOCK(object);

	return (error);
	}

	/*
	* Return the specified range of pages from the given object. For each
	* page offset within the range, if a page already exists within the object
	* at that offset and it is busy, then wait for it to change state. If,
	* instead, the page doesn't exist, then allocate it.
	*
	* The caller must always specify an allocation class.
	*
	* allocation classes:
	* VM_ALLOC_NORMAL normal process request
	* VM_ALLOC_SYSTEM system really needs the pages
	*
	* The caller must always specify that the pages are to be busied and/or
	* wired.
	*
	* optional allocation flags:
	* VM_ALLOC_IGN_SBUSY do not sleep on soft busy pages
	* VM_ALLOC_NOBUSY do not exclusive busy the page
	* VM_ALLOC_NOWAIT do not sleep
	* VM_ALLOC_SBUSY set page to sbusy state
	* VM_ALLOC_WIRED wire the pages
	* VM_ALLOC_ZERO zero and validate any invalid pages
	*
	* If VM_ALLOC_NOWAIT is not specified, this routine may sleep. Otherwise, it
	* may return a partial prefix of the requested range.
	*/
	int
	vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
	vm_page_t *ma, int count)
	{
	vm_page_t m, mpred;
	int pflags;
	int i;

	VM_OBJECT_ASSERT_WLOCKED(object);
	KASSERT(((u_int)allocflags >> VM_ALLOC_COUNT_SHIFT) == 0,
	("vm_page_grap_pages: VM_ALLOC_COUNT() is not allowed"));
	vm_page_grab_check(allocflags);

	pflags = vm_page_grab_pflags(allocflags);
	if (count == 0)
	return (0);

	i = 0;
	retrylookup:
	m = vm_radix_lookup_le(&object->rtree, pindex + i);
	if (m == NULL \|\| m->pindex != pindex + i) {
	mpred = m;
	m = NULL;
	} else
	mpred = TAILQ_PREV(m, pglist, listq);
	for (; i < count; i++) {
	if (m != NULL) {
	if (!vm_page_tryacquire(m, allocflags)) {
	if (vm_page_grab_sleep(object, m, pindex,
	"grbmaw", allocflags, true))
	goto retrylookup;
	break;
	}
	} else {
	if ((allocflags & VM_ALLOC_NOCREAT) != 0)
	break;
	m = vm_page_alloc_after(object, pindex + i,
	pflags \| VM_ALLOC_COUNT(count - i), mpred);
	if (m == NULL) {
	if ((allocflags & (VM_ALLOC_NOWAIT \|
	VM_ALLOC_WAITFAIL)) != 0)
	break;
	goto retrylookup;
	}
	}
	if (vm_page_none_valid(m) &&
	(allocflags & VM_ALLOC_ZERO) != 0) {
	if ((m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);
	vm_page_valid(m);
	}
	vm_page_grab_release(m, allocflags);
	ma[i] = mpred = m;
	m = vm_page_next(m);
	}
	return (i);
	}

	/*
	* Unlocked variant of vm_page_grab_pages(). This accepts the same flags
	* and will fall back to the locked variant to handle allocation.
	*/
	int
	vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex,
	int allocflags, vm_page_t *ma, int count)
	{
	vm_page_t m, pred;
	int flags;
	int i;

	vm_page_grab_check(allocflags);

	/*
	* Modify flags for lockless acquire to hold the page until we
	* set it valid if necessary.
	*/
	flags = allocflags & ~VM_ALLOC_NOBUSY;
	pred = NULL;
	for (i = 0; i < count; i++, pindex++) {
	if (!vm_page_acquire_unlocked(object, pindex, pred, &m, flags))
	return (i);
	if (m == NULL)
	break;
	if ((flags & VM_ALLOC_ZERO) != 0 && vm_page_none_valid(m)) {
	if ((m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);
	vm_page_valid(m);
	}
	/* m will still be wired or busy according to flags. */
	vm_page_grab_release(m, allocflags);
	pred = ma[i] = m;
	}
	if ((allocflags & VM_ALLOC_NOCREAT) != 0)
	return (i);
	count -= i;
	VM_OBJECT_WLOCK(object);
	i += vm_page_grab_pages(object, pindex, allocflags, &ma[i], count);
	VM_OBJECT_WUNLOCK(object);

	return (i);
	}

	/*
	* Mapping function for valid or dirty bits in a page.
	*
	* Inputs are required to range within a page.
	*/
	vm_page_bits_t
	vm_page_bits(int base, int size)
	{
	int first_bit;
	int last_bit;

	KASSERT(
	base + size <= PAGE_SIZE,
	("vm_page_bits: illegal base/size %d/%d", base, size)
	);

	if (size == 0) /* handle degenerate case */
	return (0);

	first_bit = base >> DEV_BSHIFT;
	last_bit = (base + size - 1) >> DEV_BSHIFT;

	return (((vm_page_bits_t)2 << last_bit) -
	((vm_page_bits_t)1 << first_bit));
	}

	void
	vm_page_bits_set(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t set)
	{

	#if PAGE_SIZE == 32768
	atomic_set_64((uint64_t *)bits, set);
	#elif PAGE_SIZE == 16384
	atomic_set_32((uint32_t *)bits, set);
	#elif (PAGE_SIZE == 8192) && defined(atomic_set_16)
	atomic_set_16((uint16_t *)bits, set);
	#elif (PAGE_SIZE == 4096) && defined(atomic_set_8)
	atomic_set_8((uint8_t *)bits, set);
	#else /* PAGE_SIZE <= 8192 */
	uintptr_t addr;
	int shift;

	addr = (uintptr_t)bits;
	/*
	* Use a trick to perform a 32-bit atomic on the
	* containing aligned word, to not depend on the existence
	* of atomic_{set, clear}_{8, 16}.
	*/
	shift = addr & (sizeof(uint32_t) - 1);
	#if BYTE_ORDER == BIG_ENDIAN
	shift = (sizeof(uint32_t) - sizeof(vm_page_bits_t) - shift) * NBBY;
	#else
	shift *= NBBY;
	#endif
	addr &= ~(sizeof(uint32_t) - 1);
	atomic_set_32((uint32_t *)addr, set << shift);
	#endif /* PAGE_SIZE */
	}

	static inline void
	vm_page_bits_clear(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t clear)
	{

	#if PAGE_SIZE == 32768
	atomic_clear_64((uint64_t *)bits, clear);
	#elif PAGE_SIZE == 16384
	atomic_clear_32((uint32_t *)bits, clear);
	#elif (PAGE_SIZE == 8192) && defined(atomic_clear_16)
	atomic_clear_16((uint16_t *)bits, clear);
	#elif (PAGE_SIZE == 4096) && defined(atomic_clear_8)
	atomic_clear_8((uint8_t *)bits, clear);
	#else /* PAGE_SIZE <= 8192 */
	uintptr_t addr;
	int shift;

	addr = (uintptr_t)bits;
	/*
	* Use a trick to perform a 32-bit atomic on the
	* containing aligned word, to not depend on the existence
	* of atomic_{set, clear}_{8, 16}.
	*/
	shift = addr & (sizeof(uint32_t) - 1);
	#if BYTE_ORDER == BIG_ENDIAN
	shift = (sizeof(uint32_t) - sizeof(vm_page_bits_t) - shift) * NBBY;
	#else
	shift *= NBBY;
	#endif
	addr &= ~(sizeof(uint32_t) - 1);
	atomic_clear_32((uint32_t *)addr, clear << shift);
	#endif /* PAGE_SIZE */
	}

	static inline vm_page_bits_t
	vm_page_bits_swap(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t newbits)
	{
	#if PAGE_SIZE == 32768
	uint64_t old;

	old = *bits;
	while (atomic_fcmpset_64(bits, &old, newbits) == 0);
	return (old);
	#elif PAGE_SIZE == 16384
	uint32_t old;

	old = *bits;
	while (atomic_fcmpset_32(bits, &old, newbits) == 0);
	return (old);
	#elif (PAGE_SIZE == 8192) && defined(atomic_fcmpset_16)
	uint16_t old;

	old = *bits;
	while (atomic_fcmpset_16(bits, &old, newbits) == 0);
	return (old);
	#elif (PAGE_SIZE == 4096) && defined(atomic_fcmpset_8)
	uint8_t old;

	old = *bits;
	while (atomic_fcmpset_8(bits, &old, newbits) == 0);
	return (old);
	#else /* PAGE_SIZE <= 4096*/
	uintptr_t addr;
	uint32_t old, new, mask;
	int shift;

	addr = (uintptr_t)bits;
	/*
	* Use a trick to perform a 32-bit atomic on the
	* containing aligned word, to not depend on the existence
	* of atomic_{set, swap, clear}_{8, 16}.
	*/
	shift = addr & (sizeof(uint32_t) - 1);
	#if BYTE_ORDER == BIG_ENDIAN
	shift = (sizeof(uint32_t) - sizeof(vm_page_bits_t) - shift) * NBBY;
	#else
	shift *= NBBY;
	#endif
	addr &= ~(sizeof(uint32_t) - 1);
	mask = VM_PAGE_BITS_ALL << shift;

	old = *bits;
	do {
	new = old & ~mask;
	new \|= newbits << shift;
	} while (atomic_fcmpset_32((uint32_t *)addr, &old, new) == 0);
	return (old >> shift);
	#endif /* PAGE_SIZE */
	}

	/*
	* vm_page_set_valid_range:
	*
	* Sets portions of a page valid. The arguments are expected
	* to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
	* of any partial chunks touched by the range. The invalid portion of
	* such chunks will be zeroed.
	*
	* (base + size) must be less then or equal to PAGE_SIZE.
	*/
	void
	vm_page_set_valid_range(vm_page_t m, int base, int size)
	{
	int endoff, frag;
	vm_page_bits_t pagebits;

	vm_page_assert_busied(m);
	if (size == 0) /* handle degenerate case */
	return;

	/*
	* If the base is not DEV_BSIZE aligned and the valid
	* bit is clear, we have to zero out a portion of the
	* first block.
	*/
	if ((frag = rounddown2(base, DEV_BSIZE)) != base &&
	(m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
	pmap_zero_page_area(m, frag, base - frag);

	/*
	* If the ending offset is not DEV_BSIZE aligned and the
	* valid bit is clear, we have to zero out a portion of
	* the last block.
	*/
	endoff = base + size;
	if ((frag = rounddown2(endoff, DEV_BSIZE)) != endoff &&
	(m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
	pmap_zero_page_area(m, endoff,
	DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));

	/*
	* Assert that no previously invalid block that is now being validated
	* is already dirty.
	*/
	KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0,
	("vm_page_set_valid_range: page %p is dirty", m));

	/*
	* Set valid bits inclusive of any overlap.
	*/
	pagebits = vm_page_bits(base, size);
	if (vm_page_xbusied(m))
	m->valid \|= pagebits;
	else
	vm_page_bits_set(m, &m->valid, pagebits);
	}

	/*
	* Set the page dirty bits and free the invalid swap space if
	* present. Returns the previous dirty bits.
	*/
	vm_page_bits_t
	vm_page_set_dirty(vm_page_t m)
	{
	vm_page_bits_t old;

	VM_PAGE_OBJECT_BUSY_ASSERT(m);

	if (vm_page_xbusied(m) && !pmap_page_is_write_mapped(m)) {
	old = m->dirty;
	m->dirty = VM_PAGE_BITS_ALL;
	} else
	old = vm_page_bits_swap(m, &m->dirty, VM_PAGE_BITS_ALL);
	if (old == 0 && (m->a.flags & PGA_SWAP_SPACE) != 0)
	vm_pager_page_unswapped(m);

	return (old);
	}

	/*
	* Clear the given bits from the specified page's dirty field.
	*/
	static __inline void
	vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits)
	{

	vm_page_assert_busied(m);

	/*
	* If the page is xbusied and not write mapped we are the
	* only thread that can modify dirty bits. Otherwise, The pmap
	* layer can call vm_page_dirty() without holding a distinguished
	* lock. The combination of page busy and atomic operations
	* suffice to guarantee consistency of the page dirty field.
	*/
	if (vm_page_xbusied(m) && !pmap_page_is_write_mapped(m))
	m->dirty &= ~pagebits;
	else
	vm_page_bits_clear(m, &m->dirty, pagebits);
	}

	/*
	* vm_page_set_validclean:
	*
	* Sets portions of a page valid and clean. The arguments are expected
	* to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
	* of any partial chunks touched by the range. The invalid portion of
	* such chunks will be zero'd.
	*
	* (base + size) must be less then or equal to PAGE_SIZE.
	*/
	void
	vm_page_set_validclean(vm_page_t m, int base, int size)
	{
	vm_page_bits_t oldvalid, pagebits;
	int endoff, frag;

	vm_page_assert_busied(m);
	if (size == 0) /* handle degenerate case */
	return;

	/*
	* If the base is not DEV_BSIZE aligned and the valid
	* bit is clear, we have to zero out a portion of the
	* first block.
	*/
	if ((frag = rounddown2(base, DEV_BSIZE)) != base &&
	(m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0)
	pmap_zero_page_area(m, frag, base - frag);

	/*
	* If the ending offset is not DEV_BSIZE aligned and the
	* valid bit is clear, we have to zero out a portion of
	* the last block.
	*/
	endoff = base + size;
	if ((frag = rounddown2(endoff, DEV_BSIZE)) != endoff &&
	(m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0)
	pmap_zero_page_area(m, endoff,
	DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));

	/*
	* Set valid, clear dirty bits. If validating the entire
	* page we can safely clear the pmap modify bit. We also
	* use this opportunity to clear the PGA_NOSYNC flag. If a process
	* takes a write fault on a MAP_NOSYNC memory area the flag will
	* be set again.
	*
	* We set valid bits inclusive of any overlap, but we can only
	* clear dirty bits for DEV_BSIZE chunks that are fully within
	* the range.
	*/
	oldvalid = m->valid;
	pagebits = vm_page_bits(base, size);
	if (vm_page_xbusied(m))
	m->valid \|= pagebits;
	else
	vm_page_bits_set(m, &m->valid, pagebits);
	#if 0 /* NOT YET */
	if ((frag = base & (DEV_BSIZE - 1)) != 0) {
	frag = DEV_BSIZE - frag;
	base += frag;
	size -= frag;
	if (size < 0)
	size = 0;
	}
	pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
	#endif
	if (base == 0 && size == PAGE_SIZE) {
	/*
	* The page can only be modified within the pmap if it is
	* mapped, and it can only be mapped if it was previously
	* fully valid.
	*/
	if (oldvalid == VM_PAGE_BITS_ALL)
	/*
	* Perform the pmap_clear_modify() first. Otherwise,
	* a concurrent pmap operation, such as
	* pmap_protect(), could clear a modification in the
	* pmap and set the dirty field on the page before
	* pmap_clear_modify() had begun and after the dirty
	* field was cleared here.
	*/
	pmap_clear_modify(m);
	m->dirty = 0;
	vm_page_aflag_clear(m, PGA_NOSYNC);
	} else if (oldvalid != VM_PAGE_BITS_ALL && vm_page_xbusied(m))
	m->dirty &= ~pagebits;
	else
	vm_page_clear_dirty_mask(m, pagebits);
	}

	void
	vm_page_clear_dirty(vm_page_t m, int base, int size)
	{

	vm_page_clear_dirty_mask(m, vm_page_bits(base, size));
	}

	/*
	* vm_page_set_invalid:
	*
	* Invalidates DEV_BSIZE'd chunks within a page. Both the
	* valid and dirty bits for the effected areas are cleared.
	*/
	void
	vm_page_set_invalid(vm_page_t m, int base, int size)
	{
	vm_page_bits_t bits;
	vm_object_t object;

	/*
	* The object lock is required so that pages can't be mapped
	* read-only while we're in the process of invalidating them.
	*/
	object = m->object;
	VM_OBJECT_ASSERT_WLOCKED(object);
	vm_page_assert_busied(m);

	if (object->type == OBJT_VNODE && base == 0 && IDX_TO_OFF(m->pindex) +
	size >= object->un_pager.vnp.vnp_size)
	bits = VM_PAGE_BITS_ALL;
	else
	bits = vm_page_bits(base, size);
	if (object->ref_count != 0 && vm_page_all_valid(m) && bits != 0)
	pmap_remove_all(m);
	KASSERT((bits == 0 && vm_page_all_valid(m)) \|\|
	!pmap_page_is_mapped(m),
	("vm_page_set_invalid: page %p is mapped", m));
	if (vm_page_xbusied(m)) {
	m->valid &= ~bits;
	m->dirty &= ~bits;
	} else {
	vm_page_bits_clear(m, &m->valid, bits);
	vm_page_bits_clear(m, &m->dirty, bits);
	}
	}

	/*
	* vm_page_invalid:
	*
	* Invalidates the entire page. The page must be busy, unmapped, and
	* the enclosing object must be locked. The object locks protects
	* against concurrent read-only pmap enter which is done without
	* busy.
	*/
	void
	vm_page_invalid(vm_page_t m)
	{

	vm_page_assert_busied(m);
	VM_OBJECT_ASSERT_LOCKED(m->object);
	MPASS(!pmap_page_is_mapped(m));

	if (vm_page_xbusied(m))
	m->valid = 0;
	else
	vm_page_bits_clear(m, &m->valid, VM_PAGE_BITS_ALL);
	}

	/*
	* vm_page_zero_invalid()
	*
	* The kernel assumes that the invalid portions of a page contain
	* garbage, but such pages can be mapped into memory by user code.
	* When this occurs, we must zero out the non-valid portions of the
	* page so user code sees what it expects.
	*
	* Pages are most often semi-valid when the end of a file is mapped
	* into memory and the file's size is not page aligned.
	*/
	void
	vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
	{
	int b;
	int i;

	/*
	* Scan the valid bits looking for invalid sections that
	* must be zeroed. Invalid sub-DEV_BSIZE'd areas ( where the
	* valid bit may be set ) have already been zeroed by
	* vm_page_set_validclean().
	*/
	for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
	if (i == (PAGE_SIZE / DEV_BSIZE) \|\|
	(m->valid & ((vm_page_bits_t)1 << i))) {
	if (i > b) {
	pmap_zero_page_area(m,
	b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
	}
	b = i + 1;
	}
	}

	/*
	* setvalid is TRUE when we can safely set the zero'd areas
	* as being valid. We can do this if there are no cache consistancy
	* issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
	*/
	if (setvalid)
	vm_page_valid(m);
	}

	/*
	* vm_page_is_valid:
	*
	* Is (partial) page valid? Note that the case where size == 0
	* will return FALSE in the degenerate case where the page is
	* entirely invalid, and TRUE otherwise.
	*
	* Some callers envoke this routine without the busy lock held and
	* handle races via higher level locks. Typical callers should
	* hold a busy lock to prevent invalidation.
	*/
	int
	vm_page_is_valid(vm_page_t m, int base, int size)
	{
	vm_page_bits_t bits;

	bits = vm_page_bits(base, size);
	return (m->valid != 0 && (m->valid & bits) == bits);
	}

	/*
	* Returns true if all of the specified predicates are true for the entire
	* (super)page and false otherwise.
	*/
	bool
	vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m)
	{
	vm_object_t object;
	int i, npages;

	object = m->object;
	if (skip_m != NULL && skip_m->object != object)
	return (false);
	VM_OBJECT_ASSERT_LOCKED(object);
	npages = atop(pagesizes[m->psind]);

	/*
	* The physically contiguous pages that make up a superpage, i.e., a
	* page with a page size index ("psind") greater than zero, will
	* occupy adjacent entries in vm_page_array[].
	*/
	for (i = 0; i < npages; i++) {
	/* Always test object consistency, including "skip_m". */
	if (m[i].object != object)
	return (false);
	if (&m[i] == skip_m)
	continue;
	if ((flags & PS_NONE_BUSY) != 0 && vm_page_busied(&m[i]))
	return (false);
	if ((flags & PS_ALL_DIRTY) != 0) {
	/*
	* Calling vm_page_test_dirty() or pmap_is_modified()
	* might stop this case from spuriously returning
	* "false". However, that would require a write lock
	* on the object containing "m[i]".
	*/
	if (m[i].dirty != VM_PAGE_BITS_ALL)
	return (false);
	}
	if ((flags & PS_ALL_VALID) != 0 &&
	m[i].valid != VM_PAGE_BITS_ALL)
	return (false);
	}
	return (true);
	}

	/*
	* Set the page's dirty bits if the page is modified.
	*/
	void
	vm_page_test_dirty(vm_page_t m)
	{

	vm_page_assert_busied(m);
	if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m))
	vm_page_dirty(m);
	}

	void
	vm_page_valid(vm_page_t m)
	{

	vm_page_assert_busied(m);
	if (vm_page_xbusied(m))
	m->valid = VM_PAGE_BITS_ALL;
	else
	vm_page_bits_set(m, &m->valid, VM_PAGE_BITS_ALL);
	}

	void
	vm_page_lock_KBI(vm_page_t m, const char *file, int line)
	{

	mtx_lock_flags_(vm_page_lockptr(m), 0, file, line);
	}

	void
	vm_page_unlock_KBI(vm_page_t m, const char *file, int line)
	{

	mtx_unlock_flags_(vm_page_lockptr(m), 0, file, line);
	}

	int
	vm_page_trylock_KBI(vm_page_t m, const char *file, int line)
	{

	return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line));
	}

	#if defined(INVARIANTS) \|\| defined(INVARIANT_SUPPORT)
	void
	vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line)
	{

	vm_page_lock_assert_KBI(m, MA_OWNED, file, line);
	}

	void
	vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line)
	{

	mtx_assert_(vm_page_lockptr(m), a, file, line);
	}
	#endif

	#ifdef INVARIANTS
	void
	vm_page_object_busy_assert(vm_page_t m)
	{

	/*
	* Certain of the page's fields may only be modified by the
	* holder of a page or object busy.
	*/
	if (m->object != NULL && !vm_page_busied(m))
	VM_OBJECT_ASSERT_BUSY(m->object);
	}

	void
	vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits)
	{

	if ((bits & PGA_WRITEABLE) == 0)
	return;

	/*
	* The PGA_WRITEABLE flag can only be set if the page is
	* managed, is exclusively busied or the object is locked.
	* Currently, this flag is only set by pmap_enter().
	*/
	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
	("PGA_WRITEABLE on unmanaged page"));
	if (!vm_page_xbusied(m))
	VM_OBJECT_ASSERT_BUSY(m->object);
	}
	#endif

	#include "opt_ddb.h"
	#ifdef DDB
	#include <sys/kernel.h>

	#include <ddb/ddb.h>

	DB_SHOW_COMMAND(page, vm_page_print_page_info)
	{

	db_printf("vm_cnt.v_free_count: %d\n", vm_free_count());
	db_printf("vm_cnt.v_inactive_count: %d\n", vm_inactive_count());
	db_printf("vm_cnt.v_active_count: %d\n", vm_active_count());
	db_printf("vm_cnt.v_laundry_count: %d\n", vm_laundry_count());
	db_printf("vm_cnt.v_wire_count: %d\n", vm_wire_count());
	db_printf("vm_cnt.v_free_reserved: %d\n", vm_cnt.v_free_reserved);
	db_printf("vm_cnt.v_free_min: %d\n", vm_cnt.v_free_min);
	db_printf("vm_cnt.v_free_target: %d\n", vm_cnt.v_free_target);
	db_printf("vm_cnt.v_inactive_target: %d\n", vm_cnt.v_inactive_target);
	}

	DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
	{
	int dom;

	db_printf("pq_free %d\n", vm_free_count());
	for (dom = 0; dom < vm_ndomains; dom++) {
	db_printf(
	"dom %d page_cnt %d free %d pq_act %d pq_inact %d pq_laund %d pq_unsw %d\n",
	dom,
	vm_dom[dom].vmd_page_count,
	vm_dom[dom].vmd_free_count,
	vm_dom[dom].vmd_pagequeues[PQ_ACTIVE].pq_cnt,
	vm_dom[dom].vmd_pagequeues[PQ_INACTIVE].pq_cnt,
	vm_dom[dom].vmd_pagequeues[PQ_LAUNDRY].pq_cnt,
	vm_dom[dom].vmd_pagequeues[PQ_UNSWAPPABLE].pq_cnt);
	}
	}

	DB_SHOW_COMMAND(pginfo, vm_page_print_pginfo)
	{
	vm_page_t m;
	boolean_t phys, virt;

	if (!have_addr) {
	db_printf("show pginfo addr\n");
	return;
	}

	phys = strchr(modif, 'p') != NULL;
	virt = strchr(modif, 'v') != NULL;
	if (virt)
	m = PHYS_TO_VM_PAGE(pmap_kextract(addr));
	else if (phys)
	m = PHYS_TO_VM_PAGE(addr);
	else
	m = (vm_page_t)addr;
	db_printf(
	"page %p obj %p pidx 0x%jx phys 0x%jx q %d ref 0x%x\n"
	" af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n",
	m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr,
	m->a.queue, m->ref_count, m->a.flags, m->oflags,
	m->flags, m->a.act_count, m->busy_lock, m->valid, m->dirty);
	}
	#endif /* DDB */
	diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
	index 4a073cec8e51..93b3dc411925 100644
	--- a/sys/vm/vm_page.h
	+++ b/sys/vm/vm_page.h
	@@ -1,987 +1,1000 @@
	/*-
	* SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
	*
	* Copyright (c) 1991, 1993
	* The Regents of the University of California. All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* The Mach Operating System project at Carnegie-Mellon University.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* from: @(#)vm_page.h 8.2 (Berkeley) 12/13/93
	*
	*
	* Copyright (c) 1987, 1990 Carnegie-Mellon University.
	* All rights reserved.
	*
	* Authors: Avadis Tevanian, Jr., Michael Wayne Young
	*
	* Permission to use, copy, modify and distribute this software and
	* its documentation is hereby granted, provided that both the copyright
	* notice and this permission notice appear in all copies of the
	* software, derivative works or modified versions, and any portions
	* thereof, and that both notices appear in supporting documentation.
	*
	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
	* FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	*
	* Carnegie Mellon requests users of this software to return to
	*
	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	* School of Computer Science
	* Carnegie Mellon University
	* Pittsburgh PA 15213-3890
	*
	* any improvements or extensions that they make and grant Carnegie the
	* rights to redistribute these changes.
	*
	* $FreeBSD$
	*/

	/*
	* Resident memory system definitions.
	*/

	#ifndef _VM_PAGE_
	#define _VM_PAGE_

	#include <vm/pmap.h>

	/*
	* Management of resident (logical) pages.
	*
	* A small structure is kept for each resident
	* page, indexed by page number. Each structure
	* is an element of several collections:
	*
	* A radix tree used to quickly
	* perform object/offset lookups
	*
	* A list of all pages for a given object,
	* so they can be quickly deactivated at
	* time of deallocation.
	*
	* An ordered list of pages due for pageout.
	*
	* In addition, the structure contains the object
	* and offset to which this page belongs (for pageout),
	* and sundry status bits.
	*
	* In general, operations on this structure's mutable fields are
	* synchronized using either one of or a combination of locks. If a
	* field is annotated with two of these locks then holding either is
	* sufficient for read access but both are required for write access.
	* The physical address of a page is used to select its page lock from
	* a pool. The queue lock for a page depends on the value of its queue
	* field and is described in detail below.
	*
	* The following annotations are possible:
	- * (A) the field is atomic and may require additional synchronization.
	+ * (A) the field must be accessed using atomic(9) and may require
	+ * additional synchronization.
	* (B) the page busy lock.
	* (C) the field is immutable.
	* (F) the per-domain lock for the free queues
	* (M) Machine dependent, defined by pmap layer.
	* (O) the object that the page belongs to.
	* (P) the page lock.
	* (Q) the page's queue lock.
	*
	* The busy lock is an embedded reader-writer lock that protects the
	* page's contents and identity (i.e., its <object, pindex> tuple) as
	* well as certain valid/dirty modifications. To avoid bloating the
	* the page structure, the busy lock lacks some of the features available
	* the kernel's general-purpose synchronization primitives. As a result,
	* busy lock ordering rules are not verified, lock recursion is not
	* detected, and an attempt to xbusy a busy page or sbusy an xbusy page
	* results will trigger a panic rather than causing the thread to block.
	* vm_page_sleep_if_busy() can be used to sleep until the page's busy
	* state changes, after which the caller must re-lookup the page and
	* re-evaluate its state. vm_page_busy_acquire() will block until
	* the lock is acquired.
	*
	* The valid field is protected by the page busy lock (B) and object
	* lock (O). Transitions from invalid to valid are generally done
	* via I/O or zero filling and do not require the object lock.
	* These must be protected with the busy lock to prevent page-in or
	* creation races. Page invalidation generally happens as a result
	* of truncate or msync. When invalidated, pages must not be present
	* in pmap and must hold the object lock to prevent concurrent
	* speculative read-only mappings that do not require busy. I/O
	* routines may check for validity without a lock if they are prepared
	* to handle invalidation races with higher level locks (vnode) or are
	* unconcerned with races so long as they hold a reference to prevent
	* recycling. When a valid bit is set while holding a shared busy
	* lock (A) atomic operations are used to protect against concurrent
	* modification.
	*
	* In contrast, the synchronization of accesses to the page's
	* dirty field is a mix of machine dependent (M) and busy (B). In
	* the machine-independent layer, the page busy must be held to
	* operate on the field. However, the pmap layer is permitted to
	* set all bits within the field without holding that lock. If the
	* underlying architecture does not support atomic read-modify-write
	* operations on the field's type, then the machine-independent
	* layer uses a 32-bit atomic on the aligned 32-bit word that
	* contains the dirty field. In the machine-independent layer,
	* the implementation of read-modify-write operations on the
	* field is encapsulated in vm_page_clear_dirty_mask(). An
	* exclusive busy lock combined with pmap_remove_{write/all}() is the
	* only way to ensure a page can not become dirty. I/O generally
	* removes the page from pmap to ensure exclusive access and atomic
	* writes.
	*
	* The ref_count field tracks references to the page. References that
	* prevent the page from being reclaimable are called wirings and are
	* counted in the low bits of ref_count. The containing object's
	* reference, if one exists, is counted using the VPRC_OBJREF bit in the
	* ref_count field. Additionally, the VPRC_BLOCKED bit is used to
	* atomically check for wirings and prevent new wirings via
	* pmap_extract_and_hold(). When a page belongs to an object, it may be
	* wired only when the object is locked, or the page is busy, or by
	* pmap_extract_and_hold(). As a result, if the object is locked and the
	* page is not busy (or is exclusively busied by the current thread), and
	* the page is unmapped, its wire count will not increase. The ref_count
	* field is updated using atomic operations in most cases, except when it
	* is known that no other references to the page exist, such as in the page
	* allocator. A page may be present in the page queues, or even actively
	* scanned by the page daemon, without an explicitly counted referenced.
	* The page daemon must therefore handle the possibility of a concurrent
	* free of the page.
	*
	* The queue state of a page consists of the queue and act_count fields of
	* its atomically updated state, and the subset of atomic flags specified
	* by PGA_QUEUE_STATE_MASK. The queue field contains the page's page queue
	* index, or PQ_NONE if it does not belong to a page queue. To modify the
	* queue field, the page queue lock corresponding to the old value must be
	* held, unless that value is PQ_NONE, in which case the queue index must
	* be updated using an atomic RMW operation. There is one exception to
	* this rule: the page daemon may transition the queue field from
	* PQ_INACTIVE to PQ_NONE immediately prior to freeing the page during an
	* inactive queue scan. At that point the page is already dequeued and no
	* other references to that vm_page structure can exist. The PGA_ENQUEUED
	* flag, when set, indicates that the page structure is physically inserted
	* into the queue corresponding to the page's queue index, and may only be
	* set or cleared with the corresponding page queue lock held.
	*
	* To avoid contention on page queue locks, page queue operations (enqueue,
	* dequeue, requeue) are batched using fixed-size per-CPU queues. A
	* deferred operation is requested by setting one of the flags in
	* PGA_QUEUE_OP_MASK and inserting an entry into a batch queue. When a
	* queue is full, an attempt to insert a new entry will lock the page
	* queues and trigger processing of the pending entries. The
	* type-stability of vm_page structures is crucial to this scheme since the
	* processing of entries in a given batch queue may be deferred
	* indefinitely. In particular, a page may be freed with pending batch
	* queue entries. The page queue operation flags must be set using atomic
	* RWM operations.
	*/

	#if PAGE_SIZE == 4096
	#define VM_PAGE_BITS_ALL 0xffu
	typedef uint8_t vm_page_bits_t;
	#elif PAGE_SIZE == 8192
	#define VM_PAGE_BITS_ALL 0xffffu
	typedef uint16_t vm_page_bits_t;
	#elif PAGE_SIZE == 16384
	#define VM_PAGE_BITS_ALL 0xffffffffu
	typedef uint32_t vm_page_bits_t;
	#elif PAGE_SIZE == 32768
	#define VM_PAGE_BITS_ALL 0xfffffffffffffffflu
	typedef uint64_t vm_page_bits_t;
	#endif

	typedef union vm_page_astate {
	struct {
	uint16_t flags;
	uint8_t queue;
	uint8_t act_count;
	};
	uint32_t _bits;
	} vm_page_astate_t;

	struct vm_page {
	union {
	TAILQ_ENTRY(vm_page) q; /* page queue or free list (Q) */
	struct {
	SLIST_ENTRY(vm_page) ss; /* private slists */
	} s;
	struct {
	u_long p;
	u_long v;
	} memguard;
	struct {
	void *slab;
	void *zone;
	} uma;
	} plinks;
	TAILQ_ENTRY(vm_page) listq; /* pages in same object (O) */
	vm_object_t object; /* which object am I in (O) */
	vm_pindex_t pindex; /* offset into object (O,P) */
	vm_paddr_t phys_addr; /* physical address of page (C) */
	struct md_page md; /* machine dependent stuff */
	u_int ref_count; /* page references (A) */
	- volatile u_int busy_lock; /* busy owners lock */
	- union vm_page_astate a; /* state accessed atomically */
	+ u_int busy_lock; /* busy owners lock (A) */
	+ union vm_page_astate a; /* state accessed atomically (A) */
	uint8_t order; /* index of the buddy queue (F) */
	uint8_t pool; /* vm_phys freepool index (F) */
	uint8_t flags; /* page PG_* flags (P) */
	uint8_t oflags; /* page VPO_* flags (O) */
	int8_t psind; /* pagesizes[] index (O) */
	int8_t segind; /* vm_phys segment index (C) */
	/* NOTE that these must support one bit per DEV_BSIZE in a page */
	/* so, on normal X86 kernels, they must be at least 8 bits wide */
	vm_page_bits_t valid; /* valid DEV_BSIZE chunk map (O,B) */
	vm_page_bits_t dirty; /* dirty DEV_BSIZE chunk map (M,B) */
	};

	/*
	* Special bits used in the ref_count field.
	*
	* ref_count is normally used to count wirings that prevent the page from being
	* reclaimed, but also supports several special types of references that do not
	* prevent reclamation. Accesses to the ref_count field must be atomic unless
	* the page is unallocated.
	*
	* VPRC_OBJREF is the reference held by the containing object. It can set or
	* cleared only when the corresponding object's write lock is held.
	*
	* VPRC_BLOCKED is used to atomically block wirings via pmap lookups while
	* attempting to tear down all mappings of a given page. The page lock and
	* object write lock must both be held in order to set or clear this bit.
	*/
	#define VPRC_BLOCKED 0x40000000u /* mappings are being removed */
	#define VPRC_OBJREF 0x80000000u /* object reference, cleared with (O) */
	#define VPRC_WIRE_COUNT(c) ((c) & ~(VPRC_BLOCKED \| VPRC_OBJREF))
	#define VPRC_WIRE_COUNT_MAX (~(VPRC_BLOCKED \| VPRC_OBJREF))

	/*
	* Page flags stored in oflags:
	*
	* Access to these page flags is synchronized by the lock on the object
	* containing the page (O).
	*
	* Note: VPO_UNMANAGED (used by OBJT_DEVICE, OBJT_PHYS and OBJT_SG)
	* indicates that the page is not under PV management but
	* otherwise should be treated as a normal page. Pages not
	* under PV management cannot be paged out via the
	* object/vm_page_t because there is no knowledge of their pte
	* mappings, and such pages are also not on any PQ queue.
	*
	*/
	#define VPO_KMEM_EXEC 0x01 /* kmem mapping allows execution */
	#define VPO_SWAPSLEEP 0x02 /* waiting for swap to finish */
	#define VPO_UNMANAGED 0x04 /* no PV management for page */
	#define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */

	/*
	* Busy page implementation details.
	* The algorithm is taken mostly by rwlock(9) and sx(9) locks implementation,
	* even if the support for owner identity is removed because of size
	* constraints. Checks on lock recursion are then not possible, while the
	* lock assertions effectiveness is someway reduced.
	*/
	#define VPB_BIT_SHARED 0x01
	#define VPB_BIT_EXCLUSIVE 0x02
	#define VPB_BIT_WAITERS 0x04
	#define VPB_BIT_FLAGMASK \
	(VPB_BIT_SHARED \| VPB_BIT_EXCLUSIVE \| VPB_BIT_WAITERS)

	#define VPB_SHARERS_SHIFT 3
	#define VPB_SHARERS(x) \
	(((x) & ~VPB_BIT_FLAGMASK) >> VPB_SHARERS_SHIFT)
	#define VPB_SHARERS_WORD(x) ((x) << VPB_SHARERS_SHIFT \| VPB_BIT_SHARED)
	#define VPB_ONE_SHARER (1 << VPB_SHARERS_SHIFT)

	#define VPB_SINGLE_EXCLUSIVE VPB_BIT_EXCLUSIVE
	#ifdef INVARIANTS
	#define VPB_CURTHREAD_EXCLUSIVE \
	(VPB_BIT_EXCLUSIVE \| ((u_int)(uintptr_t)curthread & ~VPB_BIT_FLAGMASK))
	#else
	#define VPB_CURTHREAD_EXCLUSIVE VPB_SINGLE_EXCLUSIVE
	#endif

	#define VPB_UNBUSIED VPB_SHARERS_WORD(0)

	/* Freed lock blocks both shared and exclusive. */
	#define VPB_FREED (0xffffffff - VPB_BIT_SHARED)

	#define PQ_NONE 255
	#define PQ_INACTIVE 0
	#define PQ_ACTIVE 1
	#define PQ_LAUNDRY 2
	#define PQ_UNSWAPPABLE 3
	#define PQ_COUNT 4

	#ifndef VM_PAGE_HAVE_PGLIST
	TAILQ_HEAD(pglist, vm_page);
	#define VM_PAGE_HAVE_PGLIST
	#endif
	SLIST_HEAD(spglist, vm_page);

	#ifdef _KERNEL
	extern vm_page_t bogus_page;
	#endif /* _KERNEL */

	extern struct mtx_padalign pa_lock[];

	#if defined(__arm__)
	#define PDRSHIFT PDR_SHIFT
	#elif !defined(PDRSHIFT)
	#define PDRSHIFT 21
	#endif

	#define pa_index(pa) ((pa) >> PDRSHIFT)
	#define PA_LOCKPTR(pa) ((struct mtx *)(&pa_lock[pa_index(pa) % PA_LOCK_COUNT]))
	#define PA_LOCKOBJPTR(pa) ((struct lock_object *)PA_LOCKPTR((pa)))
	#define PA_LOCK(pa) mtx_lock(PA_LOCKPTR(pa))
	#define PA_TRYLOCK(pa) mtx_trylock(PA_LOCKPTR(pa))
	#define PA_UNLOCK(pa) mtx_unlock(PA_LOCKPTR(pa))
	#define PA_UNLOCK_COND(pa) \
	do { \
	if ((pa) != 0) { \
	PA_UNLOCK((pa)); \
	(pa) = 0; \
	} \
	} while (0)

	#define PA_LOCK_ASSERT(pa, a) mtx_assert(PA_LOCKPTR(pa), (a))

	#if defined(KLD_MODULE) && !defined(KLD_TIED)
	#define vm_page_lock(m) vm_page_lock_KBI((m), LOCK_FILE, LOCK_LINE)
	#define vm_page_unlock(m) vm_page_unlock_KBI((m), LOCK_FILE, LOCK_LINE)
	#define vm_page_trylock(m) vm_page_trylock_KBI((m), LOCK_FILE, LOCK_LINE)
	#else /* !KLD_MODULE */
	#define vm_page_lockptr(m) (PA_LOCKPTR(VM_PAGE_TO_PHYS((m))))
	#define vm_page_lock(m) mtx_lock(vm_page_lockptr((m)))
	#define vm_page_unlock(m) mtx_unlock(vm_page_lockptr((m)))
	#define vm_page_trylock(m) mtx_trylock(vm_page_lockptr((m)))
	#endif
	#if defined(INVARIANTS)
	#define vm_page_assert_locked(m) \
	vm_page_assert_locked_KBI((m), __FILE__, __LINE__)
	#define vm_page_lock_assert(m, a) \
	vm_page_lock_assert_KBI((m), (a), __FILE__, __LINE__)
	#else
	#define vm_page_assert_locked(m)
	#define vm_page_lock_assert(m, a)
	#endif

	/*
	* The vm_page's aflags are updated using atomic operations. To set or clear
	* these flags, the functions vm_page_aflag_set() and vm_page_aflag_clear()
	* must be used. Neither these flags nor these functions are part of the KBI.
	*
	* PGA_REFERENCED may be cleared only if the page is locked. It is set by
	* both the MI and MD VM layers. However, kernel loadable modules should not
	* directly set this flag. They should call vm_page_reference() instead.
	*
	* PGA_WRITEABLE is set exclusively on managed pages by pmap_enter().
	* When it does so, the object must be locked, or the page must be
	* exclusive busied. The MI VM layer must never access this flag
	* directly. Instead, it should call pmap_page_is_write_mapped().
	*
	* PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has
	* at least one executable mapping. It is not consumed by the MI VM layer.
	*
	* PGA_NOSYNC must be set and cleared with the page busy lock held.
	*
	* PGA_ENQUEUED is set and cleared when a page is inserted into or removed
	* from a page queue, respectively. It determines whether the plinks.q field
	* of the page is valid. To set or clear this flag, the queue lock for the
	* page must be held: the page queue lock corresponding to the page's "queue"
	* field if its value is not PQ_NONE, and the page lock otherwise.
	*
	* PGA_DEQUEUE is set when the page is scheduled to be dequeued from a page
	* queue, and cleared when the dequeue request is processed. A page may
	* have PGA_DEQUEUE set and PGA_ENQUEUED cleared, for instance if a dequeue
	* is requested after the page is scheduled to be enqueued but before it is
	* actually inserted into the page queue. For allocated pages, the page lock
	* must be held to set this flag, but it may be set by vm_page_free_prep()
	* without the page lock held. The page queue lock must be held to clear the
	* PGA_DEQUEUE flag.
	*
	* PGA_REQUEUE is set when the page is scheduled to be enqueued or requeued
	* in its page queue. The page lock must be held to set this flag, and the
	* queue lock for the page must be held to clear it.
	*
	* PGA_REQUEUE_HEAD is a special flag for enqueuing pages near the head of
	* the inactive queue, thus bypassing LRU. The page lock must be held to
	* set this flag, and the queue lock for the page must be held to clear it.
	*
	* PGA_SWAP_FREE is used to defer freeing swap space to the pageout daemon
	* when the context that dirties the page does not have the object write lock
	* held.
	*/
	#define PGA_WRITEABLE 0x0001 /* page may be mapped writeable */
	#define PGA_REFERENCED 0x0002 /* page has been referenced */
	#define PGA_EXECUTABLE 0x0004 /* page may be mapped executable */
	#define PGA_ENQUEUED 0x0008 /* page is enqueued in a page queue */
	#define PGA_DEQUEUE 0x0010 /* page is due to be dequeued */
	#define PGA_REQUEUE 0x0020 /* page is due to be requeued */
	#define PGA_REQUEUE_HEAD 0x0040 /* page requeue should bypass LRU */
	#define PGA_NOSYNC 0x0080 /* do not collect for syncer */
	#define PGA_SWAP_FREE 0x0100 /* page with swap space was dirtied */
	#define PGA_SWAP_SPACE 0x0200 /* page has allocated swap space */

	#define PGA_QUEUE_OP_MASK (PGA_DEQUEUE \| PGA_REQUEUE \| PGA_REQUEUE_HEAD)
	#define PGA_QUEUE_STATE_MASK (PGA_ENQUEUED \| PGA_QUEUE_OP_MASK)

	/*
	* Page flags. If changed at any other time than page allocation or
	* freeing, the modification must be protected by the vm_page lock.
	*
	* The PG_PCPU_CACHE flag is set at allocation time if the page was
	* allocated from a per-CPU cache. It is cleared the next time that the
	* page is allocated from the physical memory allocator.
	*/
	#define PG_PCPU_CACHE 0x01 /* was allocated from per-CPU caches */
	#define PG_FICTITIOUS 0x02 /* physical page doesn't exist */
	#define PG_ZERO 0x04 /* page is zeroed */
	#define PG_MARKER 0x08 /* special queue marker page */
	#define PG_NODUMP 0x10 /* don't include this page in a dump */

	/*
	* Misc constants.
	*/
	#define ACT_DECLINE 1
	#define ACT_ADVANCE 3
	#define ACT_INIT 5
	#define ACT_MAX 64

	#ifdef _KERNEL

	#include <sys/systm.h>

	#include <machine/atomic.h>

	/*
	* Each pageable resident page falls into one of five lists:
	*
	* free
	* Available for allocation now.
	*
	* inactive
	* Low activity, candidates for reclamation.
	* This list is approximately LRU ordered.
	*
	* laundry
	* This is the list of pages that should be
	* paged out next.
	*
	* unswappable
	* Dirty anonymous pages that cannot be paged
	* out because no swap device is configured.
	*
	* active
	* Pages that are "active", i.e., they have been
	* recently referenced.
	*
	*/

	extern vm_page_t vm_page_array; /* First resident page in table */
	extern long vm_page_array_size; /* number of vm_page_t's */
	extern long first_page; /* first physical page number */

	#define VM_PAGE_TO_PHYS(entry) ((entry)->phys_addr)

	/*
	* PHYS_TO_VM_PAGE() returns the vm_page_t object that represents a memory
	* page to which the given physical address belongs. The correct vm_page_t
	* object is returned for addresses that are not page-aligned.
	*/
	vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);

	/*
	* Page allocation parameters for vm_page for the functions
	* vm_page_alloc(), vm_page_grab(), vm_page_alloc_contig() and
	* vm_page_alloc_freelist(). Some functions support only a subset
	* of the flags, and ignore others, see the flags legend.
	*
	* The meaning of VM_ALLOC_ZERO differs slightly between the vm_page_alloc*()
	* and the vm_page_grab*() functions. See these functions for details.
	*
	* Bits 0 - 1 define class.
	* Bits 2 - 15 dedicated for flags.
	* Legend:
	* (a) - vm_page_alloc() supports the flag.
	* (c) - vm_page_alloc_contig() supports the flag.
	* (f) - vm_page_alloc_freelist() supports the flag.
	* (g) - vm_page_grab() supports the flag.
	* (p) - vm_page_grab_pages() supports the flag.
	* Bits above 15 define the count of additional pages that the caller
	* intends to allocate.
	*/
	#define VM_ALLOC_NORMAL 0
	#define VM_ALLOC_INTERRUPT 1
	#define VM_ALLOC_SYSTEM 2
	#define VM_ALLOC_CLASS_MASK 3
	#define VM_ALLOC_WAITOK 0x0008 /* (acf) Sleep and retry */
	#define VM_ALLOC_WAITFAIL 0x0010 /* (acf) Sleep and return error */
	#define VM_ALLOC_WIRED 0x0020 /* (acfgp) Allocate a wired page */
	#define VM_ALLOC_ZERO 0x0040 /* (acfgp) Allocate a prezeroed page */
	#define VM_ALLOC_NOOBJ 0x0100 /* (acg) No associated object */
	#define VM_ALLOC_NOBUSY 0x0200 /* (acgp) Do not excl busy the page */
	#define VM_ALLOC_NOCREAT 0x0400 /* (gp) Don't create a page */
	#define VM_ALLOC_IGN_SBUSY 0x1000 /* (gp) Ignore shared busy flag */
	#define VM_ALLOC_NODUMP 0x2000 /* (ag) don't include in dump */
	#define VM_ALLOC_SBUSY 0x4000 /* (acgp) Shared busy the page */
	#define VM_ALLOC_NOWAIT 0x8000 /* (acfgp) Do not sleep */
	#define VM_ALLOC_COUNT_SHIFT 16
	#define VM_ALLOC_COUNT(count) ((count) << VM_ALLOC_COUNT_SHIFT)

	#ifdef M_NOWAIT
	static inline int
	malloc2vm_flags(int malloc_flags)
	{
	int pflags;

	KASSERT((malloc_flags & M_USE_RESERVE) == 0 \|\|
	(malloc_flags & M_NOWAIT) != 0,
	("M_USE_RESERVE requires M_NOWAIT"));
	pflags = (malloc_flags & M_USE_RESERVE) != 0 ? VM_ALLOC_INTERRUPT :
	VM_ALLOC_SYSTEM;
	if ((malloc_flags & M_ZERO) != 0)
	pflags \|= VM_ALLOC_ZERO;
	if ((malloc_flags & M_NODUMP) != 0)
	pflags \|= VM_ALLOC_NODUMP;
	if ((malloc_flags & M_NOWAIT))
	pflags \|= VM_ALLOC_NOWAIT;
	if ((malloc_flags & M_WAITOK))
	pflags \|= VM_ALLOC_WAITOK;
	return (pflags);
	}
	#endif

	/*
	* Predicates supported by vm_page_ps_test():
	*
	* PS_ALL_DIRTY is true only if the entire (super)page is dirty.
	* However, it can be spuriously false when the (super)page has become
	* dirty in the pmap but that information has not been propagated to the
	* machine-independent layer.
	*/
	#define PS_ALL_DIRTY 0x1
	#define PS_ALL_VALID 0x2
	#define PS_NONE_BUSY 0x4

	bool vm_page_busy_acquire(vm_page_t m, int allocflags);
	void vm_page_busy_downgrade(vm_page_t m);
	int vm_page_busy_tryupgrade(vm_page_t m);
	void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
	void vm_page_busy_sleep_unlocked(vm_object_t obj, vm_page_t m,
	vm_pindex_t pindex, const char *wmesg, bool nonshared);
	void vm_page_free(vm_page_t m);
	void vm_page_free_zero(vm_page_t m);

	void vm_page_activate (vm_page_t);
	void vm_page_advise(vm_page_t m, int advice);
	vm_page_t vm_page_alloc(vm_object_t, vm_pindex_t, int);
	vm_page_t vm_page_alloc_domain(vm_object_t, vm_pindex_t, int, int);
	vm_page_t vm_page_alloc_after(vm_object_t, vm_pindex_t, int, vm_page_t);
	vm_page_t vm_page_alloc_domain_after(vm_object_t, vm_pindex_t, int, int,
	vm_page_t);
	vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
	u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
	vm_paddr_t boundary, vm_memattr_t memattr);
	vm_page_t vm_page_alloc_contig_domain(vm_object_t object,
	vm_pindex_t pindex, int domain, int req, u_long npages, vm_paddr_t low,
	vm_paddr_t high, u_long alignment, vm_paddr_t boundary,
	vm_memattr_t memattr);
	vm_page_t vm_page_alloc_freelist(int, int);
	vm_page_t vm_page_alloc_freelist_domain(int, int, int);
	void vm_page_bits_set(vm_page_t m, vm_page_bits_t *bits, vm_page_bits_t set);
	bool vm_page_blacklist_add(vm_paddr_t pa, bool verbose);
	vm_page_t vm_page_grab(vm_object_t, vm_pindex_t, int);
	vm_page_t vm_page_grab_unlocked(vm_object_t, vm_pindex_t, int);
	int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
	vm_page_t *ma, int count);
	int vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex,
	int allocflags, vm_page_t *ma, int count);
	int vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex,
	int allocflags);
	int vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object,
	vm_pindex_t pindex, int allocflags);
	void vm_page_deactivate(vm_page_t);
	void vm_page_deactivate_noreuse(vm_page_t);
	void vm_page_dequeue(vm_page_t m);
	void vm_page_dequeue_deferred(vm_page_t m);
	vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t);
	void vm_page_free_invalid(vm_page_t);
	vm_page_t vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr);
	void vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
	int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
	void vm_page_invalid(vm_page_t m);
	void vm_page_launder(vm_page_t m);
	vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t);
	vm_page_t vm_page_next(vm_page_t m);
	void vm_page_pqbatch_drain(void);
	void vm_page_pqbatch_submit(vm_page_t m, uint8_t queue);
	bool vm_page_pqstate_commit(vm_page_t m, vm_page_astate_t *old,
	vm_page_astate_t new);
	vm_page_t vm_page_prev(vm_page_t m);
	bool vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m);
	void vm_page_putfake(vm_page_t m);
	void vm_page_readahead_finish(vm_page_t m);
	bool vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low,
	vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
	bool vm_page_reclaim_contig_domain(int domain, int req, u_long npages,
	vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
	void vm_page_reference(vm_page_t m);
	#define VPR_TRYFREE 0x01
	#define VPR_NOREUSE 0x02
	void vm_page_release(vm_page_t m, int flags);
	void vm_page_release_locked(vm_page_t m, int flags);
	vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t);
	bool vm_page_remove(vm_page_t);
	bool vm_page_remove_xbusy(vm_page_t);
	int vm_page_rename(vm_page_t, vm_object_t, vm_pindex_t);
	void vm_page_replace(vm_page_t mnew, vm_object_t object,
	vm_pindex_t pindex, vm_page_t mold);
	int vm_page_sbusied(vm_page_t m);
	vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
	vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
	vm_page_bits_t vm_page_set_dirty(vm_page_t m);
	void vm_page_set_valid_range(vm_page_t m, int base, int size);
	int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
	int vm_page_sleep_if_xbusy(vm_page_t m, const char *msg);
	vm_offset_t vm_page_startup(vm_offset_t vaddr);
	void vm_page_sunbusy(vm_page_t m);
	bool vm_page_try_remove_all(vm_page_t m);
	bool vm_page_try_remove_write(vm_page_t m);
	int vm_page_trysbusy(vm_page_t m);
	int vm_page_tryxbusy(vm_page_t m);
	void vm_page_unhold_pages(vm_page_t *ma, int count);
	void vm_page_unswappable(vm_page_t m);
	void vm_page_unwire(vm_page_t m, uint8_t queue);
	bool vm_page_unwire_noq(vm_page_t m);
	void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
	void vm_page_wire(vm_page_t);
	bool vm_page_wire_mapped(vm_page_t m);
	void vm_page_xunbusy_hard(vm_page_t m);
	void vm_page_xunbusy_hard_unchecked(vm_page_t m);
	void vm_page_set_validclean (vm_page_t, int, int);
	void vm_page_clear_dirty(vm_page_t, int, int);
	void vm_page_set_invalid(vm_page_t, int, int);
	void vm_page_valid(vm_page_t m);
	int vm_page_is_valid(vm_page_t, int, int);
	void vm_page_test_dirty(vm_page_t);
	vm_page_bits_t vm_page_bits(int base, int size);
	void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid);
	void vm_page_free_pages_toq(struct spglist *free, bool update_wire_count);

	void vm_page_dirty_KBI(vm_page_t m);
	void vm_page_lock_KBI(vm_page_t m, const char *file, int line);
	void vm_page_unlock_KBI(vm_page_t m, const char *file, int line);
	int vm_page_trylock_KBI(vm_page_t m, const char *file, int line);
	#if defined(INVARIANTS) \|\| defined(INVARIANT_SUPPORT)
	void vm_page_assert_locked_KBI(vm_page_t m, const char *file, int line);
	void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line);
	#endif

	+#define vm_page_busy_fetch(m) atomic_load_int(&(m)->busy_lock)
	+
	#define vm_page_assert_busied(m) \
	KASSERT(vm_page_busied(m), \
	("vm_page_assert_busied: page %p not busy @ %s:%d", \
	(m), __FILE__, __LINE__))

	#define vm_page_assert_sbusied(m) \
	KASSERT(vm_page_sbusied(m), \
	("vm_page_assert_sbusied: page %p not shared busy @ %s:%d", \
	(m), __FILE__, __LINE__))

	#define vm_page_assert_unbusied(m) \
	- KASSERT((m->busy_lock & ~VPB_BIT_WAITERS) != \
	+ KASSERT((vm_page_busy_fetch(m) & ~VPB_BIT_WAITERS) != \
	VPB_CURTHREAD_EXCLUSIVE, \
	("vm_page_assert_xbusied: page %p busy_lock %#x owned" \
	" by me @ %s:%d", \
	(m), (m)->busy_lock, __FILE__, __LINE__)); \

	#define vm_page_assert_xbusied_unchecked(m) do { \
	KASSERT(vm_page_xbusied(m), \
	("vm_page_assert_xbusied: page %p not exclusive busy @ %s:%d", \
	(m), __FILE__, __LINE__)); \
	} while (0)
	#define vm_page_assert_xbusied(m) do { \
	vm_page_assert_xbusied_unchecked(m); \
	- KASSERT((m->busy_lock & ~VPB_BIT_WAITERS) == \
	+ KASSERT((vm_page_busy_fetch(m) & ~VPB_BIT_WAITERS) == \
	VPB_CURTHREAD_EXCLUSIVE, \
	("vm_page_assert_xbusied: page %p busy_lock %#x not owned" \
	" by me @ %s:%d", \
	(m), (m)->busy_lock, __FILE__, __LINE__)); \
	} while (0)

	#define vm_page_busied(m) \
	- ((m)->busy_lock != VPB_UNBUSIED)
	+ (vm_page_busy_fetch(m) != VPB_UNBUSIED)

	#define vm_page_sbusy(m) do { \
	if (!vm_page_trysbusy(m)) \
	panic("%s: page %p failed shared busying", __func__, \
	(m)); \
	} while (0)

	#define vm_page_xbusied(m) \
	- (((m)->busy_lock & VPB_SINGLE_EXCLUSIVE) != 0)
	+ ((vm_page_busy_fetch(m) & VPB_SINGLE_EXCLUSIVE) != 0)

	#define vm_page_busy_freed(m) \
	- ((m)->busy_lock == VPB_FREED)
	+ (vm_page_busy_fetch(m) == VPB_FREED)

	#define vm_page_xbusy(m) do { \
	if (!vm_page_tryxbusy(m)) \
	panic("%s: page %p failed exclusive busying", __func__, \
	(m)); \
	} while (0)

	/* Note: page m's lock must not be owned by the caller. */
	#define vm_page_xunbusy(m) do { \
	if (!atomic_cmpset_rel_int(&(m)->busy_lock, \
	VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \
	vm_page_xunbusy_hard(m); \
	} while (0)
	#define vm_page_xunbusy_unchecked(m) do { \
	if (!atomic_cmpset_rel_int(&(m)->busy_lock, \
	VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \
	vm_page_xunbusy_hard_unchecked(m); \
	} while (0)

	#ifdef INVARIANTS
	void vm_page_object_busy_assert(vm_page_t m);
	#define VM_PAGE_OBJECT_BUSY_ASSERT(m) vm_page_object_busy_assert(m)
	void vm_page_assert_pga_writeable(vm_page_t m, uint16_t bits);
	#define VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits) \
	vm_page_assert_pga_writeable(m, bits)
	+/*
	+ * Claim ownership of a page's xbusy state. In non-INVARIANTS kernels this
	+ * operation is a no-op since ownership is not tracked. In particular
	+ * this macro does not provide any synchronization with the previous owner.
	+ */
	#define vm_page_xbusy_claim(m) do { \
	+ u_int _busy_lock; \
	+ \
	vm_page_assert_xbusied_unchecked((m)); \
	- (m)->busy_lock = VPB_CURTHREAD_EXCLUSIVE; \
	+ do { \
	+ _busy_lock = vm_page_busy_fetch(m); \
	+ } while (!atomic_cmpset_int(&(m)->busy_lock, _busy_lock, \
	+ (_busy_lock & VPB_BIT_FLAGMASK) \| VPB_CURTHREAD_EXCLUSIVE)); \
	} while (0)
	#else
	#define VM_PAGE_OBJECT_BUSY_ASSERT(m) (void)0
	#define VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits) (void)0
	#define vm_page_xbusy_claim(m)
	#endif

	#if BYTE_ORDER == BIG_ENDIAN
	#define VM_PAGE_AFLAG_SHIFT 16
	#else
	#define VM_PAGE_AFLAG_SHIFT 0
	#endif

	/*
	* Load a snapshot of a page's 32-bit atomic state.
	*/
	static inline vm_page_astate_t
	vm_page_astate_load(vm_page_t m)
	{
	vm_page_astate_t a;

	a._bits = atomic_load_32(&m->a._bits);
	return (a);
	}

	/*
	* Atomically compare and set a page's atomic state.
	*/
	static inline bool
	vm_page_astate_fcmpset(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new)
	{

	KASSERT(new.queue == PQ_INACTIVE \|\| (new.flags & PGA_REQUEUE_HEAD) == 0,
	("%s: invalid head requeue request for page %p", __func__, m));
	KASSERT((new.flags & PGA_ENQUEUED) == 0 \|\| new.queue != PQ_NONE,
	("%s: setting PGA_ENQUEUED with PQ_NONE in page %p", __func__, m));
	KASSERT(new._bits != old->_bits,
	("%s: bits are unchanged", __func__));

	return (atomic_fcmpset_32(&m->a._bits, &old->_bits, new._bits) != 0);
	}

	/*
	* Clear the given bits in the specified page.
	*/
	static inline void
	vm_page_aflag_clear(vm_page_t m, uint16_t bits)
	{
	uint32_t *addr, val;

	/*
	* Access the whole 32-bit word containing the aflags field with an
	* atomic update. Parallel non-atomic updates to the other fields
	* within this word are handled properly by the atomic update.
	*/
	addr = (void *)&m->a;
	val = bits << VM_PAGE_AFLAG_SHIFT;
	atomic_clear_32(addr, val);
	}

	/*
	* Set the given bits in the specified page.
	*/
	static inline void
	vm_page_aflag_set(vm_page_t m, uint16_t bits)
	{
	uint32_t *addr, val;

	VM_PAGE_ASSERT_PGA_WRITEABLE(m, bits);

	/*
	* Access the whole 32-bit word containing the aflags field with an
	* atomic update. Parallel non-atomic updates to the other fields
	* within this word are handled properly by the atomic update.
	*/
	addr = (void *)&m->a;
	val = bits << VM_PAGE_AFLAG_SHIFT;
	atomic_set_32(addr, val);
	}

	/*
	* vm_page_dirty:
	*
	* Set all bits in the page's dirty field.
	*
	* The object containing the specified page must be locked if the
	* call is made from the machine-independent layer.
	*
	* See vm_page_clear_dirty_mask().
	*/
	static __inline void
	vm_page_dirty(vm_page_t m)
	{

	/* Use vm_page_dirty_KBI() under INVARIANTS to save memory. */
	#if (defined(KLD_MODULE) && !defined(KLD_TIED)) \|\| defined(INVARIANTS)
	vm_page_dirty_KBI(m);
	#else
	m->dirty = VM_PAGE_BITS_ALL;
	#endif
	}

	/*
	* vm_page_undirty:
	*
	* Set page to not be dirty. Note: does not clear pmap modify bits
	*/
	static __inline void
	vm_page_undirty(vm_page_t m)
	{

	VM_PAGE_OBJECT_BUSY_ASSERT(m);
	m->dirty = 0;
	}

	static inline uint8_t
	_vm_page_queue(vm_page_astate_t as)
	{

	if ((as.flags & PGA_DEQUEUE) != 0)
	return (PQ_NONE);
	return (as.queue);
	}

	/*
	* vm_page_queue:
	*
	* Return the index of the queue containing m.
	*/
	static inline uint8_t
	vm_page_queue(vm_page_t m)
	{

	return (_vm_page_queue(vm_page_astate_load(m)));
	}

	static inline bool
	vm_page_active(vm_page_t m)
	{

	return (vm_page_queue(m) == PQ_ACTIVE);
	}

	static inline bool
	vm_page_inactive(vm_page_t m)
	{

	return (vm_page_queue(m) == PQ_INACTIVE);
	}

	static inline bool
	vm_page_in_laundry(vm_page_t m)
	{
	uint8_t queue;

	queue = vm_page_queue(m);
	return (queue == PQ_LAUNDRY \|\| queue == PQ_UNSWAPPABLE);
	}

	/*
	* vm_page_drop:
	*
	* Release a reference to a page and return the old reference count.
	*/
	static inline u_int
	vm_page_drop(vm_page_t m, u_int val)
	{
	u_int old;

	/*
	* Synchronize with vm_page_free_prep(): ensure that all updates to the
	* page structure are visible before it is freed.
	*/
	atomic_thread_fence_rel();
	old = atomic_fetchadd_int(&m->ref_count, -val);
	KASSERT(old != VPRC_BLOCKED,
	("vm_page_drop: page %p has an invalid refcount value", m));
	return (old);
	}

	/*
	* vm_page_wired:
	*
	* Perform a racy check to determine whether a reference prevents the page
	* from being reclaimable. If the page's object is locked, and the page is
	* unmapped and exclusively busied by the current thread, no new wirings
	* may be created.
	*/
	static inline bool
	vm_page_wired(vm_page_t m)
	{

	return (VPRC_WIRE_COUNT(m->ref_count) > 0);
	}

	static inline bool
	vm_page_all_valid(vm_page_t m)
	{

	return (m->valid == VM_PAGE_BITS_ALL);
	}

	static inline bool
	vm_page_none_valid(vm_page_t m)
	{

	return (m->valid == 0);
	}

	#endif /* _KERNEL */
	#endif /* !_VM_PAGE_ */
	diff --git a/sys/x86/include/acpica_machdep.h b/sys/x86/include/acpica_machdep.h
	index 89012b3f4bc4..05e8711c16fa 100644
	--- a/sys/x86/include/acpica_machdep.h
	+++ b/sys/x86/include/acpica_machdep.h
	@@ -1,90 +1,91 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2002 Mitsuru IWASAKI
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	/******************************************************************************
	*
	* Name: acpica_machdep.h - arch-specific defines, etc.
	* $Revision$
	*
	*****************************************************************************/

	#ifndef __ACPICA_MACHDEP_H__
	#define __ACPICA_MACHDEP_H__

	#ifdef _KERNEL
	/*
	* Calling conventions:
	*
	* ACPI_SYSTEM_XFACE - Interfaces to host OS (handlers, threads)
	* ACPI_EXTERNAL_XFACE - External ACPI interfaces
	* ACPI_INTERNAL_XFACE - Internal ACPI interfaces
	* ACPI_INTERNAL_VAR_XFACE - Internal variable-parameter list interfaces
	*/
	#define ACPI_SYSTEM_XFACE
	#define ACPI_EXTERNAL_XFACE
	#define ACPI_INTERNAL_XFACE
	#define ACPI_INTERNAL_VAR_XFACE

	/* Asm macros */

	#define ACPI_ASM_MACROS
	#define BREAKPOINT3
	#define ACPI_DISABLE_IRQS() disable_intr()
	#define ACPI_ENABLE_IRQS() enable_intr()

	#define ACPI_FLUSH_CPU_CACHE() wbinvd()

	/* Section 5.2.10.1: global lock acquire/release functions */
	int acpi_acquire_global_lock(volatile uint32_t *);
	int acpi_release_global_lock(volatile uint32_t *);
	#define ACPI_ACQUIRE_GLOBAL_LOCK(GLptr, Acq) do { \
	(Acq) = acpi_acquire_global_lock(&((GLptr)->GlobalLock)); \
	} while (0)
	#define ACPI_RELEASE_GLOBAL_LOCK(GLptr, Acq) do { \
	(Acq) = acpi_release_global_lock(&((GLptr)->GlobalLock)); \
	} while (0)

	enum intr_trigger;
	enum intr_polarity;

	void acpi_SetDefaultIntrModel(int model);
	void acpi_cpu_c1(void);
	void acpi_cpu_idle_mwait(uint32_t mwait_hint);
	void acpi_map_table(vm_paddr_t pa, const char sig);
	void acpi_unmap_table(void *table);
	vm_paddr_t acpi_find_table(const char *sig);
	void madt_parse_interrupt_values(void *entry,
	enum intr_trigger trig, enum intr_polarity pol);

	extern int madt_found_sci_override;
	+extern int (*apei_nmi)(void);

	#endif /* _KERNEL */

	#endif /* __ACPICA_MACHDEP_H__ */
	diff --git a/sys/x86/include/bus_dma.h b/sys/x86/include/bus_dma.h
	index 5bb70419726a..b4da787e9cfc 100644
	--- a/sys/x86/include/bus_dma.h
	+++ b/sys/x86/include/bus_dma.h
	@@ -1,201 +1,195 @@
	/*-
	* Copyright (c) 2017 Jason A. Harmening.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef _X86_BUS_DMA_H_
	#define _X86_BUS_DMA_H_

	#define WANT_INLINE_DMAMAP
	#include <sys/bus_dma.h>
	#include <sys/_null.h>

	#include <x86/busdma_impl.h>

	/*
	* Is DMA address 1:1 mapping of physical address
	*/
	static inline bool
	bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->id_mapped(dmat, buf, buflen));
	}

	/*
	* Allocate a handle for mapping from kva/uva/physical
	* address space into bus device space.
	*/
	static inline int
	bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->map_create(dmat, flags, mapp));
	}

	/*
	* Destroy a handle for mapping from kva/uva/physical
	* address space into bus device space.
	*/
	static inline int
	bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->map_destroy(dmat, map));
	}

	/*
	* Allocate a piece of memory that can be efficiently mapped into
	* bus device space based on the constraints lited in the dma tag.
	* A dmamap to for use with dmamap_load is also allocated.
	*/
	static inline int
	bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
	bus_dmamap_t *mapp)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp));
	}

	/*
	* Free a piece of memory and it's allociated dmamap, that was allocated
	* via bus_dmamem_alloc. Make the same choice for free/contigfree.
	*/
	static inline void
	bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	tc->impl->mem_free(dmat, vaddr, map);
	}

	/*
	* Release the mapping held by map.
	*/
	static inline void
	bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
	{
	struct bus_dma_tag_common *tc;

	if (map != NULL) {
	tc = (struct bus_dma_tag_common *)dmat;
	tc->impl->map_unload(dmat, map);
	}
	}

	static inline void
	bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
	{
	struct bus_dma_tag_common *tc;

	if (map != NULL) {
	tc = (struct bus_dma_tag_common *)dmat;
	tc->impl->map_sync(dmat, map, op);
	}
	}

	/*
	* Utility function to load a physical buffer. segp contains
	* the starting segment on entrace, and the ending segment on exit.
	*/
	static inline int
	_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
	bus_size_t buflen, int flags, bus_dma_segment_t segs, int segp)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs,
	segp));
	}

	static inline int
	_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma,
	bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs,
	int *segp)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags,
	segs, segp));
	}

	/*
	* Utility function to load a linear buffer. segp contains
	* the starting segment on entrace, and the ending segment on exit.
	*/
	static inline int
	_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
	bus_size_t buflen, struct pmap pmap, int flags, bus_dma_segment_t segs,
	int *segp)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,
	segp));
	}

	static inline void
	_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
	struct memdesc mem, bus_dmamap_callback_t callback, void *callback_arg)
	{
	struct bus_dma_tag_common *tc;

	if (map != NULL) {
	tc = (struct bus_dma_tag_common *)dmat;
	tc->impl->map_waitok(dmat, map, mem, callback, callback_arg);
	}
	}

	static inline bus_dma_segment_t *
	_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
	bus_dma_segment_t *segs, int nsegs, int error)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->map_complete(dmat, map, segs, nsegs, error));
	}

	-#ifdef _KERNEL
	-bool bus_dma_dmar_set_buswide(device_t dev);
	-int bus_dma_dmar_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map,
	- vm_paddr_t start, vm_size_t length, int flags);
	-#endif
	-
	#endif /* !_X86_BUS_DMA_H_ */

	diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c
	index 45151c8dd355..234a920d1ded 100644
	--- a/sys/x86/iommu/intel_ctx.c
	+++ b/sys/x86/iommu/intel_ctx.c
	@@ -1,949 +1,951 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/malloc.h>
	#include <sys/bus.h>
	#include <sys/interrupt.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/limits.h>
	#include <sys/lock.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/proc.h>
	#include <sys/rwlock.h>
	#include <sys/rman.h>
	#include <sys/sysctl.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	#include <sys/vmem.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_map.h>
	+#include <contrib/dev/acpica/include/acpi.h>
	+#include <contrib/dev/acpica/include/accommon.h>
	+#include <dev/pci/pcireg.h>
	+#include <dev/pci/pcivar.h>
	#include <machine/atomic.h>
	#include <machine/bus.h>
	#include <machine/md_var.h>
	#include <machine/specialreg.h>
	-#include <contrib/dev/acpica/include/acpi.h>
	-#include <contrib/dev/acpica/include/accommon.h>
	#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	-#include <dev/pci/pcireg.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>
	-#include <dev/pci/pcivar.h>

	static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
	static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain");

	static void dmar_domain_unload_task(void *arg, int pending);
	static void dmar_unref_domain_locked(struct dmar_unit *dmar,
	struct dmar_domain *domain);
	static void dmar_domain_destroy(struct dmar_domain *domain);

	static void
	dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
	{
	struct sf_buf *sf;
	dmar_root_entry_t *re;
	vm_page_t ctxm;

	/*
	* Allocated context page must be linked.
	*/
	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC);
	if (ctxm != NULL)
	return;

	/*
	* Page not present, allocate and link. Note that other
	* thread might execute this sequence in parallel. This
	* should be safe, because the context entries written by both
	* threads are equal.
	*/
	TD_PREP_PINNED_ASSERT;
	ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO \|
	IOMMU_PGF_WAITOK);
	re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf);
	re += bus;
	dmar_pte_store(&re->r1, DMAR_ROOT_R1_P \| (DMAR_ROOT_R1_CTP_MASK &
	VM_PAGE_TO_PHYS(ctxm)));
	dmar_flush_root_to_ram(dmar, re);
	dmar_unmap_pgtbl(sf);
	TD_PINNED_ASSERT;
	}

	static dmar_ctx_entry_t *
	dmar_map_ctx_entry(struct dmar_ctx ctx, struct sf_buf *sfp)
	{
	struct dmar_unit *dmar;
	dmar_ctx_entry_t *ctxp;

	dmar = (struct dmar_unit *)ctx->context.domain->iommu;

	ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 +
	PCI_RID2BUS(ctx->rid), IOMMU_PGF_NOALLOC \| IOMMU_PGF_WAITOK, sfp);
	ctxp += ctx->rid & 0xff;
	return (ctxp);
	}

	static void
	device_tag_init(struct dmar_ctx *ctx, device_t dev)
	{
	struct dmar_domain *domain;
	bus_addr_t maxaddr;

	domain = (struct dmar_domain *)ctx->context.domain;
	maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR);
	ctx->context.tag->common.ref_count = 1; /* Prevent free */
	ctx->context.tag->common.impl = &bus_dma_iommu_impl;
	ctx->context.tag->common.boundary = 0;
	ctx->context.tag->common.lowaddr = maxaddr;
	ctx->context.tag->common.highaddr = maxaddr;
	ctx->context.tag->common.maxsize = maxaddr;
	ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED;
	ctx->context.tag->common.maxsegsz = maxaddr;
	ctx->context.tag->ctx = (struct iommu_ctx *)ctx;
	ctx->context.tag->owner = dev;
	}

	static void
	ctx_id_entry_init_one(dmar_ctx_entry_t ctxp, struct dmar_domain domain,
	vm_page_t ctx_root)
	{
	/*
	* For update due to move, the store is not atomic. It is
	* possible that DMAR read upper doubleword, while low
	* doubleword is not yet updated. The domain id is stored in
	* the upper doubleword, while the table pointer in the lower.
	*
	* There is no good solution, for the same reason it is wrong
	* to clear P bit in the ctx entry for update.
	*/
	dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) \|
	domain->awlvl);
	if (ctx_root == NULL) {
	dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS \| DMAR_CTX1_P);
	} else {
	dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR \|
	(DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) \|
	DMAR_CTX1_P);
	}
	}

	static void
	ctx_id_entry_init(struct dmar_ctx ctx, dmar_ctx_entry_t ctxp, bool move,
	int busno)
	{
	struct dmar_unit *unit;
	struct dmar_domain *domain;
	vm_page_t ctx_root;
	int i;

	domain = (struct dmar_domain *)ctx->context.domain;
	unit = (struct dmar_unit *)domain->iodom.iommu;
	KASSERT(move \|\| (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),
	("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
	unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner),
	pci_get_function(ctx->context.tag->owner),
	ctxp->ctx1, ctxp->ctx2));

	if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 &&
	(unit->hw_ecap & DMAR_ECAP_PT) != 0) {
	KASSERT(domain->pgtbl_obj == NULL,
	("ctx %p non-null pgtbl_obj", ctx));
	ctx_root = NULL;
	} else {
	ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0,
	IOMMU_PGF_NOALLOC);
	}

	- if (dmar_is_buswide_ctx(unit, busno)) {
	+ if (iommu_is_buswide_ctx((struct iommu_unit *)unit, busno)) {
	MPASS(!move);
	for (i = 0; i <= PCI_BUSMAX; i++) {
	ctx_id_entry_init_one(&ctxp[i], domain, ctx_root);
	}
	} else {
	ctx_id_entry_init_one(ctxp, domain, ctx_root);
	}
	dmar_flush_ctx_to_ram(unit, ctxp);
	}

	static int
	dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force)
	{
	int error;

	/*
	* If dmar declares Caching Mode as Set, follow 11.5 "Caching
	* Mode Consideration" and do the (global) invalidation of the
	* negative TLB entries.
	*/
	if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force)
	return (0);
	if (dmar->qi_enabled) {
	dmar_qi_invalidate_ctx_glob_locked(dmar);
	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 \|\| force)
	dmar_qi_invalidate_iotlb_glob_locked(dmar);
	return (0);
	}
	error = dmar_inv_ctx_glob(dmar);
	if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 \|\| force))
	error = dmar_inv_iotlb_glob(dmar);
	return (error);
	}

	static int
	domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus,
	int slot, int func, int dev_domain, int dev_busno,
	const void *dev_path, int dev_path_len)
	{
	struct iommu_map_entries_tailq rmrr_entries;
	struct iommu_map_entry entry, entry1;
	vm_page_t *ma;
	iommu_gaddr_t start, end;
	vm_pindex_t size, i;
	int error, error1;

	error = 0;
	TAILQ_INIT(&rmrr_entries);
	dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path,
	dev_path_len, &rmrr_entries);
	TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
	/*
	* VT-d specification requires that the start of an
	* RMRR entry is 4k-aligned. Buggy BIOSes put
	* anything into the start and end fields. Truncate
	* and round as neccesary.
	*
	* We also allow the overlapping RMRR entries, see
	* iommu_gas_alloc_region().
	*/
	start = entry->start;
	end = entry->end;
	if (bootverbose)
	printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n",
	domain->iodom.iommu->unit, bus, slot, func,
	(uintmax_t)start, (uintmax_t)end);
	entry->start = trunc_page(start);
	entry->end = round_page(end);
	if (entry->start == entry->end) {
	/* Workaround for some AMI (?) BIOSes */
	if (bootverbose) {
	if (dev != NULL)
	device_printf(dev, "");
	printf("pci%d:%d:%d ", bus, slot, func);
	printf("BIOS bug: dmar%d RMRR "
	"region (%jx, %jx) corrected\n",
	domain->iodom.iommu->unit, start, end);
	}
	entry->end += DMAR_PAGE_SIZE * 0x20;
	}
	size = OFF_TO_IDX(entry->end - entry->start);
	ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
	for (i = 0; i < size; i++) {
	ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
	VM_MEMATTR_DEFAULT);
	}
	error1 = iommu_gas_map_region((struct iommu_domain *)domain,
	entry,
	IOMMU_MAP_ENTRY_READ \| IOMMU_MAP_ENTRY_WRITE,
	IOMMU_MF_CANWAIT \| IOMMU_MF_RMRR, ma);
	/*
	* Non-failed RMRR entries are owned by context rb
	* tree. Get rid of the failed entry, but do not stop
	* the loop. Rest of the parsed RMRR entries are
	* loaded and removed on the context destruction.
	*/
	if (error1 == 0 && entry->end != entry->start) {
	IOMMU_LOCK(domain->iodom.iommu);
	domain->refs++; /* XXXKIB prevent free */
	domain->iodom.flags \|= IOMMU_DOMAIN_RMRR;
	IOMMU_UNLOCK(domain->iodom.iommu);
	} else {
	if (error1 != 0) {
	if (dev != NULL)
	device_printf(dev, "");
	printf("pci%d:%d:%d ", bus, slot, func);
	printf(
	"dmar%d failed to map RMRR region (%jx, %jx) %d\n",
	domain->iodom.iommu->unit, start, end,
	error1);
	error = error1;
	}
	TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
	iommu_gas_free_entry((struct iommu_domain *)domain,
	entry);
	}
	for (i = 0; i < size; i++)
	vm_page_putfake(ma[i]);
	free(ma, M_TEMP);
	}
	return (error);
	}

	static struct dmar_domain *
	dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)
	{
	struct iommu_domain *iodom;
	struct dmar_domain *domain;
	int error, id, mgaw;

	id = alloc_unr(dmar->domids);
	if (id == -1)
	return (NULL);
	domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK \| M_ZERO);
	iodom = (struct iommu_domain *)domain;
	domain->domain = id;
	LIST_INIT(&domain->contexts);
	RB_INIT(&domain->iodom.rb_root);
	TAILQ_INIT(&domain->iodom.unload_entries);
	TASK_INIT(&domain->iodom.unload_task, 0, dmar_domain_unload_task,
	domain);
	mtx_init(&domain->iodom.lock, "dmardom", NULL, MTX_DEF);
	domain->dmar = dmar;
	domain->iodom.iommu = &dmar->iommu;

	/*
	* For now, use the maximal usable physical address of the
	* installed memory to calculate the mgaw on id_mapped domain.
	* It is useful for the identity mapping, and less so for the
	* virtualized bus address space.
	*/
	domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;
	mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped);
	error = domain_set_agaw(domain, mgaw);
	if (error != 0)
	goto fail;
	if (!id_mapped)
	/* Use all supported address space for remapping. */
	domain->iodom.end = 1ULL << (domain->agaw - 1);

	iommu_gas_init_domain((struct iommu_domain *)domain);

	if (id_mapped) {
	if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
	domain->pgtbl_obj = domain_get_idmap_pgtbl(domain,
	domain->iodom.end);
	}
	domain->iodom.flags \|= IOMMU_DOMAIN_IDMAP;
	} else {
	error = domain_alloc_pgtbl(domain);
	if (error != 0)
	goto fail;
	/* Disable local apic region access */
	error = iommu_gas_reserve_region(iodom, 0xfee00000,
	0xfeefffff + 1);
	if (error != 0)
	goto fail;
	}
	return (domain);

	fail:
	dmar_domain_destroy(domain);
	return (NULL);
	}

	static struct dmar_ctx *
	dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid)
	{
	struct dmar_ctx *ctx;

	ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK \| M_ZERO);
	ctx->context.domain = (struct iommu_domain *)domain;
	ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu),
	M_DMAR_CTX, M_WAITOK \| M_ZERO);
	ctx->rid = rid;
	ctx->refs = 1;
	return (ctx);
	}

	static void
	dmar_ctx_link(struct dmar_ctx *ctx)
	{
	struct dmar_domain *domain;

	domain = (struct dmar_domain *)ctx->context.domain;
	IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
	KASSERT(domain->refs >= domain->ctx_cnt,
	("dom %p ref underflow %d %d", domain, domain->refs,
	domain->ctx_cnt));
	domain->refs++;
	domain->ctx_cnt++;
	LIST_INSERT_HEAD(&domain->contexts, ctx, link);
	}

	static void
	dmar_ctx_unlink(struct dmar_ctx *ctx)
	{
	struct dmar_domain *domain;

	domain = (struct dmar_domain *)ctx->context.domain;
	IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
	KASSERT(domain->refs > 0,
	("domain %p ctx dtr refs %d", domain, domain->refs));
	KASSERT(domain->ctx_cnt >= domain->refs,
	("domain %p ctx dtr refs %d ctx_cnt %d", domain,
	domain->refs, domain->ctx_cnt));
	domain->refs--;
	domain->ctx_cnt--;
	LIST_REMOVE(ctx, link);
	}

	static void
	dmar_domain_destroy(struct dmar_domain *domain)
	{
	struct dmar_unit *dmar;

	KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),
	("unfinished unloads %p", domain));
	KASSERT(LIST_EMPTY(&domain->contexts),
	("destroying dom %p with contexts", domain));
	KASSERT(domain->ctx_cnt == 0,
	("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));
	KASSERT(domain->refs == 0,
	("destroying dom %p with refs %d", domain, domain->refs));
	if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
	DMAR_DOMAIN_LOCK(domain);
	iommu_gas_fini_domain((struct iommu_domain *)domain);
	DMAR_DOMAIN_UNLOCK(domain);
	}
	if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) {
	if (domain->pgtbl_obj != NULL)
	DMAR_DOMAIN_PGLOCK(domain);
	domain_free_pgtbl(domain);
	}
	mtx_destroy(&domain->iodom.lock);
	dmar = (struct dmar_unit *)domain->iodom.iommu;
	free_unr(dmar->domids, domain->domain);
	free(domain, M_DMAR_DOMAIN);
	}

	static struct dmar_ctx *
	dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid,
	int dev_domain, int dev_busno, const void *dev_path, int dev_path_len,
	bool id_mapped, bool rmrr_init)
	{
	struct dmar_domain domain, domain1;
	struct dmar_ctx ctx, ctx1;
	+ struct iommu_unit *unit;
	dmar_ctx_entry_t *ctxp;
	struct sf_buf *sf;
	int bus, slot, func, error;
	bool enable;

	if (dev != NULL) {
	bus = pci_get_bus(dev);
	slot = pci_get_slot(dev);
	func = pci_get_function(dev);
	} else {
	bus = PCI_RID2BUS(rid);
	slot = PCI_RID2SLOT(rid);
	func = PCI_RID2FUNC(rid);
	}
	enable = false;
	TD_PREP_PINNED_ASSERT;
	+ unit = (struct iommu_unit *)dmar;
	DMAR_LOCK(dmar);
	- KASSERT(!dmar_is_buswide_ctx(dmar, bus) \|\| (slot == 0 && func == 0),
	- ("dmar%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus,
	+ KASSERT(!iommu_is_buswide_ctx(unit, bus) \|\| (slot == 0 && func == 0),
	+ ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus,
	slot, func));
	ctx = dmar_find_ctx_locked(dmar, rid);
	error = 0;
	if (ctx == NULL) {
	/*
	* Perform the allocations which require sleep or have
	* higher chance to succeed if the sleep is allowed.
	*/
	DMAR_UNLOCK(dmar);
	dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid));
	domain1 = dmar_domain_alloc(dmar, id_mapped);
	if (domain1 == NULL) {
	TD_PINNED_ASSERT;
	return (NULL);
	}
	if (!id_mapped) {
	error = domain_init_rmrr(domain1, dev, bus,
	slot, func, dev_domain, dev_busno, dev_path,
	dev_path_len);
	if (error != 0) {
	dmar_domain_destroy(domain1);
	TD_PINNED_ASSERT;
	return (NULL);
	}
	}
	ctx1 = dmar_ctx_alloc(domain1, rid);
	ctxp = dmar_map_ctx_entry(ctx1, &sf);
	DMAR_LOCK(dmar);

	/*
	* Recheck the contexts, other thread might have
	* already allocated needed one.
	*/
	ctx = dmar_find_ctx_locked(dmar, rid);
	if (ctx == NULL) {
	domain = domain1;
	ctx = ctx1;
	dmar_ctx_link(ctx);
	ctx->context.tag->owner = dev;
	device_tag_init(ctx, dev);

	/*
	* This is the first activated context for the
	* DMAR unit. Enable the translation after
	* everything is set up.
	*/
	if (LIST_EMPTY(&dmar->domains))
	enable = true;
	LIST_INSERT_HEAD(&dmar->domains, domain, link);
	ctx_id_entry_init(ctx, ctxp, false, bus);
	if (dev != NULL) {
	device_printf(dev,
	"dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d "
	"agaw %d %s-mapped\n",
	dmar->iommu.unit, dmar->segment, bus, slot,
	func, rid, domain->domain, domain->mgaw,
	domain->agaw, id_mapped ? "id" : "re");
	}
	dmar_unmap_pgtbl(sf);
	} else {
	dmar_unmap_pgtbl(sf);
	dmar_domain_destroy(domain1);
	/* Nothing needs to be done to destroy ctx1. */
	free(ctx1, M_DMAR_CTX);
	domain = (struct dmar_domain *)ctx->context.domain;
	ctx->refs++; /* tag referenced us */
	}
	} else {
	domain = (struct dmar_domain *)ctx->context.domain;
	if (ctx->context.tag->owner == NULL)
	ctx->context.tag->owner = dev;
	ctx->refs++; /* tag referenced us */
	}

	error = dmar_flush_for_ctx_entry(dmar, enable);
	if (error != 0) {
	dmar_free_ctx_locked(dmar, ctx);
	TD_PINNED_ASSERT;
	return (NULL);
	}

	/*
	* The dmar lock was potentially dropped between check for the
	* empty context list and now. Recheck the state of GCMD_TE
	* to avoid unneeded command.
	*/
	if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
	error = dmar_enable_translation(dmar);
	if (error == 0) {
	if (bootverbose) {
	printf("dmar%d: enabled translation\n",
	dmar->iommu.unit);
	}
	} else {
	printf("dmar%d: enabling translation failed, "
	"error %d\n", dmar->iommu.unit, error);
	dmar_free_ctx_locked(dmar, ctx);
	TD_PINNED_ASSERT;
	return (NULL);
	}
	}
	DMAR_UNLOCK(dmar);
	TD_PINNED_ASSERT;
	return (ctx);
	}

	struct dmar_ctx *
	dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid,
	bool id_mapped, bool rmrr_init)
	{
	int dev_domain, dev_path_len, dev_busno;

	dev_domain = pci_get_domain(dev);
	dev_path_len = dmar_dev_depth(dev);
	ACPI_DMAR_PCI_PATH dev_path[dev_path_len];
	dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len);
	return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno,
	dev_path, dev_path_len, id_mapped, rmrr_init));
	}

	struct dmar_ctx *
	dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid,
	int dev_domain, int dev_busno,
	const void *dev_path, int dev_path_len,
	bool id_mapped, bool rmrr_init)
	{

	return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno,
	dev_path, dev_path_len, id_mapped, rmrr_init));
	}

	int
	dmar_move_ctx_to_domain(struct dmar_domain domain, struct dmar_ctx ctx)
	{
	struct dmar_unit *dmar;
	struct dmar_domain *old_domain;
	dmar_ctx_entry_t *ctxp;
	struct sf_buf *sf;
	int error;

	dmar = domain->dmar;
	old_domain = (struct dmar_domain *)ctx->context.domain;
	if (domain == old_domain)
	return (0);
	KASSERT(old_domain->iodom.iommu == domain->iodom.iommu,
	("domain %p %u moving between dmars %u %u", domain,
	domain->domain, old_domain->iodom.iommu->unit,
	domain->iodom.iommu->unit));
	TD_PREP_PINNED_ASSERT;

	ctxp = dmar_map_ctx_entry(ctx, &sf);
	DMAR_LOCK(dmar);
	dmar_ctx_unlink(ctx);
	ctx->context.domain = &domain->iodom;
	dmar_ctx_link(ctx);
	ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100);
	dmar_unmap_pgtbl(sf);
	error = dmar_flush_for_ctx_entry(dmar, true);
	/* If flush failed, rolling back would not work as well. */
	printf("dmar%d rid %x domain %d->%d %s-mapped\n",
	dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain,
	(domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ? "id" : "re");
	dmar_unref_domain_locked(dmar, old_domain);
	TD_PINNED_ASSERT;
	return (error);
	}

	static void
	dmar_unref_domain_locked(struct dmar_unit dmar, struct dmar_domain domain)
	{

	DMAR_ASSERT_LOCKED(dmar);
	KASSERT(domain->refs >= 1,
	("dmar %d domain %p refs %u", dmar->iommu.unit, domain,
	domain->refs));
	KASSERT(domain->refs > domain->ctx_cnt,
	("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain,
	domain->refs, domain->ctx_cnt));

	if (domain->refs > 1) {
	domain->refs--;
	DMAR_UNLOCK(dmar);
	return;
	}

	KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0,
	("lost ref on RMRR domain %p", domain));

	LIST_REMOVE(domain, link);
	DMAR_UNLOCK(dmar);

	taskqueue_drain(dmar->iommu.delayed_taskqueue,
	&domain->iodom.unload_task);
	dmar_domain_destroy(domain);
	}

	void
	dmar_free_ctx_locked(struct dmar_unit dmar, struct dmar_ctx ctx)
	{
	struct sf_buf *sf;
	dmar_ctx_entry_t *ctxp;
	struct dmar_domain *domain;

	DMAR_ASSERT_LOCKED(dmar);
	KASSERT(ctx->refs >= 1,
	("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));

	/*
	* If our reference is not last, only the dereference should
	* be performed.
	*/
	if (ctx->refs > 1) {
	ctx->refs--;
	DMAR_UNLOCK(dmar);
	return;
	}

	KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
	("lost ref on disabled ctx %p", ctx));

	/*
	* Otherwise, the context entry must be cleared before the
	* page table is destroyed. The mapping of the context
	* entries page could require sleep, unlock the dmar.
	*/
	DMAR_UNLOCK(dmar);
	TD_PREP_PINNED_ASSERT;
	ctxp = dmar_map_ctx_entry(ctx, &sf);
	DMAR_LOCK(dmar);
	KASSERT(ctx->refs >= 1,
	("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs));

	/*
	* Other thread might have referenced the context, in which
	* case again only the dereference should be performed.
	*/
	if (ctx->refs > 1) {
	ctx->refs--;
	DMAR_UNLOCK(dmar);
	dmar_unmap_pgtbl(sf);
	TD_PINNED_ASSERT;
	return;
	}

	KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
	("lost ref on disabled ctx %p", ctx));

	/*
	* Clear the context pointer and flush the caches.
	* XXXKIB: cannot do this if any RMRR entries are still present.
	*/
	dmar_pte_clear(&ctxp->ctx1);
	ctxp->ctx2 = 0;
	dmar_flush_ctx_to_ram(dmar, ctxp);
	dmar_inv_ctx_glob(dmar);
	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
	if (dmar->qi_enabled)
	dmar_qi_invalidate_iotlb_glob_locked(dmar);
	else
	dmar_inv_iotlb_glob(dmar);
	}
	dmar_unmap_pgtbl(sf);
	domain = (struct dmar_domain *)ctx->context.domain;
	dmar_ctx_unlink(ctx);
	free(ctx->context.tag, M_DMAR_CTX);
	free(ctx, M_DMAR_CTX);
	dmar_unref_domain_locked(dmar, domain);
	TD_PINNED_ASSERT;
	}

	void
	dmar_free_ctx(struct dmar_ctx *ctx)
	{
	struct dmar_unit *dmar;

	dmar = (struct dmar_unit *)ctx->context.domain->iommu;
	DMAR_LOCK(dmar);
	dmar_free_ctx_locked(dmar, ctx);
	}

	/*
	* Returns with the domain locked.
	*/
	struct dmar_ctx *
	dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
	{
	struct dmar_domain *domain;
	struct dmar_ctx *ctx;

	DMAR_ASSERT_LOCKED(dmar);

	LIST_FOREACH(domain, &dmar->domains, link) {
	LIST_FOREACH(ctx, &domain->contexts, link) {
	if (ctx->rid == rid)
	return (ctx);
	}
	}
	return (NULL);
	}

	void
	dmar_domain_free_entry(struct iommu_map_entry *entry, bool free)
	{
	struct iommu_domain *domain;

	domain = entry->domain;
	IOMMU_DOMAIN_LOCK(domain);
	if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
	iommu_gas_free_region(domain, entry);
	else
	iommu_gas_free_space(domain, entry);
	IOMMU_DOMAIN_UNLOCK(domain);
	if (free)
	iommu_gas_free_entry(domain, entry);
	else
	entry->flags = 0;
	}

	void
	dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free)
	{
	struct dmar_domain *domain;
	struct dmar_unit *unit;

	domain = (struct dmar_domain *)entry->domain;
	unit = (struct dmar_unit *)domain->iodom.iommu;
	if (unit->qi_enabled) {
	DMAR_LOCK(unit);
	dmar_qi_invalidate_locked((struct dmar_domain *)entry->domain,
	entry->start, entry->end - entry->start, &entry->gseq,
	true);
	if (!free)
	entry->flags \|= IOMMU_MAP_ENTRY_QI_NF;
	TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
	DMAR_UNLOCK(unit);
	} else {
	domain_flush_iotlb_sync((struct dmar_domain *)entry->domain,
	entry->start, entry->end - entry->start);
	dmar_domain_free_entry(entry, free);
	}
	}

	static bool
	dmar_domain_unload_emit_wait(struct dmar_domain *domain,
	struct iommu_map_entry *entry)
	{

	if (TAILQ_NEXT(entry, dmamap_link) == NULL)
	return (true);
	return (domain->batch_no++ % dmar_batch_coalesce == 0);
	}

	void
	dmar_domain_unload(struct dmar_domain *domain,
	struct iommu_map_entries_tailq *entries, bool cansleep)
	{
	struct dmar_unit *unit;
	struct iommu_map_entry entry, entry1;
	int error;

	unit = (struct dmar_unit *)domain->iodom.iommu;

	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
	KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
	("not mapped entry %p %p", domain, entry));
	error = domain_unmap_buf(domain, entry->start, entry->end -
	entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
	KASSERT(error == 0, ("unmap %p error %d", domain, error));
	if (!unit->qi_enabled) {
	domain_flush_iotlb_sync(domain, entry->start,
	entry->end - entry->start);
	TAILQ_REMOVE(entries, entry, dmamap_link);
	dmar_domain_free_entry(entry, true);
	}
	}
	if (TAILQ_EMPTY(entries))
	return;

	KASSERT(unit->qi_enabled, ("loaded entry left"));
	DMAR_LOCK(unit);
	TAILQ_FOREACH(entry, entries, dmamap_link) {
	dmar_qi_invalidate_locked(domain, entry->start, entry->end -
	entry->start, &entry->gseq,
	dmar_domain_unload_emit_wait(domain, entry));
	}
	TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
	DMAR_UNLOCK(unit);
	}

	static void
	dmar_domain_unload_task(void *arg, int pending)
	{
	struct dmar_domain *domain;
	struct iommu_map_entries_tailq entries;

	domain = arg;
	TAILQ_INIT(&entries);

	for (;;) {
	DMAR_DOMAIN_LOCK(domain);
	TAILQ_SWAP(&domain->iodom.unload_entries, &entries,
	iommu_map_entry, dmamap_link);
	DMAR_DOMAIN_UNLOCK(domain);
	if (TAILQ_EMPTY(&entries))
	break;
	dmar_domain_unload(domain, &entries, true);
	}
	}

	struct iommu_ctx *
	iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
	bool id_mapped, bool rmrr_init)
	{
	struct dmar_unit *dmar;
	struct dmar_ctx *ret;

	dmar = (struct dmar_unit *)iommu;

	ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init);

	return ((struct iommu_ctx *)ret);
	}

	void
	iommu_free_ctx_locked(struct iommu_unit iommu, struct iommu_ctx context)
	{
	struct dmar_unit *dmar;
	struct dmar_ctx *ctx;

	dmar = (struct dmar_unit *)iommu;
	ctx = (struct dmar_ctx *)context;

	dmar_free_ctx_locked(dmar, ctx);
	}

	void
	iommu_free_ctx(struct iommu_ctx *context)
	{
	struct dmar_unit *dmar;
	struct dmar_ctx *ctx;

	ctx = (struct dmar_ctx *)context;
	dmar = (struct dmar_unit *)ctx->context.domain->iommu;

	dmar_free_ctx(ctx);
	}

	void
	iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free)
	{

	dmar_domain_unload_entry(entry, free);
	}

	void
	iommu_domain_unload(struct iommu_domain *iodom,
	struct iommu_map_entries_tailq *entries, bool cansleep)
	{
	struct dmar_domain *domain;

	domain = (struct dmar_domain *)iodom;

	dmar_domain_unload(domain, entries, cansleep);
	}
	diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h
	index fba5e36f95cd..eae01bcc134d 100644
	--- a/sys/x86/iommu/intel_dmar.h
	+++ b/sys/x86/iommu/intel_dmar.h
	@@ -1,458 +1,446 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013-2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#ifndef __X86_IOMMU_INTEL_DMAR_H
	#define __X86_IOMMU_INTEL_DMAR_H

	#include <dev/iommu/iommu.h>

	struct dmar_unit;

	/*
	* Locking annotations:
	* (u) - Protected by iommu unit lock
	* (d) - Protected by domain lock
	* (c) - Immutable after initialization
	*/

	/*
	* The domain abstraction. Most non-constant members of the domain
	* are protected by owning dmar unit lock, not by the domain lock.
	* Most important, the dmar lock protects the contexts list.
	*
	* The domain lock protects the address map for the domain, and list
	* of unload entries delayed.
	*
	* Page tables pages and pages content is protected by the vm object
	* lock pgtbl_obj, which contains the page tables pages.
	*/
	struct dmar_domain {
	struct iommu_domain iodom;
	int domain; /* (c) DID, written in context entry */
	int mgaw; /* (c) Real max address width */
	int agaw; /* (c) Adjusted guest address width */
	int pglvl; /* (c) The pagelevel */
	int awlvl; /* (c) The pagelevel as the bitmask,
	to set in context entry */
	u_int ctx_cnt; /* (u) Number of contexts owned */
	u_int refs; /* (u) Refs, including ctx */
	struct dmar_unit dmar; / (c) */
	LIST_ENTRY(dmar_domain) link; /* (u) Member in the dmar list */
	LIST_HEAD(, dmar_ctx) contexts; /* (u) */
	vm_object_t pgtbl_obj; /* (c) Page table pages */
	u_int batch_no;
	};

	struct dmar_ctx {
	struct iommu_ctx context;
	uint16_t rid; /* (c) pci RID */
	uint64_t last_fault_rec[2]; /* Last fault reported */
	LIST_ENTRY(dmar_ctx) link; /* (u) Member in the domain list */
	u_int refs; /* (u) References from tags */
	};

	#define DMAR_DOMAIN_PGLOCK(dom) VM_OBJECT_WLOCK((dom)->pgtbl_obj)
	#define DMAR_DOMAIN_PGTRYLOCK(dom) VM_OBJECT_TRYWLOCK((dom)->pgtbl_obj)
	#define DMAR_DOMAIN_PGUNLOCK(dom) VM_OBJECT_WUNLOCK((dom)->pgtbl_obj)
	#define DMAR_DOMAIN_ASSERT_PGLOCKED(dom) \
	VM_OBJECT_ASSERT_WLOCKED((dom)->pgtbl_obj)

	#define DMAR_DOMAIN_LOCK(dom) mtx_lock(&(dom)->iodom.lock)
	#define DMAR_DOMAIN_UNLOCK(dom) mtx_unlock(&(dom)->iodom.lock)
	#define DMAR_DOMAIN_ASSERT_LOCKED(dom) mtx_assert(&(dom)->iodom.lock, MA_OWNED)

	struct dmar_msi_data {
	int irq;
	int irq_rid;
	struct resource *irq_res;
	void *intr_handle;
	int (handler)(void );
	int msi_data_reg;
	int msi_addr_reg;
	int msi_uaddr_reg;
	void (enable_intr)(struct dmar_unit );
	void (disable_intr)(struct dmar_unit );
	const char *name;
	};

	#define DMAR_INTR_FAULT 0
	#define DMAR_INTR_QI 1
	#define DMAR_INTR_TOTAL 2

	struct dmar_unit {
	struct iommu_unit iommu;
	device_t dev;
	uint16_t segment;
	uint64_t base;

	/* Resources */
	int reg_rid;
	struct resource *regs;

	struct dmar_msi_data intrs[DMAR_INTR_TOTAL];

	/* Hardware registers cache */
	uint32_t hw_ver;
	uint64_t hw_cap;
	uint64_t hw_ecap;
	uint32_t hw_gcmd;

	/* Data for being a dmar */
	LIST_HEAD(, dmar_domain) domains;
	struct unrhdr *domids;
	vm_object_t ctx_obj;
	u_int barrier_flags;

	/* Fault handler data */
	struct mtx fault_lock;
	uint64_t *fault_log;
	int fault_log_head;
	int fault_log_tail;
	int fault_log_size;
	struct task fault_task;
	struct taskqueue *fault_taskqueue;

	/* QI */
	int qi_enabled;
	vm_offset_t inv_queue;
	vm_size_t inv_queue_size;
	uint32_t inv_queue_avail;
	uint32_t inv_queue_tail;
	volatile uint32_t inv_waitd_seq_hw; /* hw writes there on wait
	descr completion */
	uint64_t inv_waitd_seq_hw_phys;
	uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */
	u_int inv_waitd_gen; /* seq number generation AKA seq overflows */
	u_int inv_seq_waiters; /* count of waiters for seq */
	u_int inv_queue_full; /* informational counter */

	/* IR */
	int ir_enabled;
	vm_paddr_t irt_phys;
	dmar_irte_t *irt;
	u_int irte_cnt;
	vmem_t *irtids;

	/* Delayed freeing of map entries queue processing */
	struct iommu_map_entries_tailq tlb_flush_entries;
	struct task qi_task;
	struct taskqueue *qi_taskqueue;
	-
	- /*
	- * Bitmap of buses for which context must ignore slot:func,
	- * duplicating the page table pointer into all context table
	- * entries. This is a client-controlled quirk to support some
	- * NTBs.
	- */
	- uint32_t buswide_ctxs[(PCI_BUSMAX + 1) / NBBY / sizeof(uint32_t)];
	-
	};

	#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->iommu.lock)
	#define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->iommu.lock)
	#define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->iommu.lock, MA_OWNED)

	#define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock)
	#define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock)
	#define DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)

	#define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
	#define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)
	#define DMAR_X2APIC(dmar) \
	(x2apic_mode && ((dmar)->hw_ecap & DMAR_ECAP_EIM) != 0)

	/* Barrier ids */
	#define DMAR_BARRIER_RMRR 0
	#define DMAR_BARRIER_USEQ 1

	struct dmar_unit *dmar_find(device_t dev, bool verbose);
	struct dmar_unit dmar_find_hpet(device_t dev, uint16_t rid);
	struct dmar_unit dmar_find_ioapic(u_int apic_id, uint16_t rid);

	u_int dmar_nd2mask(u_int nd);
	bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
	int domain_set_agaw(struct dmar_domain *domain, int mgaw);
	int dmar_maxaddr2mgaw(struct dmar_unit *unit, iommu_gaddr_t maxaddr,
	bool allow_less);
	vm_pindex_t pglvl_max_pages(int pglvl);
	int domain_is_sp_lvl(struct dmar_domain *domain, int lvl);
	iommu_gaddr_t pglvl_page_size(int total_pglvl, int lvl);
	iommu_gaddr_t domain_page_size(struct dmar_domain *domain, int lvl);
	int calc_am(struct dmar_unit *unit, iommu_gaddr_t base, iommu_gaddr_t size,
	iommu_gaddr_t *isizep);
	struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags);
	void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags);
	void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
	struct sf_buf **sf);
	void dmar_unmap_pgtbl(struct sf_buf *sf);
	int dmar_load_root_entry_ptr(struct dmar_unit *unit);
	int dmar_inv_ctx_glob(struct dmar_unit *unit);
	int dmar_inv_iotlb_glob(struct dmar_unit *unit);
	int dmar_flush_write_bufs(struct dmar_unit *unit);
	void dmar_flush_pte_to_ram(struct dmar_unit unit, dmar_pte_t dst);
	void dmar_flush_ctx_to_ram(struct dmar_unit unit, dmar_ctx_entry_t dst);
	void dmar_flush_root_to_ram(struct dmar_unit unit, dmar_root_entry_t dst);
	int dmar_enable_translation(struct dmar_unit *unit);
	int dmar_disable_translation(struct dmar_unit *unit);
	int dmar_load_irt_ptr(struct dmar_unit *unit);
	int dmar_enable_ir(struct dmar_unit *unit);
	int dmar_disable_ir(struct dmar_unit *unit);
	bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
	void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);
	uint64_t dmar_get_timeout(void);
	void dmar_update_timeout(uint64_t newval);

	int dmar_fault_intr(void *arg);
	void dmar_enable_fault_intr(struct dmar_unit *unit);
	void dmar_disable_fault_intr(struct dmar_unit *unit);
	int dmar_init_fault_log(struct dmar_unit *unit);
	void dmar_fini_fault_log(struct dmar_unit *unit);

	int dmar_qi_intr(void *arg);
	void dmar_enable_qi_intr(struct dmar_unit *unit);
	void dmar_disable_qi_intr(struct dmar_unit *unit);
	int dmar_init_qi(struct dmar_unit *unit);
	void dmar_fini_qi(struct dmar_unit *unit);
	void dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t start,
	iommu_gaddr_t size, struct iommu_qi_genseq *psec, bool emit_wait);
	void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
	void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
	void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
	void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt);

	vm_object_t domain_get_idmap_pgtbl(struct dmar_domain *domain,
	iommu_gaddr_t maxaddr);
	void put_idmap_pgtbl(vm_object_t obj);
	int domain_map_buf(struct iommu_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags);
	int domain_unmap_buf(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size, int flags);
	void domain_flush_iotlb_sync(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size);
	int domain_alloc_pgtbl(struct dmar_domain *domain);
	void domain_free_pgtbl(struct dmar_domain *domain);

	int dmar_dev_depth(device_t child);
	void dmar_dev_path(device_t child, int busno, void path1, int depth);

	struct dmar_ctx dmar_get_ctx_for_dev(struct dmar_unit dmar, device_t dev,
	uint16_t rid, bool id_mapped, bool rmrr_init);
	struct dmar_ctx dmar_get_ctx_for_devpath(struct dmar_unit dmar, uint16_t rid,
	int dev_domain, int dev_busno, const void *dev_path, int dev_path_len,
	bool id_mapped, bool rmrr_init);
	int dmar_move_ctx_to_domain(struct dmar_domain domain, struct dmar_ctx ctx);
	void dmar_free_ctx_locked(struct dmar_unit dmar, struct dmar_ctx ctx);
	void dmar_free_ctx(struct dmar_ctx *ctx);
	struct dmar_ctx dmar_find_ctx_locked(struct dmar_unit dmar, uint16_t rid);
	void dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free);
	void dmar_domain_unload(struct dmar_domain *domain,
	struct iommu_map_entries_tailq *entries, bool cansleep);
	void dmar_domain_free_entry(struct iommu_map_entry *entry, bool free);

	void dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain,
	int dev_busno, const void *dev_path, int dev_path_len,
	struct iommu_map_entries_tailq *rmrr_entries);
	int dmar_instantiate_rmrr_ctxs(struct iommu_unit *dmar);

	void dmar_quirks_post_ident(struct dmar_unit *dmar);
	void dmar_quirks_pre_use(struct iommu_unit *dmar);

	int dmar_init_irt(struct dmar_unit *unit);
	void dmar_fini_irt(struct dmar_unit *unit);

	-void dmar_set_buswide_ctx(struct iommu_unit *unit, u_int busno);
	-bool dmar_is_buswide_ctx(struct dmar_unit *unit, u_int busno);
	-
	extern iommu_haddr_t dmar_high;
	extern int haw;
	extern int dmar_tbl_pagecnt;
	extern int dmar_batch_coalesce;

	static inline uint32_t
	dmar_read4(const struct dmar_unit *unit, int reg)
	{

	return (bus_read_4(unit->regs, reg));
	}

	static inline uint64_t
	dmar_read8(const struct dmar_unit *unit, int reg)
	{
	#ifdef __i386__
	uint32_t high, low;

	low = bus_read_4(unit->regs, reg);
	high = bus_read_4(unit->regs, reg + 4);
	return (low \| ((uint64_t)high << 32));
	#else
	return (bus_read_8(unit->regs, reg));
	#endif
	}

	static inline void
	dmar_write4(const struct dmar_unit *unit, int reg, uint32_t val)
	{

	KASSERT(reg != DMAR_GCMD_REG \|\| (val & DMAR_GCMD_TE) ==
	(unit->hw_gcmd & DMAR_GCMD_TE),
	("dmar%d clearing TE 0x%08x 0x%08x", unit->iommu.unit,
	unit->hw_gcmd, val));
	bus_write_4(unit->regs, reg, val);
	}

	static inline void
	dmar_write8(const struct dmar_unit *unit, int reg, uint64_t val)
	{

	KASSERT(reg != DMAR_GCMD_REG, ("8byte GCMD write"));
	#ifdef __i386__
	uint32_t high, low;

	low = val;
	high = val >> 32;
	bus_write_4(unit->regs, reg, low);
	bus_write_4(unit->regs, reg + 4, high);
	#else
	bus_write_8(unit->regs, reg, val);
	#endif
	}

	/*
	* dmar_pte_store and dmar_pte_clear ensure that on i386, 32bit writes
	* are issued in the correct order. For store, the lower word,
	* containing the P or R and W bits, is set only after the high word
	* is written. For clear, the P bit is cleared first, then the high
	* word is cleared.
	*
	* dmar_pte_update updates the pte. For amd64, the update is atomic.
	* For i386, it first disables the entry by clearing the word
	* containing the P bit, and then defer to dmar_pte_store. The locked
	* cmpxchg8b is probably available on any machine having DMAR support,
	* but interrupt translation table may be mapped uncached.
	*/
	static inline void
	dmar_pte_store1(volatile uint64_t *dst, uint64_t val)
	{
	#ifdef __i386__
	volatile uint32_t *p;
	uint32_t hi, lo;

	hi = val >> 32;
	lo = val;
	p = (volatile uint32_t *)dst;
	*(p + 1) = hi;
	*p = lo;
	#else
	*dst = val;
	#endif
	}

	static inline void
	dmar_pte_store(volatile uint64_t *dst, uint64_t val)
	{

	KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
	dst, (uintmax_t)*dst, (uintmax_t)val));
	dmar_pte_store1(dst, val);
	}

	static inline void
	dmar_pte_update(volatile uint64_t *dst, uint64_t val)
	{

	#ifdef __i386__
	volatile uint32_t *p;

	p = (volatile uint32_t *)dst;
	*p = 0;
	#endif
	dmar_pte_store1(dst, val);
	}

	static inline void
	dmar_pte_clear(volatile uint64_t *dst)
	{
	#ifdef __i386__
	volatile uint32_t *p;

	p = (volatile uint32_t *)dst;
	*p = 0;
	*(p + 1) = 0;
	#else
	*dst = 0;
	#endif
	}

	extern struct timespec dmar_hw_timeout;

	#define DMAR_WAIT_UNTIL(cond) \
	{ \
	struct timespec last, curr; \
	bool forever; \
	\
	if (dmar_hw_timeout.tv_sec == 0 && \
	dmar_hw_timeout.tv_nsec == 0) { \
	forever = true; \
	} else { \
	forever = false; \
	nanouptime(&curr); \
	timespecadd(&curr, &dmar_hw_timeout, &last); \
	} \
	for (;;) { \
	if (cond) { \
	error = 0; \
	break; \
	} \
	nanouptime(&curr); \
	if (!forever && timespeccmp(&last, &curr, <)) { \
	error = ETIMEDOUT; \
	break; \
	} \
	cpu_spinwait(); \
	} \
	}

	#ifdef INVARIANTS
	#define TD_PREP_PINNED_ASSERT \
	int old_td_pinned; \
	old_td_pinned = curthread->td_pinned
	#define TD_PINNED_ASSERT \
	KASSERT(curthread->td_pinned == old_td_pinned, \
	("pin count leak: %d %d %s:%d", curthread->td_pinned, \
	old_td_pinned, __FILE__, __LINE__))
	#else
	#define TD_PREP_PINNED_ASSERT
	#define TD_PINNED_ASSERT
	#endif

	#endif
	diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c
	index a8a2bc7cf770..17fea13d1387 100644
	--- a/sys/x86/iommu/intel_drv.c
	+++ b/sys/x86/iommu/intel_drv.c
	@@ -1,1363 +1,1340 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013-2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_acpi.h"
	#if defined(__amd64__)
	#define DEV_APIC
	#else
	#include "opt_apic.h"
	#endif
	#include "opt_ddb.h"

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/memdesc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/rwlock.h>
	#include <sys/smp.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/vmem.h>
	-#include <machine/bus.h>
	-#include <machine/pci_cfgreg.h>
	-#include <contrib/dev/acpica/include/acpi.h>
	-#include <contrib/dev/acpica/include/accommon.h>
	-#include <dev/acpica/acpivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_map.h>
	-#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	-#include <dev/iommu/busdma_iommu.h>
	+#include <contrib/dev/acpica/include/acpi.h>
	+#include <contrib/dev/acpica/include/accommon.h>
	+#include <dev/acpica/acpivar.h>
	#include <dev/pci/pcireg.h>
	#include <dev/pci/pcivar.h>
	+#include <machine/bus.h>
	+#include <machine/pci_cfgreg.h>
	+#include <x86/include/busdma_impl.h>
	+#include <dev/iommu/busdma_iommu.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>

	#ifdef DEV_APIC
	#include "pcib_if.h"
	#include <machine/intr_machdep.h>
	#include <x86/apicreg.h>
	#include <x86/apicvar.h>
	#endif

	#define DMAR_FAULT_IRQ_RID 0
	#define DMAR_QI_IRQ_RID 1
	#define DMAR_REG_RID 2

	static devclass_t dmar_devclass;
	static device_t *dmar_devs;
	static int dmar_devcnt;

	typedef int (dmar_iter_t)(ACPI_DMAR_HEADER , void *);

	static void
	dmar_iterate_tbl(dmar_iter_t iter, void *arg)
	{
	ACPI_TABLE_DMAR *dmartbl;
	ACPI_DMAR_HEADER *dmarh;
	char ptr, ptrend;
	ACPI_STATUS status;

	status = AcpiGetTable(ACPI_SIG_DMAR, 1, (ACPI_TABLE_HEADER **)&dmartbl);
	if (ACPI_FAILURE(status))
	return;
	ptr = (char )dmartbl + sizeof(dmartbl);
	ptrend = (char *)dmartbl + dmartbl->Header.Length;
	for (;;) {
	if (ptr >= ptrend)
	break;
	dmarh = (ACPI_DMAR_HEADER *)ptr;
	if (dmarh->Length <= 0) {
	printf("dmar_identify: corrupted DMAR table, l %d\n",
	dmarh->Length);
	break;
	}
	ptr += dmarh->Length;
	if (!iter(dmarh, arg))
	break;
	}
	AcpiPutTable((ACPI_TABLE_HEADER *)dmartbl);
	}

	struct find_iter_args {
	int i;
	ACPI_DMAR_HARDWARE_UNIT *res;
	};

	static int
	dmar_find_iter(ACPI_DMAR_HEADER dmarh, void arg)
	{
	struct find_iter_args *fia;

	if (dmarh->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT)
	return (1);

	fia = arg;
	if (fia->i == 0) {
	fia->res = (ACPI_DMAR_HARDWARE_UNIT *)dmarh;
	return (0);
	}
	fia->i--;
	return (1);
	}

	static ACPI_DMAR_HARDWARE_UNIT *
	dmar_find_by_index(int idx)
	{
	struct find_iter_args fia;

	fia.i = idx;
	fia.res = NULL;
	dmar_iterate_tbl(dmar_find_iter, &fia);
	return (fia.res);
	}

	static int
	dmar_count_iter(ACPI_DMAR_HEADER dmarh, void arg)
	{

	if (dmarh->Type == ACPI_DMAR_TYPE_HARDWARE_UNIT)
	dmar_devcnt++;
	return (1);
	}

	static int dmar_enable = 0;
	static void
	dmar_identify(driver_t *driver, device_t parent)
	{
	ACPI_TABLE_DMAR *dmartbl;
	ACPI_DMAR_HARDWARE_UNIT *dmarh;
	ACPI_STATUS status;
	int i, error;

	if (acpi_disabled("dmar"))
	return;
	TUNABLE_INT_FETCH("hw.dmar.enable", &dmar_enable);
	if (!dmar_enable)
	return;
	status = AcpiGetTable(ACPI_SIG_DMAR, 1, (ACPI_TABLE_HEADER **)&dmartbl);
	if (ACPI_FAILURE(status))
	return;
	haw = dmartbl->Width + 1;
	if ((1ULL << (haw + 1)) > BUS_SPACE_MAXADDR)
	dmar_high = BUS_SPACE_MAXADDR;
	else
	dmar_high = 1ULL << (haw + 1);
	if (bootverbose) {
	printf("DMAR HAW=%d flags=<%b>\n", dmartbl->Width,
	(unsigned)dmartbl->Flags,
	"\020\001INTR_REMAP\002X2APIC_OPT_OUT");
	}
	AcpiPutTable((ACPI_TABLE_HEADER *)dmartbl);

	dmar_iterate_tbl(dmar_count_iter, NULL);
	if (dmar_devcnt == 0)
	return;
	dmar_devs = malloc(sizeof(device_t) * dmar_devcnt, M_DEVBUF,
	M_WAITOK \| M_ZERO);
	for (i = 0; i < dmar_devcnt; i++) {
	dmarh = dmar_find_by_index(i);
	if (dmarh == NULL) {
	printf("dmar_identify: cannot find HWUNIT %d\n", i);
	continue;
	}
	dmar_devs[i] = BUS_ADD_CHILD(parent, 1, "dmar", i);
	if (dmar_devs[i] == NULL) {
	printf("dmar_identify: cannot create instance %d\n", i);
	continue;
	}
	error = bus_set_resource(dmar_devs[i], SYS_RES_MEMORY,
	DMAR_REG_RID, dmarh->Address, PAGE_SIZE);
	if (error != 0) {
	printf(
	"dmar%d: unable to alloc register window at 0x%08jx: error %d\n",
	i, (uintmax_t)dmarh->Address, error);
	device_delete_child(parent, dmar_devs[i]);
	dmar_devs[i] = NULL;
	}
	}
	}

	static int
	dmar_probe(device_t dev)
	{

	if (acpi_get_handle(dev) != NULL)
	return (ENXIO);
	device_set_desc(dev, "DMA remap");
	return (BUS_PROBE_NOWILDCARD);
	}

	static void
	dmar_release_intr(device_t dev, struct dmar_unit *unit, int idx)
	{
	struct dmar_msi_data *dmd;

	dmd = &unit->intrs[idx];
	if (dmd->irq == -1)
	return;
	bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
	bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
	bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
	PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)),
	dev, dmd->irq);
	dmd->irq = -1;
	}

	static void
	dmar_release_resources(device_t dev, struct dmar_unit *unit)
	{
	int i;

	iommu_fini_busdma(&unit->iommu);
	dmar_fini_irt(unit);
	dmar_fini_qi(unit);
	dmar_fini_fault_log(unit);
	for (i = 0; i < DMAR_INTR_TOTAL; i++)
	dmar_release_intr(dev, unit, i);
	if (unit->regs != NULL) {
	bus_deactivate_resource(dev, SYS_RES_MEMORY, unit->reg_rid,
	unit->regs);
	bus_release_resource(dev, SYS_RES_MEMORY, unit->reg_rid,
	unit->regs);
	unit->regs = NULL;
	}
	if (unit->domids != NULL) {
	delete_unrhdr(unit->domids);
	unit->domids = NULL;
	}
	if (unit->ctx_obj != NULL) {
	vm_object_deallocate(unit->ctx_obj);
	unit->ctx_obj = NULL;
	}
	}

	static int
	dmar_alloc_irq(device_t dev, struct dmar_unit *unit, int idx)
	{
	device_t pcib;
	struct dmar_msi_data *dmd;
	uint64_t msi_addr;
	uint32_t msi_data;
	int error;

	dmd = &unit->intrs[idx];
	pcib = device_get_parent(device_get_parent(dev)); /* Really not pcib */
	error = PCIB_ALLOC_MSIX(pcib, dev, &dmd->irq);
	if (error != 0) {
	device_printf(dev, "cannot allocate %s interrupt, %d\n",
	dmd->name, error);
	goto err1;
	}
	error = bus_set_resource(dev, SYS_RES_IRQ, dmd->irq_rid,
	dmd->irq, 1);
	if (error != 0) {
	device_printf(dev, "cannot set %s interrupt resource, %d\n",
	dmd->name, error);
	goto err2;
	}
	dmd->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
	&dmd->irq_rid, RF_ACTIVE);
	if (dmd->irq_res == NULL) {
	device_printf(dev,
	"cannot allocate resource for %s interrupt\n", dmd->name);
	error = ENXIO;
	goto err3;
	}
	error = bus_setup_intr(dev, dmd->irq_res, INTR_TYPE_MISC,
	dmd->handler, NULL, unit, &dmd->intr_handle);
	if (error != 0) {
	device_printf(dev, "cannot setup %s interrupt, %d\n",
	dmd->name, error);
	goto err4;
	}
	bus_describe_intr(dev, dmd->irq_res, dmd->intr_handle, "%s", dmd->name);
	error = PCIB_MAP_MSI(pcib, dev, dmd->irq, &msi_addr, &msi_data);
	if (error != 0) {
	device_printf(dev, "cannot map %s interrupt, %d\n",
	dmd->name, error);
	goto err5;
	}
	dmar_write4(unit, dmd->msi_data_reg, msi_data);
	dmar_write4(unit, dmd->msi_addr_reg, msi_addr);
	/* Only for xAPIC mode */
	dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32);
	return (0);

	err5:
	bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
	err4:
	bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
	err3:
	bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
	err2:
	PCIB_RELEASE_MSIX(pcib, dev, dmd->irq);
	dmd->irq = -1;
	err1:
	return (error);
	}

	#ifdef DEV_APIC
	static int
	dmar_remap_intr(device_t dev, device_t child, u_int irq)
	{
	struct dmar_unit *unit;
	struct dmar_msi_data *dmd;
	uint64_t msi_addr;
	uint32_t msi_data;
	int i, error;

	unit = device_get_softc(dev);
	for (i = 0; i < DMAR_INTR_TOTAL; i++) {
	dmd = &unit->intrs[i];
	if (irq == dmd->irq) {
	error = PCIB_MAP_MSI(device_get_parent(
	device_get_parent(dev)),
	dev, irq, &msi_addr, &msi_data);
	if (error != 0)
	return (error);
	DMAR_LOCK(unit);
	(dmd->disable_intr)(unit);
	dmar_write4(unit, dmd->msi_data_reg, msi_data);
	dmar_write4(unit, dmd->msi_addr_reg, msi_addr);
	dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32);
	(dmd->enable_intr)(unit);
	DMAR_UNLOCK(unit);
	return (0);
	}
	}
	return (ENOENT);
	}
	#endif

	static void
	dmar_print_caps(device_t dev, struct dmar_unit *unit,
	ACPI_DMAR_HARDWARE_UNIT *dmaru)
	{
	uint32_t caphi, ecaphi;

	device_printf(dev, "regs@0x%08jx, ver=%d.%d, seg=%d, flags=<%b>\n",
	(uintmax_t)dmaru->Address, DMAR_MAJOR_VER(unit->hw_ver),
	DMAR_MINOR_VER(unit->hw_ver), dmaru->Segment,
	dmaru->Flags, "\020\001INCLUDE_ALL_PCI");
	caphi = unit->hw_cap >> 32;
	device_printf(dev, "cap=%b,", (u_int)unit->hw_cap,
	"\020\004AFL\005WBF\006PLMR\007PHMR\010CM\027ZLR\030ISOCH");
	printf("%b, ", caphi, "\020\010PSI\027DWD\030DRD\031FL1GP\034PSI");
	printf("ndoms=%d, sagaw=%d, mgaw=%d, fro=%d, nfr=%d, superp=%d",
	DMAR_CAP_ND(unit->hw_cap), DMAR_CAP_SAGAW(unit->hw_cap),
	DMAR_CAP_MGAW(unit->hw_cap), DMAR_CAP_FRO(unit->hw_cap),
	DMAR_CAP_NFR(unit->hw_cap), DMAR_CAP_SPS(unit->hw_cap));
	if ((unit->hw_cap & DMAR_CAP_PSI) != 0)
	printf(", mamv=%d", DMAR_CAP_MAMV(unit->hw_cap));
	printf("\n");
	ecaphi = unit->hw_ecap >> 32;
	device_printf(dev, "ecap=%b,", (u_int)unit->hw_ecap,
	"\020\001C\002QI\003DI\004IR\005EIM\007PT\010SC\031ECS\032MTS"
	"\033NEST\034DIS\035PASID\036PRS\037ERS\040SRS");
	printf("%b, ", ecaphi, "\020\002NWFS\003EAFS");
	printf("mhmw=%d, iro=%d\n", DMAR_ECAP_MHMV(unit->hw_ecap),
	DMAR_ECAP_IRO(unit->hw_ecap));
	}

	static int
	dmar_attach(device_t dev)
	{
	struct dmar_unit *unit;
	ACPI_DMAR_HARDWARE_UNIT *dmaru;
	uint64_t timeout;
	int i, error;

	unit = device_get_softc(dev);
	unit->dev = dev;
	unit->iommu.unit = device_get_unit(dev);
	dmaru = dmar_find_by_index(unit->iommu.unit);
	if (dmaru == NULL)
	return (EINVAL);
	unit->segment = dmaru->Segment;
	unit->base = dmaru->Address;
	unit->reg_rid = DMAR_REG_RID;
	unit->regs = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	&unit->reg_rid, RF_ACTIVE);
	if (unit->regs == NULL) {
	device_printf(dev, "cannot allocate register window\n");
	return (ENOMEM);
	}
	unit->hw_ver = dmar_read4(unit, DMAR_VER_REG);
	unit->hw_cap = dmar_read8(unit, DMAR_CAP_REG);
	unit->hw_ecap = dmar_read8(unit, DMAR_ECAP_REG);
	if (bootverbose)
	dmar_print_caps(dev, unit, dmaru);
	dmar_quirks_post_ident(unit);

	timeout = dmar_get_timeout();
	TUNABLE_UINT64_FETCH("hw.dmar.timeout", &timeout);
	dmar_update_timeout(timeout);

	for (i = 0; i < DMAR_INTR_TOTAL; i++)
	unit->intrs[i].irq = -1;

	unit->intrs[DMAR_INTR_FAULT].name = "fault";
	unit->intrs[DMAR_INTR_FAULT].irq_rid = DMAR_FAULT_IRQ_RID;
	unit->intrs[DMAR_INTR_FAULT].handler = dmar_fault_intr;
	unit->intrs[DMAR_INTR_FAULT].msi_data_reg = DMAR_FEDATA_REG;
	unit->intrs[DMAR_INTR_FAULT].msi_addr_reg = DMAR_FEADDR_REG;
	unit->intrs[DMAR_INTR_FAULT].msi_uaddr_reg = DMAR_FEUADDR_REG;
	unit->intrs[DMAR_INTR_FAULT].enable_intr = dmar_enable_fault_intr;
	unit->intrs[DMAR_INTR_FAULT].disable_intr = dmar_disable_fault_intr;
	error = dmar_alloc_irq(dev, unit, DMAR_INTR_FAULT);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	return (error);
	}
	if (DMAR_HAS_QI(unit)) {
	unit->intrs[DMAR_INTR_QI].name = "qi";
	unit->intrs[DMAR_INTR_QI].irq_rid = DMAR_QI_IRQ_RID;
	unit->intrs[DMAR_INTR_QI].handler = dmar_qi_intr;
	unit->intrs[DMAR_INTR_QI].msi_data_reg = DMAR_IEDATA_REG;
	unit->intrs[DMAR_INTR_QI].msi_addr_reg = DMAR_IEADDR_REG;
	unit->intrs[DMAR_INTR_QI].msi_uaddr_reg = DMAR_IEUADDR_REG;
	unit->intrs[DMAR_INTR_QI].enable_intr = dmar_enable_qi_intr;
	unit->intrs[DMAR_INTR_QI].disable_intr = dmar_disable_qi_intr;
	error = dmar_alloc_irq(dev, unit, DMAR_INTR_QI);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	return (error);
	}
	}

	mtx_init(&unit->iommu.lock, "dmarhw", NULL, MTX_DEF);
	unit->domids = new_unrhdr(0, dmar_nd2mask(DMAR_CAP_ND(unit->hw_cap)),
	&unit->iommu.lock);
	LIST_INIT(&unit->domains);

	/*
	* 9.2 "Context Entry":
	* When Caching Mode (CM) field is reported as Set, the
	* domain-id value of zero is architecturally reserved.
	* Software must not use domain-id value of zero
	* when CM is Set.
	*/
	if ((unit->hw_cap & DMAR_CAP_CM) != 0)
	alloc_unr_specific(unit->domids, 0);

	unit->ctx_obj = vm_pager_allocate(OBJT_PHYS, NULL, IDX_TO_OFF(1 +
	DMAR_CTX_CNT), 0, 0, NULL);

	/*
	* Allocate and load the root entry table pointer. Enable the
	* address translation after the required invalidations are
	* done.
	*/
	dmar_pgalloc(unit->ctx_obj, 0, IOMMU_PGF_WAITOK \| IOMMU_PGF_ZERO);
	DMAR_LOCK(unit);
	error = dmar_load_root_entry_ptr(unit);
	if (error != 0) {
	DMAR_UNLOCK(unit);
	dmar_release_resources(dev, unit);
	return (error);
	}
	error = dmar_inv_ctx_glob(unit);
	if (error != 0) {
	DMAR_UNLOCK(unit);
	dmar_release_resources(dev, unit);
	return (error);
	}
	if ((unit->hw_ecap & DMAR_ECAP_DI) != 0) {
	error = dmar_inv_iotlb_glob(unit);
	if (error != 0) {
	DMAR_UNLOCK(unit);
	dmar_release_resources(dev, unit);
	return (error);
	}
	}

	DMAR_UNLOCK(unit);
	error = dmar_init_fault_log(unit);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	return (error);
	}
	error = dmar_init_qi(unit);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	return (error);
	}
	error = dmar_init_irt(unit);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	return (error);
	}
	error = iommu_init_busdma(&unit->iommu);
	if (error != 0) {
	dmar_release_resources(dev, unit);
	return (error);
	}

	#ifdef NOTYET
	DMAR_LOCK(unit);
	error = dmar_enable_translation(unit);
	if (error != 0) {
	DMAR_UNLOCK(unit);
	dmar_release_resources(dev, unit);
	return (error);
	}
	DMAR_UNLOCK(unit);
	#endif

	return (0);
	}

	static int
	dmar_detach(device_t dev)
	{

	return (EBUSY);
	}

	static int
	dmar_suspend(device_t dev)
	{

	return (0);
	}

	static int
	dmar_resume(device_t dev)
	{

	/* XXXKIB */
	return (0);
	}

	static device_method_t dmar_methods[] = {
	DEVMETHOD(device_identify, dmar_identify),
	DEVMETHOD(device_probe, dmar_probe),
	DEVMETHOD(device_attach, dmar_attach),
	DEVMETHOD(device_detach, dmar_detach),
	DEVMETHOD(device_suspend, dmar_suspend),
	DEVMETHOD(device_resume, dmar_resume),
	#ifdef DEV_APIC
	DEVMETHOD(bus_remap_intr, dmar_remap_intr),
	#endif
	DEVMETHOD_END
	};

	static driver_t dmar_driver = {
	"dmar",
	dmar_methods,
	sizeof(struct dmar_unit),
	};

	DRIVER_MODULE(dmar, acpi, dmar_driver, dmar_devclass, 0, 0);
	MODULE_DEPEND(dmar, acpi, 1, 1, 1);

	-void
	-dmar_set_buswide_ctx(struct iommu_unit *unit, u_int busno)
	-{
	- struct dmar_unit *dmar;
	-
	- dmar = (struct dmar_unit *)unit;
	-
	- MPASS(busno <= PCI_BUSMAX);
	- DMAR_LOCK(dmar);
	- dmar->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] \|=
	- 1 << (busno % (NBBY * sizeof(uint32_t)));
	- DMAR_UNLOCK(dmar);
	-}
	-
	-bool
	-dmar_is_buswide_ctx(struct dmar_unit *unit, u_int busno)
	-{
	-
	- MPASS(busno <= PCI_BUSMAX);
	- return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] &
	- (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0);
	-}
	-
	static void
	dmar_print_path(int busno, int depth, const ACPI_DMAR_PCI_PATH *path)
	{
	int i;

	printf("[%d, ", busno);
	for (i = 0; i < depth; i++) {
	if (i != 0)
	printf(", ");
	printf("(%d, %d)", path[i].Device, path[i].Function);
	}
	printf("]");
	}

	int
	dmar_dev_depth(device_t child)
	{
	devclass_t pci_class;
	device_t bus, pcib;
	int depth;

	pci_class = devclass_find("pci");
	for (depth = 1; ; depth++) {
	bus = device_get_parent(child);
	pcib = device_get_parent(bus);
	if (device_get_devclass(device_get_parent(pcib)) !=
	pci_class)
	return (depth);
	child = pcib;
	}
	}

	void
	dmar_dev_path(device_t child, int busno, void path1, int depth)
	{
	devclass_t pci_class;
	device_t bus, pcib;
	ACPI_DMAR_PCI_PATH *path;

	pci_class = devclass_find("pci");
	path = path1;
	for (depth--; depth != -1; depth--) {
	path[depth].Device = pci_get_slot(child);
	path[depth].Function = pci_get_function(child);
	bus = device_get_parent(child);
	pcib = device_get_parent(bus);
	if (device_get_devclass(device_get_parent(pcib)) !=
	pci_class) {
	/* reached a host bridge */
	*busno = pcib_get_bus(bus);
	return;
	}
	child = pcib;
	}
	panic("wrong depth");
	}

	static int
	dmar_match_pathes(int busno1, const ACPI_DMAR_PCI_PATH *path1, int depth1,
	int busno2, const ACPI_DMAR_PCI_PATH *path2, int depth2,
	enum AcpiDmarScopeType scope_type)
	{
	int i, depth;

	if (busno1 != busno2)
	return (0);
	if (scope_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && depth1 != depth2)
	return (0);
	depth = depth1;
	if (depth2 < depth)
	depth = depth2;
	for (i = 0; i < depth; i++) {
	if (path1[i].Device != path2[i].Device \|\|
	path1[i].Function != path2[i].Function)
	return (0);
	}
	return (1);
	}

	static int
	dmar_match_devscope(ACPI_DMAR_DEVICE_SCOPE *devscope, int dev_busno,
	const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len)
	{
	ACPI_DMAR_PCI_PATH *path;
	int path_len;

	if (devscope->Length < sizeof(*devscope)) {
	printf("dmar_match_devscope: corrupted DMAR table, dl %d\n",
	devscope->Length);
	return (-1);
	}
	if (devscope->EntryType != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
	devscope->EntryType != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
	return (0);
	path_len = devscope->Length - sizeof(*devscope);
	if (path_len % 2 != 0) {
	printf("dmar_match_devscope: corrupted DMAR table, dl %d\n",
	devscope->Length);
	return (-1);
	}
	path_len /= 2;
	path = (ACPI_DMAR_PCI_PATH *)(devscope + 1);
	if (path_len == 0) {
	printf("dmar_match_devscope: corrupted DMAR table, dl %d\n",
	devscope->Length);
	return (-1);
	}

	return (dmar_match_pathes(devscope->Bus, path, path_len, dev_busno,
	dev_path, dev_path_len, devscope->EntryType));
	}

	static bool
	dmar_match_by_path(struct dmar_unit *unit, int dev_domain, int dev_busno,
	const ACPI_DMAR_PCI_PATH dev_path, int dev_path_len, const char *banner)
	{
	ACPI_DMAR_HARDWARE_UNIT *dmarh;
	ACPI_DMAR_DEVICE_SCOPE *devscope;
	char ptr, ptrend;
	int match;

	dmarh = dmar_find_by_index(unit->iommu.unit);
	if (dmarh == NULL)
	return (false);
	if (dmarh->Segment != dev_domain)
	return (false);
	if ((dmarh->Flags & ACPI_DMAR_INCLUDE_ALL) != 0) {
	if (banner != NULL)
	*banner = "INCLUDE_ALL";
	return (true);
	}
	ptr = (char )dmarh + sizeof(dmarh);
	ptrend = (char *)dmarh + dmarh->Header.Length;
	while (ptr < ptrend) {
	devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr;
	ptr += devscope->Length;
	match = dmar_match_devscope(devscope, dev_busno, dev_path,
	dev_path_len);
	if (match == -1)
	return (false);
	if (match == 1) {
	if (banner != NULL)
	*banner = "specific match";
	return (true);
	}
	}
	return (false);
	}

	static struct dmar_unit *
	dmar_find_by_scope(int dev_domain, int dev_busno,
	const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len)
	{
	struct dmar_unit *unit;
	int i;

	for (i = 0; i < dmar_devcnt; i++) {
	if (dmar_devs[i] == NULL)
	continue;
	unit = device_get_softc(dmar_devs[i]);
	if (dmar_match_by_path(unit, dev_domain, dev_busno, dev_path,
	dev_path_len, NULL))
	return (unit);
	}
	return (NULL);
	}

	struct dmar_unit *
	dmar_find(device_t dev, bool verbose)
	{
	device_t dmar_dev;
	struct dmar_unit *unit;
	const char *banner;
	int i, dev_domain, dev_busno, dev_path_len;

	/*
	* This function can only handle PCI(e) devices.
	*/
	if (device_get_devclass(device_get_parent(dev)) !=
	devclass_find("pci"))
	return (NULL);

	dmar_dev = NULL;
	dev_domain = pci_get_domain(dev);
	dev_path_len = dmar_dev_depth(dev);
	ACPI_DMAR_PCI_PATH dev_path[dev_path_len];
	dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len);
	banner = "";

	for (i = 0; i < dmar_devcnt; i++) {
	if (dmar_devs[i] == NULL)
	continue;
	unit = device_get_softc(dmar_devs[i]);
	if (dmar_match_by_path(unit, dev_domain, dev_busno,
	dev_path, dev_path_len, &banner))
	break;
	}
	if (i == dmar_devcnt)
	return (NULL);

	if (verbose) {
	device_printf(dev, "pci%d:%d:%d:%d matched dmar%d by %s",
	dev_domain, pci_get_bus(dev), pci_get_slot(dev),
	pci_get_function(dev), unit->iommu.unit, banner);
	printf(" scope path ");
	dmar_print_path(dev_busno, dev_path_len, dev_path);
	printf("\n");
	}
	return (unit);
	}

	static struct dmar_unit *
	dmar_find_nonpci(u_int id, u_int entry_type, uint16_t *rid)
	{
	device_t dmar_dev;
	struct dmar_unit *unit;
	ACPI_DMAR_HARDWARE_UNIT *dmarh;
	ACPI_DMAR_DEVICE_SCOPE *devscope;
	ACPI_DMAR_PCI_PATH *path;
	char ptr, ptrend;
	#ifdef DEV_APIC
	int error;
	#endif
	int i;

	for (i = 0; i < dmar_devcnt; i++) {
	dmar_dev = dmar_devs[i];
	if (dmar_dev == NULL)
	continue;
	unit = (struct dmar_unit *)device_get_softc(dmar_dev);
	dmarh = dmar_find_by_index(i);
	if (dmarh == NULL)
	continue;
	ptr = (char )dmarh + sizeof(dmarh);
	ptrend = (char *)dmarh + dmarh->Header.Length;
	for (;;) {
	if (ptr >= ptrend)
	break;
	devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr;
	ptr += devscope->Length;
	if (devscope->EntryType != entry_type)
	continue;
	if (devscope->EnumerationId != id)
	continue;
	#ifdef DEV_APIC
	if (entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
	error = ioapic_get_rid(id, rid);
	/*
	* If our IOAPIC has PCI bindings then
	* use the PCI device rid.
	*/
	if (error == 0)
	return (unit);
	}
	#endif
	if (devscope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE)
	== 2) {
	if (rid != NULL) {
	path = (ACPI_DMAR_PCI_PATH *)
	(devscope + 1);
	*rid = PCI_RID(devscope->Bus,
	path->Device, path->Function);
	}
	return (unit);
	}
	printf(
	"dmar_find_nonpci: id %d type %d path length != 2\n",
	id, entry_type);
	break;
	}
	}
	return (NULL);
	}


	struct dmar_unit *
	dmar_find_hpet(device_t dev, uint16_t *rid)
	{

	return (dmar_find_nonpci(hpet_get_uid(dev), ACPI_DMAR_SCOPE_TYPE_HPET,
	rid));
	}

	struct dmar_unit *
	dmar_find_ioapic(u_int apic_id, uint16_t *rid)
	{

	return (dmar_find_nonpci(apic_id, ACPI_DMAR_SCOPE_TYPE_IOAPIC, rid));
	}

	struct rmrr_iter_args {
	struct dmar_domain *domain;
	int dev_domain;
	int dev_busno;
	const ACPI_DMAR_PCI_PATH *dev_path;
	int dev_path_len;
	struct iommu_map_entries_tailq *rmrr_entries;
	};

	static int
	dmar_rmrr_iter(ACPI_DMAR_HEADER dmarh, void arg)
	{
	struct rmrr_iter_args *ria;
	ACPI_DMAR_RESERVED_MEMORY *resmem;
	ACPI_DMAR_DEVICE_SCOPE *devscope;
	struct iommu_map_entry *entry;
	char ptr, ptrend;
	int match;

	if (dmarh->Type != ACPI_DMAR_TYPE_RESERVED_MEMORY)
	return (1);

	ria = arg;
	resmem = (ACPI_DMAR_RESERVED_MEMORY *)dmarh;
	if (resmem->Segment != ria->dev_domain)
	return (1);

	ptr = (char )resmem + sizeof(resmem);
	ptrend = (char *)resmem + resmem->Header.Length;
	for (;;) {
	if (ptr >= ptrend)
	break;
	devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr;
	ptr += devscope->Length;
	match = dmar_match_devscope(devscope, ria->dev_busno,
	ria->dev_path, ria->dev_path_len);
	if (match == 1) {
	entry = iommu_gas_alloc_entry(
	(struct iommu_domain *)ria->domain,
	IOMMU_PGF_WAITOK);
	entry->start = resmem->BaseAddress;
	/* The RMRR entry end address is inclusive. */
	entry->end = resmem->EndAddress;
	TAILQ_INSERT_TAIL(ria->rmrr_entries, entry,
	unroll_link);
	}
	}

	return (1);
	}

	void
	dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain, int dev_busno,
	const void *dev_path, int dev_path_len,
	struct iommu_map_entries_tailq *rmrr_entries)
	{
	struct rmrr_iter_args ria;

	ria.domain = domain;
	ria.dev_domain = dev_domain;
	ria.dev_busno = dev_busno;
	ria.dev_path = (const ACPI_DMAR_PCI_PATH *)dev_path;
	ria.dev_path_len = dev_path_len;
	ria.rmrr_entries = rmrr_entries;
	dmar_iterate_tbl(dmar_rmrr_iter, &ria);
	}

	struct inst_rmrr_iter_args {
	struct dmar_unit *dmar;
	};

	static device_t
	dmar_path_dev(int segment, int path_len, int busno,
	const ACPI_DMAR_PCI_PATH path, uint16_t rid)
	{
	device_t dev;
	int i;

	dev = NULL;
	for (i = 0; i < path_len; i++) {
	dev = pci_find_dbsf(segment, busno, path->Device,
	path->Function);
	if (i != path_len - 1) {
	busno = pci_cfgregread(busno, path->Device,
	path->Function, PCIR_SECBUS_1, 1);
	path++;
	}
	}
	*rid = PCI_RID(busno, path->Device, path->Function);
	return (dev);
	}

	static int
	dmar_inst_rmrr_iter(ACPI_DMAR_HEADER dmarh, void arg)
	{
	const ACPI_DMAR_RESERVED_MEMORY *resmem;
	const ACPI_DMAR_DEVICE_SCOPE *devscope;
	struct inst_rmrr_iter_args *iria;
	const char ptr, ptrend;
	device_t dev;
	struct dmar_unit *unit;
	int dev_path_len;
	uint16_t rid;

	iria = arg;

	if (dmarh->Type != ACPI_DMAR_TYPE_RESERVED_MEMORY)
	return (1);

	resmem = (ACPI_DMAR_RESERVED_MEMORY *)dmarh;
	if (resmem->Segment != iria->dmar->segment)
	return (1);

	ptr = (const char )resmem + sizeof(resmem);
	ptrend = (const char *)resmem + resmem->Header.Length;
	for (;;) {
	if (ptr >= ptrend)
	break;
	devscope = (const ACPI_DMAR_DEVICE_SCOPE *)ptr;
	ptr += devscope->Length;
	/* XXXKIB bridge */
	if (devscope->EntryType != ACPI_DMAR_SCOPE_TYPE_ENDPOINT)
	continue;
	rid = 0;
	dev_path_len = (devscope->Length -
	sizeof(ACPI_DMAR_DEVICE_SCOPE)) / 2;
	dev = dmar_path_dev(resmem->Segment, dev_path_len,
	devscope->Bus,
	(const ACPI_DMAR_PCI_PATH *)(devscope + 1), &rid);
	if (dev == NULL) {
	if (bootverbose) {
	printf("dmar%d no dev found for RMRR "
	"[%#jx, %#jx] rid %#x scope path ",
	iria->dmar->iommu.unit,
	(uintmax_t)resmem->BaseAddress,
	(uintmax_t)resmem->EndAddress,
	rid);
	dmar_print_path(devscope->Bus, dev_path_len,
	(const ACPI_DMAR_PCI_PATH *)(devscope + 1));
	printf("\n");
	}
	unit = dmar_find_by_scope(resmem->Segment,
	devscope->Bus,
	(const ACPI_DMAR_PCI_PATH *)(devscope + 1),
	dev_path_len);
	if (iria->dmar != unit)
	continue;
	dmar_get_ctx_for_devpath(iria->dmar, rid,
	resmem->Segment, devscope->Bus,
	(const ACPI_DMAR_PCI_PATH *)(devscope + 1),
	dev_path_len, false, true);
	} else {
	unit = dmar_find(dev, false);
	if (iria->dmar != unit)
	continue;
	iommu_instantiate_ctx(&(iria)->dmar->iommu,
	dev, true);
	}
	}

	return (1);

	}

	/*
	* Pre-create all contexts for the DMAR which have RMRR entries.
	*/
	int
	dmar_instantiate_rmrr_ctxs(struct iommu_unit *unit)
	{
	struct dmar_unit *dmar;
	struct inst_rmrr_iter_args iria;
	int error;

	dmar = (struct dmar_unit *)unit;

	if (!dmar_barrier_enter(dmar, DMAR_BARRIER_RMRR))
	return (0);

	error = 0;
	iria.dmar = dmar;
	dmar_iterate_tbl(dmar_inst_rmrr_iter, &iria);
	DMAR_LOCK(dmar);
	if (!LIST_EMPTY(&dmar->domains)) {
	KASSERT((dmar->hw_gcmd & DMAR_GCMD_TE) == 0,
	("dmar%d: RMRR not handled but translation is already enabled",
	dmar->iommu.unit));
	error = dmar_enable_translation(dmar);
	if (bootverbose) {
	if (error == 0) {
	printf("dmar%d: enabled translation\n",
	dmar->iommu.unit);
	} else {
	printf("dmar%d: enabling translation failed, "
	"error %d\n", dmar->iommu.unit, error);
	}
	}
	}
	dmar_barrier_exit(dmar, DMAR_BARRIER_RMRR);
	return (error);
	}

	#ifdef DDB
	#include <ddb/ddb.h>
	#include <ddb/db_lex.h>

	static void
	dmar_print_domain_entry(const struct iommu_map_entry *entry)
	{
	struct iommu_map_entry l, r;

	db_printf(
	" start %jx end %jx first %jx last %jx free_down %jx flags %x ",
	entry->start, entry->end, entry->first, entry->last,
	entry->free_down, entry->flags);
	db_printf("left ");
	l = RB_LEFT(entry, rb_entry);
	if (l == NULL)
	db_printf("NULL ");
	else
	db_printf("%jx ", l->start);
	db_printf("right ");
	r = RB_RIGHT(entry, rb_entry);
	if (r == NULL)
	db_printf("NULL");
	else
	db_printf("%jx", r->start);
	db_printf("\n");
	}

	static void
	dmar_print_ctx(struct dmar_ctx *ctx)
	{

	db_printf(
	" @%p pci%d:%d:%d refs %d flags %x loads %lu unloads %lu\n",
	ctx, pci_get_bus(ctx->context.tag->owner),
	pci_get_slot(ctx->context.tag->owner),
	pci_get_function(ctx->context.tag->owner), ctx->refs,
	ctx->context.flags, ctx->context.loads, ctx->context.unloads);
	}

	static void
	dmar_print_domain(struct dmar_domain *domain, bool show_mappings)
	{
	struct iommu_domain *iodom;
	struct iommu_map_entry *entry;
	struct dmar_ctx *ctx;

	iodom = (struct iommu_domain *)domain;

	db_printf(
	" @%p dom %d mgaw %d agaw %d pglvl %d end %jx refs %d\n"
	" ctx_cnt %d flags %x pgobj %p map_ents %u\n",
	domain, domain->domain, domain->mgaw, domain->agaw, domain->pglvl,
	(uintmax_t)domain->iodom.end, domain->refs, domain->ctx_cnt,
	domain->iodom.flags, domain->pgtbl_obj, domain->iodom.entries_cnt);
	if (!LIST_EMPTY(&domain->contexts)) {
	db_printf(" Contexts:\n");
	LIST_FOREACH(ctx, &domain->contexts, link)
	dmar_print_ctx(ctx);
	}
	if (!show_mappings)
	return;
	db_printf(" mapped:\n");
	RB_FOREACH(entry, iommu_gas_entries_tree, &iodom->rb_root) {
	dmar_print_domain_entry(entry);
	if (db_pager_quit)
	break;
	}
	if (db_pager_quit)
	return;
	db_printf(" unloading:\n");
	TAILQ_FOREACH(entry, &domain->iodom.unload_entries, dmamap_link) {
	dmar_print_domain_entry(entry);
	if (db_pager_quit)
	break;
	}
	}

	DB_FUNC(dmar_domain, db_dmar_print_domain, db_show_table, CS_OWN, NULL)
	{
	struct dmar_unit *unit;
	struct dmar_domain *domain;
	struct dmar_ctx *ctx;
	bool show_mappings, valid;
	int pci_domain, bus, device, function, i, t;
	db_expr_t radix;

	valid = false;
	radix = db_radix;
	db_radix = 10;
	t = db_read_token();
	if (t == tSLASH) {
	t = db_read_token();
	if (t != tIDENT) {
	db_printf("Bad modifier\n");
	db_radix = radix;
	db_skip_to_eol();
	return;
	}
	show_mappings = strchr(db_tok_string, 'm') != NULL;
	t = db_read_token();
	} else {
	show_mappings = false;
	}
	if (t == tNUMBER) {
	pci_domain = db_tok_number;
	t = db_read_token();
	if (t == tNUMBER) {
	bus = db_tok_number;
	t = db_read_token();
	if (t == tNUMBER) {
	device = db_tok_number;
	t = db_read_token();
	if (t == tNUMBER) {
	function = db_tok_number;
	valid = true;
	}
	}
	}
	}
	db_radix = radix;
	db_skip_to_eol();
	if (!valid) {
	db_printf("usage: show dmar_domain [/m] "
	"<domain> <bus> <device> <func>\n");
	return;
	}
	for (i = 0; i < dmar_devcnt; i++) {
	unit = device_get_softc(dmar_devs[i]);
	LIST_FOREACH(domain, &unit->domains, link) {
	LIST_FOREACH(ctx, &domain->contexts, link) {
	if (pci_domain == unit->segment &&
	bus == pci_get_bus(ctx->context.tag->owner) &&
	device ==
	pci_get_slot(ctx->context.tag->owner) &&
	function ==
	pci_get_function(ctx->context.tag->owner)) {
	dmar_print_domain(domain,
	show_mappings);
	goto out;
	}
	}
	}
	}
	out:;
	}

	static void
	dmar_print_one(int idx, bool show_domains, bool show_mappings)
	{
	struct dmar_unit *unit;
	struct dmar_domain *domain;
	int i, frir;

	unit = device_get_softc(dmar_devs[idx]);
	db_printf("dmar%d at %p, root at 0x%jx, ver 0x%x\n", unit->iommu.unit,
	unit, dmar_read8(unit, DMAR_RTADDR_REG),
	dmar_read4(unit, DMAR_VER_REG));
	db_printf("cap 0x%jx ecap 0x%jx gsts 0x%x fsts 0x%x fectl 0x%x\n",
	(uintmax_t)dmar_read8(unit, DMAR_CAP_REG),
	(uintmax_t)dmar_read8(unit, DMAR_ECAP_REG),
	dmar_read4(unit, DMAR_GSTS_REG),
	dmar_read4(unit, DMAR_FSTS_REG),
	dmar_read4(unit, DMAR_FECTL_REG));
	if (unit->ir_enabled) {
	db_printf("ir is enabled; IRT @%p phys 0x%jx maxcnt %d\n",
	unit->irt, (uintmax_t)unit->irt_phys, unit->irte_cnt);
	}
	db_printf("fed 0x%x fea 0x%x feua 0x%x\n",
	dmar_read4(unit, DMAR_FEDATA_REG),
	dmar_read4(unit, DMAR_FEADDR_REG),
	dmar_read4(unit, DMAR_FEUADDR_REG));
	db_printf("primary fault log:\n");
	for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) {
	frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16;
	db_printf(" %d at 0x%x: %jx %jx\n", i, frir,
	(uintmax_t)dmar_read8(unit, frir),
	(uintmax_t)dmar_read8(unit, frir + 8));
	}
	if (DMAR_HAS_QI(unit)) {
	db_printf("ied 0x%x iea 0x%x ieua 0x%x\n",
	dmar_read4(unit, DMAR_IEDATA_REG),
	dmar_read4(unit, DMAR_IEADDR_REG),
	dmar_read4(unit, DMAR_IEUADDR_REG));
	if (unit->qi_enabled) {
	db_printf("qi is enabled: queue @0x%jx (IQA 0x%jx) "
	"size 0x%jx\n"
	" head 0x%x tail 0x%x avail 0x%x status 0x%x ctrl 0x%x\n"
	" hw compl 0x%x@%p/phys@%jx next seq 0x%x gen 0x%x\n",
	(uintmax_t)unit->inv_queue,
	(uintmax_t)dmar_read8(unit, DMAR_IQA_REG),
	(uintmax_t)unit->inv_queue_size,
	dmar_read4(unit, DMAR_IQH_REG),
	dmar_read4(unit, DMAR_IQT_REG),
	unit->inv_queue_avail,
	dmar_read4(unit, DMAR_ICS_REG),
	dmar_read4(unit, DMAR_IECTL_REG),
	unit->inv_waitd_seq_hw,
	&unit->inv_waitd_seq_hw,
	(uintmax_t)unit->inv_waitd_seq_hw_phys,
	unit->inv_waitd_seq,
	unit->inv_waitd_gen);
	} else {
	db_printf("qi is disabled\n");
	}
	}
	if (show_domains) {
	db_printf("domains:\n");
	LIST_FOREACH(domain, &unit->domains, link) {
	dmar_print_domain(domain, show_mappings);
	if (db_pager_quit)
	break;
	}
	}
	}

	DB_SHOW_COMMAND(dmar, db_dmar_print)
	{
	bool show_domains, show_mappings;

	show_domains = strchr(modif, 'd') != NULL;
	show_mappings = strchr(modif, 'm') != NULL;
	if (!have_addr) {
	db_printf("usage: show dmar [/d] [/m] index\n");
	return;
	}
	dmar_print_one((int)addr, show_domains, show_mappings);
	}

	DB_SHOW_ALL_COMMAND(dmars, db_show_all_dmars)
	{
	int i;
	bool show_domains, show_mappings;

	show_domains = strchr(modif, 'd') != NULL;
	show_mappings = strchr(modif, 'm') != NULL;

	for (i = 0; i < dmar_devcnt; i++) {
	dmar_print_one(i, show_domains, show_mappings);
	if (db_pager_quit)
	break;
	}
	}
	#endif

	struct iommu_unit *
	iommu_find(device_t dev, bool verbose)
	{
	struct dmar_unit *dmar;

	dmar = dmar_find(dev, verbose);

	return (&dmar->iommu);
	}
	diff --git a/sys/x86/iommu/intel_idpgtbl.c b/sys/x86/iommu/intel_idpgtbl.c
	index d85abefc54c4..4c151ffffef8 100644
	--- a/sys/x86/iommu/intel_idpgtbl.c
	+++ b/sys/x86/iommu/intel_idpgtbl.c
	@@ -1,811 +1,811 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/malloc.h>
	#include <sys/bus.h>
	#include <sys/interrupt.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/proc.h>
	#include <sys/rwlock.h>
	#include <sys/rman.h>
	#include <sys/sf_buf.h>
	#include <sys/sysctl.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/uio.h>
	#include <sys/vmem.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_map.h>
	+#include <dev/pci/pcireg.h>
	#include <machine/atomic.h>
	#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <machine/md_var.h>
	#include <machine/specialreg.h>
	#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	-#include <dev/pci/pcireg.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>

	static int domain_unmap_buf_locked(struct dmar_domain *domain,
	iommu_gaddr_t base, iommu_gaddr_t size, int flags);

	/*
	* The cache of the identity mapping page tables for the DMARs. Using
	* the cache saves significant amount of memory for page tables by
	* reusing the page tables, since usually DMARs are identical and have
	* the same capabilities. Still, cache records the information needed
	* to match DMAR capabilities and page table format, to correctly
	* handle different DMARs.
	*/

	struct idpgtbl {
	iommu_gaddr_t maxaddr; /* Page table covers the guest address
	range [0..maxaddr) */
	int pglvl; /* Total page table levels ignoring
	superpages */
	int leaf; /* The last materialized page table
	level, it is non-zero if superpages
	are supported */
	vm_object_t pgtbl_obj; /* The page table pages */
	LIST_ENTRY(idpgtbl) link;
	};

	static struct sx idpgtbl_lock;
	SX_SYSINIT(idpgtbl, &idpgtbl_lock, "idpgtbl");
	static LIST_HEAD(, idpgtbl) idpgtbls = LIST_HEAD_INITIALIZER(idpgtbls);
	static MALLOC_DEFINE(M_DMAR_IDPGTBL, "dmar_idpgtbl",
	"Intel DMAR Identity mappings cache elements");

	/*
	* Build the next level of the page tables for the identity mapping.
	* - lvl is the level to build;
	* - idx is the index of the page table page in the pgtbl_obj, which is
	* being allocated filled now;
	* - addr is the starting address in the bus address space which is
	* mapped by the page table page.
	*/
	static void
	domain_idmap_nextlvl(struct idpgtbl *tbl, int lvl, vm_pindex_t idx,
	iommu_gaddr_t addr)
	{
	vm_page_t m1;
	dmar_pte_t *pte;
	struct sf_buf *sf;
	iommu_gaddr_t f, pg_sz;
	vm_pindex_t base;
	int i;

	VM_OBJECT_ASSERT_LOCKED(tbl->pgtbl_obj);
	if (addr >= tbl->maxaddr)
	return;
	(void)dmar_pgalloc(tbl->pgtbl_obj, idx, IOMMU_PGF_OBJL \|
	IOMMU_PGF_WAITOK \| IOMMU_PGF_ZERO);
	base = idx * DMAR_NPTEPG + 1; /* Index of the first child page of idx */
	pg_sz = pglvl_page_size(tbl->pglvl, lvl);
	if (lvl != tbl->leaf) {
	for (i = 0, f = addr; i < DMAR_NPTEPG; i++, f += pg_sz)
	domain_idmap_nextlvl(tbl, lvl + 1, base + i, f);
	}
	VM_OBJECT_WUNLOCK(tbl->pgtbl_obj);
	pte = dmar_map_pgtbl(tbl->pgtbl_obj, idx, IOMMU_PGF_WAITOK, &sf);
	if (lvl == tbl->leaf) {
	for (i = 0, f = addr; i < DMAR_NPTEPG; i++, f += pg_sz) {
	if (f >= tbl->maxaddr)
	break;
	pte[i].pte = (DMAR_PTE_ADDR_MASK & f) \|
	DMAR_PTE_R \| DMAR_PTE_W;
	}
	} else {
	for (i = 0, f = addr; i < DMAR_NPTEPG; i++, f += pg_sz) {
	if (f >= tbl->maxaddr)
	break;
	m1 = dmar_pgalloc(tbl->pgtbl_obj, base + i,
	IOMMU_PGF_NOALLOC);
	KASSERT(m1 != NULL, ("lost page table page"));
	pte[i].pte = (DMAR_PTE_ADDR_MASK &
	VM_PAGE_TO_PHYS(m1)) \| DMAR_PTE_R \| DMAR_PTE_W;
	}
	}
	/* domain_get_idmap_pgtbl flushes CPU cache if needed. */
	dmar_unmap_pgtbl(sf);
	VM_OBJECT_WLOCK(tbl->pgtbl_obj);
	}

	/*
	* Find a ready and compatible identity-mapping page table in the
	* cache. If not found, populate the identity-mapping page table for
	* the context, up to the maxaddr. The maxaddr byte is allowed to be
	* not mapped, which is aligned with the definition of Maxmem as the
	* highest usable physical address + 1. If superpages are used, the
	* maxaddr is typically mapped.
	*/
	vm_object_t
	domain_get_idmap_pgtbl(struct dmar_domain *domain, iommu_gaddr_t maxaddr)
	{
	struct dmar_unit *unit;
	struct idpgtbl *tbl;
	vm_object_t res;
	vm_page_t m;
	int leaf, i;

	leaf = 0; /* silence gcc */

	/*
	* First, determine where to stop the paging structures.
	*/
	for (i = 0; i < domain->pglvl; i++) {
	if (i == domain->pglvl - 1 \|\| domain_is_sp_lvl(domain, i)) {
	leaf = i;
	break;
	}
	}

	/*
	* Search the cache for a compatible page table. Qualified
	* page table must map up to maxaddr, its level must be
	* supported by the DMAR and leaf should be equal to the
	* calculated value. The later restriction could be lifted
	* but I believe it is currently impossible to have any
	* deviations for existing hardware.
	*/
	sx_slock(&idpgtbl_lock);
	LIST_FOREACH(tbl, &idpgtbls, link) {
	if (tbl->maxaddr >= maxaddr &&
	dmar_pglvl_supported(domain->dmar, tbl->pglvl) &&
	tbl->leaf == leaf) {
	res = tbl->pgtbl_obj;
	vm_object_reference(res);
	sx_sunlock(&idpgtbl_lock);
	domain->pglvl = tbl->pglvl; /* XXXKIB ? */
	goto end;
	}
	}

	/*
	* Not found in cache, relock the cache into exclusive mode to
	* be able to add element, and recheck cache again after the
	* relock.
	*/
	sx_sunlock(&idpgtbl_lock);
	sx_xlock(&idpgtbl_lock);
	LIST_FOREACH(tbl, &idpgtbls, link) {
	if (tbl->maxaddr >= maxaddr &&
	dmar_pglvl_supported(domain->dmar, tbl->pglvl) &&
	tbl->leaf == leaf) {
	res = tbl->pgtbl_obj;
	vm_object_reference(res);
	sx_xunlock(&idpgtbl_lock);
	domain->pglvl = tbl->pglvl; /* XXXKIB ? */
	return (res);
	}
	}

	/*
	* Still not found, create new page table.
	*/
	tbl = malloc(sizeof(*tbl), M_DMAR_IDPGTBL, M_WAITOK);
	tbl->pglvl = domain->pglvl;
	tbl->leaf = leaf;
	tbl->maxaddr = maxaddr;
	tbl->pgtbl_obj = vm_pager_allocate(OBJT_PHYS, NULL,
	IDX_TO_OFF(pglvl_max_pages(tbl->pglvl)), 0, 0, NULL);
	VM_OBJECT_WLOCK(tbl->pgtbl_obj);
	domain_idmap_nextlvl(tbl, 0, 0, 0);
	VM_OBJECT_WUNLOCK(tbl->pgtbl_obj);
	LIST_INSERT_HEAD(&idpgtbls, tbl, link);
	res = tbl->pgtbl_obj;
	vm_object_reference(res);
	sx_xunlock(&idpgtbl_lock);

	end:
	/*
	* Table was found or created.
	*
	* If DMAR does not snoop paging structures accesses, flush
	* CPU cache to memory. Note that dmar_unmap_pgtbl() coherent
	* argument was possibly invalid at the time of the identity
	* page table creation, since DMAR which was passed at the
	* time of creation could be coherent, while current DMAR is
	* not.
	*
	* If DMAR cannot look into the chipset write buffer, flush it
	* as well.
	*/
	unit = domain->dmar;
	if (!DMAR_IS_COHERENT(unit)) {
	VM_OBJECT_WLOCK(res);
	for (m = vm_page_lookup(res, 0); m != NULL;
	m = vm_page_next(m))
	pmap_invalidate_cache_pages(&m, 1);
	VM_OBJECT_WUNLOCK(res);
	}
	if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) {
	DMAR_LOCK(unit);
	dmar_flush_write_bufs(unit);
	DMAR_UNLOCK(unit);
	}

	return (res);
	}

	/*
	* Return a reference to the identity mapping page table to the cache.
	*/
	void
	put_idmap_pgtbl(vm_object_t obj)
	{
	struct idpgtbl tbl, tbl1;
	vm_object_t rmobj;

	sx_slock(&idpgtbl_lock);
	KASSERT(obj->ref_count >= 2, ("lost cache reference"));
	vm_object_deallocate(obj);

	/*
	* Cache always owns one last reference on the page table object.
	* If there is an additional reference, object must stay.
	*/
	if (obj->ref_count > 1) {
	sx_sunlock(&idpgtbl_lock);
	return;
	}

	/*
	* Cache reference is the last, remove cache element and free
	* page table object, returning the page table pages to the
	* system.
	*/
	sx_sunlock(&idpgtbl_lock);
	sx_xlock(&idpgtbl_lock);
	LIST_FOREACH_SAFE(tbl, &idpgtbls, link, tbl1) {
	rmobj = tbl->pgtbl_obj;
	if (rmobj->ref_count == 1) {
	LIST_REMOVE(tbl, link);
	atomic_subtract_int(&dmar_tbl_pagecnt,
	rmobj->resident_page_count);
	vm_object_deallocate(rmobj);
	free(tbl, M_DMAR_IDPGTBL);
	}
	}
	sx_xunlock(&idpgtbl_lock);
	}

	/*
	* The core routines to map and unmap host pages at the given guest
	* address. Support superpages.
	*/

	/*
	* Index of the pte for the guest address base in the page table at
	* the level lvl.
	*/
	static int
	domain_pgtbl_pte_off(struct dmar_domain *domain, iommu_gaddr_t base, int lvl)
	{

	base >>= DMAR_PAGE_SHIFT + (domain->pglvl - lvl - 1) *
	DMAR_NPTEPGSHIFT;
	return (base & DMAR_PTEMASK);
	}

	/*
	* Returns the page index of the page table page in the page table
	* object, which maps the given address base at the page table level
	* lvl.
	*/
	static vm_pindex_t
	domain_pgtbl_get_pindex(struct dmar_domain *domain, iommu_gaddr_t base, int lvl)
	{
	vm_pindex_t idx, pidx;
	int i;

	KASSERT(lvl >= 0 && lvl < domain->pglvl,
	("wrong lvl %p %d", domain, lvl));

	for (pidx = idx = 0, i = 0; i < lvl; i++, pidx = idx) {
	idx = domain_pgtbl_pte_off(domain, base, i) +
	pidx * DMAR_NPTEPG + 1;
	}
	return (idx);
	}

	static dmar_pte_t *
	domain_pgtbl_map_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl,
	int flags, vm_pindex_t idxp, struct sf_buf *sf)
	{
	vm_page_t m;
	struct sf_buf *sfp;
	dmar_pte_t pte, ptep;
	vm_pindex_t idx, idx1;

	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);
	KASSERT((flags & IOMMU_PGF_OBJL) != 0, ("lost PGF_OBJL"));

	idx = domain_pgtbl_get_pindex(domain, base, lvl);
	if (sf != NULL && idx == idxp) {
	pte = (dmar_pte_t )sf_buf_kva(sf);
	} else {
	if (*sf != NULL)
	dmar_unmap_pgtbl(*sf);
	*idxp = idx;
	retry:
	pte = dmar_map_pgtbl(domain->pgtbl_obj, idx, flags, sf);
	if (pte == NULL) {
	KASSERT(lvl > 0,
	("lost root page table page %p", domain));
	/*
	* Page table page does not exist, allocate
	* it and create a pte in the preceeding page level
	* to reference the allocated page table page.
	*/
	m = dmar_pgalloc(domain->pgtbl_obj, idx, flags \|
	IOMMU_PGF_ZERO);
	if (m == NULL)
	return (NULL);

	/*
	* Prevent potential free while pgtbl_obj is
	* unlocked in the recursive call to
	* domain_pgtbl_map_pte(), if other thread did
	* pte write and clean while the lock is
	* dropped.
	*/
	m->ref_count++;

	sfp = NULL;
	ptep = domain_pgtbl_map_pte(domain, base, lvl - 1,
	flags, &idx1, &sfp);
	if (ptep == NULL) {
	KASSERT(m->pindex != 0,
	("loosing root page %p", domain));
	m->ref_count--;
	dmar_pgfree(domain->pgtbl_obj, m->pindex,
	flags);
	return (NULL);
	}
	dmar_pte_store(&ptep->pte, DMAR_PTE_R \| DMAR_PTE_W \|
	VM_PAGE_TO_PHYS(m));
	dmar_flush_pte_to_ram(domain->dmar, ptep);
	sf_buf_page(sfp)->ref_count += 1;
	m->ref_count--;
	dmar_unmap_pgtbl(sfp);
	/* Only executed once. */
	goto retry;
	}
	}
	pte += domain_pgtbl_pte_off(domain, base, lvl);
	return (pte);
	}

	static int
	domain_map_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags)
	{
	dmar_pte_t *pte;
	struct sf_buf *sf;
	iommu_gaddr_t pg_sz, base1, size1;
	vm_pindex_t pi, c, idx, run_sz;
	int lvl;
	bool superpage;

	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);

	base1 = base;
	size1 = size;
	flags \|= IOMMU_PGF_OBJL;
	TD_PREP_PINNED_ASSERT;

	for (sf = NULL, pi = 0; size > 0; base += pg_sz, size -= pg_sz,
	pi += run_sz) {
	for (lvl = 0, c = 0, superpage = false;; lvl++) {
	pg_sz = domain_page_size(domain, lvl);
	run_sz = pg_sz >> DMAR_PAGE_SHIFT;
	if (lvl == domain->pglvl - 1)
	break;
	/*
	* Check if the current base suitable for the
	* superpage mapping. First, verify the level.
	*/
	if (!domain_is_sp_lvl(domain, lvl))
	continue;
	/*
	* Next, look at the size of the mapping and
	* alignment of both guest and host addresses.
	*/
	if (size < pg_sz \|\| (base & (pg_sz - 1)) != 0 \|\|
	(VM_PAGE_TO_PHYS(ma[pi]) & (pg_sz - 1)) != 0)
	continue;
	/* All passed, check host pages contiguouty. */
	if (c == 0) {
	for (c = 1; c < run_sz; c++) {
	if (VM_PAGE_TO_PHYS(ma[pi + c]) !=
	VM_PAGE_TO_PHYS(ma[pi + c - 1]) +
	PAGE_SIZE)
	break;
	}
	}
	if (c >= run_sz) {
	superpage = true;
	break;
	}
	}
	KASSERT(size >= pg_sz,
	("mapping loop overflow %p %jx %jx %jx", domain,
	(uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz));
	KASSERT(pg_sz > 0, ("pg_sz 0 lvl %d", lvl));
	pte = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf);
	if (pte == NULL) {
	KASSERT((flags & IOMMU_PGF_WAITOK) == 0,
	("failed waitable pte alloc %p", domain));
	if (sf != NULL)
	dmar_unmap_pgtbl(sf);
	domain_unmap_buf_locked(domain, base1, base - base1,
	flags);
	TD_PINNED_ASSERT;
	return (ENOMEM);
	}
	dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) \| pflags \|
	(superpage ? DMAR_PTE_SP : 0));
	dmar_flush_pte_to_ram(domain->dmar, pte);
	sf_buf_page(sf)->ref_count += 1;
	}
	if (sf != NULL)
	dmar_unmap_pgtbl(sf);
	TD_PINNED_ASSERT;
	return (0);
	}

	int
	domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
	iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags)
	{
	struct dmar_domain *domain;
	struct dmar_unit *unit;
	uint64_t pflags;
	int error;

	pflags = ((eflags & IOMMU_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) \|
	((eflags & IOMMU_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) \|
	((eflags & IOMMU_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) \|
	((eflags & IOMMU_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0);

	domain = (struct dmar_domain *)iodom;
	unit = domain->dmar;

	KASSERT((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) == 0,
	("modifying idmap pagetable domain %p", domain));
	KASSERT((base & DMAR_PAGE_MASK) == 0,
	("non-aligned base %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT((size & DMAR_PAGE_MASK) == 0,
	("non-aligned size %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT(size > 0, ("zero size %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT(base < (1ULL << domain->agaw),
	("base too high %p %jx %jx agaw %d", domain, (uintmax_t)base,
	(uintmax_t)size, domain->agaw));
	KASSERT(base + size < (1ULL << domain->agaw),
	("end too high %p %jx %jx agaw %d", domain, (uintmax_t)base,
	(uintmax_t)size, domain->agaw));
	KASSERT(base + size > base,
	("size overflow %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT((pflags & (DMAR_PTE_R \| DMAR_PTE_W)) != 0,
	("neither read nor write %jx", (uintmax_t)pflags));
	KASSERT((pflags & ~(DMAR_PTE_R \| DMAR_PTE_W \| DMAR_PTE_SNP \|
	DMAR_PTE_TM)) == 0,
	("invalid pte flags %jx", (uintmax_t)pflags));
	KASSERT((pflags & DMAR_PTE_SNP) == 0 \|\|
	(unit->hw_ecap & DMAR_ECAP_SC) != 0,
	("PTE_SNP for dmar without snoop control %p %jx",
	domain, (uintmax_t)pflags));
	KASSERT((pflags & DMAR_PTE_TM) == 0 \|\|
	(unit->hw_ecap & DMAR_ECAP_DI) != 0,
	("PTE_TM for dmar without DIOTLB %p %jx",
	domain, (uintmax_t)pflags));
	KASSERT((flags & ~IOMMU_PGF_WAITOK) == 0, ("invalid flags %x", flags));

	DMAR_DOMAIN_PGLOCK(domain);
	error = domain_map_buf_locked(domain, base, size, ma, pflags, flags);
	DMAR_DOMAIN_PGUNLOCK(domain);
	if (error != 0)
	return (error);

	if ((unit->hw_cap & DMAR_CAP_CM) != 0)
	domain_flush_iotlb_sync(domain, base, size);
	else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) {
	/* See 11.1 Write Buffer Flushing. */
	DMAR_LOCK(unit);
	dmar_flush_write_bufs(unit);
	DMAR_UNLOCK(unit);
	}
	return (0);
	}

	static void domain_unmap_clear_pte(struct dmar_domain *domain,
	iommu_gaddr_t base, int lvl, int flags, dmar_pte_t *pte,
	struct sf_buf **sf, bool free_fs);

	static void
	domain_free_pgtbl_pde(struct dmar_domain *domain, iommu_gaddr_t base,
	int lvl, int flags)
	{
	struct sf_buf *sf;
	dmar_pte_t *pde;
	vm_pindex_t idx;

	sf = NULL;
	pde = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf);
	domain_unmap_clear_pte(domain, base, lvl, flags, pde, &sf, true);
	}

	static void
	domain_unmap_clear_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl,
	int flags, dmar_pte_t pte, struct sf_buf *sf, bool free_sf)
	{
	vm_page_t m;

	dmar_pte_clear(&pte->pte);
	dmar_flush_pte_to_ram(domain->dmar, pte);
	m = sf_buf_page(*sf);
	if (free_sf) {
	dmar_unmap_pgtbl(*sf);
	*sf = NULL;
	}
	m->ref_count--;
	if (m->ref_count != 0)
	return;
	KASSERT(lvl != 0,
	("lost reference (lvl) on root pg domain %p base %jx lvl %d",
	domain, (uintmax_t)base, lvl));
	KASSERT(m->pindex != 0,
	("lost reference (idx) on root pg domain %p base %jx lvl %d",
	domain, (uintmax_t)base, lvl));
	dmar_pgfree(domain->pgtbl_obj, m->pindex, flags);
	domain_free_pgtbl_pde(domain, base, lvl - 1, flags);
	}

	/*
	* Assumes that the unmap is never partial.
	*/
	static int
	domain_unmap_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size, int flags)
	{
	dmar_pte_t *pte;
	struct sf_buf *sf;
	vm_pindex_t idx;
	iommu_gaddr_t pg_sz;
	int lvl;

	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);
	if (size == 0)
	return (0);

	KASSERT((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) == 0,
	("modifying idmap pagetable domain %p", domain));
	KASSERT((base & DMAR_PAGE_MASK) == 0,
	("non-aligned base %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT((size & DMAR_PAGE_MASK) == 0,
	("non-aligned size %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT(base < (1ULL << domain->agaw),
	("base too high %p %jx %jx agaw %d", domain, (uintmax_t)base,
	(uintmax_t)size, domain->agaw));
	KASSERT(base + size < (1ULL << domain->agaw),
	("end too high %p %jx %jx agaw %d", domain, (uintmax_t)base,
	(uintmax_t)size, domain->agaw));
	KASSERT(base + size > base,
	("size overflow %p %jx %jx", domain, (uintmax_t)base,
	(uintmax_t)size));
	KASSERT((flags & ~IOMMU_PGF_WAITOK) == 0, ("invalid flags %x", flags));

	pg_sz = 0; /* silence gcc */
	flags \|= IOMMU_PGF_OBJL;
	TD_PREP_PINNED_ASSERT;

	for (sf = NULL; size > 0; base += pg_sz, size -= pg_sz) {
	for (lvl = 0; lvl < domain->pglvl; lvl++) {
	if (lvl != domain->pglvl - 1 &&
	!domain_is_sp_lvl(domain, lvl))
	continue;
	pg_sz = domain_page_size(domain, lvl);
	if (pg_sz > size)
	continue;
	pte = domain_pgtbl_map_pte(domain, base, lvl, flags,
	&idx, &sf);
	KASSERT(pte != NULL,
	("sleeping or page missed %p %jx %d 0x%x",
	domain, (uintmax_t)base, lvl, flags));
	if ((pte->pte & DMAR_PTE_SP) != 0 \|\|
	lvl == domain->pglvl - 1) {
	domain_unmap_clear_pte(domain, base, lvl,
	flags, pte, &sf, false);
	break;
	}
	}
	KASSERT(size >= pg_sz,
	("unmapping loop overflow %p %jx %jx %jx", domain,
	(uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz));
	}
	if (sf != NULL)
	dmar_unmap_pgtbl(sf);
	/*
	* See 11.1 Write Buffer Flushing for an explanation why RWBF
	* can be ignored there.
	*/

	TD_PINNED_ASSERT;
	return (0);
	}

	int
	domain_unmap_buf(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size, int flags)
	{
	int error;

	DMAR_DOMAIN_PGLOCK(domain);
	error = domain_unmap_buf_locked(domain, base, size, flags);
	DMAR_DOMAIN_PGUNLOCK(domain);
	return (error);
	}

	int
	domain_alloc_pgtbl(struct dmar_domain *domain)
	{
	vm_page_t m;

	KASSERT(domain->pgtbl_obj == NULL,
	("already initialized %p", domain));

	domain->pgtbl_obj = vm_pager_allocate(OBJT_PHYS, NULL,
	IDX_TO_OFF(pglvl_max_pages(domain->pglvl)), 0, 0, NULL);
	DMAR_DOMAIN_PGLOCK(domain);
	m = dmar_pgalloc(domain->pgtbl_obj, 0, IOMMU_PGF_WAITOK \|
	IOMMU_PGF_ZERO \| IOMMU_PGF_OBJL);
	/* No implicit free of the top level page table page. */
	m->ref_count = 1;
	DMAR_DOMAIN_PGUNLOCK(domain);
	DMAR_LOCK(domain->dmar);
	domain->iodom.flags \|= IOMMU_DOMAIN_PGTBL_INITED;
	DMAR_UNLOCK(domain->dmar);
	return (0);
	}

	void
	domain_free_pgtbl(struct dmar_domain *domain)
	{
	vm_object_t obj;
	vm_page_t m;

	obj = domain->pgtbl_obj;
	if (obj == NULL) {
	KASSERT((domain->dmar->hw_ecap & DMAR_ECAP_PT) != 0 &&
	(domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0,
	("lost pagetable object domain %p", domain));
	return;
	}
	DMAR_DOMAIN_ASSERT_PGLOCKED(domain);
	domain->pgtbl_obj = NULL;

	if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0) {
	put_idmap_pgtbl(obj);
	domain->iodom.flags &= ~IOMMU_DOMAIN_IDMAP;
	return;
	}

	/* Obliterate ref_counts */
	VM_OBJECT_ASSERT_WLOCKED(obj);
	for (m = vm_page_lookup(obj, 0); m != NULL; m = vm_page_next(m))
	m->ref_count = 0;
	VM_OBJECT_WUNLOCK(obj);
	vm_object_deallocate(obj);
	}

	static inline uint64_t
	domain_wait_iotlb_flush(struct dmar_unit *unit, uint64_t wt, int iro)
	{
	uint64_t iotlbr;

	dmar_write8(unit, iro + DMAR_IOTLB_REG_OFF, DMAR_IOTLB_IVT \|
	DMAR_IOTLB_DR \| DMAR_IOTLB_DW \| wt);
	for (;;) {
	iotlbr = dmar_read8(unit, iro + DMAR_IOTLB_REG_OFF);
	if ((iotlbr & DMAR_IOTLB_IVT) == 0)
	break;
	cpu_spinwait();
	}
	return (iotlbr);
	}

	void
	domain_flush_iotlb_sync(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size)
	{
	struct dmar_unit *unit;
	iommu_gaddr_t isize;
	uint64_t iotlbr;
	int am, iro;

	unit = domain->dmar;
	KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call",
	unit->iommu.unit));
	iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16;
	DMAR_LOCK(unit);
	if ((unit->hw_cap & DMAR_CAP_PSI) == 0 \|\| size > 2 * 1024 * 1024) {
	iotlbr = domain_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM \|
	DMAR_IOTLB_DID(domain->domain), iro);
	KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
	DMAR_IOTLB_IAIG_INVLD,
	("dmar%d: invalidation failed %jx", unit->iommu.unit,
	(uintmax_t)iotlbr));
	} else {
	for (; size > 0; base += isize, size -= isize) {
	am = calc_am(unit, base, size, &isize);
	dmar_write8(unit, iro, base \| am);
	iotlbr = domain_wait_iotlb_flush(unit,
	DMAR_IOTLB_IIRG_PAGE \|
	DMAR_IOTLB_DID(domain->domain), iro);
	KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
	DMAR_IOTLB_IAIG_INVLD,
	("dmar%d: PSI invalidation failed "
	"iotlbr 0x%jx base 0x%jx size 0x%jx am %d",
	unit->iommu.unit, (uintmax_t)iotlbr,
	(uintmax_t)base, (uintmax_t)size, am));
	/*
	* Any non-page granularity covers whole guest
	* address space for the domain.
	*/
	if ((iotlbr & DMAR_IOTLB_IAIG_MASK) !=
	DMAR_IOTLB_IAIG_PAGE)
	break;
	}
	}
	DMAR_UNLOCK(unit);
	}
	diff --git a/sys/x86/iommu/intel_intrmap.c b/sys/x86/iommu/intel_intrmap.c
	index d2bce59c4c2e..e95d8a8090b3 100644
	--- a/sys/x86/iommu/intel_intrmap.c
	+++ b/sys/x86/iommu/intel_intrmap.c
	@@ -1,384 +1,385 @@
	/*-
	* Copyright (c) 2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/rwlock.h>
	+#include <sys/sysctl.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/vmem.h>
	-#include <machine/bus.h>
	-#include <machine/intr_machdep.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	+#include <dev/pci/pcireg.h>
	+#include <dev/pci/pcivar.h>
	+#include <machine/bus.h>
	+#include <machine/intr_machdep.h>
	#include <x86/include/apicreg.h>
	#include <x86/include/apicvar.h>
	#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	-#include <dev/pci/pcireg.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>
	-#include <dev/pci/pcivar.h>
	#include <x86/iommu/iommu_intrmap.h>

	static struct dmar_unit dmar_ir_find(device_t src, uint16_t rid,
	int *is_dmar);
	static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,
	uint64_t low, uint16_t rid);
	static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);

	int
	iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
	{
	struct dmar_unit *unit;
	vmem_addr_t vmem_res;
	u_int idx, i;
	int error;

	unit = dmar_ir_find(src, NULL, NULL);
	if (unit == NULL \|\| !unit->ir_enabled) {
	for (i = 0; i < count; i++)
	cookies[i] = -1;
	return (EOPNOTSUPP);
	}

	error = vmem_alloc(unit->irtids, count, M_FIRSTFIT \| M_NOWAIT,
	&vmem_res);
	if (error != 0) {
	KASSERT(error != EOPNOTSUPP,
	("impossible EOPNOTSUPP from vmem"));
	return (error);
	}
	idx = vmem_res;
	for (i = 0; i < count; i++)
	cookies[i] = idx + i;
	return (0);
	}

	int
	iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
	uint64_t addr, uint32_t data)
	{
	struct dmar_unit *unit;
	uint64_t low;
	uint16_t rid;
	int is_dmar;

	unit = dmar_ir_find(src, &rid, &is_dmar);
	if (is_dmar) {
	KASSERT(unit == NULL, ("DMAR cannot translate itself"));

	/*
	* See VT-d specification, 5.1.6 Remapping Hardware -
	* Interrupt Programming.
	*/
	*data = vector;
	*addr = MSI_INTEL_ADDR_BASE \| ((cpu & 0xff) << 12);
	if (x2apic_mode)
	*addr \|= ((uint64_t)cpu & 0xffffff00) << 32;
	else
	KASSERT(cpu <= 0xff, ("cpu id too big %d", cpu));
	return (0);
	}
	if (unit == NULL \|\| !unit->ir_enabled \|\| cookie == -1)
	return (EOPNOTSUPP);

	low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :
	DMAR_IRTE1_DST_xAPIC(cpu)) \| DMAR_IRTE1_V(vector) \|
	DMAR_IRTE1_DLM_FM \| DMAR_IRTE1_TM_EDGE \| DMAR_IRTE1_RH_DIRECT \|
	DMAR_IRTE1_DM_PHYSICAL \| DMAR_IRTE1_P;
	dmar_ir_program_irte(unit, cookie, low, rid);

	if (addr != NULL) {
	/*
	* See VT-d specification, 5.1.5.2 MSI and MSI-X
	* Register Programming.
	*/
	*addr = MSI_INTEL_ADDR_BASE \| ((cookie & 0x7fff) << 5) \|
	((cookie & 0x8000) << 2) \| 0x18;
	*data = 0;
	}
	return (0);
	}

	int
	iommu_unmap_msi_intr(device_t src, u_int cookie)
	{
	struct dmar_unit *unit;

	if (cookie == -1)
	return (0);
	unit = dmar_ir_find(src, NULL, NULL);
	return (dmar_ir_free_irte(unit, cookie));
	}

	int
	iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
	bool activehi, int irq, u_int cookie, uint32_t hi, uint32_t *lo)
	{
	struct dmar_unit *unit;
	vmem_addr_t vmem_res;
	uint64_t low, iorte;
	u_int idx;
	int error;
	uint16_t rid;

	unit = dmar_find_ioapic(ioapic_id, &rid);
	if (unit == NULL \|\| !unit->ir_enabled) {
	*cookie = -1;
	return (EOPNOTSUPP);
	}

	error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT \| M_NOWAIT, &vmem_res);
	if (error != 0) {
	KASSERT(error != EOPNOTSUPP,
	("impossible EOPNOTSUPP from vmem"));
	return (error);
	}
	idx = vmem_res;
	low = 0;
	switch (irq) {
	case IRQ_EXTINT:
	low \|= DMAR_IRTE1_DLM_ExtINT;
	break;
	case IRQ_NMI:
	low \|= DMAR_IRTE1_DLM_NMI;
	break;
	case IRQ_SMI:
	low \|= DMAR_IRTE1_DLM_SMI;
	break;
	default:
	KASSERT(vector != 0, ("No vector for IRQ %u", irq));
	low \|= DMAR_IRTE1_DLM_FM \| DMAR_IRTE1_V(vector);
	break;
	}
	low \|= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :
	DMAR_IRTE1_DST_xAPIC(cpu)) \|
	(edge ? DMAR_IRTE1_TM_EDGE : DMAR_IRTE1_TM_LEVEL) \|
	DMAR_IRTE1_RH_DIRECT \| DMAR_IRTE1_DM_PHYSICAL \| DMAR_IRTE1_P;
	dmar_ir_program_irte(unit, idx, low, rid);

	if (hi != NULL) {
	/*
	* See VT-d specification, 5.1.5.1 I/OxAPIC
	* Programming.
	*/
	iorte = (1ULL << 48) \| ((uint64_t)(idx & 0x7fff) << 49) \|
	((idx & 0x8000) != 0 ? (1 << 11) : 0) \|
	(edge ? IOART_TRGREDG : IOART_TRGRLVL) \|
	(activehi ? IOART_INTAHI : IOART_INTALO) \|
	IOART_DELFIXED \| vector;
	*hi = iorte >> 32;
	*lo = iorte;
	}
	*cookie = idx;
	return (0);
	}

	int
	iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
	{
	struct dmar_unit *unit;
	u_int idx;

	idx = *cookie;
	if (idx == -1)
	return (0);
	*cookie = -1;
	unit = dmar_find_ioapic(ioapic_id, NULL);
	KASSERT(unit != NULL && unit->ir_enabled,
	("unmap: cookie %d unit %p", idx, unit));
	return (dmar_ir_free_irte(unit, idx));
	}

	static struct dmar_unit *
	dmar_ir_find(device_t src, uint16_t rid, int is_dmar)
	{
	devclass_t src_class;
	struct dmar_unit *unit;

	/*
	* We need to determine if the interrupt source generates FSB
	* interrupts. If yes, it is either DMAR, in which case
	* interrupts are not remapped. Or it is HPET, and interrupts
	* are remapped. For HPET, source id is reported by HPET
	* record in DMAR ACPI table.
	*/
	if (is_dmar != NULL)
	*is_dmar = FALSE;
	src_class = device_get_devclass(src);
	if (src_class == devclass_find("dmar")) {
	unit = NULL;
	if (is_dmar != NULL)
	*is_dmar = TRUE;
	} else if (src_class == devclass_find("hpet")) {
	unit = dmar_find_hpet(src, rid);
	} else {
	unit = dmar_find(src, bootverbose);
	if (unit != NULL && rid != NULL)
	iommu_get_requester(src, rid);
	}
	return (unit);
	}

	static void
	dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low,
	uint16_t rid)
	{
	dmar_irte_t *irte;
	uint64_t high;

	KASSERT(idx < unit->irte_cnt,
	("bad cookie %d %d", idx, unit->irte_cnt));
	irte = &(unit->irt[idx]);
	high = DMAR_IRTE2_SVT_RID \| DMAR_IRTE2_SQ_RID \|
	DMAR_IRTE2_SID_RID(rid);
	if (bootverbose) {
	device_printf(unit->dev,
	"programming irte[%d] rid %#x high %#jx low %#jx\n",
	idx, rid, (uintmax_t)high, (uintmax_t)low);
	}
	DMAR_LOCK(unit);
	if ((irte->irte1 & DMAR_IRTE1_P) != 0) {
	/*
	* The rte is already valid. Assume that the request
	* is to remap the interrupt for balancing. Only low
	* word of rte needs to be changed. Assert that the
	* high word contains expected value.
	*/
	KASSERT(irte->irte2 == high,
	("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2,
	(uintmax_t)high));
	dmar_pte_update(&irte->irte1, low);
	} else {
	dmar_pte_store(&irte->irte2, high);
	dmar_pte_store(&irte->irte1, low);
	}
	dmar_qi_invalidate_iec(unit, idx, 1);
	DMAR_UNLOCK(unit);

	}

	static int
	dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie)
	{
	dmar_irte_t *irte;

	KASSERT(unit != NULL && unit->ir_enabled,
	("unmap: cookie %d unit %p", cookie, unit));
	KASSERT(cookie < unit->irte_cnt,
	("bad cookie %u %u", cookie, unit->irte_cnt));
	irte = &(unit->irt[cookie]);
	dmar_pte_clear(&irte->irte1);
	dmar_pte_clear(&irte->irte2);
	DMAR_LOCK(unit);
	dmar_qi_invalidate_iec(unit, cookie, 1);
	DMAR_UNLOCK(unit);
	vmem_free(unit->irtids, cookie, 1);
	return (0);
	}

	static u_int
	clp2(u_int v)
	{

	return (powerof2(v) ? v : 1 << fls(v));
	}

	int
	dmar_init_irt(struct dmar_unit *unit)
	{

	if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)
	return (0);
	unit->ir_enabled = 1;
	TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);
	if (!unit->ir_enabled)
	return (0);
	if (!unit->qi_enabled) {
	unit->ir_enabled = 0;
	if (bootverbose)
	device_printf(unit->dev,
	"QI disabled, disabling interrupt remapping\n");
	return (0);
	}
	unit->irte_cnt = clp2(num_io_irqs);
	unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig(
	unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO \| M_WAITOK, 0,
	dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ?
	VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE);
	if (unit->irt == NULL)
	return (ENOMEM);
	unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);
	unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,
	M_FIRSTFIT \| M_NOWAIT);
	DMAR_LOCK(unit);
	dmar_load_irt_ptr(unit);
	dmar_qi_invalidate_iec_glob(unit);
	DMAR_UNLOCK(unit);

	/*
	* Initialize mappings for already configured interrupt pins.
	* Required, because otherwise the interrupts fault without
	* irtes.
	*/
	intr_reprogram();

	DMAR_LOCK(unit);
	dmar_enable_ir(unit);
	DMAR_UNLOCK(unit);
	return (0);
	}

	void
	dmar_fini_irt(struct dmar_unit *unit)
	{

	unit->ir_enabled = 0;
	if (unit->irt != NULL) {
	dmar_disable_ir(unit);
	dmar_qi_invalidate_iec_glob(unit);
	vmem_destroy(unit->irtids);
	kmem_free((vm_offset_t)unit->irt, unit->irte_cnt *
	sizeof(dmar_irte_t));
	}
	}
	diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c
	index 5377ac448df8..ab2c4d4e80fc 100644
	--- a/sys/x86/iommu/intel_qi.c
	+++ b/sys/x86/iommu/intel_qi.c
	@@ -1,478 +1,478 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_acpi.h"

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/malloc.h>
	#include <sys/memdesc.h>
	#include <sys/module.h>
	#include <sys/rman.h>
	#include <sys/taskqueue.h>
	#include <sys/time.h>
	#include <sys/tree.h>
	#include <sys/vmem.h>
	-#include <machine/bus.h>
	-#include <contrib/dev/acpica/include/acpi.h>
	-#include <contrib/dev/acpica/include/accommon.h>
	-#include <dev/acpica/acpivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_page.h>
	#include <vm/vm_map.h>
	+#include <contrib/dev/acpica/include/acpi.h>
	+#include <contrib/dev/acpica/include/accommon.h>
	+#include <dev/acpica/acpivar.h>
	+#include <dev/pci/pcireg.h>
	+#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	-#include <dev/pci/pcireg.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>

	static bool
	dmar_qi_seq_processed(const struct dmar_unit *unit,
	const struct iommu_qi_genseq *pseq)
	{

	return (pseq->gen < unit->inv_waitd_gen \|\|
	(pseq->gen == unit->inv_waitd_gen &&
	pseq->seq <= unit->inv_waitd_seq_hw));
	}

	static int
	dmar_enable_qi(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);
	unit->hw_gcmd \|= DMAR_GCMD_QIE;
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
	!= 0));
	return (error);
	}

	static int
	dmar_disable_qi(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);
	unit->hw_gcmd &= ~DMAR_GCMD_QIE;
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
	== 0));
	return (error);
	}

	static void
	dmar_qi_advance_tail(struct dmar_unit *unit)
	{

	DMAR_ASSERT_LOCKED(unit);
	dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
	}

	static void
	dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
	{
	uint32_t head;
	int bytes;

	DMAR_ASSERT_LOCKED(unit);
	bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
	for (;;) {
	if (bytes <= unit->inv_queue_avail)
	break;
	/* refill */
	head = dmar_read4(unit, DMAR_IQH_REG);
	head &= DMAR_IQH_MASK;
	unit->inv_queue_avail = head - unit->inv_queue_tail -
	DMAR_IQ_DESCR_SZ;
	if (head <= unit->inv_queue_tail)
	unit->inv_queue_avail += unit->inv_queue_size;
	if (bytes <= unit->inv_queue_avail)
	break;

	/*
	* No space in the queue, do busy wait. Hardware must
	* make a progress. But first advance the tail to
	* inform the descriptor streamer about entries we
	* might have already filled, otherwise they could
	* clog the whole queue..
	*/
	dmar_qi_advance_tail(unit);
	unit->inv_queue_full++;
	cpu_spinwait();
	}
	unit->inv_queue_avail -= bytes;
	}

	static void
	dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
	{

	DMAR_ASSERT_LOCKED(unit);
	(volatile uint64_t )(unit->inv_queue + unit->inv_queue_tail) = data1;
	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
	("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
	(uintmax_t)unit->inv_queue_size));
	unit->inv_queue_tail &= unit->inv_queue_size - 1;
	(volatile uint64_t )(unit->inv_queue + unit->inv_queue_tail) = data2;
	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
	("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
	(uintmax_t)unit->inv_queue_size));
	unit->inv_queue_tail &= unit->inv_queue_size - 1;
	}

	static void
	dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
	bool memw, bool fence)
	{

	DMAR_ASSERT_LOCKED(unit);
	dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID \|
	(intr ? DMAR_IQ_DESCR_WAIT_IF : 0) \|
	(memw ? DMAR_IQ_DESCR_WAIT_SW : 0) \|
	(fence ? DMAR_IQ_DESCR_WAIT_FN : 0) \|
	(memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
	memw ? unit->inv_waitd_seq_hw_phys : 0);
	}

	static void
	dmar_qi_emit_wait_seq(struct dmar_unit unit, struct iommu_qi_genseq pseq,
	bool emit_wait)
	{
	struct iommu_qi_genseq gsec;
	uint32_t seq;

	KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
	DMAR_ASSERT_LOCKED(unit);
	if (unit->inv_waitd_seq == 0xffffffff) {
	gsec.gen = unit->inv_waitd_gen;
	gsec.seq = unit->inv_waitd_seq;
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
	dmar_qi_advance_tail(unit);
	while (!dmar_qi_seq_processed(unit, &gsec))
	cpu_spinwait();
	unit->inv_waitd_gen++;
	unit->inv_waitd_seq = 1;
	}
	seq = unit->inv_waitd_seq++;
	pseq->gen = unit->inv_waitd_gen;
	pseq->seq = seq;
	if (emit_wait) {
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit_wait_descr(unit, seq, true, true, false);
	}
	}

	static void
	dmar_qi_wait_for_seq(struct dmar_unit unit, const struct iommu_qi_genseq gseq,
	bool nowait)
	{

	DMAR_ASSERT_LOCKED(unit);
	unit->inv_seq_waiters++;
	while (!dmar_qi_seq_processed(unit, gseq)) {
	if (cold \|\| nowait) {
	cpu_spinwait();
	} else {
	msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0,
	"dmarse", hz);
	}
	}
	unit->inv_seq_waiters--;
	}

	void
	dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base,
	iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
	{
	struct dmar_unit *unit;
	iommu_gaddr_t isize;
	int am;

	unit = domain->dmar;
	DMAR_ASSERT_LOCKED(unit);
	for (; size > 0; base += isize, size -= isize) {
	am = calc_am(unit, base, size, &isize);
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \|
	DMAR_IQ_DESCR_IOTLB_PAGE \| DMAR_IQ_DESCR_IOTLB_DW \|
	DMAR_IQ_DESCR_IOTLB_DR \|
	DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
	base \| am);
	}
	dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
	dmar_qi_advance_tail(unit);
	}

	void
	dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
	{
	struct iommu_qi_genseq gseq;

	DMAR_ASSERT_LOCKED(unit);
	dmar_qi_ensure(unit, 2);
	dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV \| DMAR_IQ_DESCR_CTX_GLOB, 0);
	dmar_qi_emit_wait_seq(unit, &gseq, true);
	dmar_qi_advance_tail(unit);
	dmar_qi_wait_for_seq(unit, &gseq, false);
	}

	void
	dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
	{
	struct iommu_qi_genseq gseq;

	DMAR_ASSERT_LOCKED(unit);
	dmar_qi_ensure(unit, 2);
	dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV \| DMAR_IQ_DESCR_IOTLB_GLOB \|
	DMAR_IQ_DESCR_IOTLB_DW \| DMAR_IQ_DESCR_IOTLB_DR, 0);
	dmar_qi_emit_wait_seq(unit, &gseq, true);
	dmar_qi_advance_tail(unit);
	dmar_qi_wait_for_seq(unit, &gseq, false);
	}

	void
	dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
	{
	struct iommu_qi_genseq gseq;

	DMAR_ASSERT_LOCKED(unit);
	dmar_qi_ensure(unit, 2);
	dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
	dmar_qi_emit_wait_seq(unit, &gseq, true);
	dmar_qi_advance_tail(unit);
	dmar_qi_wait_for_seq(unit, &gseq, false);
	}

	void
	dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
	{
	struct iommu_qi_genseq gseq;
	u_int c, l;

	DMAR_ASSERT_LOCKED(unit);
	KASSERT(start < unit->irte_cnt && start < start + cnt &&
	start + cnt <= unit->irte_cnt,
	("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
	for (; cnt > 0; cnt -= c, start += c) {
	l = ffs(start \| cnt) - 1;
	c = 1 << l;
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV \|
	DMAR_IQ_DESCR_IEC_IDX \| DMAR_IQ_DESCR_IEC_IIDX(start) \|
	DMAR_IQ_DESCR_IEC_IM(l), 0);
	}
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit_wait_seq(unit, &gseq, true);
	dmar_qi_advance_tail(unit);

	/*
	* The caller of the function, in particular,
	* dmar_ir_program_irte(), may be called from the context
	* where the sleeping is forbidden (in fact, the
	* intr_table_lock mutex may be held, locked from
	* intr_shuffle_irqs()). Wait for the invalidation completion
	* using the busy wait.
	*
	* The impact on the interrupt input setup code is small, the
	* expected overhead is comparable with the chipset register
	* read. It is more harmful for the parallel DMA operations,
	* since we own the dmar unit lock until whole invalidation
	* queue is processed, which includes requests possibly issued
	* before our request.
	*/
	dmar_qi_wait_for_seq(unit, &gseq, true);
	}

	int
	dmar_qi_intr(void *arg)
	{
	struct dmar_unit *unit;

	unit = arg;
	KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
	unit->iommu.unit));
	taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
	return (FILTER_HANDLED);
	}

	static void
	dmar_qi_task(void *arg, int pending __unused)
	{
	struct dmar_unit *unit;
	struct iommu_map_entry *entry;
	uint32_t ics;

	unit = arg;

	DMAR_LOCK(unit);
	for (;;) {
	entry = TAILQ_FIRST(&unit->tlb_flush_entries);
	if (entry == NULL)
	break;
	if (!dmar_qi_seq_processed(unit, &entry->gseq))
	break;
	TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
	DMAR_UNLOCK(unit);
	dmar_domain_free_entry(entry, (entry->flags &
	IOMMU_MAP_ENTRY_QI_NF) == 0);
	DMAR_LOCK(unit);
	}
	ics = dmar_read4(unit, DMAR_ICS_REG);
	if ((ics & DMAR_ICS_IWC) != 0) {
	ics = DMAR_ICS_IWC;
	dmar_write4(unit, DMAR_ICS_REG, ics);
	}
	if (unit->inv_seq_waiters > 0)
	wakeup(&unit->inv_seq_waiters);
	DMAR_UNLOCK(unit);
	}

	int
	dmar_init_qi(struct dmar_unit *unit)
	{
	uint64_t iqa;
	uint32_t ics;
	int qi_sz;

	if (!DMAR_HAS_QI(unit) \|\| (unit->hw_cap & DMAR_CAP_CM) != 0)
	return (0);
	unit->qi_enabled = 1;
	TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
	if (!unit->qi_enabled)
	return (0);

	TAILQ_INIT(&unit->tlb_flush_entries);
	TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
	unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
	taskqueue_thread_enqueue, &unit->qi_taskqueue);
	taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
	"dmar%d qi taskq", unit->iommu.unit);

	unit->inv_waitd_gen = 0;
	unit->inv_waitd_seq = 1;

	qi_sz = DMAR_IQA_QS_DEF;
	TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
	if (qi_sz > DMAR_IQA_QS_MAX)
	qi_sz = DMAR_IQA_QS_MAX;
	unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
	/* Reserve one descriptor to prevent wraparound. */
	unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;

	/* The invalidation queue reads by DMARs are always coherent. */
	unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK \|
	M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
	unit->inv_waitd_seq_hw_phys = pmap_kextract(
	(vm_offset_t)&unit->inv_waitd_seq_hw);

	DMAR_LOCK(unit);
	dmar_write8(unit, DMAR_IQT_REG, 0);
	iqa = pmap_kextract(unit->inv_queue);
	iqa \|= qi_sz;
	dmar_write8(unit, DMAR_IQA_REG, iqa);
	dmar_enable_qi(unit);
	ics = dmar_read4(unit, DMAR_ICS_REG);
	if ((ics & DMAR_ICS_IWC) != 0) {
	ics = DMAR_ICS_IWC;
	dmar_write4(unit, DMAR_ICS_REG, ics);
	}
	dmar_enable_qi_intr(unit);
	DMAR_UNLOCK(unit);

	return (0);
	}

	void
	dmar_fini_qi(struct dmar_unit *unit)
	{
	struct iommu_qi_genseq gseq;

	if (!unit->qi_enabled)
	return;
	taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
	taskqueue_free(unit->qi_taskqueue);
	unit->qi_taskqueue = NULL;

	DMAR_LOCK(unit);
	/* quisce */
	dmar_qi_ensure(unit, 1);
	dmar_qi_emit_wait_seq(unit, &gseq, true);
	dmar_qi_advance_tail(unit);
	dmar_qi_wait_for_seq(unit, &gseq, false);
	/* only after the quisce, disable queue */
	dmar_disable_qi_intr(unit);
	dmar_disable_qi(unit);
	KASSERT(unit->inv_seq_waiters == 0,
	("dmar%d: waiters on disabled queue", unit->iommu.unit));
	DMAR_UNLOCK(unit);

	kmem_free(unit->inv_queue, unit->inv_queue_size);
	unit->inv_queue = 0;
	unit->inv_queue_size = 0;
	unit->qi_enabled = 0;
	}

	void
	dmar_enable_qi_intr(struct dmar_unit *unit)
	{
	uint32_t iectl;

	DMAR_ASSERT_LOCKED(unit);
	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
	unit->iommu.unit));
	iectl = dmar_read4(unit, DMAR_IECTL_REG);
	iectl &= ~DMAR_IECTL_IM;
	dmar_write4(unit, DMAR_IECTL_REG, iectl);
	}

	void
	dmar_disable_qi_intr(struct dmar_unit *unit)
	{
	uint32_t iectl;

	DMAR_ASSERT_LOCKED(unit);
	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
	unit->iommu.unit));
	iectl = dmar_read4(unit, DMAR_IECTL_REG);
	dmar_write4(unit, DMAR_IECTL_REG, iectl \| DMAR_IECTL_IM);
	}
	diff --git a/sys/x86/iommu/intel_quirks.c b/sys/x86/iommu/intel_quirks.c
	index d0eac82e7298..1a025b3419eb 100644
	--- a/sys/x86/iommu/intel_quirks.c
	+++ b/sys/x86/iommu/intel_quirks.c
	@@ -1,245 +1,245 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013, 2015 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/memdesc.h>
	#include <sys/module.h>
	#include <sys/mutex.h>
	#include <sys/rman.h>
	#include <sys/rwlock.h>
	#include <sys/smp.h>
	#include <sys/taskqueue.h>
	#include <sys/tree.h>
	#include <sys/vmem.h>
	-#include <machine/bus.h>
	-#include <contrib/dev/acpica/include/acpi.h>
	-#include <contrib/dev/acpica/include/accommon.h>
	-#include <dev/acpica/acpivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_map.h>
	+#include <contrib/dev/acpica/include/acpi.h>
	+#include <contrib/dev/acpica/include/accommon.h>
	+#include <dev/acpica/acpivar.h>
	+#include <dev/pci/pcireg.h>
	+#include <dev/pci/pcivar.h>
	+#include <machine/bus.h>
	#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	-#include <dev/pci/pcireg.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>
	-#include <dev/pci/pcivar.h>

	typedef void (dmar_quirk_cpu_fun)(struct dmar_unit );

	struct intel_dmar_quirk_cpu {
	u_int ext_family;
	u_int ext_model;
	u_int family_code;
	u_int model;
	u_int stepping;
	dmar_quirk_cpu_fun quirk;
	const char *descr;
	};

	typedef void (dmar_quirk_nb_fun)(struct dmar_unit , device_t nb);

	struct intel_dmar_quirk_nb {
	u_int dev_id;
	u_int rev_no;
	dmar_quirk_nb_fun quirk;
	const char *descr;
	};

	#define QUIRK_NB_ALL_REV 0xffffffff

	static void
	dmar_match_quirks(struct dmar_unit *dmar,
	const struct intel_dmar_quirk_nb *nb_quirks, int nb_quirks_len,
	const struct intel_dmar_quirk_cpu *cpu_quirks, int cpu_quirks_len)
	{
	device_t nb;
	const struct intel_dmar_quirk_nb *nb_quirk;
	const struct intel_dmar_quirk_cpu *cpu_quirk;
	u_int p[4];
	u_int dev_id, rev_no;
	u_int ext_family, ext_model, family_code, model, stepping;
	int i;

	if (nb_quirks != NULL) {
	nb = pci_find_bsf(0, 0, 0);
	if (nb != NULL) {
	dev_id = pci_get_device(nb);
	rev_no = pci_get_revid(nb);
	for (i = 0; i < nb_quirks_len; i++) {
	nb_quirk = &nb_quirks[i];
	if (nb_quirk->dev_id == dev_id &&
	(nb_quirk->rev_no == rev_no \|\|
	nb_quirk->rev_no == QUIRK_NB_ALL_REV)) {
	if (bootverbose) {
	device_printf(dmar->dev,
	"NB IOMMU quirk %s\n",
	nb_quirk->descr);
	}
	nb_quirk->quirk(dmar, nb);
	}
	}
	} else {
	device_printf(dmar->dev, "cannot find northbridge\n");
	}
	}
	if (cpu_quirks != NULL) {
	do_cpuid(1, p);
	ext_family = (p[0] & CPUID_EXT_FAMILY) >> 20;
	ext_model = (p[0] & CPUID_EXT_MODEL) >> 16;
	family_code = (p[0] & CPUID_FAMILY) >> 8;
	model = (p[0] & CPUID_MODEL) >> 4;
	stepping = p[0] & CPUID_STEPPING;
	for (i = 0; i < cpu_quirks_len; i++) {
	cpu_quirk = &cpu_quirks[i];
	if (cpu_quirk->ext_family == ext_family &&
	cpu_quirk->ext_model == ext_model &&
	cpu_quirk->family_code == family_code &&
	cpu_quirk->model == model &&
	(cpu_quirk->stepping == -1 \|\|
	cpu_quirk->stepping == stepping)) {
	if (bootverbose) {
	device_printf(dmar->dev,
	"CPU IOMMU quirk %s\n",
	cpu_quirk->descr);
	}
	cpu_quirk->quirk(dmar);
	}
	}
	}
	}

	static void
	nb_5400_no_low_high_prot_mem(struct dmar_unit *unit, device_t nb __unused)
	{

	unit->hw_cap &= ~(DMAR_CAP_PHMR \| DMAR_CAP_PLMR);
	}

	static void
	nb_no_ir(struct dmar_unit *unit, device_t nb __unused)
	{

	unit->hw_ecap &= ~(DMAR_ECAP_IR \| DMAR_ECAP_EIM);
	}

	static void
	nb_5500_no_ir_rev13(struct dmar_unit *unit, device_t nb)
	{
	u_int rev_no;

	rev_no = pci_get_revid(nb);
	if (rev_no <= 0x13)
	nb_no_ir(unit, nb);
	}

	static const struct intel_dmar_quirk_nb pre_use_nb[] = {
	{
	.dev_id = 0x4001, .rev_no = 0x20,
	.quirk = nb_5400_no_low_high_prot_mem,
	.descr = "5400 E23" /* no low/high protected memory */
	},
	{
	.dev_id = 0x4003, .rev_no = 0x20,
	.quirk = nb_5400_no_low_high_prot_mem,
	.descr = "5400 E23" /* no low/high protected memory */
	},
	{
	.dev_id = 0x3403, .rev_no = QUIRK_NB_ALL_REV,
	.quirk = nb_5500_no_ir_rev13,
	.descr = "5500 E47, E53" /* interrupt remapping does not work */
	},
	{
	.dev_id = 0x3405, .rev_no = QUIRK_NB_ALL_REV,
	.quirk = nb_5500_no_ir_rev13,
	.descr = "5500 E47, E53" /* interrupt remapping does not work */
	},
	{
	.dev_id = 0x3405, .rev_no = 0x22,
	.quirk = nb_no_ir,
	.descr = "5500 E47, E53" /* interrupt remapping does not work */
	},
	{
	.dev_id = 0x3406, .rev_no = QUIRK_NB_ALL_REV,
	.quirk = nb_5500_no_ir_rev13,
	.descr = "5500 E47, E53" /* interrupt remapping does not work */
	},
	};

	static void
	cpu_e5_am9(struct dmar_unit *unit)
	{

	unit->hw_cap &= ~(0x3fULL << 48);
	unit->hw_cap \|= (9ULL << 48);
	}

	static const struct intel_dmar_quirk_cpu post_ident_cpu[] = {
	{
	.ext_family = 0, .ext_model = 2, .family_code = 6, .model = 13,
	.stepping = 6, .quirk = cpu_e5_am9,
	.descr = "E5 BT176" /* AM should be at most 9 */
	},
	};

	void
	dmar_quirks_pre_use(struct iommu_unit *unit)
	{
	struct dmar_unit *dmar;

	dmar = (struct dmar_unit *)unit;

	if (!dmar_barrier_enter(dmar, DMAR_BARRIER_USEQ))
	return;
	DMAR_LOCK(dmar);
	dmar_match_quirks(dmar, pre_use_nb, nitems(pre_use_nb),
	NULL, 0);
	dmar_barrier_exit(dmar, DMAR_BARRIER_USEQ);
	}

	void
	dmar_quirks_post_ident(struct dmar_unit *dmar)
	{

	dmar_match_quirks(dmar, NULL, 0, post_ident_cpu,
	nitems(post_ident_cpu));
	}
	diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c
	index 7e89465240af..43a00428c46c 100644
	--- a/sys/x86/iommu/intel_utils.c
	+++ b/sys/x86/iommu/intel_utils.c
	@@ -1,669 +1,669 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/proc.h>
	#include <sys/queue.h>
	#include <sys/rman.h>
	#include <sys/rwlock.h>
	#include <sys/sched.h>
	#include <sys/sf_buf.h>
	#include <sys/sysctl.h>
	#include <sys/systm.h>
	#include <sys/taskqueue.h>
	#include <sys/time.h>
	#include <sys/tree.h>
	#include <sys/vmem.h>
	-#include <dev/pci/pcivar.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_page.h>
	#include <vm/vm_map.h>
	#include <vm/vm_pageout.h>
	+#include <dev/pci/pcireg.h>
	+#include <dev/pci/pcivar.h>
	#include <machine/bus.h>
	#include <machine/cpu.h>
	#include <machine/intr_machdep.h>
	#include <x86/include/apicvar.h>
	#include <x86/include/busdma_impl.h>
	-#include <x86/iommu/intel_reg.h>
	#include <dev/iommu/busdma_iommu.h>
	-#include <dev/pci/pcireg.h>
	+#include <x86/iommu/intel_reg.h>
	#include <x86/iommu/intel_dmar.h>

	u_int
	dmar_nd2mask(u_int nd)
	{
	static const u_int masks[] = {
	0x000f, /* nd == 0 */
	0x002f, /* nd == 1 */
	0x00ff, /* nd == 2 */
	0x02ff, /* nd == 3 */
	0x0fff, /* nd == 4 */
	0x2fff, /* nd == 5 */
	0xffff, /* nd == 6 */
	0x0000, /* nd == 7 reserved */
	};

	KASSERT(nd <= 6, ("number of domains %d", nd));
	return (masks[nd]);
	}

	static const struct sagaw_bits_tag {
	int agaw;
	int cap;
	int awlvl;
	int pglvl;
	} sagaw_bits[] = {
	{.agaw = 30, .cap = DMAR_CAP_SAGAW_2LVL, .awlvl = DMAR_CTX2_AW_2LVL,
	.pglvl = 2},
	{.agaw = 39, .cap = DMAR_CAP_SAGAW_3LVL, .awlvl = DMAR_CTX2_AW_3LVL,
	.pglvl = 3},
	{.agaw = 48, .cap = DMAR_CAP_SAGAW_4LVL, .awlvl = DMAR_CTX2_AW_4LVL,
	.pglvl = 4},
	{.agaw = 57, .cap = DMAR_CAP_SAGAW_5LVL, .awlvl = DMAR_CTX2_AW_5LVL,
	.pglvl = 5},
	{.agaw = 64, .cap = DMAR_CAP_SAGAW_6LVL, .awlvl = DMAR_CTX2_AW_6LVL,
	.pglvl = 6}
	};

	bool
	dmar_pglvl_supported(struct dmar_unit *unit, int pglvl)
	{
	int i;

	for (i = 0; i < nitems(sagaw_bits); i++) {
	if (sagaw_bits[i].pglvl != pglvl)
	continue;
	if ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0)
	return (true);
	}
	return (false);
	}

	int
	domain_set_agaw(struct dmar_domain *domain, int mgaw)
	{
	int sagaw, i;

	domain->mgaw = mgaw;
	sagaw = DMAR_CAP_SAGAW(domain->dmar->hw_cap);
	for (i = 0; i < nitems(sagaw_bits); i++) {
	if (sagaw_bits[i].agaw >= mgaw) {
	domain->agaw = sagaw_bits[i].agaw;
	domain->pglvl = sagaw_bits[i].pglvl;
	domain->awlvl = sagaw_bits[i].awlvl;
	return (0);
	}
	}
	device_printf(domain->dmar->dev,
	"context request mgaw %d: no agaw found, sagaw %x\n",
	mgaw, sagaw);
	return (EINVAL);
	}

	/*
	* Find a best fit mgaw for the given maxaddr:
	* - if allow_less is false, must find sagaw which maps all requested
	* addresses (used by identity mappings);
	* - if allow_less is true, and no supported sagaw can map all requested
	* address space, accept the biggest sagaw, whatever is it.
	*/
	int
	dmar_maxaddr2mgaw(struct dmar_unit *unit, iommu_gaddr_t maxaddr, bool allow_less)
	{
	int i;

	for (i = 0; i < nitems(sagaw_bits); i++) {
	if ((1ULL << sagaw_bits[i].agaw) >= maxaddr &&
	(DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0)
	break;
	}
	if (allow_less && i == nitems(sagaw_bits)) {
	do {
	i--;
	} while ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap)
	== 0);
	}
	if (i < nitems(sagaw_bits))
	return (sagaw_bits[i].agaw);
	KASSERT(0, ("no mgaw for maxaddr %jx allow_less %d",
	(uintmax_t) maxaddr, allow_less));
	return (-1);
	}

	/*
	* Calculate the total amount of page table pages needed to map the
	* whole bus address space on the context with the selected agaw.
	*/
	vm_pindex_t
	pglvl_max_pages(int pglvl)
	{
	vm_pindex_t res;
	int i;

	for (res = 0, i = pglvl; i > 0; i--) {
	res *= DMAR_NPTEPG;
	res++;
	}
	return (res);
	}

	/*
	* Return true if the page table level lvl supports the superpage for
	* the context ctx.
	*/
	int
	domain_is_sp_lvl(struct dmar_domain *domain, int lvl)
	{
	int alvl, cap_sps;
	static const int sagaw_sp[] = {
	DMAR_CAP_SPS_2M,
	DMAR_CAP_SPS_1G,
	DMAR_CAP_SPS_512G,
	DMAR_CAP_SPS_1T
	};

	alvl = domain->pglvl - lvl - 1;
	cap_sps = DMAR_CAP_SPS(domain->dmar->hw_cap);
	return (alvl < nitems(sagaw_sp) && (sagaw_sp[alvl] & cap_sps) != 0);
	}

	iommu_gaddr_t
	pglvl_page_size(int total_pglvl, int lvl)
	{
	int rlvl;
	static const iommu_gaddr_t pg_sz[] = {
	(iommu_gaddr_t)DMAR_PAGE_SIZE,
	(iommu_gaddr_t)DMAR_PAGE_SIZE << DMAR_NPTEPGSHIFT,
	(iommu_gaddr_t)DMAR_PAGE_SIZE << (2 * DMAR_NPTEPGSHIFT),
	(iommu_gaddr_t)DMAR_PAGE_SIZE << (3 * DMAR_NPTEPGSHIFT),
	(iommu_gaddr_t)DMAR_PAGE_SIZE << (4 * DMAR_NPTEPGSHIFT),
	(iommu_gaddr_t)DMAR_PAGE_SIZE << (5 * DMAR_NPTEPGSHIFT)
	};

	KASSERT(lvl >= 0 && lvl < total_pglvl,
	("total %d lvl %d", total_pglvl, lvl));
	rlvl = total_pglvl - lvl - 1;
	KASSERT(rlvl < nitems(pg_sz), ("sizeof pg_sz lvl %d", lvl));
	return (pg_sz[rlvl]);
	}

	iommu_gaddr_t
	domain_page_size(struct dmar_domain *domain, int lvl)
	{

	return (pglvl_page_size(domain->pglvl, lvl));
	}

	int
	calc_am(struct dmar_unit *unit, iommu_gaddr_t base, iommu_gaddr_t size,
	iommu_gaddr_t *isizep)
	{
	iommu_gaddr_t isize;
	int am;

	for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) {
	isize = 1ULL << (am + DMAR_PAGE_SHIFT);
	if ((base & (isize - 1)) == 0 && size >= isize)
	break;
	if (am == 0)
	break;
	}
	*isizep = isize;
	return (am);
	}

	iommu_haddr_t dmar_high;
	int haw;
	int dmar_tbl_pagecnt;

	vm_page_t
	dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags)
	{
	vm_page_t m;
	int zeroed, aflags;

	zeroed = (flags & IOMMU_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0;
	aflags = zeroed \| VM_ALLOC_NOBUSY \| VM_ALLOC_SYSTEM \| VM_ALLOC_NODUMP \|
	((flags & IOMMU_PGF_WAITOK) != 0 ? VM_ALLOC_WAITFAIL :
	VM_ALLOC_NOWAIT);
	for (;;) {
	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WLOCK(obj);
	m = vm_page_lookup(obj, idx);
	if ((flags & IOMMU_PGF_NOALLOC) != 0 \|\| m != NULL) {
	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WUNLOCK(obj);
	break;
	}
	m = vm_page_alloc_contig(obj, idx, aflags, 1, 0,
	dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WUNLOCK(obj);
	if (m != NULL) {
	if (zeroed && (m->flags & PG_ZERO) == 0)
	pmap_zero_page(m);
	atomic_add_int(&dmar_tbl_pagecnt, 1);
	break;
	}
	if ((flags & IOMMU_PGF_WAITOK) == 0)
	break;
	}
	return (m);
	}

	void
	dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags)
	{
	vm_page_t m;

	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WLOCK(obj);
	m = vm_page_grab(obj, idx, VM_ALLOC_NOCREAT);
	if (m != NULL) {
	vm_page_free(m);
	atomic_subtract_int(&dmar_tbl_pagecnt, 1);
	}
	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WUNLOCK(obj);
	}

	void *
	dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
	struct sf_buf **sf)
	{
	vm_page_t m;
	bool allocated;

	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WLOCK(obj);
	m = vm_page_lookup(obj, idx);
	if (m == NULL && (flags & IOMMU_PGF_ALLOC) != 0) {
	m = dmar_pgalloc(obj, idx, flags \| IOMMU_PGF_OBJL);
	allocated = true;
	} else
	allocated = false;
	if (m == NULL) {
	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WUNLOCK(obj);
	return (NULL);
	}
	/* Sleepable allocations cannot fail. */
	if ((flags & IOMMU_PGF_WAITOK) != 0)
	VM_OBJECT_WUNLOCK(obj);
	sched_pin();
	*sf = sf_buf_alloc(m, SFB_CPUPRIVATE \| ((flags & IOMMU_PGF_WAITOK)
	== 0 ? SFB_NOWAIT : 0));
	if (*sf == NULL) {
	sched_unpin();
	if (allocated) {
	VM_OBJECT_ASSERT_WLOCKED(obj);
	dmar_pgfree(obj, m->pindex, flags \| IOMMU_PGF_OBJL);
	}
	if ((flags & IOMMU_PGF_OBJL) == 0)
	VM_OBJECT_WUNLOCK(obj);
	return (NULL);
	}
	if ((flags & (IOMMU_PGF_WAITOK \| IOMMU_PGF_OBJL)) ==
	(IOMMU_PGF_WAITOK \| IOMMU_PGF_OBJL))
	VM_OBJECT_WLOCK(obj);
	else if ((flags & (IOMMU_PGF_WAITOK \| IOMMU_PGF_OBJL)) == 0)
	VM_OBJECT_WUNLOCK(obj);
	return ((void )sf_buf_kva(sf));
	}

	void
	dmar_unmap_pgtbl(struct sf_buf *sf)
	{

	sf_buf_free(sf);
	sched_unpin();
	}

	static void
	dmar_flush_transl_to_ram(struct dmar_unit unit, void dst, size_t sz)
	{

	if (DMAR_IS_COHERENT(unit))
	return;
	/*
	* If DMAR does not snoop paging structures accesses, flush
	* CPU cache to memory.
	*/
	pmap_force_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz);
	}

	void
	dmar_flush_pte_to_ram(struct dmar_unit unit, dmar_pte_t dst)
	{

	dmar_flush_transl_to_ram(unit, dst, sizeof(*dst));
	}

	void
	dmar_flush_ctx_to_ram(struct dmar_unit unit, dmar_ctx_entry_t dst)
	{

	dmar_flush_transl_to_ram(unit, dst, sizeof(*dst));
	}

	void
	dmar_flush_root_to_ram(struct dmar_unit unit, dmar_root_entry_t dst)
	{

	dmar_flush_transl_to_ram(unit, dst, sizeof(*dst));
	}

	/*
	* Load the root entry pointer into the hardware, busily waiting for
	* the completion.
	*/
	int
	dmar_load_root_entry_ptr(struct dmar_unit *unit)
	{
	vm_page_t root_entry;
	int error;

	/*
	* Access to the GCMD register must be serialized while the
	* command is submitted.
	*/
	DMAR_ASSERT_LOCKED(unit);

	VM_OBJECT_RLOCK(unit->ctx_obj);
	root_entry = vm_page_lookup(unit->ctx_obj, 0);
	VM_OBJECT_RUNLOCK(unit->ctx_obj);
	dmar_write8(unit, DMAR_RTADDR_REG, VM_PAGE_TO_PHYS(root_entry));
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd \| DMAR_GCMD_SRTP);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_RTPS)
	!= 0));
	return (error);
	}

	/*
	* Globally invalidate the context entries cache, busily waiting for
	* the completion.
	*/
	int
	dmar_inv_ctx_glob(struct dmar_unit *unit)
	{
	int error;

	/*
	* Access to the CCMD register must be serialized while the
	* command is submitted.
	*/
	DMAR_ASSERT_LOCKED(unit);
	KASSERT(!unit->qi_enabled, ("QI enabled"));

	/*
	* The DMAR_CCMD_ICC bit in the upper dword should be written
	* after the low dword write is completed. Amd64
	* dmar_write8() does not have this issue, i386 dmar_write8()
	* writes the upper dword last.
	*/
	dmar_write8(unit, DMAR_CCMD_REG, DMAR_CCMD_ICC \| DMAR_CCMD_CIRG_GLOB);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_CCMD_REG + 4) & DMAR_CCMD_ICC32)
	== 0));
	return (error);
	}

	/*
	* Globally invalidate the IOTLB, busily waiting for the completion.
	*/
	int
	dmar_inv_iotlb_glob(struct dmar_unit *unit)
	{
	int error, reg;

	DMAR_ASSERT_LOCKED(unit);
	KASSERT(!unit->qi_enabled, ("QI enabled"));

	reg = 16 * DMAR_ECAP_IRO(unit->hw_ecap);
	/* See a comment about DMAR_CCMD_ICC in dmar_inv_ctx_glob. */
	dmar_write8(unit, reg + DMAR_IOTLB_REG_OFF, DMAR_IOTLB_IVT \|
	DMAR_IOTLB_IIRG_GLB \| DMAR_IOTLB_DR \| DMAR_IOTLB_DW);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, reg + DMAR_IOTLB_REG_OFF + 4) &
	DMAR_IOTLB_IVT32) == 0));
	return (error);
	}

	/*
	* Flush the chipset write buffers. See 11.1 "Write Buffer Flushing"
	* in the architecture specification.
	*/
	int
	dmar_flush_write_bufs(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);

	/*
	* DMAR_GCMD_WBF is only valid when CAP_RWBF is reported.
	*/
	KASSERT((unit->hw_cap & DMAR_CAP_RWBF) != 0,
	("dmar%d: no RWBF", unit->iommu.unit));

	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd \| DMAR_GCMD_WBF);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_WBFS)
	!= 0));
	return (error);
	}

	int
	dmar_enable_translation(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);
	unit->hw_gcmd \|= DMAR_GCMD_TE;
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES)
	!= 0));
	return (error);
	}

	int
	dmar_disable_translation(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);
	unit->hw_gcmd &= ~DMAR_GCMD_TE;
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES)
	== 0));
	return (error);
	}

	int
	dmar_load_irt_ptr(struct dmar_unit *unit)
	{
	uint64_t irta, s;
	int error;

	DMAR_ASSERT_LOCKED(unit);
	irta = unit->irt_phys;
	if (DMAR_X2APIC(unit))
	irta \|= DMAR_IRTA_EIME;
	s = fls(unit->irte_cnt) - 2;
	KASSERT(unit->irte_cnt >= 2 && s <= DMAR_IRTA_S_MASK &&
	powerof2(unit->irte_cnt),
	("IRTA_REG_S overflow %x", unit->irte_cnt));
	irta \|= s;
	dmar_write8(unit, DMAR_IRTA_REG, irta);
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd \| DMAR_GCMD_SIRTP);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRTPS)
	!= 0));
	return (error);
	}

	int
	dmar_enable_ir(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);
	unit->hw_gcmd \|= DMAR_GCMD_IRE;
	unit->hw_gcmd &= ~DMAR_GCMD_CFI;
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES)
	!= 0));
	return (error);
	}

	int
	dmar_disable_ir(struct dmar_unit *unit)
	{
	int error;

	DMAR_ASSERT_LOCKED(unit);
	unit->hw_gcmd &= ~DMAR_GCMD_IRE;
	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES)
	== 0));
	return (error);
	}

	#define BARRIER_F \
	u_int f_done, f_inproc, f_wakeup; \
	\
	f_done = 1 << (barrier_id * 3); \
	f_inproc = 1 << (barrier_id * 3 + 1); \
	f_wakeup = 1 << (barrier_id * 3 + 2)

	bool
	dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id)
	{
	BARRIER_F;

	DMAR_LOCK(dmar);
	if ((dmar->barrier_flags & f_done) != 0) {
	DMAR_UNLOCK(dmar);
	return (false);
	}

	if ((dmar->barrier_flags & f_inproc) != 0) {
	while ((dmar->barrier_flags & f_inproc) != 0) {
	dmar->barrier_flags \|= f_wakeup;
	msleep(&dmar->barrier_flags, &dmar->iommu.lock, 0,
	"dmarb", 0);
	}
	KASSERT((dmar->barrier_flags & f_done) != 0,
	("dmar%d barrier %d missing done", dmar->iommu.unit,
	barrier_id));
	DMAR_UNLOCK(dmar);
	return (false);
	}

	dmar->barrier_flags \|= f_inproc;
	DMAR_UNLOCK(dmar);
	return (true);
	}

	void
	dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id)
	{
	BARRIER_F;

	DMAR_ASSERT_LOCKED(dmar);
	KASSERT((dmar->barrier_flags & (f_done \| f_inproc)) == f_inproc,
	("dmar%d barrier %d missed entry", dmar->iommu.unit, barrier_id));
	dmar->barrier_flags \|= f_done;
	if ((dmar->barrier_flags & f_wakeup) != 0)
	wakeup(&dmar->barrier_flags);
	dmar->barrier_flags &= ~(f_inproc \| f_wakeup);
	DMAR_UNLOCK(dmar);
	}

	int dmar_batch_coalesce = 100;
	struct timespec dmar_hw_timeout = {
	.tv_sec = 0,
	.tv_nsec = 1000000
	};

	static const uint64_t d = 1000000000;

	void
	dmar_update_timeout(uint64_t newval)
	{

	/* XXXKIB not atomic */
	dmar_hw_timeout.tv_sec = newval / d;
	dmar_hw_timeout.tv_nsec = newval % d;
	}

	uint64_t
	dmar_get_timeout(void)
	{

	return ((uint64_t)dmar_hw_timeout.tv_sec * d +
	dmar_hw_timeout.tv_nsec);
	}

	static int
	dmar_timeout_sysctl(SYSCTL_HANDLER_ARGS)
	{
	uint64_t val;
	int error;

	val = dmar_get_timeout();
	error = sysctl_handle_long(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	dmar_update_timeout(val);
	return (error);
	}

	static SYSCTL_NODE(_hw_iommu, OID_AUTO, dmar, CTLFLAG_RD \| CTLFLAG_MPSAFE,
	NULL, "");
	SYSCTL_INT(_hw_iommu_dmar, OID_AUTO, tbl_pagecnt, CTLFLAG_RD,
	&dmar_tbl_pagecnt, 0,
	"Count of pages used for DMAR pagetables");
	SYSCTL_INT(_hw_iommu_dmar, OID_AUTO, batch_coalesce, CTLFLAG_RWTUN,
	&dmar_batch_coalesce, 0,
	"Number of qi batches between interrupt");
	SYSCTL_PROC(_hw_iommu_dmar, OID_AUTO, timeout,
	CTLTYPE_U64 \| CTLFLAG_RW \| CTLFLAG_MPSAFE, 0, 0,
	dmar_timeout_sysctl, "QU",
	"Timeout for command wait, in nanoseconds");
	diff --git a/sys/x86/x86/busdma_machdep.c b/sys/x86/x86/busdma_machdep.c
	index 3442313f55c8..168645702bc5 100644
	--- a/sys/x86/x86/busdma_machdep.c
	+++ b/sys/x86/x86/busdma_machdep.c
	@@ -1,315 +1,319 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 1997, 1998 Justin T. Gibbs.
	* Copyright (c) 2013 The FreeBSD Foundation
	* All rights reserved.
	*
	* This software was developed by Konstantin Belousov <kib@FreeBSD.org>
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions, and the following disclaimer,
	* without modification, immediately at the beginning of the file.
	* 2. The name of the author may not be used to endorse or promote products
	* derived from this software without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
	* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_acpi.h"

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/malloc.h>
	#include <sys/bus.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/memdesc.h>
	#include <sys/mutex.h>
	#include <sys/uio.h>
	#include <sys/vmmeter.h>
	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_param.h>
	#include <vm/vm_page.h>
	#include <vm/vm_phys.h>
	#include <vm/pmap.h>
	#include <machine/bus.h>
	#include <x86/include/busdma_impl.h>

	/*
	* Convenience function for manipulating driver locks from busdma (during
	* busdma_swi, for example). Drivers that don't provide their own locks
	* should specify &Giant to dmat->lockfuncarg. Drivers that use their own
	* non-mutex locking scheme don't have to use this at all.
	*/
	void
	busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
	{
	struct mtx *dmtx;

	dmtx = (struct mtx *)arg;
	switch (op) {
	case BUS_DMA_LOCK:
	mtx_lock(dmtx);
	break;
	case BUS_DMA_UNLOCK:
	mtx_unlock(dmtx);
	break;
	default:
	panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
	}
	}

	/*
	* dflt_lock should never get called. It gets put into the dma tag when
	* lockfunc == NULL, which is only valid if the maps that are associated
	* with the tag are meant to never be defered.
	* XXX Should have a way to identify which driver is responsible here.
	*/
	void
	bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op)
	{

	panic("driver error: busdma dflt_lock called");
	}

	/*
	* Return true if a match is made.
	*
	* To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
	*
	* If paddr is within the bounds of the dma tag then call the filter callback
	* to check for a match, if there is no filter callback then assume a match.
	*/
	int
	bus_dma_run_filter(struct bus_dma_tag_common *tc, vm_paddr_t paddr)
	{
	int retval;

	retval = 0;
	do {
	if ((paddr >= BUS_SPACE_MAXADDR \|\|
	(paddr > tc->lowaddr && paddr <= tc->highaddr) \|\|
	(paddr & (tc->alignment - 1)) != 0) &&
	(tc->filter == NULL \|\|
	(*tc->filter)(tc->filterarg, paddr) != 0))
	retval = 1;

	tc = tc->parent;
	} while (retval == 0 && tc != NULL);
	return (retval);
	}

	int
	common_bus_dma_tag_create(struct bus_dma_tag_common *parent,
	bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr,
	bus_addr_t highaddr, bus_dma_filter_t filter, void filterarg,
	bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags,
	bus_dma_lock_t lockfunc, void lockfuncarg, size_t sz, void **dmat)
	{
	void *newtag;
	struct bus_dma_tag_common *common;

	KASSERT(sz >= sizeof(struct bus_dma_tag_common), ("sz"));
	/* Basic sanity checking */
	if (boundary != 0 && boundary < maxsegsz)
	maxsegsz = boundary;
	if (maxsegsz == 0)
	return (EINVAL);
	/* Return a NULL tag on failure */
	*dmat = NULL;

	newtag = malloc(sz, M_DEVBUF, M_ZERO \| M_NOWAIT);
	if (newtag == NULL) {
	CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
	__func__, newtag, 0, ENOMEM);
	return (ENOMEM);
	}

	common = newtag;
	common->impl = &bus_dma_bounce_impl;
	common->parent = parent;
	common->alignment = alignment;
	common->boundary = boundary;
	common->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
	common->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
	common->filter = filter;
	common->filterarg = filterarg;
	common->maxsize = maxsize;
	common->nsegments = nsegments;
	common->maxsegsz = maxsegsz;
	common->flags = flags;
	common->ref_count = 1; /* Count ourself */
	if (lockfunc != NULL) {
	common->lockfunc = lockfunc;
	common->lockfuncarg = lockfuncarg;
	} else {
	common->lockfunc = bus_dma_dflt_lock;
	common->lockfuncarg = NULL;
	}

	/* Take into account any restrictions imposed by our parent tag */
	if (parent != NULL) {
	common->impl = parent->impl;
	common->lowaddr = MIN(parent->lowaddr, common->lowaddr);
	common->highaddr = MAX(parent->highaddr, common->highaddr);
	if (common->boundary == 0)
	common->boundary = parent->boundary;
	else if (parent->boundary != 0) {
	common->boundary = MIN(parent->boundary,
	common->boundary);
	}
	if (common->filter == NULL) {
	/*
	* Short circuit looking at our parent directly
	* since we have encapsulated all of its information
	*/
	common->filter = parent->filter;
	common->filterarg = parent->filterarg;
	common->parent = parent->parent;
	}
	common->domain = parent->domain;
	atomic_add_int(&parent->ref_count, 1);
	}
	common->domain = vm_phys_domain_match(common->domain, 0ul,
	common->lowaddr);
	*dmat = common;
	return (0);
	}

	int
	bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	domain = vm_phys_domain_match(domain, 0ul, tc->lowaddr);
	/* Only call the callback if it changes. */
	if (domain == tc->domain)
	return (0);
	tc->domain = domain;
	return (tc->impl->tag_set_domain(dmat));
	}

	/*
	* Allocate a device specific dma_tag.
	*/
	int
	bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
	bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
	bus_dma_filter_t filter, void filterarg, bus_size_t maxsize,
	int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
	void lockfuncarg, bus_dma_tag_t dmat)
	{
	struct bus_dma_tag_common *tc;
	int error;

	if (parent == NULL) {
	error = bus_dma_bounce_impl.tag_create(parent, alignment,
	boundary, lowaddr, highaddr, filter, filterarg, maxsize,
	nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat);
	} else {
	tc = (struct bus_dma_tag_common *)parent;
	error = tc->impl->tag_create(parent, alignment,
	boundary, lowaddr, highaddr, filter, filterarg, maxsize,
	nsegments, maxsegsz, flags, lockfunc, lockfuncarg, dmat);
	}
	return (error);
	}

	void
	bus_dma_template_init(bus_dma_tag_template_t *t, bus_dma_tag_t parent)
	{

	if (t == NULL)
	return;

	t->parent = parent;
	t->alignment = 1;
	t->boundary = 0;
	t->lowaddr = t->highaddr = BUS_SPACE_MAXADDR;
	t->maxsize = t->maxsegsize = BUS_SPACE_MAXSIZE;
	t->nsegments = BUS_SPACE_UNRESTRICTED;
	t->lockfunc = NULL;
	t->lockfuncarg = NULL;
	t->flags = 0;
	}

	int
	bus_dma_template_tag(bus_dma_tag_template_t t, bus_dma_tag_t dmat)
	{

	if (t == NULL \|\| dmat == NULL)
	return (EINVAL);

	return (bus_dma_tag_create(t->parent, t->alignment, t->boundary,
	t->lowaddr, t->highaddr, NULL, NULL, t->maxsize,
	t->nsegments, t->maxsegsize, t->flags, t->lockfunc, t->lockfuncarg,
	dmat));
	}

	void
	bus_dma_template_clone(bus_dma_tag_template_t *t, bus_dma_tag_t dmat)
	{
	struct bus_dma_tag_common *common;

	if (t == NULL \|\| dmat == NULL)
	return;

	common = (struct bus_dma_tag_common *)dmat;

	t->parent = (bus_dma_tag_t)common->parent;
	t->alignment = common->alignment;
	t->boundary = common->boundary;
	t->lowaddr = common->lowaddr;
	t->highaddr = common->highaddr;
	t->maxsize = common->maxsize;
	t->nsegments = common->nsegments;
	t->maxsegsize = common->maxsegsz;
	t->flags = common->flags;
	t->lockfunc = common->lockfunc;
	t->lockfuncarg = common->lockfuncarg;
	}

	int
	bus_dma_tag_destroy(bus_dma_tag_t dmat)
	{
	struct bus_dma_tag_common *tc;

	tc = (struct bus_dma_tag_common *)dmat;
	return (tc->impl->tag_destroy(dmat));
	}

	#ifndef ACPI_DMAR
	+bool bus_dma_iommu_set_buswide(device_t dev);
	+int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map,
	+ vm_paddr_t start, vm_size_t length, int flags);
	+
	bool
	-bus_dma_dmar_set_buswide(device_t dev)
	+bus_dma_iommu_set_buswide(device_t dev)
	{
	return (false);
	}

	int
	-bus_dma_dmar_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map,
	+bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map,
	vm_paddr_t start, vm_size_t length, int flags)
	{
	return (0);
	}
	#endif
	diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c
	index 74a5261f9112..21efb6d6b122 100644
	--- a/sys/x86/x86/cpu_machdep.c
	+++ b/sys/x86/x86/cpu_machdep.c
	@@ -1,1503 +1,1508 @@
	/*-
	* Copyright (c) 2003 Peter Wemm.
	* Copyright (c) 1992 Terrence R. Lambert.
	* Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
	* All rights reserved.
	*
	* This code is derived from software contributed to Berkeley by
	* William Jolitz.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	* 3. All advertising materials mentioning features or use of this software
	* must display the following acknowledgement:
	* This product includes software developed by the University of
	* California, Berkeley and its contributors.
	* 4. Neither the name of the University nor the names of its contributors
	* may be used to endorse or promote products derived from this software
	* without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include "opt_acpi.h"
	#include "opt_atpic.h"
	#include "opt_cpu.h"
	#include "opt_ddb.h"
	#include "opt_inet.h"
	#include "opt_isa.h"
	#include "opt_kdb.h"
	#include "opt_kstack_pages.h"
	#include "opt_maxmem.h"
	#include "opt_mp_watchdog.h"
	#include "opt_platform.h"
	#ifdef __i386__
	#include "opt_apic.h"
	#endif

	#include <sys/param.h>
	#include <sys/proc.h>
	#include <sys/systm.h>
	#include <sys/bus.h>
	#include <sys/cpu.h>
	#include <sys/domainset.h>
	#include <sys/kdb.h>
	#include <sys/kernel.h>
	#include <sys/ktr.h>
	#include <sys/lock.h>
	#include <sys/malloc.h>
	#include <sys/mutex.h>
	#include <sys/pcpu.h>
	#include <sys/rwlock.h>
	#include <sys/sched.h>
	#include <sys/smp.h>
	#include <sys/sysctl.h>

	#include <machine/clock.h>
	#include <machine/cpu.h>
	#include <machine/cputypes.h>
	#include <machine/specialreg.h>
	#include <machine/md_var.h>
	#include <machine/mp_watchdog.h>
	#include <machine/tss.h>
	#ifdef SMP
	#include <machine/smp.h>
	#endif
	#ifdef CPU_ELAN
	#include <machine/elan_mmcr.h>
	#endif
	#include <x86/acpica_machdep.h>

	#include <vm/vm.h>
	#include <vm/vm_extern.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_page.h>
	#include <vm/vm_map.h>
	#include <vm/vm_object.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_param.h>

	#include <isa/isareg.h>

	#include <contrib/dev/acpica/include/acpi.h>

	#define STATE_RUNNING 0x0
	#define STATE_MWAIT 0x1
	#define STATE_SLEEPING 0x2

	#ifdef SMP
	static u_int cpu_reset_proxyid;
	static volatile u_int cpu_reset_proxy_active;
	#endif

	struct msr_op_arg {
	u_int msr;
	int op;
	uint64_t arg1;
	};

	static void
	x86_msr_op_one(void *argp)
	{
	struct msr_op_arg *a;
	uint64_t v;

	a = argp;
	switch (a->op) {
	case MSR_OP_ANDNOT:
	v = rdmsr(a->msr);
	v &= ~a->arg1;
	wrmsr(a->msr, v);
	break;
	case MSR_OP_OR:
	v = rdmsr(a->msr);
	v \|= a->arg1;
	wrmsr(a->msr, v);
	break;
	case MSR_OP_WRITE:
	wrmsr(a->msr, a->arg1);
	break;
	}
	}

	#define MSR_OP_EXMODE_MASK 0xf0000000
	#define MSR_OP_OP_MASK 0x000000ff

	void
	x86_msr_op(u_int msr, u_int op, uint64_t arg1)
	{
	struct thread *td;
	struct msr_op_arg a;
	u_int exmode;
	int bound_cpu, i, is_bound;

	a.op = op & MSR_OP_OP_MASK;
	MPASS(a.op == MSR_OP_ANDNOT \|\| a.op == MSR_OP_OR \|\|
	a.op == MSR_OP_WRITE);
	exmode = op & MSR_OP_EXMODE_MASK;
	MPASS(exmode == MSR_OP_LOCAL \|\| exmode == MSR_OP_SCHED \|\|
	exmode == MSR_OP_RENDEZVOUS);
	a.msr = msr;
	a.arg1 = arg1;
	switch (exmode) {
	case MSR_OP_LOCAL:
	x86_msr_op_one(&a);
	break;
	case MSR_OP_SCHED:
	td = curthread;
	thread_lock(td);
	is_bound = sched_is_bound(td);
	bound_cpu = td->td_oncpu;
	CPU_FOREACH(i) {
	sched_bind(td, i);
	x86_msr_op_one(&a);
	}
	if (is_bound)
	sched_bind(td, bound_cpu);
	else
	sched_unbind(td);
	thread_unlock(td);
	break;
	case MSR_OP_RENDEZVOUS:
	smp_rendezvous(NULL, x86_msr_op_one, NULL, &a);
	break;
	}
	}

	/*
	* Automatically initialized per CPU errata in cpu_idle_tun below.
	*/
	bool mwait_cpustop_broken = false;
	SYSCTL_BOOL(_machdep, OID_AUTO, mwait_cpustop_broken, CTLFLAG_RDTUN,
	&mwait_cpustop_broken, 0,
	"Can not reliably wake MONITOR/MWAIT cpus without interrupts");

	/*
	* Machine dependent boot() routine
	*
	* I haven't seen anything to put here yet
	* Possibly some stuff might be grafted back here from boot()
	*/
	void
	cpu_boot(int howto)
	{
	}

	/*
	* Flush the D-cache for non-DMA I/O so that the I-cache can
	* be made coherent later.
	*/
	void
	cpu_flush_dcache(void *ptr, size_t len)
	{
	/* Not applicable */
	}

	void
	acpi_cpu_c1(void)
	{

	__asm __volatile("sti; hlt");
	}

	/*
	* Use mwait to pause execution while waiting for an interrupt or
	* another thread to signal that there is more work.
	*
	* NOTE: Interrupts will cause a wakeup; however, this function does
	* not enable interrupt handling. The caller is responsible to enable
	* interrupts.
	*/
	void
	acpi_cpu_idle_mwait(uint32_t mwait_hint)
	{
	int *state;
	uint64_t v;

	/*
	* A comment in Linux patch claims that 'CPUs run faster with
	* speculation protection disabled. All CPU threads in a core
	* must disable speculation protection for it to be
	* disabled. Disable it while we are idle so the other
	* hyperthread can run fast.'
	*
	* XXXKIB. Software coordination mode should be supported,
	* but all Intel CPUs provide hardware coordination.
	*/

	state = &PCPU_PTR(monitorbuf)->idle_state;
	KASSERT(atomic_load_int(state) == STATE_SLEEPING,
	("cpu_mwait_cx: wrong monitorbuf state"));
	atomic_store_int(state, STATE_MWAIT);
	if (PCPU_GET(ibpb_set) \|\| hw_ssb_active) {
	v = rdmsr(MSR_IA32_SPEC_CTRL);
	wrmsr(MSR_IA32_SPEC_CTRL, v & ~(IA32_SPEC_CTRL_IBRS \|
	IA32_SPEC_CTRL_STIBP \| IA32_SPEC_CTRL_SSBD));
	} else {
	v = 0;
	}
	cpu_monitor(state, 0, 0);
	if (atomic_load_int(state) == STATE_MWAIT)
	cpu_mwait(MWAIT_INTRBREAK, mwait_hint);

	/*
	* SSB cannot be disabled while we sleep, or rather, if it was
	* disabled, the sysctl thread will bind to our cpu to tweak
	* MSR.
	*/
	if (v != 0)
	wrmsr(MSR_IA32_SPEC_CTRL, v);

	/*
	* We should exit on any event that interrupts mwait, because
	* that event might be a wanted interrupt.
	*/
	atomic_store_int(state, STATE_RUNNING);
	}

	/* Get current clock frequency for the given cpu id. */
	int
	cpu_est_clockrate(int cpu_id, uint64_t *rate)
	{
	uint64_t tsc1, tsc2;
	uint64_t acnt, mcnt, perf;
	register_t reg;

	if (pcpu_find(cpu_id) == NULL \|\| rate == NULL)
	return (EINVAL);
	#ifdef __i386__
	if ((cpu_feature & CPUID_TSC) == 0)
	return (EOPNOTSUPP);
	#endif

	/*
	* If TSC is P-state invariant and APERF/MPERF MSRs do not exist,
	* DELAY(9) based logic fails.
	*/
	if (tsc_is_invariant && !tsc_perf_stat)
	return (EOPNOTSUPP);

	#ifdef SMP
	if (smp_cpus > 1) {
	/* Schedule ourselves on the indicated cpu. */
	thread_lock(curthread);
	sched_bind(curthread, cpu_id);
	thread_unlock(curthread);
	}
	#endif

	/* Calibrate by measuring a short delay. */
	reg = intr_disable();
	if (tsc_is_invariant) {
	wrmsr(MSR_MPERF, 0);
	wrmsr(MSR_APERF, 0);
	tsc1 = rdtsc();
	DELAY(1000);
	mcnt = rdmsr(MSR_MPERF);
	acnt = rdmsr(MSR_APERF);
	tsc2 = rdtsc();
	intr_restore(reg);
	perf = 1000 * acnt / mcnt;
	rate = (tsc2 - tsc1) perf;
	} else {
	tsc1 = rdtsc();
	DELAY(1000);
	tsc2 = rdtsc();
	intr_restore(reg);
	rate = (tsc2 - tsc1) 1000;
	}

	#ifdef SMP
	if (smp_cpus > 1) {
	thread_lock(curthread);
	sched_unbind(curthread);
	thread_unlock(curthread);
	}
	#endif

	return (0);
	}

	/*
	* Shutdown the CPU as much as possible
	*/
	void
	cpu_halt(void)
	{
	for (;;)
	halt();
	}

	static void
	cpu_reset_real(void)
	{
	struct region_descriptor null_idt;
	int b;

	disable_intr();
	#ifdef CPU_ELAN
	if (elan_mmcr != NULL)
	elan_mmcr->RESCFG = 1;
	#endif
	#ifdef __i386__
	if (cpu == CPU_GEODE1100) {
	/* Attempt Geode's own reset */
	outl(0xcf8, 0x80009044ul);
	outl(0xcfc, 0xf);
	}
	#endif
	#if !defined(BROKEN_KEYBOARD_RESET)
	/*
	* Attempt to do a CPU reset via the keyboard controller,
	* do not turn off GateA20, as any machine that fails
	* to do the reset here would then end up in no man's land.
	*/
	outb(IO_KBD + 4, 0xFE);
	DELAY(500000); /* wait 0.5 sec to see if that did it */
	#endif

	/*
	* Attempt to force a reset via the Reset Control register at
	* I/O port 0xcf9. Bit 2 forces a system reset when it
	* transitions from 0 to 1. Bit 1 selects the type of reset
	* to attempt: 0 selects a "soft" reset, and 1 selects a
	* "hard" reset. We try a "hard" reset. The first write sets
	* bit 1 to select a "hard" reset and clears bit 2. The
	* second write forces a 0 -> 1 transition in bit 2 to trigger
	* a reset.
	*/
	outb(0xcf9, 0x2);
	outb(0xcf9, 0x6);
	DELAY(500000); /* wait 0.5 sec to see if that did it */

	/*
	* Attempt to force a reset via the Fast A20 and Init register
	* at I/O port 0x92. Bit 1 serves as an alternate A20 gate.
	* Bit 0 asserts INIT# when set to 1. We are careful to only
	* preserve bit 1 while setting bit 0. We also must clear bit
	* 0 before setting it if it isn't already clear.
	*/
	b = inb(0x92);
	if (b != 0xff) {
	if ((b & 0x1) != 0)
	outb(0x92, b & 0xfe);
	outb(0x92, b \| 0x1);
	DELAY(500000); /* wait 0.5 sec to see if that did it */
	}

	printf("No known reset method worked, attempting CPU shutdown\n");
	DELAY(1000000); /* wait 1 sec for printf to complete */

	/* Wipe the IDT. */
	null_idt.rd_limit = 0;
	null_idt.rd_base = 0;
	lidt(&null_idt);

	/* "good night, sweet prince .... <THUNK!>" */
	breakpoint();

	/* NOTREACHED */
	while(1);
	}

	#ifdef SMP
	static void
	cpu_reset_proxy(void)
	{

	cpu_reset_proxy_active = 1;
	while (cpu_reset_proxy_active == 1)
	ia32_pause(); /* Wait for other cpu to see that we've started */

	printf("cpu_reset_proxy: Stopped CPU %d\n", cpu_reset_proxyid);
	DELAY(1000000);
	cpu_reset_real();
	}
	#endif

	void
	cpu_reset(void)
	{
	#ifdef SMP
	struct monitorbuf *mb;
	cpuset_t map;
	u_int cnt;

	if (smp_started) {
	map = all_cpus;
	CPU_CLR(PCPU_GET(cpuid), &map);
	CPU_ANDNOT(&map, &stopped_cpus);
	if (!CPU_EMPTY(&map)) {
	printf("cpu_reset: Stopping other CPUs\n");
	stop_cpus(map);
	}

	if (PCPU_GET(cpuid) != 0) {
	cpu_reset_proxyid = PCPU_GET(cpuid);
	cpustop_restartfunc = cpu_reset_proxy;
	cpu_reset_proxy_active = 0;
	printf("cpu_reset: Restarting BSP\n");

	/* Restart CPU #0. */
	CPU_SETOF(0, &started_cpus);
	mb = &pcpu_find(0)->pc_monitorbuf;
	atomic_store_int(&mb->stop_state,
	MONITOR_STOPSTATE_RUNNING);

	cnt = 0;
	while (cpu_reset_proxy_active == 0 && cnt < 10000000) {
	ia32_pause();
	cnt++; /* Wait for BSP to announce restart */
	}
	if (cpu_reset_proxy_active == 0) {
	printf("cpu_reset: Failed to restart BSP\n");
	} else {
	cpu_reset_proxy_active = 2;
	while (1)
	ia32_pause();
	/* NOTREACHED */
	}
	}

	DELAY(1000000);
	}
	#endif
	cpu_reset_real();
	/* NOTREACHED */
	}

	bool
	cpu_mwait_usable(void)
	{

	return ((cpu_feature2 & CPUID2_MON) != 0 && ((cpu_mon_mwait_flags &
	(CPUID5_MON_MWAIT_EXT \| CPUID5_MWAIT_INTRBREAK)) ==
	(CPUID5_MON_MWAIT_EXT \| CPUID5_MWAIT_INTRBREAK)));
	}

	void (cpu_idle_hook)(sbintime_t) = NULL; / ACPI idle hook. */
	static int cpu_ident_amdc1e = 0; /* AMD C1E supported. */
	static int idle_mwait = 1; /* Use MONITOR/MWAIT for short idle. */
	SYSCTL_INT(_machdep, OID_AUTO, idle_mwait, CTLFLAG_RWTUN, &idle_mwait,
	0, "Use MONITOR/MWAIT for short idle");

	static void
	cpu_idle_acpi(sbintime_t sbt)
	{
	int *state;

	state = &PCPU_PTR(monitorbuf)->idle_state;
	atomic_store_int(state, STATE_SLEEPING);

	/* See comments in cpu_idle_hlt(). */
	disable_intr();
	if (sched_runnable())
	enable_intr();
	else if (cpu_idle_hook)
	cpu_idle_hook(sbt);
	else
	acpi_cpu_c1();
	atomic_store_int(state, STATE_RUNNING);
	}

	static void
	cpu_idle_hlt(sbintime_t sbt)
	{
	int *state;

	state = &PCPU_PTR(monitorbuf)->idle_state;
	atomic_store_int(state, STATE_SLEEPING);

	/*
	* Since we may be in a critical section from cpu_idle(), if
	* an interrupt fires during that critical section we may have
	* a pending preemption. If the CPU halts, then that thread
	* may not execute until a later interrupt awakens the CPU.
	* To handle this race, check for a runnable thread after
	* disabling interrupts and immediately return if one is
	* found. Also, we must absolutely guarentee that hlt is
	* the next instruction after sti. This ensures that any
	* interrupt that fires after the call to disable_intr() will
	* immediately awaken the CPU from hlt. Finally, please note
	* that on x86 this works fine because of interrupts enabled only
	* after the instruction following sti takes place, while IF is set
	* to 1 immediately, allowing hlt instruction to acknowledge the
	* interrupt.
	*/
	disable_intr();
	if (sched_runnable())
	enable_intr();
	else
	acpi_cpu_c1();
	atomic_store_int(state, STATE_RUNNING);
	}

	static void
	cpu_idle_mwait(sbintime_t sbt)
	{
	int *state;

	state = &PCPU_PTR(monitorbuf)->idle_state;
	atomic_store_int(state, STATE_MWAIT);

	/* See comments in cpu_idle_hlt(). */
	disable_intr();
	if (sched_runnable()) {
	atomic_store_int(state, STATE_RUNNING);
	enable_intr();
	return;
	}

	cpu_monitor(state, 0, 0);
	if (atomic_load_int(state) == STATE_MWAIT)
	__asm __volatile("sti; mwait" : : "a" (MWAIT_C1), "c" (0));
	else
	enable_intr();
	atomic_store_int(state, STATE_RUNNING);
	}

	static void
	cpu_idle_spin(sbintime_t sbt)
	{
	int *state;
	int i;

	state = &PCPU_PTR(monitorbuf)->idle_state;
	atomic_store_int(state, STATE_RUNNING);

	/*
	* The sched_runnable() call is racy but as long as there is
	* a loop missing it one time will have just a little impact if any
	* (and it is much better than missing the check at all).
	*/
	for (i = 0; i < 1000; i++) {
	if (sched_runnable())
	return;
	cpu_spinwait();
	}
	}

	/*
	* C1E renders the local APIC timer dead, so we disable it by
	* reading the Interrupt Pending Message register and clearing
	* both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27).
	*
	* Reference:
	* "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors"
	* #32559 revision 3.00+
	*/
	#define MSR_AMDK8_IPM 0xc0010055
	#define AMDK8_SMIONCMPHALT (1ULL << 27)
	#define AMDK8_C1EONCMPHALT (1ULL << 28)
	#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT \| AMDK8_C1EONCMPHALT)

	void
	cpu_probe_amdc1e(void)
	{

	/*
	* Detect the presence of C1E capability mostly on latest
	* dual-cores (or future) k8 family.
	*/
	if (cpu_vendor_id == CPU_VENDOR_AMD &&
	(cpu_id & 0x00000f00) == 0x00000f00 &&
	(cpu_id & 0x0fff0000) >= 0x00040000) {
	cpu_ident_amdc1e = 1;
	}
	}

	void (*cpu_idle_fn)(sbintime_t) = cpu_idle_acpi;

	void
	cpu_idle(int busy)
	{
	uint64_t msr;
	sbintime_t sbt = -1;

	CTR2(KTR_SPARE2, "cpu_idle(%d) at %d",
	busy, curcpu);
	#ifdef MP_WATCHDOG
	ap_watchdog(PCPU_GET(cpuid));
	#endif

	/* If we are busy - try to use fast methods. */
	if (busy) {
	if ((cpu_feature2 & CPUID2_MON) && idle_mwait) {
	cpu_idle_mwait(busy);
	goto out;
	}
	}

	/* If we have time - switch timers into idle mode. */
	if (!busy) {
	critical_enter();
	sbt = cpu_idleclock();
	}

	/* Apply AMD APIC timer C1E workaround. */
	if (cpu_ident_amdc1e && cpu_disable_c3_sleep) {
	msr = rdmsr(MSR_AMDK8_IPM);
	if (msr & AMDK8_CMPHALT)
	wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT);
	}

	/* Call main idle method. */
	cpu_idle_fn(sbt);

	/* Switch timers back into active mode. */
	if (!busy) {
	cpu_activeclock();
	critical_exit();
	}
	out:
	CTR2(KTR_SPARE2, "cpu_idle(%d) at %d done",
	busy, curcpu);
	}

	static int cpu_idle_apl31_workaround;
	SYSCTL_INT(_machdep, OID_AUTO, idle_apl31, CTLFLAG_RW,
	&cpu_idle_apl31_workaround, 0,
	"Apollo Lake APL31 MWAIT bug workaround");

	int
	cpu_idle_wakeup(int cpu)
	{
	struct monitorbuf *mb;
	int *state;

	mb = &pcpu_find(cpu)->pc_monitorbuf;
	state = &mb->idle_state;
	switch (atomic_load_int(state)) {
	case STATE_SLEEPING:
	return (0);
	case STATE_MWAIT:
	atomic_store_int(state, STATE_RUNNING);
	return (cpu_idle_apl31_workaround ? 0 : 1);
	case STATE_RUNNING:
	return (1);
	default:
	panic("bad monitor state");
	return (1);
	}
	}

	/*
	* Ordered by speed/power consumption.
	*/
	static struct {
	void *id_fn;
	char *id_name;
	int id_cpuid2_flag;
	} idle_tbl[] = {
	{ .id_fn = cpu_idle_spin, .id_name = "spin" },
	{ .id_fn = cpu_idle_mwait, .id_name = "mwait",
	.id_cpuid2_flag = CPUID2_MON },
	{ .id_fn = cpu_idle_hlt, .id_name = "hlt" },
	{ .id_fn = cpu_idle_acpi, .id_name = "acpi" },
	};

	static int
	idle_sysctl_available(SYSCTL_HANDLER_ARGS)
	{
	char avail, p;
	int error;
	int i;

	avail = malloc(256, M_TEMP, M_WAITOK);
	p = avail;
	for (i = 0; i < nitems(idle_tbl); i++) {
	if (idle_tbl[i].id_cpuid2_flag != 0 &&
	(cpu_feature2 & idle_tbl[i].id_cpuid2_flag) == 0)
	continue;
	if (strcmp(idle_tbl[i].id_name, "acpi") == 0 &&
	cpu_idle_hook == NULL)
	continue;
	p += sprintf(p, "%s%s", p != avail ? ", " : "",
	idle_tbl[i].id_name);
	}
	error = sysctl_handle_string(oidp, avail, 0, req);
	free(avail, M_TEMP);
	return (error);
	}

	SYSCTL_PROC(_machdep, OID_AUTO, idle_available,
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_NEEDGIANT,
	0, 0, idle_sysctl_available, "A",
	"list of available idle functions");

	static bool
	cpu_idle_selector(const char *new_idle_name)
	{
	int i;

	for (i = 0; i < nitems(idle_tbl); i++) {
	if (idle_tbl[i].id_cpuid2_flag != 0 &&
	(cpu_feature2 & idle_tbl[i].id_cpuid2_flag) == 0)
	continue;
	if (strcmp(idle_tbl[i].id_name, "acpi") == 0 &&
	cpu_idle_hook == NULL)
	continue;
	if (strcmp(idle_tbl[i].id_name, new_idle_name))
	continue;
	cpu_idle_fn = idle_tbl[i].id_fn;
	if (bootverbose)
	printf("CPU idle set to %s\n", idle_tbl[i].id_name);
	return (true);
	}
	return (false);
	}

	static int
	cpu_idle_sysctl(SYSCTL_HANDLER_ARGS)
	{
	char buf[16], *p;
	int error, i;

	p = "unknown";
	for (i = 0; i < nitems(idle_tbl); i++) {
	if (idle_tbl[i].id_fn == cpu_idle_fn) {
	p = idle_tbl[i].id_name;
	break;
	}
	}
	strncpy(buf, p, sizeof(buf));
	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	return (cpu_idle_selector(buf) ? 0 : EINVAL);
	}

	SYSCTL_PROC(_machdep, OID_AUTO, idle,
	CTLTYPE_STRING \| CTLFLAG_RW \| CTLFLAG_NEEDGIANT,
	0, 0, cpu_idle_sysctl, "A",
	"currently selected idle function");

	static void
	cpu_idle_tun(void *unused __unused)
	{
	char tunvar[16];

	if (TUNABLE_STR_FETCH("machdep.idle", tunvar, sizeof(tunvar)))
	cpu_idle_selector(tunvar);
	else if (cpu_vendor_id == CPU_VENDOR_AMD &&
	CPUID_TO_FAMILY(cpu_id) == 0x17 && CPUID_TO_MODEL(cpu_id) == 0x1) {
	/* Ryzen erratas 1057, 1109. */
	cpu_idle_selector("hlt");
	idle_mwait = 0;
	mwait_cpustop_broken = true;
	}

	if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_id == 0x506c9) {
	/*
	* Apollo Lake errata APL31 (public errata APL30).
	* Stores to the armed address range may not trigger
	* MWAIT to resume execution. OS needs to use
	* interrupts to wake processors from MWAIT-induced
	* sleep states.
	*/
	cpu_idle_apl31_workaround = 1;
	mwait_cpustop_broken = true;
	}
	TUNABLE_INT_FETCH("machdep.idle_apl31", &cpu_idle_apl31_workaround);
	}
	SYSINIT(cpu_idle_tun, SI_SUB_CPU, SI_ORDER_MIDDLE, cpu_idle_tun, NULL);

	static int panic_on_nmi = 0xff;
	SYSCTL_INT(_machdep, OID_AUTO, panic_on_nmi, CTLFLAG_RWTUN,
	&panic_on_nmi, 0,
	"Panic on NMI: 1 = H/W failure; 2 = unknown; 0xff = all");
	int nmi_is_broadcast = 1;
	SYSCTL_INT(_machdep, OID_AUTO, nmi_is_broadcast, CTLFLAG_RWTUN,
	&nmi_is_broadcast, 0,
	"Chipset NMI is broadcast");
	+int (*apei_nmi)(void);

	void
	nmi_call_kdb(u_int cpu, u_int type, struct trapframe *frame)
	{
	bool claimed = false;

	#ifdef DEV_ISA
	/* machine/parity/power fail/"kitchen sink" faults */
	if (isa_nmi(frame->tf_err)) {
	claimed = true;
	if ((panic_on_nmi & 1) != 0)
	panic("NMI indicates hardware failure");
	}
	#endif /* DEV_ISA */

	+ /* ACPI Platform Error Interfaces callback. */
	+ if (apei_nmi != NULL && (*apei_nmi)())
	+ claimed = true;
	+
	/*
	* NMIs can be useful for debugging. They can be hooked up to a
	* pushbutton, usually on an ISA, PCI, or PCIe card. They can also be
	* generated by an IPMI BMC, either manually or in response to a
	* watchdog timeout. For example, see the "power diag" command in
	* ports/sysutils/ipmitool. They can also be generated by a
	* hypervisor; see "bhyvectl --inject-nmi".
	*/

	#ifdef KDB
	if (!claimed && (panic_on_nmi & 2) != 0) {
	if (debugger_on_panic) {
	printf("NMI/cpu%d ... going to debugger\n", cpu);
	claimed = kdb_trap(type, 0, frame);
	}
	}
	#endif /* KDB */

	if (!claimed && panic_on_nmi != 0)
	panic("NMI");
	}

	void
	nmi_handle_intr(u_int type, struct trapframe *frame)
	{

	#ifdef SMP
	if (nmi_is_broadcast) {
	nmi_call_kdb_smp(type, frame);
	return;
	}
	#endif
	nmi_call_kdb(PCPU_GET(cpuid), type, frame);
	}

	static int hw_ibrs_active;
	int hw_ibrs_ibpb_active;
	int hw_ibrs_disable = 1;

	SYSCTL_INT(_hw, OID_AUTO, ibrs_active, CTLFLAG_RD, &hw_ibrs_active, 0,
	"Indirect Branch Restricted Speculation active");

	SYSCTL_NODE(_machdep_mitigations, OID_AUTO, ibrs,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Indirect Branch Restricted Speculation active");

	SYSCTL_INT(_machdep_mitigations_ibrs, OID_AUTO, active, CTLFLAG_RD,
	&hw_ibrs_active, 0, "Indirect Branch Restricted Speculation active");

	void
	hw_ibrs_recalculate(bool for_all_cpus)
	{
	if ((cpu_ia32_arch_caps & IA32_ARCH_CAP_IBRS_ALL) != 0) {
	x86_msr_op(MSR_IA32_SPEC_CTRL, (for_all_cpus ?
	MSR_OP_RENDEZVOUS : MSR_OP_LOCAL) \|
	(hw_ibrs_disable != 0 ? MSR_OP_ANDNOT : MSR_OP_OR),
	IA32_SPEC_CTRL_IBRS);
	hw_ibrs_active = hw_ibrs_disable == 0;
	hw_ibrs_ibpb_active = 0;
	} else {
	hw_ibrs_active = hw_ibrs_ibpb_active = (cpu_stdext_feature3 &
	CPUID_STDEXT3_IBPB) != 0 && !hw_ibrs_disable;
	}
	}

	static int
	hw_ibrs_disable_handler(SYSCTL_HANDLER_ARGS)
	{
	int error, val;

	val = hw_ibrs_disable;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	hw_ibrs_disable = val != 0;
	hw_ibrs_recalculate(true);
	return (0);
	}
	SYSCTL_PROC(_hw, OID_AUTO, ibrs_disable, CTLTYPE_INT \| CTLFLAG_RWTUN \|
	CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0, hw_ibrs_disable_handler, "I",
	"Disable Indirect Branch Restricted Speculation");

	SYSCTL_PROC(_machdep_mitigations_ibrs, OID_AUTO, disable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	hw_ibrs_disable_handler, "I",
	"Disable Indirect Branch Restricted Speculation");

	int hw_ssb_active;
	int hw_ssb_disable;

	SYSCTL_INT(_hw, OID_AUTO, spec_store_bypass_disable_active, CTLFLAG_RD,
	&hw_ssb_active, 0,
	"Speculative Store Bypass Disable active");

	SYSCTL_NODE(_machdep_mitigations, OID_AUTO, ssb,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Speculative Store Bypass Disable active");

	SYSCTL_INT(_machdep_mitigations_ssb, OID_AUTO, active, CTLFLAG_RD,
	&hw_ssb_active, 0, "Speculative Store Bypass Disable active");

	static void
	hw_ssb_set(bool enable, bool for_all_cpus)
	{

	if ((cpu_stdext_feature3 & CPUID_STDEXT3_SSBD) == 0) {
	hw_ssb_active = 0;
	return;
	}
	hw_ssb_active = enable;
	x86_msr_op(MSR_IA32_SPEC_CTRL,
	(enable ? MSR_OP_OR : MSR_OP_ANDNOT) \|
	(for_all_cpus ? MSR_OP_SCHED : MSR_OP_LOCAL), IA32_SPEC_CTRL_SSBD);
	}

	void
	hw_ssb_recalculate(bool all_cpus)
	{

	switch (hw_ssb_disable) {
	default:
	hw_ssb_disable = 0;
	/* FALLTHROUGH */
	case 0: /* off */
	hw_ssb_set(false, all_cpus);
	break;
	case 1: /* on */
	hw_ssb_set(true, all_cpus);
	break;
	case 2: /* auto */
	hw_ssb_set((cpu_ia32_arch_caps & IA32_ARCH_CAP_SSB_NO) != 0 ?
	false : true, all_cpus);
	break;
	}
	}

	static int
	hw_ssb_disable_handler(SYSCTL_HANDLER_ARGS)
	{
	int error, val;

	val = hw_ssb_disable;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	hw_ssb_disable = val;
	hw_ssb_recalculate(true);
	return (0);
	}
	SYSCTL_PROC(_hw, OID_AUTO, spec_store_bypass_disable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	hw_ssb_disable_handler, "I",
	"Speculative Store Bypass Disable (0 - off, 1 - on, 2 - auto");

	SYSCTL_PROC(_machdep_mitigations_ssb, OID_AUTO, disable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	hw_ssb_disable_handler, "I",
	"Speculative Store Bypass Disable (0 - off, 1 - on, 2 - auto");

	int hw_mds_disable;

	/*
	* Handler for Microarchitectural Data Sampling issues. Really not a
	* pointer to C function: on amd64 the code must not change any CPU
	* architectural state except possibly %rflags. Also, it is always
	* called with interrupts disabled.
	*/
	void mds_handler_void(void);
	void mds_handler_verw(void);
	void mds_handler_ivb(void);
	void mds_handler_bdw(void);
	void mds_handler_skl_sse(void);
	void mds_handler_skl_avx(void);
	void mds_handler_skl_avx512(void);
	void mds_handler_silvermont(void);
	void (*mds_handler)(void) = mds_handler_void;

	static int
	sysctl_hw_mds_disable_state_handler(SYSCTL_HANDLER_ARGS)
	{
	const char *state;

	if (mds_handler == mds_handler_void)
	state = "inactive";
	else if (mds_handler == mds_handler_verw)
	state = "VERW";
	else if (mds_handler == mds_handler_ivb)
	state = "software IvyBridge";
	else if (mds_handler == mds_handler_bdw)
	state = "software Broadwell";
	else if (mds_handler == mds_handler_skl_sse)
	state = "software Skylake SSE";
	else if (mds_handler == mds_handler_skl_avx)
	state = "software Skylake AVX";
	else if (mds_handler == mds_handler_skl_avx512)
	state = "software Skylake AVX512";
	else if (mds_handler == mds_handler_silvermont)
	state = "software Silvermont";
	else
	state = "unknown";
	return (SYSCTL_OUT(req, state, strlen(state)));
	}

	SYSCTL_PROC(_hw, OID_AUTO, mds_disable_state,
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_hw_mds_disable_state_handler, "A",
	"Microarchitectural Data Sampling Mitigation state");

	SYSCTL_NODE(_machdep_mitigations, OID_AUTO, mds,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"Microarchitectural Data Sampling Mitigation state");

	SYSCTL_PROC(_machdep_mitigations_mds, OID_AUTO, state,
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_hw_mds_disable_state_handler, "A",
	"Microarchitectural Data Sampling Mitigation state");

	_Static_assert(__offsetof(struct pcpu, pc_mds_tmp) % 64 == 0, "MDS AVX512");

	void
	hw_mds_recalculate(void)
	{
	struct pcpu *pc;
	vm_offset_t b64;
	u_long xcr0;
	int i;

	/*
	* Allow user to force VERW variant even if MD_CLEAR is not
	* reported. For instance, hypervisor might unknowingly
	* filter the cap out.
	* For the similar reasons, and for testing, allow to enable
	* mitigation even when MDS_NO cap is set.
	*/
	if (cpu_vendor_id != CPU_VENDOR_INTEL \|\| hw_mds_disable == 0 \|\|
	((cpu_ia32_arch_caps & IA32_ARCH_CAP_MDS_NO) != 0 &&
	hw_mds_disable == 3)) {
	mds_handler = mds_handler_void;
	} else if (((cpu_stdext_feature3 & CPUID_STDEXT3_MD_CLEAR) != 0 &&
	hw_mds_disable == 3) \|\| hw_mds_disable == 1) {
	mds_handler = mds_handler_verw;
	} else if (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
	(CPUID_TO_MODEL(cpu_id) == 0x2e \|\| CPUID_TO_MODEL(cpu_id) == 0x1e \|\|
	CPUID_TO_MODEL(cpu_id) == 0x1f \|\| CPUID_TO_MODEL(cpu_id) == 0x1a \|\|
	CPUID_TO_MODEL(cpu_id) == 0x2f \|\| CPUID_TO_MODEL(cpu_id) == 0x25 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x2c \|\| CPUID_TO_MODEL(cpu_id) == 0x2d \|\|
	CPUID_TO_MODEL(cpu_id) == 0x2a \|\| CPUID_TO_MODEL(cpu_id) == 0x3e \|\|
	CPUID_TO_MODEL(cpu_id) == 0x3a) &&
	(hw_mds_disable == 2 \|\| hw_mds_disable == 3)) {
	/*
	* Nehalem, SandyBridge, IvyBridge
	*/
	CPU_FOREACH(i) {
	pc = pcpu_find(i);
	if (pc->pc_mds_buf == NULL) {
	pc->pc_mds_buf = malloc_domainset(672, M_TEMP,
	DOMAINSET_PREF(pc->pc_domain), M_WAITOK);
	bzero(pc->pc_mds_buf, 16);
	}
	}
	mds_handler = mds_handler_ivb;
	} else if (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
	(CPUID_TO_MODEL(cpu_id) == 0x3f \|\| CPUID_TO_MODEL(cpu_id) == 0x3c \|\|
	CPUID_TO_MODEL(cpu_id) == 0x45 \|\| CPUID_TO_MODEL(cpu_id) == 0x46 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x56 \|\| CPUID_TO_MODEL(cpu_id) == 0x4f \|\|
	CPUID_TO_MODEL(cpu_id) == 0x47 \|\| CPUID_TO_MODEL(cpu_id) == 0x3d) &&
	(hw_mds_disable == 2 \|\| hw_mds_disable == 3)) {
	/*
	* Haswell, Broadwell
	*/
	CPU_FOREACH(i) {
	pc = pcpu_find(i);
	if (pc->pc_mds_buf == NULL) {
	pc->pc_mds_buf = malloc_domainset(1536, M_TEMP,
	DOMAINSET_PREF(pc->pc_domain), M_WAITOK);
	bzero(pc->pc_mds_buf, 16);
	}
	}
	mds_handler = mds_handler_bdw;
	} else if (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
	((CPUID_TO_MODEL(cpu_id) == 0x55 && (cpu_id &
	CPUID_STEPPING) <= 5) \|\|
	CPUID_TO_MODEL(cpu_id) == 0x4e \|\| CPUID_TO_MODEL(cpu_id) == 0x5e \|\|
	(CPUID_TO_MODEL(cpu_id) == 0x8e && (cpu_id &
	CPUID_STEPPING) <= 0xb) \|\|
	(CPUID_TO_MODEL(cpu_id) == 0x9e && (cpu_id &
	CPUID_STEPPING) <= 0xc)) &&
	(hw_mds_disable == 2 \|\| hw_mds_disable == 3)) {
	/*
	* Skylake, KabyLake, CoffeeLake, WhiskeyLake,
	* CascadeLake
	*/
	CPU_FOREACH(i) {
	pc = pcpu_find(i);
	if (pc->pc_mds_buf == NULL) {
	pc->pc_mds_buf = malloc_domainset(6 * 1024,
	M_TEMP, DOMAINSET_PREF(pc->pc_domain),
	M_WAITOK);
	b64 = (vm_offset_t)malloc_domainset(64 + 63,
	M_TEMP, DOMAINSET_PREF(pc->pc_domain),
	M_WAITOK);
	pc->pc_mds_buf64 = (void *)roundup2(b64, 64);
	bzero(pc->pc_mds_buf64, 64);
	}
	}
	xcr0 = rxcr(0);
	if ((xcr0 & XFEATURE_ENABLED_ZMM_HI256) != 0 &&
	(cpu_stdext_feature & CPUID_STDEXT_AVX512DQ) != 0)
	mds_handler = mds_handler_skl_avx512;
	else if ((xcr0 & XFEATURE_ENABLED_AVX) != 0 &&
	(cpu_feature2 & CPUID2_AVX) != 0)
	mds_handler = mds_handler_skl_avx;
	else
	mds_handler = mds_handler_skl_sse;
	} else if (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
	((CPUID_TO_MODEL(cpu_id) == 0x37 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x4a \|\|
	CPUID_TO_MODEL(cpu_id) == 0x4c \|\|
	CPUID_TO_MODEL(cpu_id) == 0x4d \|\|
	CPUID_TO_MODEL(cpu_id) == 0x5a \|\|
	CPUID_TO_MODEL(cpu_id) == 0x5d \|\|
	CPUID_TO_MODEL(cpu_id) == 0x6e \|\|
	CPUID_TO_MODEL(cpu_id) == 0x65 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x75 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x1c \|\|
	CPUID_TO_MODEL(cpu_id) == 0x26 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x27 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x35 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x36 \|\|
	CPUID_TO_MODEL(cpu_id) == 0x7a))) {
	/* Silvermont, Airmont */
	CPU_FOREACH(i) {
	pc = pcpu_find(i);
	if (pc->pc_mds_buf == NULL)
	pc->pc_mds_buf = malloc(256, M_TEMP, M_WAITOK);
	}
	mds_handler = mds_handler_silvermont;
	} else {
	hw_mds_disable = 0;
	mds_handler = mds_handler_void;
	}
	}

	static void
	hw_mds_recalculate_boot(void *arg __unused)
	{

	hw_mds_recalculate();
	}
	SYSINIT(mds_recalc, SI_SUB_SMP, SI_ORDER_ANY, hw_mds_recalculate_boot, NULL);

	static int
	sysctl_mds_disable_handler(SYSCTL_HANDLER_ARGS)
	{
	int error, val;

	val = hw_mds_disable;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	if (val < 0 \|\| val > 3)
	return (EINVAL);
	hw_mds_disable = val;
	hw_mds_recalculate();
	return (0);
	}

	SYSCTL_PROC(_hw, OID_AUTO, mds_disable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_mds_disable_handler, "I",
	"Microarchitectural Data Sampling Mitigation "
	"(0 - off, 1 - on VERW, 2 - on SW, 3 - on AUTO");

	SYSCTL_PROC(_machdep_mitigations_mds, OID_AUTO, disable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_mds_disable_handler, "I",
	"Microarchitectural Data Sampling Mitigation "
	"(0 - off, 1 - on VERW, 2 - on SW, 3 - on AUTO");

	/*
	* Intel Transactional Memory Asynchronous Abort Mitigation
	* CVE-2019-11135
	*/
	int x86_taa_enable;
	int x86_taa_state;
	enum {
	TAA_NONE = 0, /* No mitigation enabled */
	TAA_TSX_DISABLE = 1, /* Disable TSX via MSR */
	TAA_VERW = 2, /* Use VERW mitigation */
	TAA_AUTO = 3, /* Automatically select the mitigation */

	/* The states below are not selectable by the operator */

	TAA_TAA_UC = 4, /* Mitigation present in microcode */
	TAA_NOT_PRESENT = 5 /* TSX is not present */
	};

	static void
	taa_set(bool enable, bool all)
	{

	x86_msr_op(MSR_IA32_TSX_CTRL,
	(enable ? MSR_OP_OR : MSR_OP_ANDNOT) \|
	(all ? MSR_OP_RENDEZVOUS : MSR_OP_LOCAL),
	IA32_TSX_CTRL_RTM_DISABLE \| IA32_TSX_CTRL_TSX_CPUID_CLEAR);
	}

	void
	x86_taa_recalculate(void)
	{
	static int taa_saved_mds_disable = 0;
	int taa_need = 0, taa_state = 0;
	int mds_disable = 0, need_mds_recalc = 0;

	/* Check CPUID.07h.EBX.HLE and RTM for the presence of TSX */
	if ((cpu_stdext_feature & CPUID_STDEXT_HLE) == 0 \|\|
	(cpu_stdext_feature & CPUID_STDEXT_RTM) == 0) {
	/* TSX is not present */
	x86_taa_state = TAA_NOT_PRESENT;
	return;
	}

	/* Check to see what mitigation options the CPU gives us */
	if (cpu_ia32_arch_caps & IA32_ARCH_CAP_TAA_NO) {
	/* CPU is not suseptible to TAA */
	taa_need = TAA_TAA_UC;
	} else if (cpu_ia32_arch_caps & IA32_ARCH_CAP_TSX_CTRL) {
	/*
	* CPU can turn off TSX. This is the next best option
	* if TAA_NO hardware mitigation isn't present
	*/
	taa_need = TAA_TSX_DISABLE;
	} else {
	/* No TSX/TAA specific remedies are available. */
	if (x86_taa_enable == TAA_TSX_DISABLE) {
	if (bootverbose)
	printf("TSX control not available\n");
	return;
	} else
	taa_need = TAA_VERW;
	}

	/* Can we automatically take action, or are we being forced? */
	if (x86_taa_enable == TAA_AUTO)
	taa_state = taa_need;
	else
	taa_state = x86_taa_enable;

	/* No state change, nothing to do */
	if (taa_state == x86_taa_state) {
	if (bootverbose)
	printf("No TSX change made\n");
	return;
	}

	/* Does the MSR need to be turned on or off? */
	if (taa_state == TAA_TSX_DISABLE)
	taa_set(true, true);
	else if (x86_taa_state == TAA_TSX_DISABLE)
	taa_set(false, true);

	/* Does MDS need to be set to turn on VERW? */
	if (taa_state == TAA_VERW) {
	taa_saved_mds_disable = hw_mds_disable;
	mds_disable = hw_mds_disable = 1;
	need_mds_recalc = 1;
	} else if (x86_taa_state == TAA_VERW) {
	mds_disable = hw_mds_disable = taa_saved_mds_disable;
	need_mds_recalc = 1;
	}
	if (need_mds_recalc) {
	hw_mds_recalculate();
	if (mds_disable != hw_mds_disable) {
	if (bootverbose)
	printf("Cannot change MDS state for TAA\n");
	/* Don't update our state */
	return;
	}
	}

	x86_taa_state = taa_state;
	return;
	}

	static void
	taa_recalculate_boot(void * arg __unused)
	{

	x86_taa_recalculate();
	}
	SYSINIT(taa_recalc, SI_SUB_SMP, SI_ORDER_ANY, taa_recalculate_boot, NULL);

	SYSCTL_NODE(_machdep_mitigations, OID_AUTO, taa,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"TSX Asynchronous Abort Mitigation");

	static int
	sysctl_taa_handler(SYSCTL_HANDLER_ARGS)
	{
	int error, val;

	val = x86_taa_enable;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	if (val < TAA_NONE \|\| val > TAA_AUTO)
	return (EINVAL);
	x86_taa_enable = val;
	x86_taa_recalculate();
	return (0);
	}

	SYSCTL_PROC(_machdep_mitigations_taa, OID_AUTO, enable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_taa_handler, "I",
	"TAA Mitigation enablement control "
	"(0 - off, 1 - disable TSX, 2 - VERW, 3 - on AUTO");

	static int
	sysctl_taa_state_handler(SYSCTL_HANDLER_ARGS)
	{
	const char *state;

	switch (x86_taa_state) {
	case TAA_NONE:
	state = "inactive";
	break;
	case TAA_TSX_DISABLE:
	state = "TSX disabled";
	break;
	case TAA_VERW:
	state = "VERW";
	break;
	case TAA_TAA_UC:
	state = "Mitigated in microcode";
	break;
	case TAA_NOT_PRESENT:
	state = "TSX not present";
	break;
	default:
	state = "unknown";
	}

	return (SYSCTL_OUT(req, state, strlen(state)));
	}

	SYSCTL_PROC(_machdep_mitigations_taa, OID_AUTO, state,
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_taa_state_handler, "A",
	"TAA Mitigation state");

	int __read_frequently cpu_flush_rsb_ctxsw;
	SYSCTL_INT(_machdep_mitigations, OID_AUTO, flush_rsb_ctxsw,
	CTLFLAG_RW \| CTLFLAG_NOFETCH, &cpu_flush_rsb_ctxsw, 0,
	"Flush Return Stack Buffer on context switch");

	SYSCTL_NODE(_machdep_mitigations, OID_AUTO, rngds,
	CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	"MCU Optimization, disable RDSEED mitigation");

	int x86_rngds_mitg_enable = 1;
	void
	x86_rngds_mitg_recalculate(bool all_cpus)
	{
	if ((cpu_stdext_feature3 & CPUID_STDEXT3_MCUOPT) == 0)
	return;
	x86_msr_op(MSR_IA32_MCU_OPT_CTRL,
	(x86_rngds_mitg_enable ? MSR_OP_OR : MSR_OP_ANDNOT) \|
	(all_cpus ? MSR_OP_RENDEZVOUS : MSR_OP_LOCAL),
	IA32_RNGDS_MITG_DIS);
	}

	static int
	sysctl_rngds_mitg_enable_handler(SYSCTL_HANDLER_ARGS)
	{
	int error, val;

	val = x86_rngds_mitg_enable;
	error = sysctl_handle_int(oidp, &val, 0, req);
	if (error != 0 \|\| req->newptr == NULL)
	return (error);
	x86_rngds_mitg_enable = val;
	x86_rngds_mitg_recalculate(true);
	return (0);
	}
	SYSCTL_PROC(_machdep_mitigations_rngds, OID_AUTO, enable, CTLTYPE_INT \|
	CTLFLAG_RWTUN \| CTLFLAG_NOFETCH \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_rngds_mitg_enable_handler, "I",
	"MCU Optimization, disabling RDSEED mitigation control "
	"(0 - mitigation disabled (RDSEED optimized), 1 - mitigation enabled");

	static int
	sysctl_rngds_state_handler(SYSCTL_HANDLER_ARGS)
	{
	const char *state;

	if ((cpu_stdext_feature3 & CPUID_STDEXT3_MCUOPT) == 0) {
	state = "Not applicable";
	} else if (x86_rngds_mitg_enable == 0) {
	state = "RDSEED not serialized";
	} else {
	state = "Mitigated";
	}
	return (SYSCTL_OUT(req, state, strlen(state)));
	}
	SYSCTL_PROC(_machdep_mitigations_rngds, OID_AUTO, state,
	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, 0,
	sysctl_rngds_state_handler, "A",
	"MCU Optimization state");

	/*
	* Enable and restore kernel text write permissions.
	* Callers must ensure that disable_wp()/restore_wp() are executed
	* without rescheduling on the same core.
	*/
	bool
	disable_wp(void)
	{
	u_int cr0;

	cr0 = rcr0();
	if ((cr0 & CR0_WP) == 0)
	return (false);
	load_cr0(cr0 & ~CR0_WP);
	return (true);
	}

	void
	restore_wp(bool old_wp)
	{

	if (old_wp)
	load_cr0(rcr0() \| CR0_WP);
	}

	bool
	acpi_get_fadt_bootflags(uint16_t *flagsp)
	{
	#ifdef DEV_ACPI
	ACPI_TABLE_FADT *fadt;
	vm_paddr_t physaddr;

	physaddr = acpi_find_table(ACPI_SIG_FADT);
	if (physaddr == 0)
	return (false);
	fadt = acpi_map_table(physaddr, ACPI_SIG_FADT);
	if (fadt == NULL)
	return (false);
	*flagsp = fadt->BootFlags;
	acpi_unmap_table(fadt);
	return (true);
	#else
	return (false);
	#endif
	}
	diff --git a/usr.bin/calendar/calendars/calendar.freebsd b/usr.bin/calendar/calendars/calendar.freebsd
	index 7becc0dc2830..93b714ede7fb 100644
	--- a/usr.bin/calendar/calendars/calendar.freebsd
	+++ b/usr.bin/calendar/calendars/calendar.freebsd
	@@ -1,488 +1,489 @@
	/*
	* FreeBSD
	*
	* $FreeBSD$
	*/

	#ifndef _calendar_freebsd_
	#define _calendar_freebsd_

	01/01 Dimitry Andric <dim@FreeBSD.org> born in Utrecht, the Netherlands, 1969
	01/01 Lev Serebryakov <lev@FreeBSD.org> born in Leningrad, USSR, 1979
	01/01 Alexander Langer <alex@FreeBSD.org> born in Duesseldorf, Nordrhein-Westfalen, Germany, 1981
	01/01 Zach Leslie <zleslie@FreeBSD.org> born in Grand Junction, Colorado, United States, 1985
	01/02 Ion-Mihai "IOnut" Tetcu <itetcu@FreeBSD.org> born in Bucharest, Romania, 1980
	01/02 Patrick Li <pat@FreeBSD.org> born in Beijing, People's Republic of China, 1985
	01/03 Tetsurou Okazaki <okazaki@FreeBSD.org> born in Mobara, Chiba, Japan, 1972
	01/04 Hiroyuki Hanai <hanai@FreeBSD.org> born in Kagawa pre., Japan, 1969
	01/05 D Scott Phillips <scottph@FreeBSD.org> born in Anderson, Indiana, 1983
	01/06 Adriaan de Groot <adridg@FreeBSD.org> born in Calgary, Canada, 1973
	01/06 Philippe Audeoud <jadawin@FreeBSD.org> born in Bretigny-Sur-Orge, France, 1980
	01/08 Michael L. Hostbaek <mich@FreeBSD.org> born in Copenhagen, Denmark, 1977
	01/10 Jean-Yves Lefort <jylefort@FreeBSD.org> born in Charleroi, Belgium, 1980
	01/10 Guangyuan Yang <ygy@FreeBSD.org> born in Yangzhou, Jiangsu, People's Republic of China, 1997
	01/12 Yen-Ming Lee <leeym@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1977
	01/12 Ying-Chieh Liao <ijliao@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1979
	01/12 Kristof Provost <kp@FreeBSD.org> born in Aalst, Belgium, 1983
	01/13 Ruslan Bukin <br@FreeBSD.org> born in Dudinka, Russian Federation, 1985
	01/14 Yi-Jheng Lin <yzlin@FreeBSD.org> born in Taichung, Taiwan, Republic of China, 1985
	01/15 Anne Dickison <anne@FreeBSD.org> born in Madison, Indiana, United States, 1976
	01/16 Ariff Abdullah <ariff@FreeBSD.org> born in Kuala Lumpur, Malaysia, 1978
	01/16 Dmitry Sivachenko <demon@FreeBSD.org> born in Moscow, USSR, 1978
	01/16 Vanilla I. Shu <vanilla@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1978
	01/17 Raphael Kubo da Costa <rakuco@FreeBSD.org> born in Sao Paulo, Sao Paulo, Brazil, 1989
	01/18 Dejan Lesjak <lesi@FreeBSD.org> born in Ljubljana, Slovenia, Yugoslavia, 1977
	01/19 Marshall Kirk McKusick <mckusick@FreeBSD.org> born in Wilmington, Delaware, United States, 1954
	01/19 Ruslan Ermilov <ru@FreeBSD.org> born in Simferopol, USSR, 1974
	01/19 Marcelo S. Araujo <araujo@FreeBSD.org> born in Joinville, Santa Catarina, Brazil, 1981
	01/20 Poul-Henning Kamp <phk@FreeBSD.org> born in Korsoer, Denmark, 1966
	01/21 Mahdi Mokhtari <mmokhi@FreeBSD.org> born in Tehran, Iran, 1995
	01/21 Mitchell Horne <mhorne@FreeBSD.org> born in Toronto, Ontario, Canada, 1997
	01/22 Johann Visagie <wjv@FreeBSD.org> born in Cape Town, South Africa, 1970
	01/23 Hideyuki KURASHINA <rushani@FreeBSD.org> born in Niigata, Japan, 1982
	01/24 Fabien Thomas <fabient@FreeBSD.org> born in Avignon, France, 1971
	01/24 Matteo Riondato <matteo@FreeBSD.org> born in Padova, Italy, 1986
	01/25 Nick Hibma <n_hibma@FreeBSD.org> born in Groningen, the Netherlands, 1972
	01/25 Bernd Walter <ticso@FreeBSD.org> born in Moers, Nordrhein-Westfalen, Germany, 1974
	01/26 Andrew Gallatin <gallatin@FreeBSD.org> born in Buffalo, New York, United States, 1970
	01/27 Nick Sayer <nsayer@FreeBSD.org> born in San Diego, California, United States, 1968
	01/27 Jacques Anthony Vidrine <nectar@FreeBSD.org> born in Baton Rouge, Louisiana, United States, 1971
	01/27 Alexandre C. Guimaraes <rigoletto@FreeBSD.org> born in Rio de Janeiro, Rio de Janeiro, Brazil, 1982
	01/27 Enji Cooper <ngie@FreeBSD.org> born in Seattle, Washington, United States, 1984
	01/31 Hidetoshi Shimokawa <simokawa@FreeBSD.org> born in Yokohama, Kanagawa, Japan, 1970
	02/01 Doug Rabson <dfr@FreeBSD.org> born in London, England, 1966
	02/01 Nicola Vitale <nivit@FreeBSD.org> born in Busto Arsizio, Varese, Italy, 1976
	02/01 Paul Saab <ps@FreeBSD.org> born in Champaign-Urbana, Illinois, United States, 1978
	02/01 Martin Wilke <miwi@FreeBSD.org> born in Ludwigsfelde, Brandenburg, Germany, 1980
	02/01 Christian Brueffer <brueffer@FreeBSD.org> born in Gronau, Nordrhein-Westfalen, Germany, 1982
	02/01 Steven Kreuzer <skreuzer@FreeBSD.org> born in Oceanside, New York, United States, 1982
	02/01 Juli Mallett <jmallett@FreeBSD.org> born in Washington, Pennsylvania, United States, 1985
	02/02 Diomidis D. Spinellis <dds@FreeBSD.org> born in Athens, Greece, 1967
	02/02 Michael W Lucas <mwlucas@FreeBSD.org> born in Detroit, Michigan, United States, 1967
	02/02 Dmitry Chagin <dchagin@FreeBSD.org> born in Stalingrad, USSR, 1976
	02/02 Yoichi Nakayama <yoichi@FreeBSD.org> born in Tsu, Mie, Japan, 1976
	02/02 Yoshihiro Takahashi <nyan@FreeBSD.org> born in Yokohama, Kanagawa, Japan, 1976
	02/03 Jason Helfman <jgh@FreeBSD.org> born in Royal Oak, Michigan, United States, 1972
	02/03 Mateusz Piotrowski <0mp@FreeBSD.org> born in Warsaw, Poland, 1995
	02/04 Eitan Adler <eadler@FreeBSD.org> born in West Hempstead, New York, United States, 1991
	02/05 Frank Laszlo <laszlof@FreeBSD.org> born in Howell, Michigan, United States, 1983
	02/06 Julien Charbon <jch@FreeBSD.org> born in Saint Etienne, Loire, France, 1978
	02/07 Bjoern Heidotting <bhd@FreeBSD.org> born in Uelsen, Germany, 1980
	02/10 David Greenman <dg@FreeBSD.org> born in Portland, Oregon, United States, 1968
	02/10 Paul Richards <paul@FreeBSD.org> born in Ammanford, Carmarthenshire, United Kingdom, 1968
	02/10 Simon Barner <barner@FreeBSD.org> born in Rosenheim, Bayern, Germany, 1980
	02/10 Jason E. Hale <jhale@FreeBSD.org> born in Pittsburgh, Pennsylvania, United States, 1982
	02/13 Jesper Skriver <jesper@FreeBSD.org> born in Aarhus, Denmark, 1975
	02/13 Steve Wills <swills@FreeBSD.org> born in Lynchburg, Virginia, United States, 1975
	02/13 Andrey Slusar <anray@FreeBSD.org> born in Odessa, USSR, 1979
	02/13 David W. Chapman Jr. <dwcjr@FreeBSD.org> born in Bethel, Connecticut, United States, 1981
	02/14 Manolis Kiagias <manolis@FreeBSD.org> born in Chania, Greece, 1970
	02/14 Erwin Lansing <erwin@FreeBSD.org> born in 's-Hertogenbosch, the Netherlands, 1975
	02/14 Martin Blapp <mbr@FreeBSD.org> born in Olten, Switzerland, 1976
	02/15 Hiren Panchasara <hiren@FreeBSD.org> born in Ahmedabad, Gujarat, India, 1984
	02/16 Justin Hibbits <jhibbits@FreeBSD.org> born in Toledo, Ohio, United States, 1983
	02/16 Tobias Christian Berner <tcberner@FreeBSD.org> born in Bern, Switzerland, 1985
	02/18 Christoph Moench-Tegeder <cmt@FreeBSD.org> born in Hannover, Niedersachsen, Germany, 1980
	02/19 Murray Stokely <murray@FreeBSD.org> born in Jacksonville, Florida, United States, 1979
	02/20 Anders Nordby <anders@FreeBSD.org> born in Oslo, Norway, 1976
	02/21 Alex Samorukov <samm@FreeBSD.org> born in Kyiv, Ukraine, 1981
	02/21 Alexey Zelkin <phantom@FreeBSD.org> born in Simferopol, Ukraine, 1978
	02/22 Brooks Davis <brooks@FreeBSD.org> born in Longview, Washington, United States, 1976
	02/22 Jake Burkholder <jake@FreeBSD.org> born in Maynooth, Ontario, Canada, 1979
	02/23 Peter Wemm <peter@FreeBSD.org> born in Perth, Western Australia, Australia, 1971
	02/23 Mathieu Arnold <mat@FreeBSD.org> born in Champigny sur Marne, Val de Marne, France, 1978
	02/23 Vinícius Zavam <egypcio@FreeBSD.org> born in Fortaleza, Ceará, Brazil, 1986
	02/24 Johan Karlsson <johan@FreeBSD.org> born in Mariannelund, Sweden, 1974
	02/24 Colin Percival <cperciva@FreeBSD.org> born in Burnaby, Canada, 1981
	02/24 Kevin Bowling <kbowling@FreeBSD.org> born in Scottsdale, Arizona, United States, 1989
	02/24 Brandon Bergren <bdragon@FreeBSD.org> born in Edmond, Oklahoma, United States, 1984
	02/26 Pietro Cerutti <gahr@FreeBSD.org> born in Faido, Switzerland, 1984
	02/28 Daichi GOTO <daichi@FreeBSD.org> born in Shimizu Suntou, Shizuoka, Japan, 1980
	02/28 Ruslan Makhmatkhanov <rm@FreeBSD.org> born in Rostov-on-Don, USSR, 1984
	03/01 Hye-Shik Chang <perky@FreeBSD.org> born in Daejeon, Republic of Korea, 1980
	03/02 Cy Schubert <cy@FreeBSD.org> born in Edmonton, Alberta, Canada, 1956
	03/03 Sergey Matveychuk <sem@FreeBSD.org> born in Moscow, Russian Federation, 1973
	03/03 Doug White <dwhite@FreeBSD.org> born in Eugene, Oregon, United States, 1977
	03/03 Gordon Tetlow <gordon@FreeBSD.org> born in Reno, Nevada, United States, 1978
	03/04 Oleksandr Tymoshenko <gonzo@FreeBSD.org> born in Chernihiv, Ukraine, 1980
	03/05 Baptiste Daroussin <bapt@FreeBSD.org> born in Beauvais, France, 1980
	03/05 Philip Paeps <philip@FreeBSD.org> born in Leuven, Belgium, 1983
	03/05 Ulf Lilleengen <lulf@FreeBSD.org> born in Hamar, Norway, 1985
	03/06 Christopher Piazza <cpiazza@FreeBSD.org> born in Kamloops, British Columbia, Canada, 1981
	03/07 Michael P. Pritchard <mpp@FreeBSD.org> born in Los Angeles, California, United States, 1964
	03/07 Giorgos Keramidas <keramida@FreeBSD.org> born in Athens, Greece, 1976
	03/10 Andreas Klemm <andreas@FreeBSD.org> born in Duesseldorf, Nordrhein-Westfalen, Germany, 1963
	03/10 Luiz Otavio O Souza <loos@FreeBSD.org> born in Bauru, Sao Paulo, Brazil, 1978
	03/10 Nikolai Lifanov <lifanov@FreeBSD.org> born in Moscow, Russian Federation, 1989
	03/11 Soeren Straarup <xride@FreeBSD.org> born in Andst, Denmark, 1978
	03/12 Greg Lewis <glewis@FreeBSD.org> born in Adelaide, South Australia, Australia, 1969
	03/13 Alexander Leidinger <netchild@FreeBSD.org> born in Neunkirchen, Saarland, Germany, 1976
	03/13 Will Andrews <will@FreeBSD.org> born in Pontiac, Michigan, United States, 1982
	03/14 Bernhard Froehlich <decke@FreeBSD.org> born in Graz, Styria, Austria, 1985
	03/14 Eric Turgeon <ericbsd@FreeBSD.org> born in Edmundston, New Brunswick, Canada, 1982
	03/15 Paolo Pisati <piso@FreeBSD.org> born in Lodi, Italy, 1977
	03/15 Brian Fundakowski Feldman <green@FreeBSD.org> born in Alexandria, Virginia, United States, 1983
	03/17 Michael Smith <msmith@FreeBSD.org> born in Bankstown, New South Wales, Australia, 1971
	03/17 Alexander Motin <mav@FreeBSD.org> born in Simferopol, Ukraine, 1979
	03/18 Koop Mast <kwm@FreeBSD.org> born in Dokkum, the Netherlands, 1981
	03/19 Mikhail Teterin <mi@FreeBSD.org> born in Kyiv, Ukraine, 1972
	03/20 Joseph S. Atkinson <jsa@FreeBSD.org> born in Batesville, Arkansas, United States, 1977
	03/20 Henrik Brix Andersen <brix@FreeBSD.org> born in Aarhus, Denmark, 1978
	03/20 MANTANI Nobutaka <nobutaka@FreeBSD.org> born in Hiroshima, Japan, 1978
	03/20 Cameron Grant <cg@FreeBSD.org> died in Hemel Hempstead, United Kingdom, 2005
	03/22 Brad Davis <brd@FreeBSD.org> born in Farmington, New Mexico, United States, 1983
	03/23 Daniel C. Sobral <dcs@FreeBSD.org> born in Brasilia, Distrito Federal, Brazil, 1971
	03/23 Benno Rice <benno@FreeBSD.org> born in Adelaide, South Australia, Australia, 1977
	03/24 Marcel Moolenaar <marcel@FreeBSD.org> born in Hilversum, the Netherlands, 1968
	03/24 Emanuel Haupt <ehaupt@FreeBSD.org> born in Zurich, Switzerland, 1979
	03/25 Andrew R. Reiter <arr@FreeBSD.org> born in Springfield, Massachusetts, United States, 1980
	03/26 Jonathan Anderson <jonathan@FreeBSD.org> born in Ottawa, Ontario, Canada, 1983
	03/27 Josef El-Rayes <josef@FreeBSD.org> born in Linz, Austria, 1982
	03/28 Sean C. Farley <scf@FreeBSD.org> born in Indianapolis, Indiana, United States, 1970
	03/29 Dave Cottlehuber <dch@FreeBSD.org> born in Christchurch, New Zealand, 1973
	03/29 Thierry Thomas <thierry@FreeBSD.org> born in Luxeuil les Bains, France, 1961
	03/30 Po-Chuan Hsieh <sunpoet@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1978
	03/31 First quarter status reports are due on 04/15
	04/01 Matthew Jacob <mjacob@FreeBSD.org> born in San Francisco, California, United States, 1958
	04/01 Alexander V. Chernikov <melifaro@FreeBSD.org> born in Moscow, Russian Federation, 1984
	04/01 Bill Fenner <fenner@FreeBSD.org> born in Bellefonte, Pennsylvania, United States, 1971
	04/01 Peter Edwards <peadar@FreeBSD.org> born in Dublin, Ireland, 1973
	04/03 Hellmuth Michaelis <hm@FreeBSD.org> born in Kiel, Schleswig-Holstein, Germany, 1958
	04/03 Tong Liu <nemoliu@FreeBSD.org> born in Beijing, People's Republic of China, 1981
	04/03 Gabor Pali <pgj@FreeBSD.org> born in Kunhegyes, Hungary, 1982
	04/04 Jason Unovitch <junovitch@FreeBSD.org> born in Scranton, Pennsylvania, United States, 1986
	04/05 Stacey Son <sson@FreeBSD.org> born in Burley, Idaho, United States, 1967
	04/06 Peter Jeremy <peterj@FreeBSD.org> born in Sydney, New South Wales, Australia, 1961
	04/07 Edward Tomasz Napierala <trasz@FreeBSD.org> born in Wolsztyn, Poland, 1981
	04/08 Jordan K. Hubbard <jkh@FreeBSD.org> born in Honolulu, Hawaii, United States, 1963
	04/09 Ceri Davies <ceri@FreeBSD.org> born in Haverfordwest, Pembrokeshire, United Kingdom, 1976
	04/11 Bruce A. Mah <bmah@FreeBSD.org> born in Fresno, California, United States, 1969
	04/12 Patrick Gardella <patrick@FreeBSD.org> born in Columbus, Ohio, United States, 1967
	04/12 Ed Schouten <ed@FreeBSD.org> born in Oss, the Netherlands, 1986
	04/12 Ruey-Cherng Yu <rcyu@FreeBSD.org> born in Keelung, Taiwan, 1978
	04/13 Oliver Braun <obraun@FreeBSD.org> born in Nuremberg, Bavaria, Germany, 1972
	04/14 Crist J. Clark <cjc@FreeBSD.org> born in Milwaukee, Wisconsin, United States, 1970
	04/14 Glen J. Barber <gjb@FreeBSD.org> born in Wilkes-Barre, Pennsylvania, United States, 1981
	04/15 David Malone <dwmalone@FreeBSD.org> born in Dublin, Ireland, 1973
	04/17 Alexey Degtyarev <alexey@FreeBSD.org> born in Ahtubinsk, Russian Federation, 1984
	04/17 Dryice Liu <dryice@FreeBSD.org> born in Jinan, Shandong, China, 1975
	04/22 Joerg Wunsch <joerg@FreeBSD.org> born in Dresden, Sachsen, Germany, 1962
	04/22 Jun Kuriyama <kuriyama@FreeBSD.org> born in Matsue, Shimane, Japan, 1973
	04/22 Jakub Klama <jceel@FreeBSD.org> born in Blachownia, Silesia, Poland, 1989
	04/25 Richard Gallamore <ultima@FreeBSD.org> born in Kissimmee, Florida, United States, 1987
	04/26 Rene Ladan <rene@FreeBSD.org> born in Geldrop, the Netherlands, 1980
	04/28 Oleg Bulyzhin <oleg@FreeBSD.org> born in Kharkov, USSR, 1976
	04/28 Andriy Voskoboinyk <avos@FreeBSD.org> born in Bila Tserkva, Ukraine, 1992
	04/29 Adam Weinberger <adamw@FreeBSD.org> born in Berkeley, California, United States, 1980
	04/29 Eric Anholt <anholt@FreeBSD.org> born in Portland, Oregon, United States, 1983
	05/01 Randall Stewart <rrs@FreeBSD.org> born in Spokane, Washington, United States, 1959
	05/02 Kai Knoblich <kai@FreeBSD.org> born in Hannover, Niedersachsen, Germany, 1982
	05/02 Danilo G. Baio <dbaio@FreeBSD.org> born in Maringa, Parana, Brazil, 1986
	05/02 Wojciech A. Koszek <wkoszek@FreeBSD.org> born in Czestochowa, Poland, 1987
	05/03 Brian Dean <bsd@FreeBSD.org> born in Elkins, West Virginia, United States, 1966
	05/03 Patrick Kelsey <pkelsey@FreeBSD.org> born in Freehold, New Jersey, United States, 1976
	05/03 Robert Nicholas Maxwell Watson <rwatson@FreeBSD.org> born in Harrow, Middlesex, United Kingdom, 1977
	05/04 Denis Peplin <den@FreeBSD.org> born in Nizhniy Novgorod, Russian Federation, 1977
	05/08 Kirill Ponomarew <krion@FreeBSD.org> born in Volgograd, Russian Federation, 1977
	05/08 Sean Kelly <smkelly@FreeBSD.org> born in Walnut Creek, California, United States, 1982
	05/09 Daniel Eischen <deischen@FreeBSD.org> born in Syracuse, New York, United States, 1963
	05/09 Aaron Dalton <aaron@FreeBSD.org> born in Boise, Idaho, United States, 1973
	05/09 Jase Thew <jase@FreeBSD.org> born in Abergavenny, Gwent, United Kingdom, 1974
	05/09 Leandro Lupori <luporl@FreeBSD.org> born in Sao Paulo, Sao Paulo, Brazil, 1983
	05/10 Markus Brueffer <markus@FreeBSD.org> born in Gronau, Nordrhein-Westfalen, Germany, 1977
	05/11 Kurt Lidl <lidl@FreeBSD.org> born in Rockville, Maryland, United States, 1968
	05/11 Jesus Rodriguez <jesusr@FreeBSD.org> born in Barcelona, Spain, 1972
	05/11 Marcin Wojtas <mw@FreeBSD.org> born in Krakow, Poland, 1986
	05/11 Roman Kurakin <rik@FreeBSD.org> born in Moscow, USSR, 1979
	05/11 Ulrich Spoerlein <uqs@FreeBSD.org> born in Schesslitz, Bayern, Germany, 1981
	05/13 Pete Fritchman <petef@FreeBSD.org> born in Lansdale, Pennsylvania, United States, 1983
	05/13 Ben Widawsky <bwidawsk@FreeBSD.org> born in New York City, New York, United States, 1982
	05/14 Tatsumi Hosokawa <hosokawa@FreeBSD.org> born in Tokyo, Japan, 1968
	05/14 Shigeyuku Fukushima <shige@FreeBSD.org> born in Osaka, Japan, 1974
	05/14 Rebecca Cran <bcran@FreeBSD.org> born in Cambridge, United Kingdom, 1981
	05/15 Hans Petter Selasky <hselasky@FreeBSD.org> born in Flekkefjord, Norway, 1982
	05/16 Johann Kois <jkois@FreeBSD.org> born in Wolfsberg, Austria, 1975
	05/16 Marcus Alves Grando <mnag@FreeBSD.org> born in Florianopolis, Santa Catarina, Brazil, 1979
	05/17 Thomas Abthorpe <tabthorpe@FreeBSD.org> born in Port Arthur, Ontario, Canada, 1968
	05/19 Philippe Charnier <charnier@FreeBSD.org> born in Fontainebleau, France, 1966
	05/19 Ian Dowse <iedowse@FreeBSD.org> born in Dublin, Ireland, 1975
	05/19 Sofian Brabez <sbz@FreeBSD.org> born in Toulouse, France, 1984
	05/20 Dan Moschuk <dan@FreeBSD.org> died in Burlington, Ontario, Canada, 2010
	05/21 Kris Kennaway <kris@FreeBSD.org> born in Winnipeg, Manitoba, Canada, 1978
	05/22 James Gritton <jamie@FreeBSD.org> born in San Francisco, California, United States, 1967
	05/22 Clive Tong-I Lin <clive@FreeBSD.org> born in Changhua, Taiwan, Republic of China, 1978
	05/22 Michael Bushkov <bushman@FreeBSD.org> born in Rostov-on-Don, Russian Federation, 1985
	05/22 Rui Paulo <rpaulo@FreeBSD.org> born in Evora, Portugal, 1986
	05/22 David Naylor <dbn@FreeBSD.org> born in Johannesburg, South Africa, 1988
	05/23 Munechika Sumikawa <sumikawa@FreeBSD.org> born in Osaka, Osaka, Japan, 1972
	05/24 Duncan McLennan Barclay <dmlb@FreeBSD.org> born in London, Middlesex, United Kingdom, 1970
	05/24 Oliver Lehmann <oliver@FreeBSD.org> born in Karlsburg, Germany, 1981
	05/25 Pawel Pekala <pawel@FreeBSD.org> born in Swidnica, Poland, 1980
	05/25 Tom Rhodes <trhodes@FreeBSD.org> born in Ellwood City, Pennsylvania, United States, 1981
	05/25 Roman Divacky <rdivacky@FreeBSD.org> born in Brno, Czech Republic, 1983
	05/26 Jim Pirzyk <pirzyk@FreeBSD.org> born in Chicago, Illinois, United States, 1968
	05/26 Florian Smeets <flo@FreeBSD.org> born in Schwerte, Nordrhein-Westfalen, Germany, 1982
	05/27 Ollivier Robert <roberto@FreeBSD.org> born in Paris, France, 1967
	05/29 Wilko Bulte <wilko@FreeBSD.org> born in Arnhem, the Netherlands, 1965
	05/29 Seigo Tanimura <tanimura@FreeBSD.org> born in Kitakyushu, Fukuoka, Japan, 1976
	05/30 Wen Heping <wen@FreeBSD.org> born in Xiangxiang, Hunan, China, 1970
	05/31 Ville Skytta <scop@FreeBSD.org> born in Helsinki, Finland, 1974
	06/02 Jean-Marc Zucconi <jmz@FreeBSD.org> born in Pontarlier, France, 1954
	06/02 Alexander Botero-Lowry <alexbl@FreeBSD.org> born in Austin, Texas, United States, 1986
	06/03 CHOI Junho <cjh@FreeBSD.org> born in Seoul, Korea, 1974
	06/03 Wesley Shields <wxs@FreeBSD.org> born in Binghamton, New York, United States, 1981
	06/04 Julian Elischer <julian@FreeBSD.org> born in Perth, Australia, 1959
	06/04 Justin Gibbs <gibbs@FreeBSD.org> born in San Pedro, California, United States, 1973
	06/04 Jason Evans <jasone@FreeBSD.org> born in Greeley, Colorado, United States, 1973
	06/04 Thomas Moestl <tmm@FreeBSD.org> born in Braunschweig, Niedersachsen, Germany, 1980
	06/04 Devin Teske <dteske@FreeBSD.org> born in Arcadia, California, United States, 1982
	06/04 Zack Kirsch <zack@FreeBSD.org> born in Memphis, Tennessee, United States, 1982
	06/04 Johannes Jost Meixner <xmj@FreeBSD.org> born in Wiesbaden, Germany, 1987
	06/05 Johannes Lundberg <johalun@FreeBSD.org> born in Ornskoldsvik, Sweden, 1975
	06/06 Sergei Kolobov <sergei@FreeBSD.org> born in Karpinsk, Russian Federation, 1972
	06/06 Ryan Libby <rlibby@FreeBSD.org> born in Kirkland, Washington, United States, 1985
	06/06 Alan Eldridge <alane@FreeBSD.org> died in Denver, Colorado, United States, 2003
	06/07 Jimmy Olgeni <olgeni@FreeBSD.org> born in Milano, Italy, 1976
	06/07 Benjamin Close <benjsc@FreeBSD.org> born in Adelaide, Australia, 1978
	06/07 Roger Pau Monne <royger@FreeBSD.org> born in Reus, Catalunya, Spain, 1986
	06/08 Ravi Pokala <rpokala@FreeBSD.org> born in Royal Oak, Michigan, United States, 1980
	06/09 Stanislav Galabov <sgalabov@FreeBSD.org> born in Sofia, Bulgaria, 1978
	06/11 Alonso Cardenas Marquez <acm@FreeBSD.org> born in Arequipa, Peru, 1979
	06/14 Josh Paetzel <jpaetzel@FreeBSD.org> born in Minneapolis, Minnesota, United States, 1973
	06/17 Tilman Linneweh <arved@FreeBSD.org> born in Weinheim, Baden-Wuerttemberg, Germany, 1978
	06/18 Li-Wen Hsu <lwhsu@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1984
	06/18 Roman Bogorodskiy <novel@FreeBSD.org> born in Saratov, Russian Federation, 1986
	06/19 Charlie Root <root@FreeBSD.org> born in Portland, Oregon, United States, 1993
	06/21 Ganbold Tsagaankhuu <ganbold@FreeBSD.org> born in Ulaanbaatar, Mongolia, 1971
	06/21 Niels Heinen <niels@FreeBSD.org> born in Markelo, the Netherlands, 1978
	06/22 Andreas Tobler <andreast@FreeBSD.org> born in Heiden, Switzerland, 1968
	06/24 Chris Faulhaber <jedgar@FreeBSD.org> born in Springfield, Illinois, United States, 1971
	06/26 Brian Somers <brian@FreeBSD.org> born in Dundrum, Dublin, Ireland, 1967
	06/28 Mark Santcroos <marks@FreeBSD.org> born in Rotterdam, the Netherlands, 1979
	06/28 Xin Li <delphij@FreeBSD.org> born in Beijing, People's Republic of China, 1982
	06/28 Bradley T. Hughes <bhughes@FreeBSD.org> born in Amarillo, Texas, United States, 1977
	06/29 Wilfredo Sanchez Vega <wsanchez@FreeBSD.org> born in Majaguez, Puerto Rico, United States, 1972
	06/29 Daniel Harris <dannyboy@FreeBSD.org> born in Lubbock, Texas, United States, 1985
	06/29 Andrew Pantyukhin <sat@FreeBSD.org> born in Moscow, Russian Federation, 1985
	06/30 Guido van Rooij <guido@FreeBSD.org> born in Best, Noord-Brabant, the Netherlands, 1965
	06/30 Second quarter status reports are due on 07/15
	07/01 Matthew Dillon <dillon@apollo.backplane.net> born in San Francisco, California, United States, 1966
	07/01 Mateusz Guzik <mjg@FreeBSD.org> born in Dołki Górne, Poland, 1986
	07/02 Mark Christopher Ovens <marko@FreeBSD.org> born in Preston, Lancashire, United Kingdom, 1958
	07/02 Vasil Venelinov Dimov <vd@FreeBSD.org> born in Shumen, Bulgaria, 1982
	07/04 Motoyuki Konno <motoyuki@FreeBSD.org> born in Musashino, Tokyo, Japan, 1969
	07/04 Florent Thoumie <flz@FreeBSD.org> born in Montmorency, Val d'Oise, France, 1982
	07/05 Olivier Cochard-Labbe <olivier@FreeBSD.org> born in Brest, France, 1977
	07/05 Sergey Kandaurov <pluknet@FreeBSD.org> born in Gubkin, Russian Federation, 1985
	07/07 Andrew Thompson <thompsa@FreeBSD.org> born in Lower Hutt, Wellington, New Zealand, 1979
	07/07 Maxime Henrion <mux@FreeBSD.org> born in Metz, France, 1981
	07/07 George Reid <greid@FreeBSD.org> born in Frimley, Hampshire, United Kingdom, 1983
	07/10 Jung-uk Kim <jkim@FreeBSD.org> born in Seoul, Korea, 1971
	07/10 Justin Seger <jseger@FreeBSD.org> born in Harvard, Massachusetts, United States, 1981
	07/10 David Schultz <das@FreeBSD.org> born in Oakland, California, United States, 1982
	07/10 Ben Woods <woodsb02@FreeBSD.org> born in Perth, Western Australia, Australia, 1984
	07/11 Jesus R. Camou <jcamou@FreeBSD.org> born in Hermosillo, Sonora, Mexico, 1983
	07/14 Fernando Apesteguia <fernape@FreeBSD.org> born in Madrid, Spain, 1981
	07/15 Gary Jennejohn <gj@FreeBSD.org> born in Rochester, New York, United States, 1950
	07/16 Suleiman Souhlal <ssouhlal@FreeBSD.org> born in Roma, Italy, 1983
	07/16 Davide Italiano <davide@FreeBSD.org> born in Milazzo, Italy, 1989
	07/17 Michael Chin-Yuan Wu <keichii@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1980
	07/19 Masafumi NAKANE <max@FreeBSD.org> born in Okazaki, Aichi, Japan, 1972
	07/19 Simon L. Nielsen <simon@FreeBSD.org> born in Copenhagen, Denmark, 1980
	07/19 Gleb Smirnoff <glebius@FreeBSD.org> born in Kharkov, USSR, 1981
	07/20 Dru Lavigne <dru@FreeBSD.org> born in Kingston, Ontario, Canada, 1965
	07/20 Andrey V. Elsukov <ae@FreeBSD.org> born in Kotelnich, Russian Federation, 1981
	07/22 James Housley <jeh@FreeBSD.org> born in Chicago, Illinois, United States, 1965
	07/22 Jens Schweikhardt <schweikh@FreeBSD.org> born in Waiblingen, Baden-Wuerttemberg, Germany, 1967
	07/22 Lukas Ertl <le@FreeBSD.org> born in Weissenbach/Enns, Steiermark, Austria, 1976
	07/23 Sergey A. Osokin <osa@FreeBSD.org> born in Krasnogorsky, Stepnogorsk, Akmolinskaya region, Kazakhstan, 1972
	07/23 Andrey Zonov <zont@FreeBSD.org> born in Kirov, Russian Federation, 1985
	07/24 Alexander Nedotsukov <bland@FreeBSD.org> born in Ulyanovsk, Russian Federation, 1974
	07/24 Alberto Villa <avilla@FreeBSD.org> born in Vercelli, Italy, 1987
	07/27 Andriy Gapon <avg@FreeBSD.org> born in Kyrykivka, Sumy region, Ukraine, 1976
	07/28 Jim Mock <jim@FreeBSD.org> born in Bethlehem, Pennsylvania, United States, 1974
	07/28 Tom Hukins <tom@FreeBSD.org> born in Manchester, United Kingdom, 1976
	07/29 Dirk Meyer <dinoex@FreeBSD.org> born in Kassel, Hessen, Germany, 1965
	07/29 Felippe M. Motta <lippe@FreeBSD.org> born in Maceio, Alagoas, Brazil, 1988
	08/02 Gabor Kovesdan <gabor@FreeBSD.org> born in Budapest, Hungary, 1987
	08/03 Peter Holm <pho@FreeBSD.org> born in Copenhagen, Denmark, 1955
	08/05 Alfred Perlstein <alfred@FreeBSD.org> born in Brooklyn, New York, United States, 1978
	08/06 Anton Berezin <tobez@FreeBSD.org> born in Dnepropetrovsk, Ukraine, 1970
	08/06 John-Mark Gurney <jmg@FreeBSD.org> born in Detroit, Michigan, United States, 1978
	08/06 Damjan Marion <dmarion@FreeBSD.org> born in Rijeka, Croatia, 1978
	08/07 Jonathan Mini <mini@FreeBSD.org> born in San Mateo, California, United States, 1979
	08/08 Mikolaj Golub <trociny@FreeBSD.org> born in Kharkov, USSR, 1977
	08/08 Juergen Lock <nox@FreeBSD.org> died in Bremen, Germany, 2015
	08/09 Stefan Farfeleder <stefanf@FreeBSD.org> died in Wien, Austria, 2015
	08/10 Julio Merino <jmmv@FreeBSD.org> born in Barcelona, Spain, 1984
	08/10 Peter Pentchev <roam@FreeBSD.org> born in Sofia, Bulgaria, 1977
	08/12 Joe Marcus Clarke <marcus@FreeBSD.org> born in Lakeland, Florida, United States, 1976
	08/12 Max Brazhnikov <makc@FreeBSD.org> born in Leningradskaya, Russian Federation, 1979
	08/14 Stefan Esser <se@FreeBSD.org> born in Cologne, Nordrhein-Westfalen, Germany, 1961
	08/16 Andrey Chernov <ache@FreeBSD.org> died in Moscow, Russian Federation, 2017
	08/17 Olivier Houchard <cognet@FreeBSD.org> born in Nancy, France, 1980
	08/19 Chin-San Huang <chinsan@FreeBSD.org> born in Yi-Lan, Taiwan, Republic of China, 1979
	08/19 Pav Lucistnik <pav@FreeBSD.org> born in Kutna Hora, Czech Republic, 1980
	08/20 Michael Heffner <mikeh@FreeBSD.org> born in Cleona, Pennsylvania, United States, 1981
	08/21 Alfredo Dal'Ava Junior <alfredo@FreeBSD.org> born in Pocos de Caldas, Minas Gerais, Brazil, 1981
	08/21 Jason A. Harmening <jah@FreeBSD.org> born in Fort Wayne, Indiana, United States, 1981
	08/22 Ilya Bakulin <kibab@FreeBSD.org> born in Tbilisi, USSR, 1986
	08/24 Mark Linimon <linimon@FreeBSD.org> born in Houston, Texas, United States, 1955
	08/24 Alexander Botero-Lowry <alexbl@FreeBSD.org> died in San Francisco, California, United States, 2012
	08/25 Beech Rintoul <beech@FreeBSD.org> born in Oakland, California, United States, 1952
	08/25 Jean Milanez Melo <jmelo@FreeBSD.org> born in Divinopolis, Minas Gerais, Brazil, 1982
	08/26 Scott Long <scottl@FreeBSD.org> born in Chicago, Illinois, United States, 1974
	08/26 Dima Ruban <dima@FreeBSD.org> born in Nalchik, USSR, 1970
	08/26 Marc Fonvieille <blackend@FreeBSD.org> born in Avignon, France, 1972
	08/26 Herve Quiroz <hq@FreeBSD.org> born in Aix-en-Provence, France, 1977
	08/27 Andrey Chernov <ache@FreeBSD.org> born in Moscow, USSR, 1966
	08/27 Tony Finch <fanf@FreeBSD.org> born in London, United Kingdom, 1974
	08/27 Michael Johnson <ahze@FreeBSD.org> born in Morganton, North Carolina, United States, 1980
	08/28 Norikatsu Shigemura <nork@FreeBSD.org> born in Fujisawa, Kanagawa, Japan, 1974
	08/29 Thomas Gellekum <tg@FreeBSD.org> born in Moenchengladbach, Nordrhein-Westfalen, Germany, 1967
	08/29 Max Laier <mlaier@FreeBSD.org> born in Karlsruhe, Germany, 1981
	08/30 Yuri Pankov <yuripv@FreeBSD.org> born in Krasnodar, USSR, 1979
	09/01 Pyun YongHyeon <yongari@FreeBSD.org> born in Kimcheon, Korea, 1968
	09/01 William Grzybowski <wg@FreeBSD.org> born in Parana, Brazil, 1988
	09/03 Max Khon <fjoe@FreeBSD.org> born in Novosibirsk, USSR, 1976
	09/03 Allan Jude <allanjude@FreeBSD.org> born in Hamilton, Ontario, Canada, 1984
	09/03 Cheng-Lung Sung <clsung@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1977
	09/05 Mark Robert Vaughan Murray <markm@FreeBSD.org> born in Harare, Mashonaland, Zimbabwe, 1961
	09/05 Adrian Harold Chadd <adrian@FreeBSD.org> born in Perth, Western Australia, Australia, 1979
	09/05 Rodrigo Osorio <rodrigo@FreeBSD.org> born in Montevideo, Uruguay, 1975
	09/06 Eric Joyner <erj@FreeBSD.org> born in Fairfax, Virginia, United States, 1991
	09/07 Tim Bishop <tdb@FreeBSD.org> born in Cornwall, United Kingdom, 1978
	09/07 Chris Rees <crees@FreeBSD.org> born in Kettering, United Kingdom, 1987
	09/08 Boris Samorodov <bsam@FreeBSD.org> born in Krasnodar, Russian Federation, 1963
	09/09 Yoshio Mita <mita@FreeBSD.org> born in Hiroshima, Japan, 1972
	09/09 Steven Hartland <smh@FreeBSD.org> born in Wordsley, United Kingdom, 1973
	09/10 Wesley R. Peters <wes@FreeBSD.org> born in Hartford, Alabama, United States, 1961
	09/11 Gordon Bergling <gbe@FreeBSD.org> born in Magdeburg, Germany, 1981
	09/12 Weongyo Jeong <weongyo@FreeBSD.org> born in Haman, Korea, 1980
	09/12 Benedict Christopher Reuschling <bcr@FreeBSD.org> born in Darmstadt, Germany, 1981
	09/12 William C. Fumerola II <billf@FreeBSD.org> born in Detroit, Michigan, United States, 1981
	09/14 Matthew Seaman <matthew@FreeBSD.org> born in Bristol, United Kingdom, 1965
	09/15 Aleksandr Rybalko <ray@FreeBSD.org> born in Odessa, Ukraine, 1977
	09/15 Dima Panov <fluffy@FreeBSD.org> born in Khabarovsk, Russian Federation, 1978
	09/16 Maksim Yevmenkin <emax@FreeBSD.org> born in Taganrog, USSR, 1974
	09/17 Maxim Bolotin <mb@FreeBSD.org> born in Rostov-on-Don, Russian Federation, 1976
	09/18 Matthew Fleming <mdf@FreeBSD.org> born in Cleveland, Ohio, United States, 1975
	09/20 Kevin Lo <kevlo@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1972
	09/21 Alex Kozlov <ak@FreeBSD.org> born in Bila Tserkva, Ukraine, 1970
	09/21 Gleb Kurtsou <gleb@FreeBSD.org> born in Minsk, Belarus, 1984
	09/22 Alan Somers <asomers@FreeBSD.org> born in San Antonio, Texas, United States, 1982
	09/22 Bryan Drewery <bdrewery@FreeBSD.org> born in San Diego, California, United States, 1984
	09/23 Michael Dexter <dexter@FreeBSD.org> born in Los Angeles, California, 1972
	09/23 Martin Matuska <mm@FreeBSD.org> born in Bratislava, Slovakia, 1979
	09/24 Larry Rosenman <ler@FreeBSD.org> born in Queens, New York, United States, 1957
	09/25 Piotr Kubaj <pkubaj@FreeBSD.org> born in Lubin, Poland, 1991
	09/27 Kyle Evans <kevans@FreeBSD.org> born in Oklahoma City, Oklahoma, United States, 1991
	09/27 Neil Blakey-Milner <nbm@FreeBSD.org> born in Port Elizabeth, South Africa, 1978
	09/27 Renato Botelho <garga@FreeBSD.org> born in Araras, Sao Paulo, Brazil, 1979
	09/28 Greg Lehey <grog@FreeBSD.org> born in Melbourne, Victoria, Australia, 1948
	09/28 Alex Dupre <ale@FreeBSD.org> born in Milano, Italy, 1980
	09/29 Matthew Hunt <mph@FreeBSD.org> born in Johnstown, Pennsylvania, United States, 1976
	09/30 Mark Felder <feld@FreeBSD.org> born in Prairie du Chien, Wisconsin, United States, 1985
	09/30 Hiten M. Pandya <hmp@FreeBSD.org> born in Dar-es-Salaam, Tanzania, East Africa, 1986
	09/30 Third quarter status reports are due on 10/15
	10/02 Beat Gaetzi <beat@FreeBSD.org> born in Zurich, Switzerland, 1980
	10/02 Grzegorz Blach <gblach@FreeBSD.org> born in Starachowice, Poland, 1985
	10/05 Hiroki Sato <hrs@FreeBSD.org> born in Yamagata, Japan, 1977
	10/05 Chris Costello <chris@FreeBSD.org> born in Houston, Texas, United States, 1985
	10/09 Stefan Walter <stefan@FreeBSD.org> born in Werne, Nordrhein-Westfalen, Germany, 1978
	10/11 Rick Macklem <rmacklem@FreeBSD.org> born in Ontario, Canada, 1955
	10/12 Pawel Jakub Dawidek <pjd@FreeBSD.org> born in Radzyn Podlaski, Poland, 1980
	10/15 Maxim Konovalov <maxim@FreeBSD.org> born in Khabarovsk, USSR, 1973
	10/15 Eugene Grosbein <eugen@FreeBSD.org> born in Novokuznetsk, Russian Republic, USSR, 1976
	10/16 Remko Lodder <remko@FreeBSD.org> born in Rotterdam, the Netherlands, 1983
	10/17 Maho NAKATA <maho@FreeBSD.org> born in Osaka, Japan, 1974
	10/18 Sheldon Hearn <sheldonh@FreeBSD.org> born in Cape Town, Western Cape, South Africa, 1974
	10/18 Vladimir Kondratyev <wulf@FreeBSD.org> born in Ryazan, USSR, 1975
	10/19 Nicholas Souchu <nsouch@FreeBSD.org> born in Suresnes, Hauts-de-Seine, France, 1972
	10/19 Nick Barkas <snb@FreeBSD.org> born in Longview, Washington, United States, 1981
	10/19 Pedro Giffuni <pfg@FreeBSD.org> born in Bogotá, Colombia, 1968
	10/20 Joel Dahl <joel@FreeBSD.org> born in Bitterna, Skaraborg, Sweden, 1983
	10/20 Dmitry Marakasov <amdmi3@FreeBSD.org> born in Moscow, Russian Federation, 1984
	10/21 Ben Smithurst <ben@FreeBSD.org> born in Sheffield, South Yorkshire, United Kingdom, 1981
	+10/21 Daniel Ebdrup Jensen <debdrup@FreeBSD.org> born in Aalborg, Denmark, 19XX
	10/22 Jean-Sebastien Pedron <dumbbell@FreeBSD.org> born in Redon, Ille-et-Vilaine, France, 1980
	10/23 Mario Sergio Fujikawa Ferreira <lioux@FreeBSD.org> born in Brasilia, Distrito Federal, Brazil, 1976
	10/23 Romain Tartière <romain@FreeBSD.org> born in Clermont-Ferrand, France, 1984
	10/25 Eric Melville <eric@FreeBSD.org> born in Los Gatos, California, United States, 1980
	10/25 Julien Laffaye <jlaffaye@FreeBSD.org> born in Toulouse, France, 1988
	10/25 Ashish SHUKLA <ashish@FreeBSD.org> born in Kanpur, India, 1985
	10/25 Toomas Soome <tsoome@FreeBSD.org> born Estonia, 1971
	10/26 Matthew Ahrens <mahrens@FreeBSD.org> born in United States, 1979
	10/26 Philip M. Gollucci <pgollucci@FreeBSD.org> born in Silver Spring, Maryland, United States, 1979
	10/27 Takanori Watanabe <takawata@FreeBSD.org> born in Numazu, Shizuoka, Japan, 1972
	10/30 Olli Hauer <ohauer@FreeBSD.org> born in Sindelfingen, Germany, 1968
	10/31 Taras Korenko <taras@FreeBSD.org> born in Cherkasy region, Ukraine, 1980
	11/03 Ryan Stone <rstone@FreeBSD.org> born in Ottawa, Ontario, Canada, 1985
	11/04 John Hixson <jhixson@FreeBSD.org> born in Burlingame, California, United States, 1974
	11/05 M. Warner Losh <imp@FreeBSD.org> born in Kansas City, Kansas, United States, 1966
	11/06 Michael Zhilin <mizhka@FreeBSD.org> born in Stary Oskol, USSR, 1985
	11/08 Joseph R. Mingrone <jrm@FreeBSD.org> born in Charlottetown, Prince Edward Island, Canada, 1976
	11/09 Coleman Kane <cokane@FreeBSD.org> born in Cincinnati, Ohio, United States, 1980
	11/09 Antoine Brodin <antoine@FreeBSD.org> born in Bagnolet, France, 1981
	11/10 Gregory Neil Shapiro <gshapiro@FreeBSD.org> born in Providence, Rhode Island, United States, 1970
	11/11 Danilo E. Gondolfo <danilo@FreeBSD.org> born in Lobato, Parana, Brazil, 1987
	11/12 Gleb Popov <arrowd@FreeBSD.org> born in Volgograd, Russia, 1991
	11/13 John Baldwin <jhb@FreeBSD.org> born in Stuart, Virginia, United States, 1977
	11/14 Jeremie Le Hen <jlh@FreeBSD.org> born in Nancy, France, 1980
	11/15 Lars Engels <lme@FreeBSD.org> born in Hilden, Nordrhein-Westfalen, Germany, 1980
	11/15 Tijl Coosemans <tijl@FreeBSD.org> born in Duffel, Belgium, 1983
	11/16 Jose Maria Alcaide Salinas <jmas@FreeBSD.org> born in Madrid, Spain, 1962
	11/16 Matt Joras <mjoras@FreeBSD.org> born in Evanston, Illinois, United States, 1992
	11/17 Ralf S. Engelschall <rse@FreeBSD.org> born in Dachau, Bavaria, Germany, 1972
	11/18 Thomas Quinot <thomas@FreeBSD.org> born in Paris, France, 1977
	11/19 Konstantin Belousov <kib@FreeBSD.org> born in Kiev, USSR, 1972
	11/20 Dmitry Morozovsky <marck@FreeBSD.org> born in Moscow, USSR, 1968
	11/20 Gavin Atkinson <gavin@FreeBSD.org> born in Middlesbrough, United Kingdom, 1979
	11/21 Shteryana Shopova <syrinx@FreeBSD.org> born in Petrich, Bulgaria, 1982
	11/21 Mark Johnston <markj@FreeBSD.org> born in Toronto, Ontario, Canada, 1989
	11/22 Frederic Culot <culot@FreeBSD.org> born in Saint-Germain-En-Laye, France, 1976
	11/23 Josef Lawrence Karthauser <joe@FreeBSD.org> born in Pembury, Kent, United Kingdom, 1972
	11/23 Sepherosa Ziehau <sephe@FreeBSD.org> born in Shanghai, China, 1980
	11/23 Luca Pizzamiglio <pizzamig@FreeBSD.org> born in Casalpusterlengo, Italy, 1978
	11/24 Andrey Zakhvatov <andy@FreeBSD.org> born in Chelyabinsk, Russian Federation, 1974
	11/24 Daniel Gerzo <danger@FreeBSD.org> born in Bratislava, Slovakia, 1986
	11/25 Fedor Uporov <fsu@FreeBSD.org> born in Yalta, Crimea, USSR, 1988
	11/28 Nik Clayton <nik@FreeBSD.org> born in Peterborough, United Kingdom, 1973
	11/28 Stanislav Sedov <stas@FreeBSD.org> born in Chelyabinsk, USSR, 1985
	11/29 Doug Moore <dougm@FreeBSD.org> born in Arlington, Texas, United States, 1960
	11/30 Dmitri Goutnik <dmgk@FreeBSD.org> born in Minsk, USSR, 1969
	12/01 Hajimu Umemoto <ume@FreeBSD.org> born in Nara, Japan, 1961
	12/01 Alexey Dokuchaev <danfe@FreeBSD.org> born in Magadan, USSR, 1980
	12/02 Ermal Luçi <eri@FreeBSD.org> born in Tirane, Albania, 1980
	12/03 Diane Bruce <db@FreeBSD.org> born in Ottawa, Ontario, Canada, 1952
	12/04 Mariusz Zaborski <oshogbo@FreeBSD.org> born in Skierniewice, Poland, 1990
	12/05 Ivan Voras <ivoras@FreeBSD.org> born in Slavonski Brod, Croatia, 1981
	12/06 Stefan Farfeleder <stefanf@FreeBSD.org> born in Wien, Austria, 1980
	12/08 Michael Tuexen <tuexen@FreeBSD.org> born in Oldenburg, Germany, 1966
	12/10 Hiroki Tagato <tagattie@FreeBSD.org> born in Shiga, Japan, 1971
	12/11 Ganael Laplanche <martymac@FreeBSD.org> born in Reims, France, 1980
	12/11 Koichiro Iwao <meta@FreeBSD.org> born in Oita, Japan, 1987
	12/15 James FitzGibbon <jfitz@FreeBSD.org> born in Amersham, Buckinghamshire, United Kingdom, 1974
	12/15 Timur I. Bakeyev <timur@FreeBSD.org> born in Kazan, Republic of Tatarstan, USSR, 1974
	12/18 Chris Timmons <cwt@FreeBSD.org> born in Ellensburg, Washington, United States, 1964
	12/18 Dag-Erling Smorgrav <des@FreeBSD.org> born in Brussels, Belgium, 1977
	12/18 Muhammad Moinur Rahman <bofh@FreeBSD.org> born in Dhaka, Bangladesh, 1983
	12/18 Semen Ustimenko <semenu@FreeBSD.org> born in Novosibirsk, Russian Federation, 1979
	12/19 Stephen Hurd <shurd@FreeBSD.org> born in Estevan, Saskatchewan, Canada, 1975
	12/19 Emmanuel Vadot <manu@FreeBSD.org> born in Decines-Charpieu, France, 1983
	12/20 Sean Bruno <sbruno@FreeBSD.org> born in Monterey, California, United States, 1974
	12/21 Rong-En Fan <rafan@FreeBSD.org> born in Taipei, Taiwan, Republic of China, 1982
	12/22 Alan L. Cox <alc@FreeBSD.org> born in Warren, Ohio, United States, 1964
	12/22 Maxim Sobolev <sobomax@FreeBSD.org> born in Dnepropetrovsk, Ukraine, 1976
	12/23 Sean Chittenden <seanc@FreeBSD.org> born in Seattle, Washington, United States, 1979
	12/23 Alejandro Pulver <alepulver@FreeBSD.org> born in Buenos Aires, Argentina, 1989
	12/24 Jochen Neumeister <joneum@FreeBSD.org> born in Heidenheim, Germany, 1975
	12/24 Guido Falsi <madpilot@FreeBSD.org> born in Firenze, Italy, 1978
	12/25 Niclas Zeising <zeising@FreeBSD.org> born in Stockholm, Sweden, 1986
	12/28 Soren Schmidt <sos@FreeBSD.org> born in Maribo, Denmark, 1960
	12/28 Ade Lovett <ade@FreeBSD.org> born in London, England, 1969
	12/28 Marius Strobl <marius@FreeBSD.org> born in Cham, Bavaria, Germany, 1978
	12/30 Sean Eric Fagan <sef@FreeBSD.org> born in Los Angeles, California, United States, 1967
	12/31 Edwin Groothuis <edwin@FreeBSD.org> born in Geldrop, the Netherlands, 1970
	12/31 Fourth quarter status reports are due on 01/15

	#endif /* !_calendar_freebsd_ */
	diff --git a/usr.bin/comm/comm.1 b/usr.bin/comm/comm.1
	index 6e258806287e..b70e08692bd6 100644
	--- a/usr.bin/comm/comm.1
	+++ b/usr.bin/comm/comm.1
	@@ -1,118 +1,145 @@
	.\" Copyright (c) 1989, 1990, 1993
	.\" The Regents of the University of California. All rights reserved.
	.\"
	.\" This code is derived from software contributed to Berkeley by
	.\" the Institute of Electrical and Electronics Engineers, Inc.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\" 3. Neither the name of the University nor the names of its contributors
	.\" may be used to endorse or promote products derived from this software
	.\" without specific prior written permission.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" From: @(#)comm.1 8.1 (Berkeley) 6/6/93
	.\" $FreeBSD$
	.\"
	-.Dd December 12, 2009
	+.Dd July 27, 2020
	.Dt COMM 1
	.Os
	.Sh NAME
	.Nm comm
	.Nd select or reject lines common to two files
	.Sh SYNOPSIS
	.Nm
	.Op Fl 123i
	.Ar file1 file2
	.Sh DESCRIPTION
	The
	.Nm
	utility reads
	.Ar file1
	and
	.Ar file2 ,
	which should be
	sorted lexically, and produces three text
	columns as output: lines only in
	.Ar file1 ;
	lines only in
	.Ar file2 ;
	and lines in both files.
	.Pp
	The filename ``-'' means the standard input.
	.Pp
	The following options are available:
	.Bl -tag -width indent
	.It Fl 1
	Suppress printing of column 1, lines only in
	.Ar file1 .
	.It Fl 2
	Suppress printing of column 2, lines only in
	.Ar file2 .
	.It Fl 3
	Suppress printing of column 3, lines common to both.
	.It Fl i
	Case insensitive comparison of lines.
	.El
	.Pp
	Each column will have a number of tab characters prepended to it
	equal to the number of lower numbered columns that are being printed.
	For example, if column number two is being suppressed, lines printed
	in column number one will not have any tabs preceding them, and lines
	printed in column number three will have one.
	.Pp
	The
	.Nm
	utility assumes that the files are lexically sorted; all characters
	participate in line comparisons.
	.Sh ENVIRONMENT
	The
	.Ev LANG ,
	.Ev LC_ALL ,
	.Ev LC_COLLATE ,
	and
	.Ev LC_CTYPE
	environment variables affect the execution of
	.Nm
	as described in
	.Xr environ 7 .
	.Sh EXIT STATUS
	.Ex -std
	+.Sh EXAMPLES
	+Assuming a file named
	+.Pa example.txt
	+with the following contents:
	+.Bd -literal -offset indent
	+a
	+b
	+c
	+d
	+.Ed
	+.Pp
	+Show lines only in
	+.Pa example.txt ,
	+lines only in stdin and common lines:
	+.Bd -literal -offset indent
	+$ echo -e "B\enc" \| comm example.txt -
	+ B
	+a
	+b
	+ c
	+d
	+.Ed
	+.Pp
	+Show only common lines doing case insensitive comparisons:
	+.Bd -literal -offset indent
	+$ echo -e "B\enc" \| comm -1 -2 -i example.txt -
	+b
	+c
	+.Ed
	.Sh SEE ALSO
	.Xr cmp 1 ,
	.Xr diff 1 ,
	.Xr sort 1 ,
	.Xr uniq 1
	.Sh STANDARDS
	The
	.Nm
	utility conforms to
	.St -p1003.2-92 .
	.Pp
	The
	.Fl i
	-option is an extension to the
	-.Tn POSIX
	-standard.
	+option is an extension to the POSIX standard.
	.Sh HISTORY
	A
	.Nm
	command appeared in
	.At v4 .
	diff --git a/usr.bin/diff/tests/diff_test.sh b/usr.bin/diff/tests/diff_test.sh
	index 67a7300f8dd3..9f460231f480 100755
	--- a/usr.bin/diff/tests/diff_test.sh
	+++ b/usr.bin/diff/tests/diff_test.sh
	@@ -1,236 +1,236 @@
	# $FreeBSD$

	atf_test_case simple
	atf_test_case unified
	atf_test_case header
	atf_test_case header_ns
	atf_test_case ifdef
	atf_test_case group_format
	atf_test_case side_by_side
	atf_test_case brief_format
	atf_test_case b230049
	atf_test_case Bflag
	atf_test_case Nflag
	atf_test_case tabsize
	atf_test_case conflicting_format
	atf_test_case label

	simple_body()
	{
	atf_check -o file:$(atf_get_srcdir)/simple.out -s eq:1 \
	diff "$(atf_get_srcdir)/input1.in" "$(atf_get_srcdir)/input2.in"

	atf_check -o file:$(atf_get_srcdir)/simple_e.out -s eq:1 \
	diff -e "$(atf_get_srcdir)/input1.in" "$(atf_get_srcdir)/input2.in"

	atf_check -o file:$(atf_get_srcdir)/simple_u.out -s eq:1 \
	diff -u -L input1 -L input2 "$(atf_get_srcdir)/input1.in" "$(atf_get_srcdir)/input2.in"

	atf_check -o file:$(atf_get_srcdir)/simple_n.out -s eq:1 \
	diff -n "$(atf_get_srcdir)/input1.in" "$(atf_get_srcdir)/input2.in"

	atf_check -o inline:"Files $(atf_get_srcdir)/input1.in and $(atf_get_srcdir)/input2.in differ\n" -s eq:1 \
	diff -q "$(atf_get_srcdir)/input1.in" "$(atf_get_srcdir)/input2.in"

	atf_check \
	diff -q "$(atf_get_srcdir)/input1.in" "$(atf_get_srcdir)/input1.in"

	atf_check -o file:$(atf_get_srcdir)/simple_i.out -s eq:1 \
	diff -i "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"

	atf_check -o file:$(atf_get_srcdir)/simple_w.out -s eq:1 \
	diff -w "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"

	atf_check -o file:$(atf_get_srcdir)/simple_b.out -s eq:1 \
	diff -b "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"

	atf_check -o file:$(atf_get_srcdir)/simple_p.out -s eq:1 \
	diff --label input_c1.in --label input_c2.in -p "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"
	}

	unified_body()
	{
	atf_check -o file:$(atf_get_srcdir)/unified_p.out -s eq:1 \
	diff -up -L input_c1.in -L input_c2.in "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"
	atf_check -o file:$(atf_get_srcdir)/unified_9999.out -s eq:1 \
	diff -u9999 -L input_c1.in -L input_c2.in "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"
	}

	b230049_body()
	{
	printf 'a\nb\r\nc\n' > b230049_a.in
	printf 'a\r\nb\r\nc\r\n' > b230049_b.in
	atf_check -o empty -s eq:0 \
	diff -up --strip-trailing-cr -L b230049_a.in -L b230049_b.in \
	b230049_a.in b230049_b.in
	}

	header_body()
	{
	export TZ=UTC
	: > empty
	echo hello > hello
	touch -d 2015-04-03T01:02:03 empty
	touch -d 2016-12-22T11:22:33 hello
	atf_check -o "file:$(atf_get_srcdir)/header.out" -s eq:1 \
	diff -u empty hello
	}

	header_ns_body()
	{
	export TZ=UTC
	: > empty
	echo hello > hello
	touch -d 2015-04-03T01:02:03.123456789 empty
	touch -d 2016-12-22T11:22:33.987654321 hello
	atf_check -o "file:$(atf_get_srcdir)/header_ns.out" -s eq:1 \
	diff -u empty hello
	}

	ifdef_body()
	{
	atf_check -o file:$(atf_get_srcdir)/ifdef.out -s eq:1 \
	diff -D PLOP "$(atf_get_srcdir)/input_c1.in" \
	"$(atf_get_srcdir)/input_c2.in"
	}

	group_format_body()
	{
	atf_check -o file:$(atf_get_srcdir)/group-format.out -s eq:1 \
	diff --changed-group-format='<<<<<<< (local)
	%<=======
	%>>>>>>>> (stock)
	' "$(atf_get_srcdir)/input_c1.in" "$(atf_get_srcdir)/input_c2.in"
	}

	side_by_side_body()
	{
	atf_check -o save:A printf "A\nB\nC\n"
	atf_check -o save:B printf "D\nB\nE\n"

	- exp_output="A[[:space:]]+\|[[:space:]]+D\nB[[:space:]]+B\nC[[:space:]]+\|[[:space:]]+E"
	- exp_output_suppressed="A[[:space:]]+\|[[:space:]]+D\nC[[:space:]]+\|[[:space:]]+E"
	+ exp_output=$(printf "A[[:space:]]+\|[[:space:]]+D\nB[[:space:]]+B\nC[[:space:]]+\|[[:space:]]+E")
	+ exp_output_suppressed=$(printf "A[[:space:]]+\|[[:space:]]+D\nC[[:space:]]+\|[[:space:]]+E")

	atf_check -o match:"$exp_output" -s exit:1 \
	diff --side-by-side A B
	atf_check -o match:"$exp_output" -s exit:1 \
	diff -y A B
	atf_check -o match:"$exp_output_suppressed" -s exit:1 \
	diff -y --suppress-common-lines A B
	atf_check -o match:"$exp_output_suppressed" -s exit:1 \
	diff -W 65 -y --suppress-common-lines A B
	}

	brief_format_body()
	{
	atf_check mkdir A B

	atf_check -x "echo 1 > A/test-file"
	atf_check -x "echo 2 > B/test-file"

	atf_check cp -Rf A C
	atf_check cp -Rf A D

	atf_check -x "echo 3 > D/another-test-file"

	atf_check \
	-s exit:1 \
	-o inline:"Files A/test-file and B/test-file differ\n" \
	diff -rq A B

	atf_check diff -rq A C

	atf_check \
	-s exit:1 \
	-o inline:"Only in D: another-test-file\n" \
	diff -rq A D

	atf_check \
	-s exit:1 \
	-o inline:"Files A/another-test-file and D/another-test-file differ\n" \
	diff -Nrq A D
	}

	Bflag_body()
	{
	atf_check -x 'printf "A\nB\n" > A'
	atf_check -x 'printf "A\n\nB\n" > B'
	atf_check -x 'printf "A\n \nB\n" > C'
	atf_check -x 'printf "A\nC\nB\n" > D'
	atf_check -x 'printf "A\nB\nC\nD\nE\nF\nG\nH" > E'
	atf_check -x 'printf "A\n\nB\nC\nD\nE\nF\nX\nH" > F'

	atf_check -s exit:0 -o inline:"" diff -B A B
	atf_check -s exit:1 -o file:"$(atf_get_srcdir)/Bflag_C.out" diff -B A C
	atf_check -s exit:1 -o file:"$(atf_get_srcdir)/Bflag_D.out" diff -B A D
	atf_check -s exit:1 -o file:"$(atf_get_srcdir)/Bflag_F.out" diff -B E F
	}

	Nflag_body()
	{
	atf_check -x 'printf "foo" > A'

	atf_check -s exit:1 -o ignore -e ignore diff -N A NOFILE
	atf_check -s exit:1 -o ignore -e ignore diff -N NOFILE A
	atf_check -s exit:2 -o ignore -e ignore diff -N NOFILE1 NOFILE2
	}

	tabsize_body()
	{
	printf "\tA\n" > A
	printf "\tB\n" > B

	atf_check -s exit:1 \
	-o inline:"1c1\n< A\n---\n> B\n" \
	diff -t --tabsize 1 A B
	}

	conflicting_format_body()
	{
	printf "\tA\n" > A
	printf "\tB\n" > B

	atf_check -s exit:2 -e ignore diff -c -u A B
	atf_check -s exit:2 -e ignore diff -e -f A B
	atf_check -s exit:2 -e ignore diff -y -q A B
	atf_check -s exit:2 -e ignore diff -q -u A B
	atf_check -s exit:2 -e ignore diff -q -c A B
	atf_check -s exit:2 -e ignore diff --normal -c A B
	atf_check -s exit:2 -e ignore diff -c --normal A B

	atf_check -s exit:1 -o ignore -e ignore diff -u -u A B
	atf_check -s exit:1 -o ignore -e ignore diff -e -e A B
	atf_check -s exit:1 -o ignore -e ignore diff -y -y A B
	atf_check -s exit:1 -o ignore -e ignore diff -q -q A B
	atf_check -s exit:1 -o ignore -e ignore diff -c -c A B
	atf_check -s exit:1 -o ignore -e ignore diff --normal --normal A B
	}

	label_body()
	{
	printf "\tA\n" > A

	atf_check -o inline:"Files hello and world are identical\n" \
	-s exit:0 diff --label hello --label world -s A A

	atf_check -o inline:"Binary files hello and world differ\n" \
	-s exit:1 diff --label hello --label world `which diff` `which ls`
	}

	atf_init_test_cases()
	{
	atf_add_test_case simple
	atf_add_test_case unified
	atf_add_test_case header
	atf_add_test_case header_ns
	atf_add_test_case ifdef
	atf_add_test_case group_format
	atf_add_test_case side_by_side
	atf_add_test_case brief_format
	atf_add_test_case b230049
	atf_add_test_case Bflag
	atf_add_test_case Nflag
	atf_add_test_case tabsize
	atf_add_test_case conflicting_format
	atf_add_test_case label
	}
	diff --git a/usr.bin/sed/tests/sed2_test.sh b/usr.bin/sed/tests/sed2_test.sh
	index 9f831e0ed673..07f1ec844814 100755
	--- a/usr.bin/sed/tests/sed2_test.sh
	+++ b/usr.bin/sed/tests/sed2_test.sh
	@@ -1,162 +1,160 @@
	#
	# Copyright 2017 Dell EMC.
	# All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions are
	# met:
	#
	# * Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	# * Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	#
	# $FreeBSD$
	#

	atf_test_case inplace_hardlink_src
	inplace_hardlink_src_head()
	{
	atf_set "descr" "Verify -i works with a symlinked source file"
	}
	inplace_hardlink_src_body()
	{
	echo foo > a
	atf_check ln a b
	atf_check sed -i '' -e 's,foo,bar,g' b
	atf_check -o 'inline:bar\n' -s exit:0 cat b
	atf_check -s not-exit:0 stat -q '.!'*
	}

	atf_test_case inplace_symlink_src
	inplace_symlink_src_head()
	{
	atf_set "descr" "Verify -i works with a symlinked source file"
	}
	inplace_symlink_src_body()
	{
	echo foo > a
	atf_check ln -s a b
	atf_check -e not-empty -s not-exit:0 sed -i '' -e 's,foo,bar,g' b
	atf_check -s not-exit:0 stat -q '.!'*
	}

	atf_test_case inplace_command_q
	inplace_command_q_head()
	{
	atf_set "descr" "Verify -i works correctly with the 'q' command"
	}
	inplace_command_q_body()
	{
	printf '1\n2\n3\n' > a
	atf_check -o 'inline:1\n2\n' sed '2q' a
	atf_check sed -i.bak '2q' a
	atf_check -o 'inline:1\n2\n' cat a
	atf_check -o 'inline:1\n2\n3\n' cat a.bak
	atf_check -s not-exit:0 stat -q '.!'*
	}

	atf_test_case escape_subst
	escape_subst_head()
	{
	atf_set "descr" "Verify functional escaping of \\n, \\r, and \\t"
	}
	escape_subst_body()
	{
	printf "a\nt\\\t\n\tb\n\t\tc\r\n" > a
	tr -d '\r' < a > b
	printf "a\tb c\rx\n" > c

	atf_check -o 'inline:a\nt\\t\n' sed '/\t/d' a
	atf_check -o 'inline:a\nt\\t\n b\n c\r\n' sed 's/\t/ /g' a
	atf_check -o 'inline:a\nt\\t\n\t\tb\n\t\t\t\tc\r\n' sed 's/\t/\t\t/g' a
	atf_check -o 'inline:a\nt\n\tb\n\t\tc\r\n' sed 's/\\t//g' a
	atf_check -o file:b sed 's/\r//' a
	atf_check -o 'inline:abcx\n' sed 's/[ \r\t]//g' c
	}

	atf_test_case hex_subst
	hex_subst_head()
	{
	atf_set "descr" "Verify proper conversion of hex escapes"
	}
	hex_subst_body()
	{
	printf "test='foo'" > a
	printf "test='27foo'" > b
	printf "\rn" > c
	printf "xx" > d

	atf_check -o 'inline:test="foo"' sed 's/\x27/"/g' a
	atf_check -o "inline:'test'='foo'" sed 's/test/\x27test\x27/g' a

	# Make sure we take trailing digits literally.
	atf_check -o "inline:test=\"foo'" sed 's/\x2727/"/g' b

	# Single digit \x should work as well.
	atf_check -o "inline:xn" sed 's/\xd/x/' c

	- # Invalid digit should cause us to ignore the sequence. This test
	- # invokes UB, escapes of an ordinary character. A future change will
	- # make regex(3) on longer tolerate this and we'll need to adjust what
	- # we're doing, but for now this will suffice.
	- atf_check -o "inline:" sed 's/\xx//' d
	+ # This should get passed through to the underlying regex engine as
	+ # \xx, which is an invalid escape of an ordinary character.
	+ atf_check -s exit:1 -e not-empty sed 's/\xx//' d
	}

	atf_test_case commands_on_stdin
	commands_on_stdin_head()
	{
	atf_set "descr" "Verify -f -"
	}
	commands_on_stdin_body()
	{
	printf "a\n" > a
	printf "s/a/b/\n" > a_to_b
	printf "s/b/c/\n" > b_to_c
	printf "s/c/d/\n" > ./-
	atf_check -o 'inline:d\n' sed -f a_to_b -f - -f ./- a < b_to_c

	# Verify that nothing is printed if there are no input files provided.
	printf 'i\\\nx' > insert_x
	atf_check -o 'empty' sed -f - < insert_x
	}

	atf_test_case bracket_y
	bracket_y_head()
	{
	atf_set "descr" "Verify '[' is ordinary character for 'y' command"
	}
	bracket_y_body()
	{
	atf_check -e empty -o ignore echo \| sed 'y/[/x/'
	atf_check -e empty -o ignore echo \| sed 'y/[]/xy/'
	atf_check -e empty -o ignore echo \| sed 'y/[a]/xyz/'
	atf_check -e empty -o "inline:zyx" echo '][a' \| sed 'y/[a]/xyz/'
	atf_check -e empty -o "inline:bracket\n" echo 'bra[ke]' \| sed 'y/[]/ct/'
	atf_check -e empty -o "inline:bracket\n" \
	echo 'bra[ke]' \| sed 'y[\[][ct['
	}

	atf_init_test_cases()
	{
	atf_add_test_case inplace_command_q
	atf_add_test_case inplace_hardlink_src
	atf_add_test_case inplace_symlink_src
	atf_add_test_case escape_subst
	atf_add_test_case commands_on_stdin
	atf_add_test_case hex_subst
	atf_add_test_case bracket_y
	}
	diff --git a/usr.bin/truncate/truncate.1 b/usr.bin/truncate/truncate.1
	index a6065b1de9f1..2058530162c5 100644
	--- a/usr.bin/truncate/truncate.1
	+++ b/usr.bin/truncate/truncate.1
	@@ -1,163 +1,200 @@
	.\"
	.\" Copyright (c) 2000 Sheldon Hearn <sheldonh@FreeBSD.org>.
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	-.Dd December 19, 2006
	+.Dd July 27, 2020
	.Dt TRUNCATE 1
	.Os
	.Sh NAME
	.Nm truncate
	.Nd truncate or extend the length of files
	.Sh SYNOPSIS
	.Nm
	.Op Fl c
	.Bk -words
	.Fl s Xo
	.Sm off
	.Op Cm + \| - \| % \| /
	.Ar size
	.Op Cm K \| k \| M \| m \| G \| g \| T \| t
	.Sm on
	.Xc
	.Ek
	.Ar
	.Nm
	.Op Fl c
	.Bk -words
	.Fl r Ar rfile
	.Ek
	.Ar
	.Sh DESCRIPTION
	The
	.Nm
	utility adjusts the length of each regular file given on the command-line.
	.Pp
	The following options are available:
	.Bl -tag -width indent
	.It Fl c
	Do not create files if they do not exist.
	The
	.Nm
	utility does not treat this as an error.
	No error messages are displayed
	and the exit value is not affected.
	.It Fl r Ar rfile
	Truncate or extend files to the length of the file
	.Ar rfile .
	.It Fl s Xo
	.Sm off
	.Op Cm + \| - \| % \| /
	.Ar size
	.Op Cm K \| k \| M \| m \| G \| g \| T \| t
	.Sm on
	.Xc
	If the
	.Ar size
	argument is preceded by a plus sign
	.Pq Cm + ,
	files will be extended by this number of bytes.
	If the
	.Ar size
	argument is preceded by a dash
	.Pq Cm - ,
	file lengths will be reduced by no more than this number of bytes,
	to a minimum length of zero bytes.
	If the
	.Ar size
	argument is preceded by a percent sign
	.Pq Cm % ,
	files will be round up to a multiple of this number of bytes.
	If the
	.Ar size
	argument is preceded by a slash sign
	.Pq Cm / ,
	files will be round down to a multiple of this number of bytes,
	to a minimum length of zero bytes.
	Otherwise, the
	.Ar size
	argument specifies an absolute length to which all files
	should be extended or reduced as appropriate.
	.Pp
	The
	.Ar size
	argument may be suffixed with one of
	.Cm K ,
	.Cm M ,
	.Cm G
	or
	.Cm T
	(either upper or lower case) to indicate a multiple of
	Kilobytes, Megabytes, Gigabytes or Terabytes
	respectively.
	.El
	.Pp
	Exactly one of the
	.Fl r
	and
	.Fl s
	options must be specified.
	.Pp
	If a file is made smaller, its extra data is lost.
	If a file is made larger,
	it will be extended as if by writing bytes with the value zero.
	If the file does not exist,
	it is created unless the
	.Fl c
	option is specified.
	.Pp
	Note that,
	while truncating a file causes space on disk to be freed,
	extending a file does not cause space to be allocated.
	To extend a file and actually allocate the space,
	it is necessary to explicitly write data to it,
	using (for example) the shell's
	.Ql >>
	redirection syntax, or
	.Xr dd 1 .
	.Sh EXIT STATUS
	.Ex -std
	If the operation fails for an argument,
	.Nm
	will issue a diagnostic
	and continue processing the remaining arguments.
	+.Sh EXAMPLES
	+Adjust the size of the file
	+.Pa test_file
	+to 10 Megabytes but do not create it if it does not exist:
	+.Bd -literal -offset indent
	+truncate -c -s +10M test_file
	+.Ed
	+.Pp
	+Same as above but create the file if it does not exist:
	+.Bd -literal -offset indent
	+truncate -s +10M test_file
	+ls -l test_file
	+-rw-r--r-- 1 root wheel 10485760 Jul 22 18:48 test_file
	+.Ed
	+.Pp
	+Adjust the size of
	+.Pa test_file
	+to the size of the kernel and create another file
	+.Pa test_file2
	+with the same size:
	+.Bd -literal -offset indent
	+truncate -r /boot/kernel/kernel test_file test_file2
	+ls -l /boot/kernel/kernel test_file*
	+-r-xr-xr-x 1 root wheel 31352552 May 15 14:18 /boot/kernel/kernel*
	+-rw-r--r-- 1 root wheel 31352552 Jul 22 19:15 test_file
	+-rw-r--r-- 1 root wheel 31352552 Jul 22 19:15 test_file2
	+.Ed
	+.Pp
	+Downsize
	+.Pa test_file
	+in 5 Megabytes:
	+.Bd -literal -offset indent
	+# truncate -s -5M test_file
	+ls -l test_file*
	+-rw-r--r-- 1 root wheel 26109672 Jul 22 19:17 test_file
	+-rw-r--r-- 1 root wheel 31352552 Jul 22 19:15 test_file2
	+.Ed
	.Sh SEE ALSO
	.Xr dd 1 ,
	.Xr touch 1 ,
	.Xr truncate 2
	.Sh STANDARDS
	The
	.Nm
	utility conforms to no known standards.
	.Sh HISTORY
	The
	.Nm
	utility first appeared in
	.Fx 4.2 .
	.Sh AUTHORS
	The
	.Nm
	utility was written by
	.An Sheldon Hearn Aq Mt sheldonh@starjuice.net .
	diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
	index e92b39cc6825..2e91ed86d3a1 100644
	--- a/usr.sbin/bhyve/bhyve.8
	+++ b/usr.sbin/bhyve/bhyve.8
	@@ -1,716 +1,728 @@
	.\" Copyright (c) 2013 Peter Grehan
	.\" All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" $FreeBSD$
	.\"
	.Dd Jun 25, 2020
	.Dt BHYVE 8
	.Os
	.Sh NAME
	.Nm bhyve
	.Nd "run a guest operating system inside a virtual machine"
	.Sh SYNOPSIS
	.Nm
	.Op Fl AabCDeHhPSuWwxY
	.Oo
	.Sm off
	.Fl c\~
	.Oo
	.Op Cm cpus=
	.Ar numcpus
	.Oc
	.Op Cm ,sockets= Ar n
	.Op Cm ,cores= Ar n
	.Op Cm ,threads= Ar n
	.Oc
	.Sm on
	.Op Fl G Ar port
	.Op Fl g Ar gdbport
	.Oo Fl l
	.Sm off
	.Cm help \| Ar lpcdev Op Cm \&, Ar conf
	.Sm on
	.Oc
	.Oo Fl m
	.Sm off
	.Ar memsize
	.Oo
	.Cm K No \| Cm k No \| Cm M No \| Cm m No \| Cm G No \| Cm g No \| Cm T No \| Cm t
	.Oc
	.Sm on
	.Oc
	.Op Fl p Ar vcpu Ns Cm \&: Ns Ar hostcpu
	.Op Fl r Ar file
	.Oo Fl s
	.Sm off
	.Cm help \| Ar slot Cm \&, Ar emulation Op Cm \&, Ar conf
	.Sm on
	.Oc
	.Op Fl U Ar uuid
	.Ar vmname
	.Sh DESCRIPTION
	.Nm
	is a hypervisor that runs guest operating systems inside a
	virtual machine.
	.Pp
	Parameters such as the number of virtual CPUs, amount of guest memory, and
	I/O connectivity can be specified with command-line parameters.
	.Pp
	If not using a boot ROM, the guest operating system must be loaded with
	.Xr bhyveload 8
	or a similar boot loader before running
	.Nm ,
	otherwise, it is enough to run
	.Nm
	with a boot ROM of choice.
	.Pp
	.Nm
	runs until the guest operating system reboots or an unhandled hypervisor
	exit is detected.
	.Sh OPTIONS
	.Bl -tag -width 10n
	.It Fl a
	The guest's local APIC is configured in xAPIC mode.
	The xAPIC mode is the default setting so this option is redundant.
	It will be deprecated in a future version.
	.It Fl A
	Generate ACPI tables.
	Required for
	.Fx Ns /amd64
	guests.
	.It Fl b
	Enable a low-level console device supported by
	.Fx
	kernels compiled with
	.Cd "device bvmconsole" .
	This option will be deprecated in a future version.
	.It Fl c Op Ar setting ...
	Number of guest virtual CPUs
	and/or the CPU topology.
	The default value for each of
	.Ar numcpus ,
	.Ar sockets ,
	.Ar cores ,
	and
	.Ar threads
	is 1.
	The current maximum number of guest virtual CPUs is 16.
	If
	.Ar numcpus
	is not specified then it will be calculated from the other arguments.
	The topology must be consistent in that the
	.Ar numcpus
	must equal the product of
	.Ar sockets ,
	.Ar cores ,
	and
	.Ar threads .
	If a
	.Ar setting
	is specified more than once the last one has precedence.
	.It Fl C
	Include guest memory in core file.
	.It Fl D
	Destroy the VM on guest initiated power-off.
	.It Fl e
	Force
	.Nm
	to exit when a guest issues an access to an I/O port that is not emulated.
	This is intended for debug purposes.
	.It Fl g Ar gdbport
	For
	.Fx
	kernels compiled with
	.Cd "device bvmdebug" ,
	allow a remote kernel kgdb to be relayed to the guest kernel gdb stub
	via a local IPv4 address and this port.
	This option will be deprecated in a future version.
	.It Fl G Ar port
	Start a debug server that uses the GDB protocol to export guest state to a
	debugger.
	An IPv4 TCP socket will be bound to the supplied
	.Ar port
	to listen for debugger connections.
	Only a single debugger may be attached to the debug server at a time.
	If
	.Ar port
	begins with
	.Sq w ,
	.Nm
	will pause execution at the first instruction waiting for a debugger to attach.
	.It Fl h
	Print help message and exit.
	.It Fl H
	Yield the virtual CPU thread when a HLT instruction is detected.
	If this option is not specified, virtual CPUs will use 100% of a host CPU.
	.It Fl l Op Ar help\|lpcdev Ns Op , Ns Ar conf
	Allow devices behind the LPC PCI-ISA bridge to be configured.
	The only supported devices are the TTY-class devices
	.Ar com1
	and
	.Ar com2
	and the boot ROM device
	.Ar bootrom .
	.Pp
	.Ar help
	print a list of supported LPC devices.
	.It Fl m Ar memsize Ns Op Ar K\|k\|M\|m\|G\|g\|T\|t
	Guest physical memory size in bytes.
	This must be the same size that was given to
	.Xr bhyveload 8 .
	.Pp
	The size argument may be suffixed with one of K, M, G or T (either upper
	or lower case) to indicate a multiple of kilobytes, megabytes, gigabytes,
	or terabytes.
	If no suffix is given, the value is assumed to be in megabytes.
	.Pp
	.Ar memsize
	defaults to 256M.
	.It Fl p Ar vcpu:hostcpu
	Pin guest's virtual CPU
	.Em vcpu
	to
	.Em hostcpu .
	.It Fl P
	Force the guest virtual CPU to exit when a PAUSE instruction is detected.
	.It Fl r Ar file
	Resume a guest from a snapshot.
	The guest memory contents are restored from
	.Ar file ,
	and the guest device and vCPU state are restored from the file
	.Dq Ar file Ns .kern .
	.Pp
	Note that the current snapshot file format requires that the configuration of
	devices in the new VM match the VM from which the snapshot was taken by specifying the
	same
	.Op Fl s
	and
	.Op Fl l
	options.
	The count of vCPUs and memory configuration are read from the snapshot.
	.It Fl s Op Ar help\|slot,emulation Ns Op , Ns Ar conf
	Configure a virtual PCI slot and function.
	.Pp
	.Nm
	provides PCI bus emulation and virtual devices that can be attached to
	slots on the bus.
	There are 32 available slots, with the option of providing up to 8 functions
	per slot.
	.Bl -tag -width 10n
	.It Ar help
	print a list of supported PCI devices.
	.It Ar slot
	.Ar pcislot[:function]
	.Ar bus:pcislot:function
	.Pp
	The
	.Ar pcislot
	value is 0 to 31.
	The optional
	.Ar function
	value is 0 to 7.
	The optional
	.Ar bus
	value is 0 to 255.
	If not specified, the
	.Ar function
	value defaults to 0.
	If not specified, the
	.Ar bus
	value defaults to 0.
	.It Ar emulation
	.Bl -tag -width 10n
	.It Li hostbridge \| Li amd_hostbridge
	.Pp
	Provide a simple host bridge.
	This is usually configured at slot 0, and is required by most guest
	operating systems.
	The
	.Li amd_hostbridge
	emulation is identical but uses a PCI vendor ID of
	.Li AMD .
	.It Li passthru
	PCI pass-through device.
	.It Li virtio-net
	Virtio network interface.
	.It Li virtio-blk
	Virtio block storage interface.
	.It Li virtio-scsi
	Virtio SCSI interface.
	.It Li virtio-rnd
	Virtio RNG interface.
	.It Li virtio-console
	Virtio console interface, which exposes multiple ports
	to the guest in the form of simple char devices for simple IO
	between the guest and host userspaces.
	.It Li ahci
	AHCI controller attached to arbitrary devices.
	.It Li ahci-cd
	AHCI controller attached to an ATAPI CD/DVD.
	.It Li ahci-hd
	AHCI controller attached to a SATA hard-drive.
	.It Li e1000
	Intel e82545 network interface.
	.It Li uart
	PCI 16550 serial device.
	.It Li lpc
	LPC PCI-ISA bridge with COM1 and COM2 16550 serial ports and a boot ROM.
	The LPC bridge emulation can only be configured on bus 0.
	.It Li fbuf
	Raw framebuffer device attached to VNC server.
	.It Li xhci
	eXtensible Host Controller Interface (xHCI) USB controller.
	.It Li nvme
	NVM Express (NVMe) controller.
	.It Li hda
	High Definition Audio Controller.
	.El
	.It Op Ar conf
	This optional parameter describes the backend for device emulations.
	If
	.Ar conf
	is not specified, the device emulation has no backend and can be
	considered unconnected.
	.Pp
	Network backends:
	.Bl -tag -width 10n
	.It Ar tapN Ns Oo , Ns Ar mac=xx:xx:xx:xx:xx:xx Oc Ns Oo , Ns Ar mtu=N Oc
	.It Ar vmnetN Ns Oo , Ns Ar mac=xx:xx:xx:xx:xx:xx Oc Ns Oo , Ns Ar mtu=N Oc
	.It Ar netgraph,path=ADDRESS,peerhook=HOOK Ns Oo , Ns Ar socket=NAME Oc Ns Oo , Ns Ar hook=HOOK Oc Ns Oo , Ns Ar mac=xx:xx:xx:xx:xx:xx Oc Ns Oo , Ns Ar mtu=N Oc
	.Pp
	If
	.Ar mac
	is not specified, the MAC address is derived from a fixed OUI and the
	remaining bytes from an MD5 hash of the slot and function numbers and
	the device name.
	.Pp
	The MAC address is an ASCII string in
	.Xr ethers 5
	format.
	.Pp
	With virtio-net devices, the
	.Ar mtu
	parameter can be specified to inform the guest about the largest MTU
	that should be allowed, expressed in bytes.
	.Pp
	With netgraph backend, the
	.Ar path
	and
	.Ar peerhook
	parameters must be specified to set the destination node and corresponding hook.
	The optional parameters
	.Ar socket
	and
	.Ar hook
	may be used to set the
	.Xr ng_socket 4
	node name and source hook.
	The
	.Ar ADDRESS ,
	.Ar HOOK
	and
	.Ar NAME
	must comply with
	.Xr netgraph 4
	addressing rules.
	.El
	.Pp
	Block storage devices:
	.Bl -tag -width 10n
	.It Pa /filename Ns Oo , Ns Ar block-device-options Oc
	.It Pa /dev/xxx Ns Oo , Ns Ar block-device-options Oc
	.El
	.Pp
	The
	.Ar block-device-options
	are:
	.Bl -tag -width 8n
	.It Li nocache
	Open the file with
	.Dv O_DIRECT .
	.It Li direct
	Open the file using
	.Dv O_SYNC .
	.It Li ro
	Force the file to be opened read-only.
	.It Li sectorsize= Ns Ar logical Ns Oo / Ns Ar physical Oc
	Specify the logical and physical sector sizes of the emulated disk.
	The physical sector size is optional and is equal to the logical sector size
	if not explicitly specified.
	.El
	.Pp
	SCSI devices:
	.Bl -tag -width 10n
	.It Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc Ns Oo , Ns Ar scsi-device-options Oc
	.El
	.Pp
	The
	.Ar scsi-device-options
	are:
	.Bl -tag -width 10n
	.It Li iid= Ns Ar IID
	Initiator ID to use when sending requests to specified CTL port.
	The default value is 0.
	.El
	.Pp
	TTY devices:
	.Bl -tag -width 10n
	.It Li stdio
	Connect the serial port to the standard input and output of
	the
	.Nm
	process.
	.It Pa /dev/xxx
	Use the host TTY device for serial port I/O.
	.El
	.Pp
	Boot ROM device:
	.Bl -tag -width 10n
	.It Pa romfile
	Map
	.Ar romfile
	in the guest address space reserved for boot firmware.
	.El
	.Pp
	Pass-through devices:
	.Bl -tag -width 10n
	.It Ns Ar slot Ns / Ns Ar bus Ns / Ns Ar function
	Connect to a PCI device on the host at the selector described by
	.Ar slot ,
	.Ar bus ,
	and
	.Ar function
	numbers.
	.El
	.Pp
	Guest memory must be wired using the
	.Fl S
	option when a pass-through device is configured.
	.Pp
	The host device must have been reserved at boot-time using the
	.Va pptdevs
	loader variable as described in
	.Xr vmm 4 .
	.Pp
	Virtio console devices:
	.Bl -tag -width 10n
	.It Li port1= Ns Pa /path/to/port1.sock Ns ,anotherport= Ns Pa ...
	A maximum of 16 ports per device can be created.
	Every port is named and corresponds to a Unix domain socket created by
	.Nm .
	.Nm
	accepts at most one connection per port at a time.
	.Pp
	Limitations:
	.Bl -bullet -offset 2n
	.It
	Due to lack of destructors in
	.Nm ,
	sockets on the filesystem must be cleaned up manually after
	.Nm
	exits.
	.It
	There is no way to use the "console port" feature, nor the console port
	resize at present.
	.It
	Emergency write is advertised, but no-op at present.
	.El
	.El
	.Pp
	Framebuffer devices:
	.Bl -tag -width 10n
	.It Xo
	.Oo rfb= Ns Oo Ar IP\&: Oc Ns Ar port Oc Ns Oo ,w= Ns Ar width Oc Ns Oo ,h= Ns
	.Ar height Oc Ns Oo ,vga= Ns Ar vgaconf Oc Ns Oo Ns ,wait Oc Ns Oo ,password= Ns
	.Ar password Oc
	.Xc
	.Bl -tag -width 8n
	.It Ar IPv4:port No or Ar [IPv6%zone]:port
	An
	.Ar IP
	address and a
	.Ar port
	VNC should listen on.
	The default is to listen on localhost IPv4 address and default VNC port 5900.
	An IPv6 address must be enclosed in square brackets and may contain an
	optional zone identifier.
	.It Ar width No and Ar height
	A display resolution, width and height, respectively.
	If not specified, a default resolution of 1024x768 pixels will be used.
	Minimal supported resolution is 640x480 pixels,
	and maximum is 1920x1200 pixels.
	.It Ar vgaconf
	Possible values for this option are
	.Dq io
	(default),
	.Dq on
	, and
	.Dq off .
	PCI graphics cards have a dual personality in that they are
	standard PCI devices with BAR addressing, but may also
	implicitly decode legacy VGA I/O space
	.Pq Ad 0x3c0-3df
	and memory space
	.Pq 64KB at Ad 0xA0000 .
	The default
	.Dq io
	option should be used for guests that attempt to issue BIOS calls which result
	in I/O port queries, and fail to boot if I/O decode is disabled.
	.Pp
	The
	.Dq on
	option should be used along with the CSM BIOS capability in UEFI
	to boot traditional BIOS guests that require the legacy VGA I/O and
	memory regions to be available.
	.Pp
	The
	.Dq off
	option should be used for the UEFI guests that assume that
	VGA adapter is present if they detect the I/O ports.
	An example of such a guest is
	.Ox
	in UEFI mode.
	.Pp
	Please refer to the
	.Nm
	.Fx
	wiki page
	.Pq Lk https://wiki.freebsd.org/bhyve
	for configuration notes of particular guests.
	.It wait
	Instruct
	.Nm
	to only boot upon the initiation of a VNC connection, simplifying the
	installation of operating systems that require immediate keyboard input.
	This can be removed for post-installation use.
	.It password
	This type of authentication is known to be cryptographically weak and is not
	intended for use on untrusted networks.
	Many implementations will want to use stronger security, such as running
	the session over an encrypted channel provided by IPsec or SSH.
	.El
	.El
	.Pp
	xHCI USB devices:
	.Bl -tag -width 10n
	.It Li tablet
	A USB tablet device which provides precise cursor synchronization
	when using VNC.
	.El
	.Pp
	NVMe devices:
	.Bl -tag -width 10n
	.It Li devpath
	Accepted device paths are:
	.Ar /dev/blockdev
	or
	.Ar /path/to/image
	or
	.Ar ram=size_in_MiB .
	.It Li maxq
	Max number of queues.
	.It Li qsz
	Max elements in each queue.
	.It Li ioslots
	Max number of concurrent I/O requests.
	.It Li sectsz
	Sector size (defaults to blockif sector size).
	.It Li ser
	Serial number with maximum 20 characters.
	.El
	.Pp
	+AHCI devices:
	+.Bl -tag -width 10n
	+.It Li nmrr
	+Nominal Media Rotation Rate, known as RPM. value 1 will indicate device as Solid State Disk. default value is 0, not report.
	+.It Li ser
	+Serial Number with maximum 20 characters.
	+.It Li rev
	+Revision Number with maximum 8 characters.
	+.It Li model
	+Model Number with maximum 40 characters.
	+.El
	+.Pp
	HD Audio devices:
	.Bl -tag -width 10n
	.It Li play
	Playback device, typically
	.Ar /dev/dsp0 .
	.It Li rec
	Recording device, typically
	.Ar /dev/dsp0 .
	.El
	.El
	.It Fl S
	Wire guest memory.
	.It Fl u
	RTC keeps UTC time.
	.It Fl U Ar uuid
	Set the universally unique identifier
	.Pq UUID
	in the guest's System Management BIOS System Information structure.
	By default a UUID is generated from the host's hostname and
	.Ar vmname .
	.It Fl w
	Ignore accesses to unimplemented Model Specific Registers (MSRs).
	This is intended for debug purposes.
	.It Fl W
	Force virtio PCI device emulations to use MSI interrupts instead of MSI-X
	interrupts.
	.It Fl x
	The guest's local APIC is configured in x2APIC mode.
	.It Fl Y
	Disable MPtable generation.
	.It Ar vmname
	Alphanumeric name of the guest.
	This should be the same as that created by
	.Xr bhyveload 8 .
	.El
	.Sh DEBUG SERVER
	The current debug server provides limited support for debuggers.
	.Ss Registers
	Each virtual CPU is exposed to the debugger as a thread.
	.Pp
	General purpose registers can be queried for each virtual CPU, but other
	registers such as floating-point and system registers cannot be queried.
	.Ss Memory
	Memory (including memory mapped I/O regions) can be read and written by the debugger.
	Memory operations use virtual addresses that are resolved to physical addresses
	via the current virtual CPU's active address translation.
	.Ss Control
	The running guest can be interrupted by the debugger at any time
	.Pq for example, by pressing Ctrl-C in the debugger .
	.Pp
	Single stepping is only supported on Intel CPUs supporting the MTRAP VM exit.
	.Pp
	Breakpoints are supported on Intel CPUs that support single stepping.
	Note that continuing from a breakpoint while interrupts are enabled in the
	guest may not work as expected due to timer interrupts firing while single
	stepping over the breakpoint.
	.Sh SIGNAL HANDLING
	.Nm
	deals with the following signals:
	.Pp
	.Bl -tag -width indent -compact
	.It SIGTERM
	Trigger ACPI poweroff for a VM
	.El
	.Sh EXIT STATUS
	Exit status indicates how the VM was terminated:
	.Pp
	.Bl -tag -width indent -compact
	.It 0
	rebooted
	.It 1
	powered off
	.It 2
	halted
	.It 3
	triple fault
	.It 4
	exited due to an error
	.El
	.Sh EXAMPLES
	If not using a boot ROM, the guest operating system must have been loaded with
	.Xr bhyveload 8
	or a similar boot loader before
	.Xr bhyve 4
	can be run.
	Otherwise, the boot loader is not needed.
	.Pp
	To run a virtual machine with 1GB of memory, two virtual CPUs, a virtio
	block device backed by the
	.Pa /my/image
	filesystem image, and a serial port for the console:
	.Bd -literal -offset indent
	bhyve -c 2 -s 0,hostbridge -s 1,lpc -s 2,virtio-blk,/my/image \\
	-l com1,stdio -A -H -P -m 1G vm1
	.Ed
	.Pp
	Run a 24GB single-CPU virtual machine with three network ports, one of which
	has a MAC address specified:
	.Bd -literal -offset indent
	bhyve -s 0,hostbridge -s 1,lpc -s 2:0,virtio-net,tap0 \\
	-s 2:1,virtio-net,tap1 \\
	-s 2:2,virtio-net,tap2,mac=00:be:fa:76:45:00 \\
	-s 3,virtio-blk,/my/image -l com1,stdio \\
	-A -H -P -m 24G bigvm
	.Ed
	.Pp
	Run an 8GB quad-CPU virtual machine with 8 AHCI SATA disks, an AHCI ATAPI
	CD-ROM, a single virtio network port, an AMD hostbridge, and the console
	port connected to an
	.Xr nmdm 4
	null-modem device.
	.Bd -literal -offset indent
	bhyve -c 4 \\
	-s 0,amd_hostbridge -s 1,lpc \\
	-s 1:0,ahci,hd:/images/disk.1,hd:/images/disk.2,\\
	hd:/images/disk.3,hd:/images/disk.4,\\
	hd:/images/disk.5,hd:/images/disk.6,\\
	hd:/images/disk.7,hd:/images/disk.8,\\
	cd:/images/install.iso \\
	-s 3,virtio-net,tap0 \\
	-l com1,/dev/nmdm0A \\
	-A -H -P -m 8G
	.Ed
	.Pp
	Run a UEFI virtual machine with a display resolution of 800 by 600 pixels
	that can be accessed via VNC at: 0.0.0.0:5900.
	.Bd -literal -offset indent
	bhyve -c 2 -m 4G -w -H \\
	-s 0,hostbridge \\
	-s 3,ahci-cd,/path/to/uefi-OS-install.iso \\
	-s 4,ahci-hd,disk.img \\
	-s 5,virtio-net,tap0 \\
	-s 29,fbuf,tcp=0.0.0.0:5900,w=800,h=600,wait \\
	-s 30,xhci,tablet \\
	-s 31,lpc -l com1,stdio \\
	-l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \\
	uefivm
	.Ed
	.Pp
	Run a UEFI virtual machine with a VNC display that is bound to all IPv6
	addresses on port 5900.
	.Bd -literal -offset indent
	bhyve -c 2 -m 4G -w -H \\
	-s 0,hostbridge \\
	-s 4,ahci-hd,disk.img \\
	-s 5,virtio-net,tap0 \\
	-s 29,fbuf,tcp=[::]:5900,w=800,h=600 \\
	-s 30,xhci,tablet \\
	-s 31,lpc -l com1,stdio \\
	-l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \\
	uefivm
	.Ed
	.Sh SEE ALSO
	.Xr bhyve 4 ,
	.Xr netgraph 4 ,
	.Xr ng_socket 4 ,
	.Xr nmdm 4 ,
	.Xr vmm 4 ,
	.Xr ethers 5 ,
	.Xr bhyvectl 8 ,
	.Xr bhyveload 8
	.Pp
	.Rs
	.%A Intel
	.%B 64 and IA-32 Architectures Software Developer’s Manual
	.%V Volume 3
	.Re
	.Sh HISTORY
	.Nm
	first appeared in
	.Fx 10.0 .
	.Sh AUTHORS
	.An Neel Natu Aq Mt neel@freebsd.org
	.An Peter Grehan Aq Mt grehan@freebsd.org
	diff --git a/usr.sbin/bhyve/pci_ahci.c b/usr.sbin/bhyve/pci_ahci.c
	index 49e6452a355d..dfdf57f664e7 100644
	--- a/usr.sbin/bhyve/pci_ahci.c
	+++ b/usr.sbin/bhyve/pci_ahci.c
	@@ -1,2770 +1,2841 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2013 Zhixiang Yu <zcore@freebsd.org>
	* Copyright (c) 2015-2016 Alexander Motin <mav@FreeBSD.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	* $FreeBSD$
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/linker_set.h>
	#include <sys/stat.h>
	#include <sys/uio.h>
	#include <sys/ioctl.h>
	#include <sys/disk.h>
	#include <sys/ata.h>
	#include <sys/endian.h>

	#include <machine/vmm_snapshot.h>

	#include <errno.h>
	#include <fcntl.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <stdint.h>
	#include <string.h>
	#include <strings.h>
	#include <unistd.h>
	#include <assert.h>
	#include <pthread.h>
	#include <pthread_np.h>
	#include <inttypes.h>
	#include <md5.h>

	#include "bhyverun.h"
	#include "pci_emul.h"
	#include "ahci.h"
	#include "block_if.h"

	#define DEF_PORTS 6 /* Intel ICH8 AHCI supports 6 ports */
	#define MAX_PORTS 32 /* AHCI supports 32 ports */

	#define PxSIG_ATA 0x00000101 /* ATA drive */
	#define PxSIG_ATAPI 0xeb140101 /* ATAPI drive */

	enum sata_fis_type {
	FIS_TYPE_REGH2D = 0x27, /* Register FIS - host to device */
	FIS_TYPE_REGD2H = 0x34, /* Register FIS - device to host */
	FIS_TYPE_DMAACT = 0x39, /* DMA activate FIS - device to host */
	FIS_TYPE_DMASETUP = 0x41, /* DMA setup FIS - bidirectional */
	FIS_TYPE_DATA = 0x46, /* Data FIS - bidirectional */
	FIS_TYPE_BIST = 0x58, /* BIST activate FIS - bidirectional */
	FIS_TYPE_PIOSETUP = 0x5F, /* PIO setup FIS - device to host */
	FIS_TYPE_SETDEVBITS = 0xA1, /* Set dev bits FIS - device to host */
	};

	/*
	* SCSI opcodes
	*/
	#define TEST_UNIT_READY 0x00
	#define REQUEST_SENSE 0x03
	#define INQUIRY 0x12
	#define START_STOP_UNIT 0x1B
	#define PREVENT_ALLOW 0x1E
	#define READ_CAPACITY 0x25
	#define READ_10 0x28
	#define POSITION_TO_ELEMENT 0x2B
	#define READ_TOC 0x43
	#define GET_EVENT_STATUS_NOTIFICATION 0x4A
	#define MODE_SENSE_10 0x5A
	#define REPORT_LUNS 0xA0
	#define READ_12 0xA8
	#define READ_CD 0xBE

	/*
	* SCSI mode page codes
	*/
	#define MODEPAGE_RW_ERROR_RECOVERY 0x01
	#define MODEPAGE_CD_CAPABILITIES 0x2A

	/*
	* ATA commands
	*/
	#define ATA_SF_ENAB_SATA_SF 0x10
	#define ATA_SATA_SF_AN 0x05
	#define ATA_SF_DIS_SATA_SF 0x90

	/*
	* Debug printf
	*/
	#ifdef AHCI_DEBUG
	static FILE *dbg;
	#define DPRINTF(format, arg...) do{fprintf(dbg, format, ##arg);fflush(dbg);}while(0)
	#else
	#define DPRINTF(format, arg...)
	#endif
	#define WPRINTF(format, arg...) printf(format, ##arg)

	#define AHCI_PORT_IDENT 20 + 1

	struct ahci_ioreq {
	struct blockif_req io_req;
	struct ahci_port *io_pr;
	STAILQ_ENTRY(ahci_ioreq) io_flist;
	TAILQ_ENTRY(ahci_ioreq) io_blist;
	uint8_t *cfis;
	uint32_t len;
	uint32_t done;
	int slot;
	int more;
	int readop;
	};

	struct ahci_port {
	struct blockif_ctxt *bctx;
	struct pci_ahci_softc *pr_sc;
	+ struct ata_params ata_ident;
	uint8_t *cmd_lst;
	uint8_t *rfis;
	- char ident[AHCI_PORT_IDENT];
	int port;
	int atapi;
	int reset;
	int waitforclear;
	int mult_sectors;
	uint8_t xfermode;
	uint8_t err_cfis[20];
	uint8_t sense_key;
	uint8_t asc;
	u_int ccs;
	uint32_t pending;

	uint32_t clb;
	uint32_t clbu;
	uint32_t fb;
	uint32_t fbu;
	uint32_t is;
	uint32_t ie;
	uint32_t cmd;
	uint32_t unused0;
	uint32_t tfd;
	uint32_t sig;
	uint32_t ssts;
	uint32_t sctl;
	uint32_t serr;
	uint32_t sact;
	uint32_t ci;
	uint32_t sntf;
	uint32_t fbs;

	/*
	* i/o request info
	*/
	struct ahci_ioreq *ioreq;
	int ioqsz;
	STAILQ_HEAD(ahci_fhead, ahci_ioreq) iofhd;
	TAILQ_HEAD(ahci_bhead, ahci_ioreq) iobhd;
	};

	struct ahci_cmd_hdr {
	uint16_t flags;
	uint16_t prdtl;
	uint32_t prdbc;
	uint64_t ctba;
	uint32_t reserved[4];
	};

	struct ahci_prdt_entry {
	uint64_t dba;
	uint32_t reserved;
	#define DBCMASK 0x3fffff
	uint32_t dbc;
	};

	struct pci_ahci_softc {
	struct pci_devinst *asc_pi;
	pthread_mutex_t mtx;
	int ports;
	uint32_t cap;
	uint32_t ghc;
	uint32_t is;
	uint32_t pi;
	uint32_t vs;
	uint32_t ccc_ctl;
	uint32_t ccc_pts;
	uint32_t em_loc;
	uint32_t em_ctl;
	uint32_t cap2;
	uint32_t bohc;
	uint32_t lintr;
	struct ahci_port port[MAX_PORTS];
	};
	#define ahci_ctx(sc) ((sc)->asc_pi->pi_vmctx)

	static void ahci_handle_port(struct ahci_port *p);

	static inline void lba_to_msf(uint8_t *buf, int lba)
	{
	lba += 150;
	buf[0] = (lba / 75) / 60;
	buf[1] = (lba / 75) % 60;
	buf[2] = lba % 75;
	}

	/*
	* Generate HBA interrupts on global IS register write.
	*/
	static void
	ahci_generate_intr(struct pci_ahci_softc *sc, uint32_t mask)
	{
	struct pci_devinst *pi = sc->asc_pi;
	struct ahci_port *p;
	int i, nmsg;
	uint32_t mmask;

	/* Update global IS from PxIS/PxIE. */
	for (i = 0; i < sc->ports; i++) {
	p = &sc->port[i];
	if (p->is & p->ie)
	sc->is \|= (1 << i);
	}
	DPRINTF("%s(%08x) %08x", __func__, mask, sc->is);

	/* If there is nothing enabled -- clear legacy interrupt and exit. */
	if (sc->is == 0 \|\| (sc->ghc & AHCI_GHC_IE) == 0) {
	if (sc->lintr) {
	pci_lintr_deassert(pi);
	sc->lintr = 0;
	}
	return;
	}

	/* If there is anything and no MSI -- assert legacy interrupt. */
	nmsg = pci_msi_maxmsgnum(pi);
	if (nmsg == 0) {
	if (!sc->lintr) {
	sc->lintr = 1;
	pci_lintr_assert(pi);
	}
	return;
	}

	/* Assert respective MSIs for ports that were touched. */
	for (i = 0; i < nmsg; i++) {
	if (sc->ports <= nmsg \|\| i < nmsg - 1)
	mmask = 1 << i;
	else
	mmask = 0xffffffff << i;
	if (sc->is & mask && mmask & mask)
	pci_generate_msi(pi, i);
	}
	}

	/*
	* Generate HBA interrupt on specific port event.
	*/
	static void
	ahci_port_intr(struct ahci_port *p)
	{
	struct pci_ahci_softc *sc = p->pr_sc;
	struct pci_devinst *pi = sc->asc_pi;
	int nmsg;

	DPRINTF("%s(%d) %08x/%08x %08x", __func__,
	p->port, p->is, p->ie, sc->is);

	/* If there is nothing enabled -- we are done. */
	if ((p->is & p->ie) == 0)
	return;

	/* In case of non-shared MSI always generate interrupt. */
	nmsg = pci_msi_maxmsgnum(pi);
	if (sc->ports <= nmsg \|\| p->port < nmsg - 1) {
	sc->is \|= (1 << p->port);
	if ((sc->ghc & AHCI_GHC_IE) == 0)
	return;
	pci_generate_msi(pi, p->port);
	return;
	}

	/* If IS for this port is already set -- do nothing. */
	if (sc->is & (1 << p->port))
	return;

	sc->is \|= (1 << p->port);

	/* If interrupts are enabled -- generate one. */
	if ((sc->ghc & AHCI_GHC_IE) == 0)
	return;
	if (nmsg > 0) {
	pci_generate_msi(pi, nmsg - 1);
	} else if (!sc->lintr) {
	sc->lintr = 1;
	pci_lintr_assert(pi);
	}
	}

	static void
	ahci_write_fis(struct ahci_port p, enum sata_fis_type ft, uint8_t fis)
	{
	int offset, len, irq;

	if (p->rfis == NULL \|\| !(p->cmd & AHCI_P_CMD_FRE))
	return;

	switch (ft) {
	case FIS_TYPE_REGD2H:
	offset = 0x40;
	len = 20;
	irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_DHR : 0;
	break;
	case FIS_TYPE_SETDEVBITS:
	offset = 0x58;
	len = 8;
	irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_SDB : 0;
	break;
	case FIS_TYPE_PIOSETUP:
	offset = 0x20;
	len = 20;
	irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_PS : 0;
	break;
	default:
	WPRINTF("unsupported fis type %d", ft);
	return;
	}
	if (fis[2] & ATA_S_ERROR) {
	p->waitforclear = 1;
	irq \|= AHCI_P_IX_TFE;
	}
	memcpy(p->rfis + offset, fis, len);
	if (irq) {
	if (~p->is & irq) {
	p->is \|= irq;
	ahci_port_intr(p);
	}
	}
	}

	static void
	ahci_write_fis_piosetup(struct ahci_port *p)
	{
	uint8_t fis[20];

	memset(fis, 0, sizeof(fis));
	fis[0] = FIS_TYPE_PIOSETUP;
	ahci_write_fis(p, FIS_TYPE_PIOSETUP, fis);
	}

	static void
	ahci_write_fis_sdb(struct ahci_port p, int slot, uint8_t cfis, uint32_t tfd)
	{
	uint8_t fis[8];
	uint8_t error;

	error = (tfd >> 8) & 0xff;
	tfd &= 0x77;
	memset(fis, 0, sizeof(fis));
	fis[0] = FIS_TYPE_SETDEVBITS;
	fis[1] = (1 << 6);
	fis[2] = tfd;
	fis[3] = error;
	if (fis[2] & ATA_S_ERROR) {
	p->err_cfis[0] = slot;
	p->err_cfis[2] = tfd;
	p->err_cfis[3] = error;
	memcpy(&p->err_cfis[4], cfis + 4, 16);
	} else {
	(uint32_t )(fis + 4) = (1 << slot);
	p->sact &= ~(1 << slot);
	}
	p->tfd &= ~0x77;
	p->tfd \|= tfd;
	ahci_write_fis(p, FIS_TYPE_SETDEVBITS, fis);
	}

	static void
	ahci_write_fis_d2h(struct ahci_port p, int slot, uint8_t cfis, uint32_t tfd)
	{
	uint8_t fis[20];
	uint8_t error;

	error = (tfd >> 8) & 0xff;
	memset(fis, 0, sizeof(fis));
	fis[0] = FIS_TYPE_REGD2H;
	fis[1] = (1 << 6);
	fis[2] = tfd & 0xff;
	fis[3] = error;
	fis[4] = cfis[4];
	fis[5] = cfis[5];
	fis[6] = cfis[6];
	fis[7] = cfis[7];
	fis[8] = cfis[8];
	fis[9] = cfis[9];
	fis[10] = cfis[10];
	fis[11] = cfis[11];
	fis[12] = cfis[12];
	fis[13] = cfis[13];
	if (fis[2] & ATA_S_ERROR) {
	p->err_cfis[0] = 0x80;
	p->err_cfis[2] = tfd & 0xff;
	p->err_cfis[3] = error;
	memcpy(&p->err_cfis[4], cfis + 4, 16);
	} else
	p->ci &= ~(1 << slot);
	p->tfd = tfd;
	ahci_write_fis(p, FIS_TYPE_REGD2H, fis);
	}

	static void
	ahci_write_fis_d2h_ncq(struct ahci_port *p, int slot)
	{
	uint8_t fis[20];

	p->tfd = ATA_S_READY \| ATA_S_DSC;
	memset(fis, 0, sizeof(fis));
	fis[0] = FIS_TYPE_REGD2H;
	fis[1] = 0; /* No interrupt */
	fis[2] = p->tfd; /* Status */
	fis[3] = 0; /* No error */
	p->ci &= ~(1 << slot);
	ahci_write_fis(p, FIS_TYPE_REGD2H, fis);
	}

	static void
	ahci_write_reset_fis_d2h(struct ahci_port *p)
	{
	uint8_t fis[20];

	memset(fis, 0, sizeof(fis));
	fis[0] = FIS_TYPE_REGD2H;
	fis[3] = 1;
	fis[4] = 1;
	if (p->atapi) {
	fis[5] = 0x14;
	fis[6] = 0xeb;
	}
	fis[12] = 1;
	ahci_write_fis(p, FIS_TYPE_REGD2H, fis);
	}

	static void
	ahci_check_stopped(struct ahci_port *p)
	{
	/*
	* If we are no longer processing the command list and nothing
	* is in-flight, clear the running bit, the current command
	* slot, the command issue and active bits.
	*/
	if (!(p->cmd & AHCI_P_CMD_ST)) {
	if (p->pending == 0) {
	p->ccs = 0;
	p->cmd &= ~(AHCI_P_CMD_CR \| AHCI_P_CMD_CCS_MASK);
	p->ci = 0;
	p->sact = 0;
	p->waitforclear = 0;
	}
	}
	}

	static void
	ahci_port_stop(struct ahci_port *p)
	{
	struct ahci_ioreq *aior;
	uint8_t *cfis;
	int slot;
	int error;

	assert(pthread_mutex_isowned_np(&p->pr_sc->mtx));

	TAILQ_FOREACH(aior, &p->iobhd, io_blist) {
	/*
	* Try to cancel the outstanding blockif request.
	*/
	error = blockif_cancel(p->bctx, &aior->io_req);
	if (error != 0)
	continue;

	slot = aior->slot;
	cfis = aior->cfis;
	if (cfis[2] == ATA_WRITE_FPDMA_QUEUED \|\|
	cfis[2] == ATA_READ_FPDMA_QUEUED \|\|
	cfis[2] == ATA_SEND_FPDMA_QUEUED)
	p->sact &= ~(1 << slot); /* NCQ */
	else
	p->ci &= ~(1 << slot);

	/*
	* This command is now done.
	*/
	p->pending &= ~(1 << slot);

	/*
	* Delete the blockif request from the busy list
	*/
	TAILQ_REMOVE(&p->iobhd, aior, io_blist);

	/*
	* Move the blockif request back to the free list
	*/
	STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist);
	}

	ahci_check_stopped(p);
	}

	static void
	ahci_port_reset(struct ahci_port *pr)
	{
	pr->serr = 0;
	pr->sact = 0;
	pr->xfermode = ATA_UDMA6;
	pr->mult_sectors = 128;

	if (!pr->bctx) {
	pr->ssts = ATA_SS_DET_NO_DEVICE;
	pr->sig = 0xFFFFFFFF;
	pr->tfd = 0x7F;
	return;
	}
	pr->ssts = ATA_SS_DET_PHY_ONLINE \| ATA_SS_IPM_ACTIVE;
	if (pr->sctl & ATA_SC_SPD_MASK)
	pr->ssts \|= (pr->sctl & ATA_SC_SPD_MASK);
	else
	pr->ssts \|= ATA_SS_SPD_GEN3;
	pr->tfd = (1 << 8) \| ATA_S_DSC \| ATA_S_DMA;
	if (!pr->atapi) {
	pr->sig = PxSIG_ATA;
	pr->tfd \|= ATA_S_READY;
	} else
	pr->sig = PxSIG_ATAPI;
	ahci_write_reset_fis_d2h(pr);
	}

	static void
	ahci_reset(struct pci_ahci_softc *sc)
	{
	int i;

	sc->ghc = AHCI_GHC_AE;
	sc->is = 0;

	if (sc->lintr) {
	pci_lintr_deassert(sc->asc_pi);
	sc->lintr = 0;
	}

	for (i = 0; i < sc->ports; i++) {
	sc->port[i].ie = 0;
	sc->port[i].is = 0;
	sc->port[i].cmd = (AHCI_P_CMD_SUD \| AHCI_P_CMD_POD);
	if (sc->port[i].bctx)
	sc->port[i].cmd \|= AHCI_P_CMD_CPS;
	sc->port[i].sctl = 0;
	ahci_port_reset(&sc->port[i]);
	}
	}

	static void
	ata_string(uint8_t dest, const char src, int len)
	{
	int i;

	for (i = 0; i < len; i++) {
	if (*src)
	dest[i ^ 1] = *src++;
	else
	dest[i ^ 1] = ' ';
	}
	}

	static void
	atapi_string(uint8_t dest, const char src, int len)
	{
	int i;

	for (i = 0; i < len; i++) {
	if (*src)
	dest[i] = *src++;
	else
	dest[i] = ' ';
	}
	}

	/*
	* Build up the iovec based on the PRDT, 'done' and 'len'.
	*/
	static void
	ahci_build_iov(struct ahci_port p, struct ahci_ioreq aior,
	struct ahci_prdt_entry *prdt, uint16_t prdtl)
	{
	struct blockif_req *breq = &aior->io_req;
	int i, j, skip, todo, left, extra;
	uint32_t dbcsz;

	/* Copy part of PRDT between 'done' and 'len' bytes into the iov. */
	skip = aior->done;
	left = aior->len - aior->done;
	todo = 0;
	for (i = 0, j = 0; i < prdtl && j < BLOCKIF_IOV_MAX && left > 0;
	i++, prdt++) {
	dbcsz = (prdt->dbc & DBCMASK) + 1;
	/* Skip already done part of the PRDT */
	if (dbcsz <= skip) {
	skip -= dbcsz;
	continue;
	}
	dbcsz -= skip;
	if (dbcsz > left)
	dbcsz = left;
	breq->br_iov[j].iov_base = paddr_guest2host(ahci_ctx(p->pr_sc),
	prdt->dba + skip, dbcsz);
	breq->br_iov[j].iov_len = dbcsz;
	todo += dbcsz;
	left -= dbcsz;
	skip = 0;
	j++;
	}

	/* If we got limited by IOV length, round I/O down to sector size. */
	if (j == BLOCKIF_IOV_MAX) {
	extra = todo % blockif_sectsz(p->bctx);
	todo -= extra;
	assert(todo > 0);
	while (extra > 0) {
	if (breq->br_iov[j - 1].iov_len > extra) {
	breq->br_iov[j - 1].iov_len -= extra;
	break;
	}
	extra -= breq->br_iov[j - 1].iov_len;
	j--;
	}
	}

	breq->br_iovcnt = j;
	breq->br_resid = todo;
	aior->done += todo;
	aior->more = (aior->done < aior->len && i < prdtl);
	}

	static void
	ahci_handle_rw(struct ahci_port p, int slot, uint8_t cfis, uint32_t done)
	{
	struct ahci_ioreq *aior;
	struct blockif_req *breq;
	struct ahci_prdt_entry *prdt;
	struct ahci_cmd_hdr *hdr;
	uint64_t lba;
	uint32_t len;
	int err, first, ncq, readop;

	prdt = (struct ahci_prdt_entry *)(cfis + 0x80);
	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	ncq = 0;
	readop = 1;
	first = (done == 0);

	if (cfis[2] == ATA_WRITE \|\| cfis[2] == ATA_WRITE48 \|\|
	cfis[2] == ATA_WRITE_MUL \|\| cfis[2] == ATA_WRITE_MUL48 \|\|
	cfis[2] == ATA_WRITE_DMA \|\| cfis[2] == ATA_WRITE_DMA48 \|\|
	cfis[2] == ATA_WRITE_FPDMA_QUEUED)
	readop = 0;

	if (cfis[2] == ATA_WRITE_FPDMA_QUEUED \|\|
	cfis[2] == ATA_READ_FPDMA_QUEUED) {
	lba = ((uint64_t)cfis[10] << 40) \|
	((uint64_t)cfis[9] << 32) \|
	((uint64_t)cfis[8] << 24) \|
	((uint64_t)cfis[6] << 16) \|
	((uint64_t)cfis[5] << 8) \|
	cfis[4];
	len = cfis[11] << 8 \| cfis[3];
	if (!len)
	len = 65536;
	ncq = 1;
	} else if (cfis[2] == ATA_READ48 \|\| cfis[2] == ATA_WRITE48 \|\|
	cfis[2] == ATA_READ_MUL48 \|\| cfis[2] == ATA_WRITE_MUL48 \|\|
	cfis[2] == ATA_READ_DMA48 \|\| cfis[2] == ATA_WRITE_DMA48) {
	lba = ((uint64_t)cfis[10] << 40) \|
	((uint64_t)cfis[9] << 32) \|
	((uint64_t)cfis[8] << 24) \|
	((uint64_t)cfis[6] << 16) \|
	((uint64_t)cfis[5] << 8) \|
	cfis[4];
	len = cfis[13] << 8 \| cfis[12];
	if (!len)
	len = 65536;
	} else {
	lba = ((cfis[7] & 0xf) << 24) \| (cfis[6] << 16) \|
	(cfis[5] << 8) \| cfis[4];
	len = cfis[12];
	if (!len)
	len = 256;
	}
	lba *= blockif_sectsz(p->bctx);
	len *= blockif_sectsz(p->bctx);

	/* Pull request off free list */
	aior = STAILQ_FIRST(&p->iofhd);
	assert(aior != NULL);
	STAILQ_REMOVE_HEAD(&p->iofhd, io_flist);

	aior->cfis = cfis;
	aior->slot = slot;
	aior->len = len;
	aior->done = done;
	aior->readop = readop;
	breq = &aior->io_req;
	breq->br_offset = lba + done;
	ahci_build_iov(p, aior, prdt, hdr->prdtl);

	/* Mark this command in-flight. */
	p->pending \|= 1 << slot;

	/* Stuff request onto busy list. */
	TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist);

	if (ncq && first)
	ahci_write_fis_d2h_ncq(p, slot);

	if (readop)
	err = blockif_read(p->bctx, breq);
	else
	err = blockif_write(p->bctx, breq);
	assert(err == 0);
	}

	static void
	ahci_handle_flush(struct ahci_port p, int slot, uint8_t cfis)
	{
	struct ahci_ioreq *aior;
	struct blockif_req *breq;
	int err;

	/*
	* Pull request off free list
	*/
	aior = STAILQ_FIRST(&p->iofhd);
	assert(aior != NULL);
	STAILQ_REMOVE_HEAD(&p->iofhd, io_flist);
	aior->cfis = cfis;
	aior->slot = slot;
	aior->len = 0;
	aior->done = 0;
	aior->more = 0;
	breq = &aior->io_req;

	/*
	* Mark this command in-flight.
	*/
	p->pending \|= 1 << slot;

	/*
	* Stuff request onto busy list
	*/
	TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist);

	err = blockif_flush(p->bctx, breq);
	assert(err == 0);
	}

	static inline void
	read_prdt(struct ahci_port p, int slot, uint8_t cfis,
	void *buf, int size)
	{
	struct ahci_cmd_hdr *hdr;
	struct ahci_prdt_entry *prdt;
	void *to;
	int i, len;

	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	len = size;
	to = buf;
	prdt = (struct ahci_prdt_entry *)(cfis + 0x80);
	for (i = 0; i < hdr->prdtl && len; i++) {
	uint8_t *ptr;
	uint32_t dbcsz;
	int sublen;

	dbcsz = (prdt->dbc & DBCMASK) + 1;
	ptr = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba, dbcsz);
	sublen = MIN(len, dbcsz);
	memcpy(to, ptr, sublen);
	len -= sublen;
	to += sublen;
	prdt++;
	}
	}

	static void
	ahci_handle_dsm_trim(struct ahci_port p, int slot, uint8_t cfis, uint32_t done)
	{
	struct ahci_ioreq *aior;
	struct blockif_req *breq;
	uint8_t *entry;
	uint64_t elba;
	uint32_t len, elen;
	int err, first, ncq;
	uint8_t buf[512];

	first = (done == 0);
	if (cfis[2] == ATA_DATA_SET_MANAGEMENT) {
	len = (uint16_t)cfis[13] << 8 \| cfis[12];
	len *= 512;
	ncq = 0;
	} else { /* ATA_SEND_FPDMA_QUEUED */
	len = (uint16_t)cfis[11] << 8 \| cfis[3];
	len *= 512;
	ncq = 1;
	}
	read_prdt(p, slot, cfis, buf, sizeof(buf));

	next:
	entry = &buf[done];
	elba = ((uint64_t)entry[5] << 40) \|
	((uint64_t)entry[4] << 32) \|
	((uint64_t)entry[3] << 24) \|
	((uint64_t)entry[2] << 16) \|
	((uint64_t)entry[1] << 8) \|
	entry[0];
	elen = (uint16_t)entry[7] << 8 \| entry[6];
	done += 8;
	if (elen == 0) {
	if (done >= len) {
	if (ncq) {
	if (first)
	ahci_write_fis_d2h_ncq(p, slot);
	ahci_write_fis_sdb(p, slot, cfis,
	ATA_S_READY \| ATA_S_DSC);
	} else {
	ahci_write_fis_d2h(p, slot, cfis,
	ATA_S_READY \| ATA_S_DSC);
	}
	p->pending &= ~(1 << slot);
	ahci_check_stopped(p);
	if (!first)
	ahci_handle_port(p);
	return;
	}
	goto next;
	}

	/*
	* Pull request off free list
	*/
	aior = STAILQ_FIRST(&p->iofhd);
	assert(aior != NULL);
	STAILQ_REMOVE_HEAD(&p->iofhd, io_flist);
	aior->cfis = cfis;
	aior->slot = slot;
	aior->len = len;
	aior->done = done;
	aior->more = (len != done);

	breq = &aior->io_req;
	breq->br_offset = elba * blockif_sectsz(p->bctx);
	breq->br_resid = elen * blockif_sectsz(p->bctx);

	/*
	* Mark this command in-flight.
	*/
	p->pending \|= 1 << slot;

	/*
	* Stuff request onto busy list
	*/
	TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist);

	if (ncq && first)
	ahci_write_fis_d2h_ncq(p, slot);

	err = blockif_delete(p->bctx, breq);
	assert(err == 0);
	}

	static inline void
	write_prdt(struct ahci_port p, int slot, uint8_t cfis,
	void *buf, int size)
	{
	struct ahci_cmd_hdr *hdr;
	struct ahci_prdt_entry *prdt;
	void *from;
	int i, len;

	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	len = size;
	from = buf;
	prdt = (struct ahci_prdt_entry *)(cfis + 0x80);
	for (i = 0; i < hdr->prdtl && len; i++) {
	uint8_t *ptr;
	uint32_t dbcsz;
	int sublen;

	dbcsz = (prdt->dbc & DBCMASK) + 1;
	ptr = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba, dbcsz);
	sublen = MIN(len, dbcsz);
	memcpy(ptr, from, sublen);
	len -= sublen;
	from += sublen;
	prdt++;
	}
	hdr->prdbc = size - len;
	}

	static void
	ahci_checksum(uint8_t *buf, int size)
	{
	int i;
	uint8_t sum = 0;

	for (i = 0; i < size - 1; i++)
	sum += buf[i];
	buf[size - 1] = 0x100 - sum;
	}

	static void
	ahci_handle_read_log(struct ahci_port p, int slot, uint8_t cfis)
	{
	struct ahci_cmd_hdr *hdr;
	uint32_t buf[128];
	uint8_t buf8 = (uint8_t )buf;
	uint16_t buf16 = (uint16_t )buf;

	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	if (p->atapi \|\| hdr->prdtl == 0 \|\| cfis[5] != 0 \|\|
	cfis[9] != 0 \|\| cfis[12] != 1 \|\| cfis[13] != 0) {
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	return;
	}

	memset(buf, 0, sizeof(buf));
	if (cfis[4] == 0x00) { /* Log directory */
	buf16[0x00] = 1; /* Version -- 1 */
	buf16[0x10] = 1; /* NCQ Command Error Log -- 1 page */
	buf16[0x13] = 1; /* SATA NCQ Send and Receive Log -- 1 page */
	} else if (cfis[4] == 0x10) { /* NCQ Command Error Log */
	memcpy(buf8, p->err_cfis, sizeof(p->err_cfis));
	ahci_checksum(buf8, sizeof(buf));
	} else if (cfis[4] == 0x13) { /* SATA NCQ Send and Receive Log */
	if (blockif_candelete(p->bctx) && !blockif_is_ro(p->bctx)) {
	buf[0x00] = 1; /* SFQ DSM supported */
	buf[0x01] = 1; /* SFQ DSM TRIM supported */
	}
	} else {
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	return;
	}

	if (cfis[2] == ATA_READ_LOG_EXT)
	ahci_write_fis_piosetup(p);
	write_prdt(p, slot, cfis, (void *)buf, sizeof(buf));
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC \| ATA_S_READY);
	}

	static void
	handle_identify(struct ahci_port p, int slot, uint8_t cfis)
	{
	struct ahci_cmd_hdr *hdr;

	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	if (p->atapi \|\| hdr->prdtl == 0) {
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	} else {
	- uint16_t buf[256];
	+ ahci_write_fis_piosetup(p);
	+ write_prdt(p, slot, cfis, (void*)&p->ata_ident, sizeof(struct ata_params));
	+ ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC \| ATA_S_READY);
	+ }
	+}
	+
	+static void
	+ata_identify_init(struct ahci_port* p, int atapi)
	+{
	+ struct ata_params* ata_ident = &p->ata_ident;
	+
	+ if (atapi) {
	+ ata_ident->config = ATA_PROTO_ATAPI \| ATA_ATAPI_TYPE_CDROM \|
	+ ATA_ATAPI_REMOVABLE \| ATA_DRQ_FAST;
	+ ata_ident->capabilities1 = ATA_SUPPORT_LBA \|
	+ ATA_SUPPORT_DMA;
	+ ata_ident->capabilities2 = (1 << 14 \| 1);
	+ ata_ident->atavalid = ATA_FLAG_54_58 \| ATA_FLAG_64_70;
	+ ata_ident->obsolete62 = 0x3f;
	+ ata_ident->mwdmamodes = 7;
	+ if (p->xfermode & ATA_WDMA0)
	+ ata_ident->mwdmamodes \|= (1 << ((p->xfermode & 7) + 8));
	+ ata_ident->apiomodes = 3;
	+ ata_ident->mwdmamin = 0x0078;
	+ ata_ident->mwdmarec = 0x0078;
	+ ata_ident->pioblind = 0x0078;
	+ ata_ident->pioiordy = 0x0078;
	+ ata_ident->satacapabilities = (ATA_SATA_GEN1 \| ATA_SATA_GEN2 \| ATA_SATA_GEN3);
	+ ata_ident->satacapabilities2 = ((p->ssts & ATA_SS_SPD_MASK) >> 3);
	+ ata_ident->satasupport = ATA_SUPPORT_NCQ_STREAM;
	+ ata_ident->version_major = 0x3f0;
	+ ata_ident->support.command1 = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_PACKET \|
	+ ATA_SUPPORT_RESET \| ATA_SUPPORT_NOP);
	+ ata_ident->support.command2 = (1 << 14);
	+ ata_ident->support.extension = (1 << 14);
	+ ata_ident->enabled.command1 = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_PACKET \|
	+ ATA_SUPPORT_RESET \| ATA_SUPPORT_NOP);
	+ ata_ident->enabled.extension = (1 << 14);
	+ ata_ident->udmamodes = 0x7f;
	+ if (p->xfermode & ATA_UDMA0)
	+ ata_ident->udmamodes \|= (1 << ((p->xfermode & 7) + 8));
	+ ata_ident->transport_major = 0x1020;
	+ ata_ident->integrity = 0x00a5;
	+ } else {
	uint64_t sectors;
	int sectsz, psectsz, psectoff, candelete, ro;
	uint16_t cyl;
	uint8_t sech, heads;

	ro = blockif_is_ro(p->bctx);
	candelete = blockif_candelete(p->bctx);
	sectsz = blockif_sectsz(p->bctx);
	sectors = blockif_size(p->bctx) / sectsz;
	blockif_chs(p->bctx, &cyl, &heads, &sech);
	blockif_psectsz(p->bctx, &psectsz, &psectoff);
	- memset(buf, 0, sizeof(buf));
	- buf[0] = 0x0040;
	- buf[1] = cyl;
	- buf[3] = heads;
	- buf[6] = sech;
	- ata_string((uint8_t *)(buf+10), p->ident, 20);
	- ata_string((uint8_t *)(buf+23), "001", 8);
	- ata_string((uint8_t *)(buf+27), "BHYVE SATA DISK", 40);
	- buf[47] = (0x8000 \| 128);
	- buf[48] = 0;
	- buf[49] = (1 << 8 \| 1 << 9 \| 1 << 11);
	- buf[50] = (1 << 14);
	- buf[53] = (1 << 1 \| 1 << 2);
	+ ata_ident->config = ATA_DRQ_FAST;
	+ ata_ident->cylinders = cyl;
	+ ata_ident->heads = heads;
	+ ata_ident->sectors = sech;
	+
	+ ata_ident->sectors_intr = (0x8000 \| 128);
	+ ata_ident->tcg = 0;
	+
	+ ata_ident->capabilities1 = ATA_SUPPORT_DMA \|
	+ ATA_SUPPORT_LBA \| ATA_SUPPORT_IORDY;
	+ ata_ident->capabilities2 = (1 << 14);
	+ ata_ident->atavalid = ATA_FLAG_54_58 \|
	+ ATA_FLAG_64_70;
	if (p->mult_sectors)
	- buf[59] = (0x100 \| p->mult_sectors);
	+ ata_ident->multi = (ATA_MULTI_VALID \| p->mult_sectors);
	if (sectors <= 0x0fffffff) {
	- buf[60] = sectors;
	- buf[61] = (sectors >> 16);
	+ ata_ident->lba_size_1 = sectors;
	+ ata_ident->lba_size_2 = (sectors >> 16);
	} else {
	- buf[60] = 0xffff;
	- buf[61] = 0x0fff;
	+ ata_ident->lba_size_1 = 0xffff;
	+ ata_ident->lba_size_2 = 0x0fff;
	}
	- buf[63] = 0x7;
	+ ata_ident->mwdmamodes = 0x7;
	if (p->xfermode & ATA_WDMA0)
	- buf[63] \|= (1 << ((p->xfermode & 7) + 8));
	- buf[64] = 0x3;
	- buf[65] = 120;
	- buf[66] = 120;
	- buf[67] = 120;
	- buf[68] = 120;
	- buf[69] = 0;
	- buf[75] = 31;
	- buf[76] = (ATA_SATA_GEN1 \| ATA_SATA_GEN2 \| ATA_SATA_GEN3 \|
	- ATA_SUPPORT_NCQ);
	- buf[77] = (ATA_SUPPORT_RCVSND_FPDMA_QUEUED \|
	- (p->ssts & ATA_SS_SPD_MASK) >> 3);
	- buf[80] = 0x3f0;
	- buf[81] = 0x28;
	- buf[82] = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_WRITECACHE\|
	- ATA_SUPPORT_LOOKAHEAD \| ATA_SUPPORT_NOP);
	- buf[83] = (ATA_SUPPORT_ADDRESS48 \| ATA_SUPPORT_FLUSHCACHE \|
	- ATA_SUPPORT_FLUSHCACHE48 \| 1 << 14);
	- buf[84] = (1 << 14);
	- buf[85] = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_WRITECACHE\|
	- ATA_SUPPORT_LOOKAHEAD \| ATA_SUPPORT_NOP);
	- buf[86] = (ATA_SUPPORT_ADDRESS48 \| ATA_SUPPORT_FLUSHCACHE \|
	- ATA_SUPPORT_FLUSHCACHE48 \| 1 << 15);
	- buf[87] = (1 << 14);
	- buf[88] = 0x7f;
	+ ata_ident->mwdmamodes \|= (1 << ((p->xfermode & 7) + 8));
	+ ata_ident->apiomodes = 0x3;
	+ ata_ident->mwdmamin = 0x0078;
	+ ata_ident->mwdmarec = 0x0078;
	+ ata_ident->pioblind = 0x0078;
	+ ata_ident->pioiordy = 0x0078;
	+ ata_ident->support3 = 0;
	+ ata_ident->queue = 31;
	+ ata_ident->satacapabilities = (ATA_SATA_GEN1 \| ATA_SATA_GEN2 \| ATA_SATA_GEN3 \|
	+ ATA_SUPPORT_NCQ);
	+ ata_ident->satacapabilities2 = (ATA_SUPPORT_RCVSND_FPDMA_QUEUED \|
	+ (p->ssts & ATA_SS_SPD_MASK) >> 3);
	+ ata_ident->version_major = 0x3f0;
	+ ata_ident->version_minor = 0x28;
	+ ata_ident->support.command1 = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_WRITECACHE \|
	+ ATA_SUPPORT_LOOKAHEAD \| ATA_SUPPORT_NOP);
	+ ata_ident->support.command2 = (ATA_SUPPORT_ADDRESS48 \| ATA_SUPPORT_FLUSHCACHE \|
	+ ATA_SUPPORT_FLUSHCACHE48 \| 1 << 14);
	+ ata_ident->support.extension = (1 << 14);
	+ ata_ident->enabled.command1 = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_WRITECACHE \|
	+ ATA_SUPPORT_LOOKAHEAD \| ATA_SUPPORT_NOP);
	+ ata_ident->enabled.command2 = (ATA_SUPPORT_ADDRESS48 \| ATA_SUPPORT_FLUSHCACHE \|
	+ ATA_SUPPORT_FLUSHCACHE48 \| 1 << 15);
	+ ata_ident->enabled.extension = (1 << 14);
	+ ata_ident->udmamodes = 0x7f;
	if (p->xfermode & ATA_UDMA0)
	- buf[88] \|= (1 << ((p->xfermode & 7) + 8));
	- buf[100] = sectors;
	- buf[101] = (sectors >> 16);
	- buf[102] = (sectors >> 32);
	- buf[103] = (sectors >> 48);
	+ ata_ident->udmamodes \|= (1 << ((p->xfermode & 7) + 8));
	+ ata_ident->lba_size48_1 = sectors;
	+ ata_ident->lba_size48_2 = (sectors >> 16);
	+ ata_ident->lba_size48_3 = (sectors >> 32);
	+ ata_ident->lba_size48_4 = (sectors >> 48);
	+
	if (candelete && !ro) {
	- buf[69] \|= ATA_SUPPORT_RZAT \| ATA_SUPPORT_DRAT;
	- buf[105] = 1;
	- buf[169] = ATA_SUPPORT_DSM_TRIM;
	+ ata_ident->support3 \|= ATA_SUPPORT_RZAT \| ATA_SUPPORT_DRAT;
	+ ata_ident->max_dsm_blocks = 1;
	+ ata_ident->support_dsm = ATA_SUPPORT_DSM_TRIM;
	}
	- buf[106] = 0x4000;
	- buf[209] = 0x4000;
	+ ata_ident->pss = ATA_PSS_VALID_VALUE;
	+ ata_ident->lsalign = 0x4000;
	if (psectsz > sectsz) {
	- buf[106] \|= 0x2000;
	- buf[106] \|= ffsl(psectsz / sectsz) - 1;
	- buf[209] \|= (psectoff / sectsz);
	+ ata_ident->pss \|= ATA_PSS_MULTLS;
	+ ata_ident->pss \|= ffsl(psectsz / sectsz) - 1;
	+ ata_ident->lsalign \|= (psectoff / sectsz);
	}
	if (sectsz > 512) {
	- buf[106] \|= 0x1000;
	- buf[117] = sectsz / 2;
	- buf[118] = ((sectsz / 2) >> 16);
	+ ata_ident->pss \|= ATA_PSS_LSSABOVE512;
	+ ata_ident->lss_1 = sectsz / 2;
	+ ata_ident->lss_2 = ((sectsz / 2) >> 16);
	}
	- buf[119] = (ATA_SUPPORT_RWLOGDMAEXT \| 1 << 14);
	- buf[120] = (ATA_SUPPORT_RWLOGDMAEXT \| 1 << 14);
	- buf[222] = 0x1020;
	- buf[255] = 0x00a5;
	- ahci_checksum((uint8_t *)buf, sizeof(buf));
	- ahci_write_fis_piosetup(p);
	- write_prdt(p, slot, cfis, (void *)buf, sizeof(buf));
	- ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC \| ATA_S_READY);
	+ ata_ident->support2 = (ATA_SUPPORT_RWLOGDMAEXT \| 1 << 14);
	+ ata_ident->enabled2 = (ATA_SUPPORT_RWLOGDMAEXT \| 1 << 14);
	+ ata_ident->transport_major = 0x1020;
	+ ata_ident->integrity = 0x00a5;
	}
	+ ahci_checksum((uint8_t*)ata_ident, sizeof(struct ata_params));
	}

	static void
	handle_atapi_identify(struct ahci_port p, int slot, uint8_t cfis)
	{
	if (!p->atapi) {
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	} else {
	- uint16_t buf[256];
	-
	- memset(buf, 0, sizeof(buf));
	- buf[0] = (2 << 14 \| 5 << 8 \| 1 << 7 \| 2 << 5);
	- ata_string((uint8_t *)(buf+10), p->ident, 20);
	- ata_string((uint8_t *)(buf+23), "001", 8);
	- ata_string((uint8_t *)(buf+27), "BHYVE SATA DVD ROM", 40);
	- buf[49] = (1 << 9 \| 1 << 8);
	- buf[50] = (1 << 14 \| 1);
	- buf[53] = (1 << 2 \| 1 << 1);
	- buf[62] = 0x3f;
	- buf[63] = 7;
	- if (p->xfermode & ATA_WDMA0)
	- buf[63] \|= (1 << ((p->xfermode & 7) + 8));
	- buf[64] = 3;
	- buf[65] = 120;
	- buf[66] = 120;
	- buf[67] = 120;
	- buf[68] = 120;
	- buf[76] = (ATA_SATA_GEN1 \| ATA_SATA_GEN2 \| ATA_SATA_GEN3);
	- buf[77] = ((p->ssts & ATA_SS_SPD_MASK) >> 3);
	- buf[78] = (1 << 5);
	- buf[80] = 0x3f0;
	- buf[82] = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_PACKET \|
	- ATA_SUPPORT_RESET \| ATA_SUPPORT_NOP);
	- buf[83] = (1 << 14);
	- buf[84] = (1 << 14);
	- buf[85] = (ATA_SUPPORT_POWERMGT \| ATA_SUPPORT_PACKET \|
	- ATA_SUPPORT_RESET \| ATA_SUPPORT_NOP);
	- buf[87] = (1 << 14);
	- buf[88] = 0x7f;
	- if (p->xfermode & ATA_UDMA0)
	- buf[88] \|= (1 << ((p->xfermode & 7) + 8));
	- buf[222] = 0x1020;
	- buf[255] = 0x00a5;
	- ahci_checksum((uint8_t *)buf, sizeof(buf));
	ahci_write_fis_piosetup(p);
	- write_prdt(p, slot, cfis, (void *)buf, sizeof(buf));
	+ write_prdt(p, slot, cfis, (void *)&p->ata_ident, sizeof(struct ata_params));
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC \| ATA_S_READY);
	}
	}

	static void
	atapi_inquiry(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t buf[36];
	uint8_t *acmd;
	int len;
	uint32_t tfd;

	acmd = cfis + 0x40;

	if (acmd[1] & 1) { /* VPD */
	if (acmd[2] == 0) { /* Supported VPD pages */
	buf[0] = 0x05;
	buf[1] = 0;
	buf[2] = 0;
	buf[3] = 1;
	buf[4] = 0;
	len = 4 + buf[3];
	} else {
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x24;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, tfd);
	return;
	}
	} else {
	buf[0] = 0x05;
	buf[1] = 0x80;
	buf[2] = 0x00;
	buf[3] = 0x21;
	buf[4] = 31;
	buf[5] = 0;
	buf[6] = 0;
	buf[7] = 0;
	atapi_string(buf + 8, "BHYVE", 8);
	atapi_string(buf + 16, "BHYVE DVD-ROM", 16);
	atapi_string(buf + 32, "001", 4);
	len = sizeof(buf);
	}

	if (len > acmd[4])
	len = acmd[4];
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	write_prdt(p, slot, cfis, buf, len);
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	}

	static void
	atapi_read_capacity(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t buf[8];
	uint64_t sectors;

	sectors = blockif_size(p->bctx) / 2048;
	be32enc(buf, sectors - 1);
	be32enc(buf + 4, 2048);
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	write_prdt(p, slot, cfis, buf, sizeof(buf));
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	}

	static void
	atapi_read_toc(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t *acmd;
	uint8_t format;
	int len;

	acmd = cfis + 0x40;

	len = be16dec(acmd + 7);
	format = acmd[9] >> 6;
	switch (format) {
	case 0:
	{
	int msf, size;
	uint64_t sectors;
	uint8_t start_track, buf[20], *bp;

	msf = (acmd[1] >> 1) & 1;
	start_track = acmd[6];
	if (start_track > 1 && start_track != 0xaa) {
	uint32_t tfd;
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x24;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, tfd);
	return;
	}
	bp = buf + 2;
	*bp++ = 1;
	*bp++ = 1;
	if (start_track <= 1) {
	*bp++ = 0;
	*bp++ = 0x14;
	*bp++ = 1;
	*bp++ = 0;
	if (msf) {
	*bp++ = 0;
	lba_to_msf(bp, 0);
	bp += 3;
	} else {
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	}
	}
	*bp++ = 0;
	*bp++ = 0x14;
	*bp++ = 0xaa;
	*bp++ = 0;
	sectors = blockif_size(p->bctx) / blockif_sectsz(p->bctx);
	sectors >>= 2;
	if (msf) {
	*bp++ = 0;
	lba_to_msf(bp, sectors);
	bp += 3;
	} else {
	be32enc(bp, sectors);
	bp += 4;
	}
	size = bp - buf;
	be16enc(buf, size - 2);
	if (len > size)
	len = size;
	write_prdt(p, slot, cfis, buf, len);
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	}
	case 1:
	{
	uint8_t buf[12];

	memset(buf, 0, sizeof(buf));
	buf[1] = 0xa;
	buf[2] = 0x1;
	buf[3] = 0x1;
	if (len > sizeof(buf))
	len = sizeof(buf);
	write_prdt(p, slot, cfis, buf, len);
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	}
	case 2:
	{
	int msf, size;
	uint64_t sectors;
	uint8_t *bp, buf[50];

	msf = (acmd[1] >> 1) & 1;
	bp = buf + 2;
	*bp++ = 1;
	*bp++ = 1;

	*bp++ = 1;
	*bp++ = 0x14;
	*bp++ = 0;
	*bp++ = 0xa0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 1;
	*bp++ = 0;
	*bp++ = 0;

	*bp++ = 1;
	*bp++ = 0x14;
	*bp++ = 0;
	*bp++ = 0xa1;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 1;
	*bp++ = 0;
	*bp++ = 0;

	*bp++ = 1;
	*bp++ = 0x14;
	*bp++ = 0;
	*bp++ = 0xa2;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	sectors = blockif_size(p->bctx) / blockif_sectsz(p->bctx);
	sectors >>= 2;
	if (msf) {
	*bp++ = 0;
	lba_to_msf(bp, sectors);
	bp += 3;
	} else {
	be32enc(bp, sectors);
	bp += 4;
	}

	*bp++ = 1;
	*bp++ = 0x14;
	*bp++ = 0;
	*bp++ = 1;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	if (msf) {
	*bp++ = 0;
	lba_to_msf(bp, 0);
	bp += 3;
	} else {
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	*bp++ = 0;
	}

	size = bp - buf;
	be16enc(buf, size - 2);
	if (len > size)
	len = size;
	write_prdt(p, slot, cfis, buf, len);
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	}
	default:
	{
	uint32_t tfd;

	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x24;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, tfd);
	break;
	}
	}
	}

	static void
	atapi_report_luns(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t buf[16];

	memset(buf, 0, sizeof(buf));
	buf[3] = 8;

	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	write_prdt(p, slot, cfis, buf, sizeof(buf));
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	}

	static void
	atapi_read(struct ahci_port p, int slot, uint8_t cfis, uint32_t done)
	{
	struct ahci_ioreq *aior;
	struct ahci_cmd_hdr *hdr;
	struct ahci_prdt_entry *prdt;
	struct blockif_req *breq;
	uint8_t *acmd;
	uint64_t lba;
	uint32_t len;
	int err;

	acmd = cfis + 0x40;
	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	prdt = (struct ahci_prdt_entry *)(cfis + 0x80);

	lba = be32dec(acmd + 2);
	if (acmd[0] == READ_10)
	len = be16dec(acmd + 7);
	else
	len = be32dec(acmd + 6);
	if (len == 0) {
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	}
	lba *= 2048;
	len *= 2048;

	/*
	* Pull request off free list
	*/
	aior = STAILQ_FIRST(&p->iofhd);
	assert(aior != NULL);
	STAILQ_REMOVE_HEAD(&p->iofhd, io_flist);
	aior->cfis = cfis;
	aior->slot = slot;
	aior->len = len;
	aior->done = done;
	aior->readop = 1;
	breq = &aior->io_req;
	breq->br_offset = lba + done;
	ahci_build_iov(p, aior, prdt, hdr->prdtl);

	/* Mark this command in-flight. */
	p->pending \|= 1 << slot;

	/* Stuff request onto busy list. */
	TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist);

	err = blockif_read(p->bctx, breq);
	assert(err == 0);
	}

	static void
	atapi_request_sense(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t buf[64];
	uint8_t *acmd;
	int len;

	acmd = cfis + 0x40;
	len = acmd[4];
	if (len > sizeof(buf))
	len = sizeof(buf);
	memset(buf, 0, len);
	buf[0] = 0x70 \| (1 << 7);
	buf[2] = p->sense_key;
	buf[7] = 10;
	buf[12] = p->asc;
	write_prdt(p, slot, cfis, buf, len);
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	}

	static void
	atapi_start_stop_unit(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t *acmd = cfis + 0x40;
	uint32_t tfd;

	switch (acmd[4] & 3) {
	case 0:
	case 1:
	case 3:
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	tfd = ATA_S_READY \| ATA_S_DSC;
	break;
	case 2:
	/* TODO eject media */
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x53;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	break;
	}
	ahci_write_fis_d2h(p, slot, cfis, tfd);
	}

	static void
	atapi_mode_sense(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t *acmd;
	uint32_t tfd;
	uint8_t pc, code;
	int len;

	acmd = cfis + 0x40;
	len = be16dec(acmd + 7);
	pc = acmd[2] >> 6;
	code = acmd[2] & 0x3f;

	switch (pc) {
	case 0:
	switch (code) {
	case MODEPAGE_RW_ERROR_RECOVERY:
	{
	uint8_t buf[16];

	if (len > sizeof(buf))
	len = sizeof(buf);

	memset(buf, 0, sizeof(buf));
	be16enc(buf, 16 - 2);
	buf[2] = 0x70;
	buf[8] = 0x01;
	buf[9] = 16 - 10;
	buf[11] = 0x05;
	write_prdt(p, slot, cfis, buf, len);
	tfd = ATA_S_READY \| ATA_S_DSC;
	break;
	}
	case MODEPAGE_CD_CAPABILITIES:
	{
	uint8_t buf[30];

	if (len > sizeof(buf))
	len = sizeof(buf);

	memset(buf, 0, sizeof(buf));
	be16enc(buf, 30 - 2);
	buf[2] = 0x70;
	buf[8] = 0x2A;
	buf[9] = 30 - 10;
	buf[10] = 0x08;
	buf[12] = 0x71;
	be16enc(&buf[18], 2);
	be16enc(&buf[20], 512);
	write_prdt(p, slot, cfis, buf, len);
	tfd = ATA_S_READY \| ATA_S_DSC;
	break;
	}
	default:
	goto error;
	break;
	}
	break;
	case 3:
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x39;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	break;
	error:
	case 1:
	case 2:
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x24;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	break;
	}
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, tfd);
	}

	static void
	atapi_get_event_status_notification(struct ahci_port *p, int slot,
	uint8_t *cfis)
	{
	uint8_t *acmd;
	uint32_t tfd;

	acmd = cfis + 0x40;

	/* we don't support asynchronous operation */
	if (!(acmd[1] & 1)) {
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x24;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	} else {
	uint8_t buf[8];
	int len;

	len = be16dec(acmd + 7);
	if (len > sizeof(buf))
	len = sizeof(buf);

	memset(buf, 0, sizeof(buf));
	be16enc(buf, 8 - 2);
	buf[2] = 0x04;
	buf[3] = 0x10;
	buf[5] = 0x02;
	write_prdt(p, slot, cfis, buf, len);
	tfd = ATA_S_READY \| ATA_S_DSC;
	}
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, tfd);
	}

	static void
	handle_packet_cmd(struct ahci_port p, int slot, uint8_t cfis)
	{
	uint8_t *acmd;

	acmd = cfis + 0x40;

	#ifdef AHCI_DEBUG
	{
	int i;
	DPRINTF("ACMD:");
	for (i = 0; i < 16; i++)
	DPRINTF("%02x ", acmd[i]);
	DPRINTF("");
	}
	#endif

	switch (acmd[0]) {
	case TEST_UNIT_READY:
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	case INQUIRY:
	atapi_inquiry(p, slot, cfis);
	break;
	case READ_CAPACITY:
	atapi_read_capacity(p, slot, cfis);
	break;
	case PREVENT_ALLOW:
	/* TODO */
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	case READ_TOC:
	atapi_read_toc(p, slot, cfis);
	break;
	case REPORT_LUNS:
	atapi_report_luns(p, slot, cfis);
	break;
	case READ_10:
	case READ_12:
	atapi_read(p, slot, cfis, 0);
	break;
	case REQUEST_SENSE:
	atapi_request_sense(p, slot, cfis);
	break;
	case START_STOP_UNIT:
	atapi_start_stop_unit(p, slot, cfis);
	break;
	case MODE_SENSE_10:
	atapi_mode_sense(p, slot, cfis);
	break;
	case GET_EVENT_STATUS_NOTIFICATION:
	atapi_get_event_status_notification(p, slot, cfis);
	break;
	default:
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x20;
	ahci_write_fis_d2h(p, slot, cfis, (p->sense_key << 12) \|
	ATA_S_READY \| ATA_S_ERROR);
	break;
	}
	}

	static void
	ahci_handle_cmd(struct ahci_port p, int slot, uint8_t cfis)
	{

	p->tfd \|= ATA_S_BUSY;
	switch (cfis[2]) {
	case ATA_ATA_IDENTIFY:
	handle_identify(p, slot, cfis);
	break;
	case ATA_SETFEATURES:
	{
	switch (cfis[3]) {
	case ATA_SF_ENAB_SATA_SF:
	switch (cfis[12]) {
	case ATA_SATA_SF_AN:
	p->tfd = ATA_S_DSC \| ATA_S_READY;
	break;
	default:
	p->tfd = ATA_S_ERROR \| ATA_S_READY;
	p->tfd \|= (ATA_ERROR_ABORT << 8);
	break;
	}
	break;
	case ATA_SF_ENAB_WCACHE:
	case ATA_SF_DIS_WCACHE:
	case ATA_SF_ENAB_RCACHE:
	case ATA_SF_DIS_RCACHE:
	p->tfd = ATA_S_DSC \| ATA_S_READY;
	break;
	case ATA_SF_SETXFER:
	{
	switch (cfis[12] & 0xf8) {
	case ATA_PIO:
	case ATA_PIO0:
	break;
	case ATA_WDMA0:
	case ATA_UDMA0:
	p->xfermode = (cfis[12] & 0x7);
	break;
	}
	p->tfd = ATA_S_DSC \| ATA_S_READY;
	break;
	}
	default:
	p->tfd = ATA_S_ERROR \| ATA_S_READY;
	p->tfd \|= (ATA_ERROR_ABORT << 8);
	break;
	}
	ahci_write_fis_d2h(p, slot, cfis, p->tfd);
	break;
	}
	case ATA_SET_MULTI:
	if (cfis[12] != 0 &&
	(cfis[12] > 128 \|\| (cfis[12] & (cfis[12] - 1)))) {
	p->tfd = ATA_S_ERROR \| ATA_S_READY;
	p->tfd \|= (ATA_ERROR_ABORT << 8);
	} else {
	p->mult_sectors = cfis[12];
	p->tfd = ATA_S_DSC \| ATA_S_READY;
	}
	ahci_write_fis_d2h(p, slot, cfis, p->tfd);
	break;
	case ATA_READ:
	case ATA_WRITE:
	case ATA_READ48:
	case ATA_WRITE48:
	case ATA_READ_MUL:
	case ATA_WRITE_MUL:
	case ATA_READ_MUL48:
	case ATA_WRITE_MUL48:
	case ATA_READ_DMA:
	case ATA_WRITE_DMA:
	case ATA_READ_DMA48:
	case ATA_WRITE_DMA48:
	case ATA_READ_FPDMA_QUEUED:
	case ATA_WRITE_FPDMA_QUEUED:
	ahci_handle_rw(p, slot, cfis, 0);
	break;
	case ATA_FLUSHCACHE:
	case ATA_FLUSHCACHE48:
	ahci_handle_flush(p, slot, cfis);
	break;
	case ATA_DATA_SET_MANAGEMENT:
	if (cfis[11] == 0 && cfis[3] == ATA_DSM_TRIM &&
	cfis[13] == 0 && cfis[12] == 1) {
	ahci_handle_dsm_trim(p, slot, cfis, 0);
	break;
	}
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	break;
	case ATA_SEND_FPDMA_QUEUED:
	if ((cfis[13] & 0x1f) == ATA_SFPDMA_DSM &&
	cfis[17] == 0 && cfis[16] == ATA_DSM_TRIM &&
	cfis[11] == 0 && cfis[3] == 1) {
	ahci_handle_dsm_trim(p, slot, cfis, 0);
	break;
	}
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	break;
	case ATA_READ_LOG_EXT:
	case ATA_READ_LOG_DMA_EXT:
	ahci_handle_read_log(p, slot, cfis);
	break;
	case ATA_SECURITY_FREEZE_LOCK:
	case ATA_SMART_CMD:
	case ATA_NOP:
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	break;
	case ATA_CHECK_POWER_MODE:
	cfis[12] = 0xff; /* always on */
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	case ATA_STANDBY_CMD:
	case ATA_STANDBY_IMMEDIATE:
	case ATA_IDLE_CMD:
	case ATA_IDLE_IMMEDIATE:
	case ATA_SLEEP:
	case ATA_READ_VERIFY:
	case ATA_READ_VERIFY48:
	ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY \| ATA_S_DSC);
	break;
	case ATA_ATAPI_IDENTIFY:
	handle_atapi_identify(p, slot, cfis);
	break;
	case ATA_PACKET_CMD:
	if (!p->atapi) {
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	} else
	handle_packet_cmd(p, slot, cfis);
	break;
	default:
	WPRINTF("Unsupported cmd:%02x", cfis[2]);
	ahci_write_fis_d2h(p, slot, cfis,
	(ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR);
	break;
	}
	}

	static void
	ahci_handle_slot(struct ahci_port *p, int slot)
	{
	struct ahci_cmd_hdr *hdr;
	#ifdef AHCI_DEBUG
	struct ahci_prdt_entry *prdt;
	#endif
	struct pci_ahci_softc *sc;
	uint8_t *cfis;
	#ifdef AHCI_DEBUG
	int cfl, i;
	#endif

	sc = p->pr_sc;
	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);
	#ifdef AHCI_DEBUG
	cfl = (hdr->flags & 0x1f) * 4;
	#endif
	cfis = paddr_guest2host(ahci_ctx(sc), hdr->ctba,
	0x80 + hdr->prdtl * sizeof(struct ahci_prdt_entry));
	#ifdef AHCI_DEBUG
	prdt = (struct ahci_prdt_entry *)(cfis + 0x80);

	DPRINTF("cfis:");
	for (i = 0; i < cfl; i++) {
	if (i % 10 == 0)
	DPRINTF("");
	DPRINTF("%02x ", cfis[i]);
	}
	DPRINTF("");

	for (i = 0; i < hdr->prdtl; i++) {
	DPRINTF("%d@%08"PRIx64"", prdt->dbc & 0x3fffff, prdt->dba);
	prdt++;
	}
	#endif

	if (cfis[0] != FIS_TYPE_REGH2D) {
	WPRINTF("Not a H2D FIS:%02x", cfis[0]);
	return;
	}

	if (cfis[1] & 0x80) {
	ahci_handle_cmd(p, slot, cfis);
	} else {
	if (cfis[15] & (1 << 2))
	p->reset = 1;
	else if (p->reset) {
	p->reset = 0;
	ahci_port_reset(p);
	}
	p->ci &= ~(1 << slot);
	}
	}

	static void
	ahci_handle_port(struct ahci_port *p)
	{

	if (!(p->cmd & AHCI_P_CMD_ST))
	return;

	/*
	* Search for any new commands to issue ignoring those that
	* are already in-flight. Stop if device is busy or in error.
	*/
	for (; (p->ci & ~p->pending) != 0; p->ccs = ((p->ccs + 1) & 31)) {
	if ((p->tfd & (ATA_S_BUSY \| ATA_S_DRQ)) != 0)
	break;
	if (p->waitforclear)
	break;
	if ((p->ci & ~p->pending & (1 << p->ccs)) != 0) {
	p->cmd &= ~AHCI_P_CMD_CCS_MASK;
	p->cmd \|= p->ccs << AHCI_P_CMD_CCS_SHIFT;
	ahci_handle_slot(p, p->ccs);
	}
	}
	}

	/*
	* blockif callback routine - this runs in the context of the blockif
	* i/o thread, so the mutex needs to be acquired.
	*/
	static void
	ata_ioreq_cb(struct blockif_req *br, int err)
	{
	struct ahci_cmd_hdr *hdr;
	struct ahci_ioreq *aior;
	struct ahci_port *p;
	struct pci_ahci_softc *sc;
	uint32_t tfd;
	uint8_t *cfis;
	int slot, ncq, dsm;

	DPRINTF("%s %d", __func__, err);

	ncq = dsm = 0;
	aior = br->br_param;
	p = aior->io_pr;
	cfis = aior->cfis;
	slot = aior->slot;
	sc = p->pr_sc;
	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + slot AHCI_CL_SIZE);

	if (cfis[2] == ATA_WRITE_FPDMA_QUEUED \|\|
	cfis[2] == ATA_READ_FPDMA_QUEUED \|\|
	cfis[2] == ATA_SEND_FPDMA_QUEUED)
	ncq = 1;
	if (cfis[2] == ATA_DATA_SET_MANAGEMENT \|\|
	(cfis[2] == ATA_SEND_FPDMA_QUEUED &&
	(cfis[13] & 0x1f) == ATA_SFPDMA_DSM))
	dsm = 1;

	pthread_mutex_lock(&sc->mtx);

	/*
	* Delete the blockif request from the busy list
	*/
	TAILQ_REMOVE(&p->iobhd, aior, io_blist);

	/*
	* Move the blockif request back to the free list
	*/
	STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist);

	if (!err)
	hdr->prdbc = aior->done;

	if (!err && aior->more) {
	if (dsm)
	ahci_handle_dsm_trim(p, slot, cfis, aior->done);
	else
	ahci_handle_rw(p, slot, cfis, aior->done);
	goto out;
	}

	if (!err)
	tfd = ATA_S_READY \| ATA_S_DSC;
	else
	tfd = (ATA_E_ABORT << 8) \| ATA_S_READY \| ATA_S_ERROR;
	if (ncq)
	ahci_write_fis_sdb(p, slot, cfis, tfd);
	else
	ahci_write_fis_d2h(p, slot, cfis, tfd);

	/*
	* This command is now complete.
	*/
	p->pending &= ~(1 << slot);

	ahci_check_stopped(p);
	ahci_handle_port(p);
	out:
	pthread_mutex_unlock(&sc->mtx);
	DPRINTF("%s exit", __func__);
	}

	static void
	atapi_ioreq_cb(struct blockif_req *br, int err)
	{
	struct ahci_cmd_hdr *hdr;
	struct ahci_ioreq *aior;
	struct ahci_port *p;
	struct pci_ahci_softc *sc;
	uint8_t *cfis;
	uint32_t tfd;
	int slot;

	DPRINTF("%s %d", __func__, err);

	aior = br->br_param;
	p = aior->io_pr;
	cfis = aior->cfis;
	slot = aior->slot;
	sc = p->pr_sc;
	hdr = (struct ahci_cmd_hdr )(p->cmd_lst + aior->slot AHCI_CL_SIZE);

	pthread_mutex_lock(&sc->mtx);

	/*
	* Delete the blockif request from the busy list
	*/
	TAILQ_REMOVE(&p->iobhd, aior, io_blist);

	/*
	* Move the blockif request back to the free list
	*/
	STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist);

	if (!err)
	hdr->prdbc = aior->done;

	if (!err && aior->more) {
	atapi_read(p, slot, cfis, aior->done);
	goto out;
	}

	if (!err) {
	tfd = ATA_S_READY \| ATA_S_DSC;
	} else {
	p->sense_key = ATA_SENSE_ILLEGAL_REQUEST;
	p->asc = 0x21;
	tfd = (p->sense_key << 12) \| ATA_S_READY \| ATA_S_ERROR;
	}
	cfis[4] = (cfis[4] & ~7) \| ATA_I_CMD \| ATA_I_IN;
	ahci_write_fis_d2h(p, slot, cfis, tfd);

	/*
	* This command is now complete.
	*/
	p->pending &= ~(1 << slot);

	ahci_check_stopped(p);
	ahci_handle_port(p);
	out:
	pthread_mutex_unlock(&sc->mtx);
	DPRINTF("%s exit", __func__);
	}

	static void
	pci_ahci_ioreq_init(struct ahci_port *pr)
	{
	struct ahci_ioreq *vr;
	int i;

	pr->ioqsz = blockif_queuesz(pr->bctx);
	pr->ioreq = calloc(pr->ioqsz, sizeof(struct ahci_ioreq));
	STAILQ_INIT(&pr->iofhd);

	/*
	* Add all i/o request entries to the free queue
	*/
	for (i = 0; i < pr->ioqsz; i++) {
	vr = &pr->ioreq[i];
	vr->io_pr = pr;
	if (!pr->atapi)
	vr->io_req.br_callback = ata_ioreq_cb;
	else
	vr->io_req.br_callback = atapi_ioreq_cb;
	vr->io_req.br_param = vr;
	STAILQ_INSERT_TAIL(&pr->iofhd, vr, io_flist);
	}

	TAILQ_INIT(&pr->iobhd);
	}

	static void
	pci_ahci_port_write(struct pci_ahci_softc *sc, uint64_t offset, uint64_t value)
	{
	int port = (offset - AHCI_OFFSET) / AHCI_STEP;
	offset = (offset - AHCI_OFFSET) % AHCI_STEP;
	struct ahci_port *p = &sc->port[port];

	DPRINTF("pci_ahci_port %d: write offset 0x%"PRIx64" value 0x%"PRIx64"",
	port, offset, value);

	switch (offset) {
	case AHCI_P_CLB:
	p->clb = value;
	break;
	case AHCI_P_CLBU:
	p->clbu = value;
	break;
	case AHCI_P_FB:
	p->fb = value;
	break;
	case AHCI_P_FBU:
	p->fbu = value;
	break;
	case AHCI_P_IS:
	p->is &= ~value;
	ahci_port_intr(p);
	break;
	case AHCI_P_IE:
	p->ie = value & 0xFDC000FF;
	ahci_port_intr(p);
	break;
	case AHCI_P_CMD:
	{
	p->cmd &= ~(AHCI_P_CMD_ST \| AHCI_P_CMD_SUD \| AHCI_P_CMD_POD \|
	AHCI_P_CMD_CLO \| AHCI_P_CMD_FRE \| AHCI_P_CMD_APSTE \|
	AHCI_P_CMD_ATAPI \| AHCI_P_CMD_DLAE \| AHCI_P_CMD_ALPE \|
	AHCI_P_CMD_ASP \| AHCI_P_CMD_ICC_MASK);
	p->cmd \|= (AHCI_P_CMD_ST \| AHCI_P_CMD_SUD \| AHCI_P_CMD_POD \|
	AHCI_P_CMD_CLO \| AHCI_P_CMD_FRE \| AHCI_P_CMD_APSTE \|
	AHCI_P_CMD_ATAPI \| AHCI_P_CMD_DLAE \| AHCI_P_CMD_ALPE \|
	AHCI_P_CMD_ASP \| AHCI_P_CMD_ICC_MASK) & value;

	if (!(value & AHCI_P_CMD_ST)) {
	ahci_port_stop(p);
	} else {
	uint64_t clb;

	p->cmd \|= AHCI_P_CMD_CR;
	clb = (uint64_t)p->clbu << 32 \| p->clb;
	p->cmd_lst = paddr_guest2host(ahci_ctx(sc), clb,
	AHCI_CL_SIZE * AHCI_MAX_SLOTS);
	}

	if (value & AHCI_P_CMD_FRE) {
	uint64_t fb;

	p->cmd \|= AHCI_P_CMD_FR;
	fb = (uint64_t)p->fbu << 32 \| p->fb;
	/* we don't support FBSCP, so rfis size is 256Bytes */
	p->rfis = paddr_guest2host(ahci_ctx(sc), fb, 256);
	} else {
	p->cmd &= ~AHCI_P_CMD_FR;
	}

	if (value & AHCI_P_CMD_CLO) {
	p->tfd &= ~(ATA_S_BUSY \| ATA_S_DRQ);
	p->cmd &= ~AHCI_P_CMD_CLO;
	}

	if (value & AHCI_P_CMD_ICC_MASK) {
	p->cmd &= ~AHCI_P_CMD_ICC_MASK;
	}

	ahci_handle_port(p);
	break;
	}
	case AHCI_P_TFD:
	case AHCI_P_SIG:
	case AHCI_P_SSTS:
	WPRINTF("pci_ahci_port: read only registers 0x%"PRIx64"", offset);
	break;
	case AHCI_P_SCTL:
	p->sctl = value;
	if (!(p->cmd & AHCI_P_CMD_ST)) {
	if (value & ATA_SC_DET_RESET)
	ahci_port_reset(p);
	}
	break;
	case AHCI_P_SERR:
	p->serr &= ~value;
	break;
	case AHCI_P_SACT:
	p->sact \|= value;
	break;
	case AHCI_P_CI:
	p->ci \|= value;
	ahci_handle_port(p);
	break;
	case AHCI_P_SNTF:
	case AHCI_P_FBS:
	default:
	break;
	}
	}

	static void
	pci_ahci_host_write(struct pci_ahci_softc *sc, uint64_t offset, uint64_t value)
	{
	DPRINTF("pci_ahci_host: write offset 0x%"PRIx64" value 0x%"PRIx64"",
	offset, value);

	switch (offset) {
	case AHCI_CAP:
	case AHCI_PI:
	case AHCI_VS:
	case AHCI_CAP2:
	DPRINTF("pci_ahci_host: read only registers 0x%"PRIx64"", offset);
	break;
	case AHCI_GHC:
	if (value & AHCI_GHC_HR) {
	ahci_reset(sc);
	break;
	}
	if (value & AHCI_GHC_IE)
	sc->ghc \|= AHCI_GHC_IE;
	else
	sc->ghc &= ~AHCI_GHC_IE;
	ahci_generate_intr(sc, 0xffffffff);
	break;
	case AHCI_IS:
	sc->is &= ~value;
	ahci_generate_intr(sc, value);
	break;
	default:
	break;
	}
	}

	static void
	pci_ahci_write(struct vmctx ctx, int vcpu, struct pci_devinst pi,
	int baridx, uint64_t offset, int size, uint64_t value)
	{
	struct pci_ahci_softc *sc = pi->pi_arg;

	assert(baridx == 5);
	assert((offset % 4) == 0 && size == 4);

	pthread_mutex_lock(&sc->mtx);

	if (offset < AHCI_OFFSET)
	pci_ahci_host_write(sc, offset, value);
	else if (offset < AHCI_OFFSET + sc->ports * AHCI_STEP)
	pci_ahci_port_write(sc, offset, value);
	else
	WPRINTF("pci_ahci: unknown i/o write offset 0x%"PRIx64"", offset);

	pthread_mutex_unlock(&sc->mtx);
	}

	static uint64_t
	pci_ahci_host_read(struct pci_ahci_softc *sc, uint64_t offset)
	{
	uint32_t value;

	switch (offset) {
	case AHCI_CAP:
	case AHCI_GHC:
	case AHCI_IS:
	case AHCI_PI:
	case AHCI_VS:
	case AHCI_CCCC:
	case AHCI_CCCP:
	case AHCI_EM_LOC:
	case AHCI_EM_CTL:
	case AHCI_CAP2:
	{
	uint32_t *p = &sc->cap;
	p += (offset - AHCI_CAP) / sizeof(uint32_t);
	value = *p;
	break;
	}
	default:
	value = 0;
	break;
	}
	DPRINTF("pci_ahci_host: read offset 0x%"PRIx64" value 0x%x",
	offset, value);

	return (value);
	}

	static uint64_t
	pci_ahci_port_read(struct pci_ahci_softc *sc, uint64_t offset)
	{
	uint32_t value;
	int port = (offset - AHCI_OFFSET) / AHCI_STEP;
	offset = (offset - AHCI_OFFSET) % AHCI_STEP;

	switch (offset) {
	case AHCI_P_CLB:
	case AHCI_P_CLBU:
	case AHCI_P_FB:
	case AHCI_P_FBU:
	case AHCI_P_IS:
	case AHCI_P_IE:
	case AHCI_P_CMD:
	case AHCI_P_TFD:
	case AHCI_P_SIG:
	case AHCI_P_SSTS:
	case AHCI_P_SCTL:
	case AHCI_P_SERR:
	case AHCI_P_SACT:
	case AHCI_P_CI:
	case AHCI_P_SNTF:
	case AHCI_P_FBS:
	{
	uint32_t *p= &sc->port[port].clb;
	p += (offset - AHCI_P_CLB) / sizeof(uint32_t);
	value = *p;
	break;
	}
	default:
	value = 0;
	break;
	}

	DPRINTF("pci_ahci_port %d: read offset 0x%"PRIx64" value 0x%x",
	port, offset, value);

	return value;
	}

	static uint64_t
	pci_ahci_read(struct vmctx ctx, int vcpu, struct pci_devinst pi, int baridx,
	uint64_t regoff, int size)
	{
	struct pci_ahci_softc *sc = pi->pi_arg;
	uint64_t offset;
	uint32_t value;

	assert(baridx == 5);
	assert(size == 1 \|\| size == 2 \|\| size == 4);
	assert((regoff & (size - 1)) == 0);

	pthread_mutex_lock(&sc->mtx);

	offset = regoff & ~0x3; /* round down to a multiple of 4 bytes */
	if (offset < AHCI_OFFSET)
	value = pci_ahci_host_read(sc, offset);
	else if (offset < AHCI_OFFSET + sc->ports * AHCI_STEP)
	value = pci_ahci_port_read(sc, offset);
	else {
	value = 0;
	WPRINTF("pci_ahci: unknown i/o read offset 0x%"PRIx64"",
	regoff);
	}
	value >>= 8 * (regoff & 0x3);

	pthread_mutex_unlock(&sc->mtx);

	return (value);
	}

	static int
	pci_ahci_init(struct vmctx ctx, struct pci_devinst pi, char *opts, int atapi)
	{
	char bident[sizeof("XX:XX:XX")];
	struct blockif_ctxt *bctxt;
	struct pci_ahci_softc *sc;
	int ret, slots, p;
	MD5_CTX mdctx;
	u_char digest[16];
	char next, next2;
	+ char bopt, uopt, xopts, config;
	+ FILE* fp;
	+ size_t block_len;
	+ int comma, optpos;

	ret = 0;

	#ifdef AHCI_DEBUG
	dbg = fopen("/tmp/log", "w+");
	#endif

	sc = calloc(1, sizeof(struct pci_ahci_softc));
	pi->pi_arg = sc;
	sc->asc_pi = pi;
	pthread_mutex_init(&sc->mtx, NULL);
	sc->ports = 0;
	sc->pi = 0;
	slots = 32;

	for (p = 0; p < MAX_PORTS && opts != NULL; p++, opts = next) {
	+ struct ata_params *ata_ident = &sc->port[p].ata_ident;
	+ memset(ata_ident, 0, sizeof(struct ata_params));
	+
	/* Identify and cut off type of present port. */
	if (strncmp(opts, "hd:", 3) == 0) {
	atapi = 0;
	opts += 3;
	} else if (strncmp(opts, "cd:", 3) == 0) {
	atapi = 1;
	opts += 3;
	}

	/* Find and cut off the next port options. */
	next = strstr(opts, ",hd:");
	next2 = strstr(opts, ",cd:");
	if (next == NULL \|\| (next2 != NULL && next2 < next))
	next = next2;
	if (next != NULL) {
	next[0] = 0;
	next++;
	}

	if (opts[0] == 0)
	continue;

	+ uopt = strdup(opts);
	+ bopt = NULL;
	+ fp = open_memstream(&bopt, &block_len);
	+ comma = 0;
	+ optpos = 0;
	+
	+ for (xopts = strtok(uopt, ",");
	+ xopts != NULL;
	+ xopts = strtok(NULL, ",")) {
	+
	+ /* First option assume as block filename. */
	+ if (optpos == 0) {
	+ /*
	+ * Create an identifier for the backing file.
	+ * Use parts of the md5 sum of the filename
	+ */
	+ char ident[AHCI_PORT_IDENT];
	+ MD5Init(&mdctx);
	+ MD5Update(&mdctx, opts, strlen(opts));
	+ MD5Final(digest, &mdctx);
	+ snprintf(ident, AHCI_PORT_IDENT,
	+ "BHYVE-%02X%02X-%02X%02X-%02X%02X",
	+ digest[0], digest[1], digest[2], digest[3], digest[4],
	+ digest[5]);
	+ ata_string((uint8_t*)&ata_ident->serial, ident, 20);
	+ ata_string((uint8_t*)&ata_ident->revision, "001", 8);
	+ if (atapi) {
	+ ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DVD ROM", 40);
	+ }
	+ else {
	+ ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DISK", 40);
	+ }
	+ }
	+
	+ if ((config = strchr(xopts, '=')) != NULL) {
	+ *config++ = '\0';
	+ if (!strcmp("nmrr", xopts)) {
	+ ata_ident->media_rotation_rate = atoi(config);
	+ }
	+ else if (!strcmp("ser", xopts)) {
	+ ata_string((uint8_t*)(&ata_ident->serial), config, 20);
	+ }
	+ else if (!strcmp("rev", xopts)) {
	+ ata_string((uint8_t*)(&ata_ident->revision), config, 8);
	+ }
	+ else if (!strcmp("model", xopts)) {
	+ ata_string((uint8_t*)(&ata_ident->model), config, 40);
	+ }
	+ else {
	+ /* Pass all other options to blockif_open. */
	+ *--config = '=';
	+ fprintf(fp, "%s%s", comma ? "," : "", xopts);
	+ comma = 1;
	+ }
	+ }
	+ else {
	+ /* Pass all other options to blockif_open. */
	+ fprintf(fp, "%s%s", comma ? "," : "", xopts);
	+ comma = 1;
	+ }
	+ optpos++;
	+ }
	+ free(uopt);
	+ fclose(fp);
	+
	+ DPRINTF("%s\n", bopt);
	+
	/*
	* Attempt to open the backing image. Use the PCI slot/func
	* and the port number for the identifier string.
	*/
	snprintf(bident, sizeof(bident), "%d:%d:%d", pi->pi_slot,
	pi->pi_func, p);
	- bctxt = blockif_open(opts, bident);
	+ bctxt = blockif_open(bopt, bident);
	+ free(bopt);
	+
	if (bctxt == NULL) {
	sc->ports = p;
	ret = 1;
	goto open_fail;
	}
	sc->port[p].bctx = bctxt;
	sc->port[p].pr_sc = sc;
	sc->port[p].port = p;
	sc->port[p].atapi = atapi;

	- /*
	- * Create an identifier for the backing file.
	- * Use parts of the md5 sum of the filename
	- */
	- MD5Init(&mdctx);
	- MD5Update(&mdctx, opts, strlen(opts));
	- MD5Final(digest, &mdctx);
	- snprintf(sc->port[p].ident, AHCI_PORT_IDENT,
	- "BHYVE-%02X%02X-%02X%02X-%02X%02X",
	- digest[0], digest[1], digest[2], digest[3], digest[4],
	- digest[5]);
	+ ata_identify_init(&sc->port[p], atapi);

	/*
	* Allocate blockif request structures and add them
	* to the free list
	*/
	pci_ahci_ioreq_init(&sc->port[p]);

	sc->pi \|= (1 << p);
	if (sc->port[p].ioqsz < slots)
	slots = sc->port[p].ioqsz;
	}
	sc->ports = p;

	/* Intel ICH8 AHCI */
	--slots;
	if (sc->ports < DEF_PORTS)
	sc->ports = DEF_PORTS;
	sc->cap = AHCI_CAP_64BIT \| AHCI_CAP_SNCQ \| AHCI_CAP_SSNTF \|
	AHCI_CAP_SMPS \| AHCI_CAP_SSS \| AHCI_CAP_SALP \|
	AHCI_CAP_SAL \| AHCI_CAP_SCLO \| (0x3 << AHCI_CAP_ISS_SHIFT)\|
	AHCI_CAP_PMD \| AHCI_CAP_SSC \| AHCI_CAP_PSC \|
	(slots << AHCI_CAP_NCS_SHIFT) \| AHCI_CAP_SXS \| (sc->ports - 1);

	sc->vs = 0x10300;
	sc->cap2 = AHCI_CAP2_APST;
	ahci_reset(sc);

	pci_set_cfgdata16(pi, PCIR_DEVICE, 0x2821);
	pci_set_cfgdata16(pi, PCIR_VENDOR, 0x8086);
	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
	pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_SATA);
	pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_STORAGE_SATA_AHCI_1_0);
	p = MIN(sc->ports, 16);
	p = flsl(p) - ((p & (p - 1)) ? 0 : 1);
	pci_emul_add_msicap(pi, 1 << p);
	pci_emul_alloc_bar(pi, 5, PCIBAR_MEM32,
	AHCI_OFFSET + sc->ports * AHCI_STEP);

	pci_lintr_request(pi);

	open_fail:
	if (ret) {
	for (p = 0; p < sc->ports; p++) {
	if (sc->port[p].bctx != NULL)
	blockif_close(sc->port[p].bctx);
	}
	free(sc);
	}

	return (ret);
	}

	static int
	pci_ahci_hd_init(struct vmctx ctx, struct pci_devinst pi, char *opts)
	{

	return (pci_ahci_init(ctx, pi, opts, 0));
	}

	static int
	pci_ahci_atapi_init(struct vmctx ctx, struct pci_devinst pi, char *opts)
	{

	return (pci_ahci_init(ctx, pi, opts, 1));
	}

	#ifdef BHYVE_SNAPSHOT
	static int
	pci_ahci_snapshot_save_queues(struct ahci_port *port,
	struct vm_snapshot_meta *meta)
	{
	int ret;
	int idx;
	struct ahci_ioreq *ioreq;

	STAILQ_FOREACH(ioreq, &port->iofhd, io_flist) {
	idx = ((void ) ioreq - (void ) port->ioreq) / sizeof(*ioreq);
	SNAPSHOT_VAR_OR_LEAVE(idx, meta, ret, done);
	}

	idx = -1;
	SNAPSHOT_VAR_OR_LEAVE(idx, meta, ret, done);

	TAILQ_FOREACH(ioreq, &port->iobhd, io_blist) {
	idx = ((void ) ioreq - (void ) port->ioreq) / sizeof(*ioreq);
	SNAPSHOT_VAR_OR_LEAVE(idx, meta, ret, done);

	/*
	* Snapshot only the busy requests; other requests are
	* not valid.
	*/
	ret = blockif_snapshot_req(&ioreq->io_req, meta);
	if (ret != 0) {
	fprintf(stderr, "%s: failed to snapshot req\r\n",
	__func__);
	goto done;
	}
	}

	idx = -1;
	SNAPSHOT_VAR_OR_LEAVE(idx, meta, ret, done);

	done:
	return (ret);
	}

	static int
	pci_ahci_snapshot_restore_queues(struct ahci_port *port,
	struct vm_snapshot_meta *meta)
	{
	int ret;
	int idx;
	struct ahci_ioreq *ioreq;

	/* Empty the free queue before restoring. */
	while (!STAILQ_EMPTY(&port->iofhd))
	STAILQ_REMOVE_HEAD(&port->iofhd, io_flist);

	/* Restore the free queue. */
	while (1) {
	SNAPSHOT_VAR_OR_LEAVE(idx, meta, ret, done);
	if (idx == -1)
	break;

	STAILQ_INSERT_TAIL(&port->iofhd, &port->ioreq[idx], io_flist);
	}

	/* Restore the busy queue. */
	while (1) {
	SNAPSHOT_VAR_OR_LEAVE(idx, meta, ret, done);
	if (idx == -1)
	break;

	ioreq = &port->ioreq[idx];
	TAILQ_INSERT_TAIL(&port->iobhd, ioreq, io_blist);

	/*
	* Restore only the busy requests; other requests are
	* not valid.
	*/
	ret = blockif_snapshot_req(&ioreq->io_req, meta);
	if (ret != 0) {
	fprintf(stderr, "%s: failed to restore request\r\n",
	__func__);
	goto done;
	}

	/* Re-enqueue the requests in the block interface. */
	if (ioreq->readop)
	ret = blockif_read(port->bctx, &ioreq->io_req);
	else
	ret = blockif_write(port->bctx, &ioreq->io_req);

	if (ret != 0) {
	fprintf(stderr,
	"%s: failed to re-enqueue request\r\n",
	__func__);
	goto done;
	}
	}

	done:
	return (ret);
	}

	static int
	pci_ahci_snapshot(struct vm_snapshot_meta *meta)
	{
	int i, j, ret;
	void *bctx;
	struct pci_devinst *pi;
	struct pci_ahci_softc *sc;
	struct ahci_port *port;
	struct ahci_cmd_hdr *hdr;
	struct ahci_ioreq *ioreq;

	pi = meta->dev_data;
	sc = pi->pi_arg;

	/* TODO: add mtx lock/unlock */

	SNAPSHOT_VAR_OR_LEAVE(sc->ports, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->cap, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->ghc, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->is, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->pi, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->vs, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->ccc_ctl, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->ccc_pts, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->em_loc, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->em_ctl, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->cap2, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->bohc, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(sc->lintr, meta, ret, done);

	for (i = 0; i < MAX_PORTS; i++) {
	port = &sc->port[i];

	if (meta->op == VM_SNAPSHOT_SAVE)
	bctx = port->bctx;

	SNAPSHOT_VAR_OR_LEAVE(bctx, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->port, meta, ret, done);

	/* Mostly for restore; save is ensured by the lines above. */
	if (((bctx == NULL) && (port->bctx != NULL)) \|\|
	((bctx != NULL) && (port->bctx == NULL))) {
	fprintf(stderr, "%s: ports not matching\r\n", __func__);
	ret = EINVAL;
	goto done;
	}

	if (port->bctx == NULL)
	continue;

	if (port->port != i) {
	fprintf(stderr, "%s: ports not matching: "
	"actual: %d expected: %d\r\n",
	__func__, port->port, i);
	ret = EINVAL;
	goto done;
	}

	SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(port->cmd_lst,
	AHCI_CL_SIZE * AHCI_MAX_SLOTS, false, meta, ret, done);
	SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(port->rfis, 256, false, meta,
	ret, done);

	SNAPSHOT_VAR_OR_LEAVE(port->ident, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->atapi, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->reset, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->waitforclear, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->mult_sectors, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->xfermode, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->err_cfis, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->sense_key, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->asc, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->ccs, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->pending, meta, ret, done);

	SNAPSHOT_VAR_OR_LEAVE(port->clb, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->clbu, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->fb, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->fbu, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->ie, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->cmd, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->unused0, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->tfd, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->sig, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->ssts, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->sctl, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->serr, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->sact, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->ci, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->sntf, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->fbs, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(port->ioqsz, meta, ret, done);

	for (j = 0; j < port->ioqsz; j++) {
	ioreq = &port->ioreq[j];

	/* blockif_req snapshot done only for busy requests. */
	hdr = (struct ahci_cmd_hdr *)(port->cmd_lst +
	ioreq->slot * AHCI_CL_SIZE);
	SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(ioreq->cfis,
	0x80 + hdr->prdtl * sizeof(struct ahci_prdt_entry),
	false, meta, ret, done);

	SNAPSHOT_VAR_OR_LEAVE(ioreq->len, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(ioreq->done, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(ioreq->slot, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(ioreq->more, meta, ret, done);
	SNAPSHOT_VAR_OR_LEAVE(ioreq->readop, meta, ret, done);
	}

	/* Perform save / restore specific operations. */
	if (meta->op == VM_SNAPSHOT_SAVE) {
	ret = pci_ahci_snapshot_save_queues(port, meta);
	if (ret != 0)
	goto done;
	} else if (meta->op == VM_SNAPSHOT_RESTORE) {
	ret = pci_ahci_snapshot_restore_queues(port, meta);
	if (ret != 0)
	goto done;
	} else {
	ret = EINVAL;
	goto done;
	}

	ret = blockif_snapshot(port->bctx, meta);
	if (ret != 0) {
	fprintf(stderr, "%s: failed to restore blockif\r\n",
	__func__);
	goto done;
	}
	}

	done:
	return (ret);
	}

	static int
	pci_ahci_pause(struct vmctx ctx, struct pci_devinst pi)
	{
	struct pci_ahci_softc *sc;
	struct blockif_ctxt *bctxt;
	int i;

	sc = pi->pi_arg;

	for (i = 0; i < MAX_PORTS; i++) {
	bctxt = sc->port[i].bctx;
	if (bctxt == NULL)
	continue;

	blockif_pause(bctxt);
	}

	return (0);
	}

	static int
	pci_ahci_resume(struct vmctx ctx, struct pci_devinst pi)
	{
	struct pci_ahci_softc *sc;
	struct blockif_ctxt *bctxt;
	int i;

	sc = pi->pi_arg;

	for (i = 0; i < MAX_PORTS; i++) {
	bctxt = sc->port[i].bctx;
	if (bctxt == NULL)
	continue;

	blockif_resume(bctxt);
	}

	return (0);
	}
	#endif

	/*
	* Use separate emulation names to distinguish drive and atapi devices
	*/
	struct pci_devemu pci_de_ahci = {
	.pe_emu = "ahci",
	.pe_init = pci_ahci_hd_init,
	.pe_barwrite = pci_ahci_write,
	.pe_barread = pci_ahci_read,
	#ifdef BHYVE_SNAPSHOT
	.pe_snapshot = pci_ahci_snapshot,
	.pe_pause = pci_ahci_pause,
	.pe_resume = pci_ahci_resume,
	#endif
	};
	PCI_EMUL_SET(pci_de_ahci);

	struct pci_devemu pci_de_ahci_hd = {
	.pe_emu = "ahci-hd",
	.pe_init = pci_ahci_hd_init,
	.pe_barwrite = pci_ahci_write,
	.pe_barread = pci_ahci_read,
	#ifdef BHYVE_SNAPSHOT
	.pe_snapshot = pci_ahci_snapshot,
	.pe_pause = pci_ahci_pause,
	.pe_resume = pci_ahci_resume,
	#endif
	};
	PCI_EMUL_SET(pci_de_ahci_hd);

	struct pci_devemu pci_de_ahci_cd = {
	.pe_emu = "ahci-cd",
	.pe_init = pci_ahci_atapi_init,
	.pe_barwrite = pci_ahci_write,
	.pe_barread = pci_ahci_read,
	#ifdef BHYVE_SNAPSHOT
	.pe_snapshot = pci_ahci_snapshot,
	.pe_pause = pci_ahci_pause,
	.pe_resume = pci_ahci_resume,
	#endif
	};
	PCI_EMUL_SET(pci_de_ahci_cd);
	diff --git a/usr.sbin/bsdinstall/partedit/partedit_sparc64.c b/usr.sbin/bsdinstall/partedit/partedit_sparc64.c
	deleted file mode 100644
	index 514c0e9ffd1d..000000000000
	--- a/usr.sbin/bsdinstall/partedit/partedit_sparc64.c
	+++ /dev/null
	@@ -1,84 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 2011 Nathan Whitehorn
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-#include <string.h>
	-
	-#include "partedit.h"
	-
	-const char *
	-default_scheme(void) {
	- return ("VTOC8");
	-}
	-
	-int
	-is_scheme_bootable(const char *part_type) {
	- if (strcmp(part_type, "VTOC8") == 0)
	- return (1);
	- return (0);
	-}
	-
	-int
	-is_fs_bootable(const char part_type, const char fs)
	-{
	- if (strcmp(fs, "freebsd-ufs") == 0 \|\| strcmp(fs, "freebsd-zfs") == 0)
	- return (1);
	- return (0);
	-}
	-
	-
	-size_t
	-bootpart_size(const char *part_type) {
	- /* No standalone boot partition */
	-
	- return (0);
	-}
	-
	-const char *
	-bootpart_type(const char scheme, const char *mountpoint) {
	- return ("freebsd-boot");
	-}
	-
	-const char *
	-bootcode_path(const char *part_type) {
	- return (NULL);
	-}
	-
	-const char *
	-partcode_path(const char part_type, const char fs_type) {
	- if (strcmp(part_type, "VTOC8") == 0) {
	- if (strcmp(fs_type, "ufs") == 0) {
	- return ("/boot/boot1");
	- } else if (strcmp(fs_type, "zfs") == 0) {
	- return ("/boot/zfsboot");
	- }
	- }
	- return (NULL);
	-}
	-
	diff --git a/usr.sbin/ctld/kernel.c b/usr.sbin/ctld/kernel.c
	index 4da7c05f1e66..ed8d46d27a0e 100644
	--- a/usr.sbin/ctld/kernel.c
	+++ b/usr.sbin/ctld/kernel.c
	@@ -1,1335 +1,1359 @@
	/*-
	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	*
	* Copyright (c) 2003, 2004 Silicon Graphics International Corp.
	* Copyright (c) 1997-2007 Kenneth D. Merry
	* Copyright (c) 2012 The FreeBSD Foundation
	* Copyright (c) 2017 Jakub Wojciech Klama <jceel@FreeBSD.org>
	* All rights reserved.
	*
	* Portions of this software were developed by Edward Tomasz Napierala
	* under sponsorship from the FreeBSD Foundation.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions, and the following disclaimer,
	* without modification.
	* 2. Redistributions in binary form must reproduce at minimum a disclaimer
	* substantially similar to the "NO WARRANTY" disclaimer below
	* ("Disclaimer") and any redistribution must be conditioned upon
	* including a substantially similar Disclaimer requirement for further
	* binary redistribution.
	*
	* NO WARRANTY
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	* HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
	* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	* POSSIBILITY OF SUCH DAMAGES.
	*
	*/

	#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");

	#include <sys/param.h>
	#include <sys/capsicum.h>
	#include <sys/callout.h>
	#include <sys/ioctl.h>
	#include <sys/linker.h>
	#include <sys/module.h>
	#include <sys/queue.h>
	#include <sys/sbuf.h>
	#include <sys/nv.h>
	#include <sys/stat.h>
	#include <assert.h>
	#include <bsdxml.h>
	#include <capsicum_helpers.h>
	#include <ctype.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <stdint.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <strings.h>
	#include <cam/scsi/scsi_all.h>
	#include <cam/scsi/scsi_message.h>
	#include <cam/ctl/ctl.h>
	#include <cam/ctl/ctl_io.h>
	#include <cam/ctl/ctl_backend.h>
	#include <cam/ctl/ctl_ioctl.h>
	#include <cam/ctl/ctl_util.h>
	#include <cam/ctl/ctl_scsi_all.h>

	#include "ctld.h"

	#ifdef ICL_KERNEL_PROXY
	#include <netdb.h>
	#endif

	#define NVLIST_BUFSIZE 1024

	extern bool proxy_mode;

	static int ctl_fd = 0;

	void
	kernel_init(void)
	{
	int retval, saved_errno;

	ctl_fd = open(CTL_DEFAULT_DEV, O_RDWR);
	if (ctl_fd < 0 && errno == ENOENT) {
	saved_errno = errno;
	retval = kldload("ctl");
	if (retval != -1)
	ctl_fd = open(CTL_DEFAULT_DEV, O_RDWR);
	else
	errno = saved_errno;
	}
	if (ctl_fd < 0)
	log_err(1, "failed to open %s", CTL_DEFAULT_DEV);
	#ifdef WANT_ISCSI
	else {
	saved_errno = errno;
	if (modfind("cfiscsi") == -1 && kldload("cfiscsi") == -1)
	log_warn("couldn't load cfiscsi");
	errno = saved_errno;
	}
	#endif
	}

	/*
	* Name/value pair used for per-LUN attributes.
	*/
	struct cctl_lun_nv {
	char *name;
	char *value;
	STAILQ_ENTRY(cctl_lun_nv) links;
	};

	/*
	* Backend LUN information.
	*/
	struct cctl_lun {
	uint64_t lun_id;
	char *backend_type;
	uint8_t device_type;
	uint64_t size_blocks;
	uint32_t blocksize;
	char *serial_number;
	char *device_id;
	char *ctld_name;
	STAILQ_HEAD(,cctl_lun_nv) attr_list;
	STAILQ_ENTRY(cctl_lun) links;
	};

	struct cctl_port {
	uint32_t port_id;
	char *port_frontend;
	char *port_name;
	int pp;
	int vp;
	int cfiscsi_state;
	char *cfiscsi_target;
	uint16_t cfiscsi_portal_group_tag;
	char *ctld_portal_group_name;
	STAILQ_HEAD(,cctl_lun_nv) attr_list;
	STAILQ_ENTRY(cctl_port) links;
	};

	struct cctl_devlist_data {
	int num_luns;
	STAILQ_HEAD(,cctl_lun) lun_list;
	struct cctl_lun *cur_lun;
	int num_ports;
	STAILQ_HEAD(,cctl_port) port_list;
	struct cctl_port *cur_port;
	int level;
	struct sbuf *cur_sb[32];
	};

	static void
	cctl_start_element(void user_data, const char name, const char **attr)
	{
	int i;
	struct cctl_devlist_data *devlist;
	struct cctl_lun *cur_lun;

	devlist = (struct cctl_devlist_data *)user_data;
	cur_lun = devlist->cur_lun;
	devlist->level++;
	if ((u_int)devlist->level >= (sizeof(devlist->cur_sb) /
	sizeof(devlist->cur_sb[0])))
	log_errx(1, "%s: too many nesting levels, %zd max", __func__,
	sizeof(devlist->cur_sb) / sizeof(devlist->cur_sb[0]));

	devlist->cur_sb[devlist->level] = sbuf_new_auto();
	if (devlist->cur_sb[devlist->level] == NULL)
	log_err(1, "%s: unable to allocate sbuf", __func__);

	if (strcmp(name, "lun") == 0) {
	if (cur_lun != NULL)
	log_errx(1, "%s: improper lun element nesting",
	__func__);

	cur_lun = calloc(1, sizeof(*cur_lun));
	if (cur_lun == NULL)
	log_err(1, "%s: cannot allocate %zd bytes", __func__,
	sizeof(*cur_lun));

	devlist->num_luns++;
	devlist->cur_lun = cur_lun;

	STAILQ_INIT(&cur_lun->attr_list);
	STAILQ_INSERT_TAIL(&devlist->lun_list, cur_lun, links);

	for (i = 0; attr[i] != NULL; i += 2) {
	if (strcmp(attr[i], "id") == 0) {
	cur_lun->lun_id = strtoull(attr[i+1], NULL, 0);
	} else {
	log_errx(1, "%s: invalid LUN attribute %s = %s",
	__func__, attr[i], attr[i+1]);
	}
	}
	}
	}

	static void
	cctl_end_element(void user_data, const char name)
	{
	struct cctl_devlist_data *devlist;
	struct cctl_lun *cur_lun;
	char *str;

	devlist = (struct cctl_devlist_data *)user_data;
	cur_lun = devlist->cur_lun;

	if ((cur_lun == NULL)
	&& (strcmp(name, "ctllunlist") != 0))
	log_errx(1, "%s: cur_lun == NULL! (name = %s)", __func__, name);

	if (devlist->cur_sb[devlist->level] == NULL)
	log_errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
	devlist->level, name);

	sbuf_finish(devlist->cur_sb[devlist->level]);
	str = checked_strdup(sbuf_data(devlist->cur_sb[devlist->level]));

	if (strlen(str) == 0) {
	free(str);
	str = NULL;
	}

	sbuf_delete(devlist->cur_sb[devlist->level]);
	devlist->cur_sb[devlist->level] = NULL;
	devlist->level--;

	if (strcmp(name, "backend_type") == 0) {
	cur_lun->backend_type = str;
	str = NULL;
	} else if (strcmp(name, "lun_type") == 0) {
	cur_lun->device_type = strtoull(str, NULL, 0);
	} else if (strcmp(name, "size") == 0) {
	cur_lun->size_blocks = strtoull(str, NULL, 0);
	} else if (strcmp(name, "blocksize") == 0) {
	cur_lun->blocksize = strtoul(str, NULL, 0);
	} else if (strcmp(name, "serial_number") == 0) {
	cur_lun->serial_number = str;
	str = NULL;
	} else if (strcmp(name, "device_id") == 0) {
	cur_lun->device_id = str;
	str = NULL;
	} else if (strcmp(name, "ctld_name") == 0) {
	cur_lun->ctld_name = str;
	str = NULL;
	} else if (strcmp(name, "lun") == 0) {
	devlist->cur_lun = NULL;
	} else if (strcmp(name, "ctllunlist") == 0) {
	/* Nothing. */
	} else {
	struct cctl_lun_nv *nv;

	nv = calloc(1, sizeof(*nv));
	if (nv == NULL)
	log_err(1, "%s: can't allocate %zd bytes for nv pair",
	__func__, sizeof(*nv));

	nv->name = checked_strdup(name);

	nv->value = str;
	str = NULL;
	STAILQ_INSERT_TAIL(&cur_lun->attr_list, nv, links);
	}

	free(str);
	}

	static void
	cctl_start_pelement(void user_data, const char name, const char **attr)
	{
	int i;
	struct cctl_devlist_data *devlist;
	struct cctl_port *cur_port;

	devlist = (struct cctl_devlist_data *)user_data;
	cur_port = devlist->cur_port;
	devlist->level++;
	if ((u_int)devlist->level >= (sizeof(devlist->cur_sb) /
	sizeof(devlist->cur_sb[0])))
	log_errx(1, "%s: too many nesting levels, %zd max", __func__,
	sizeof(devlist->cur_sb) / sizeof(devlist->cur_sb[0]));

	devlist->cur_sb[devlist->level] = sbuf_new_auto();
	if (devlist->cur_sb[devlist->level] == NULL)
	log_err(1, "%s: unable to allocate sbuf", __func__);

	if (strcmp(name, "targ_port") == 0) {
	if (cur_port != NULL)
	log_errx(1, "%s: improper port element nesting (%s)",
	__func__, name);

	cur_port = calloc(1, sizeof(*cur_port));
	if (cur_port == NULL)
	log_err(1, "%s: cannot allocate %zd bytes", __func__,
	sizeof(*cur_port));

	devlist->num_ports++;
	devlist->cur_port = cur_port;

	STAILQ_INIT(&cur_port->attr_list);
	STAILQ_INSERT_TAIL(&devlist->port_list, cur_port, links);

	for (i = 0; attr[i] != NULL; i += 2) {
	if (strcmp(attr[i], "id") == 0) {
	cur_port->port_id = strtoul(attr[i+1], NULL, 0);
	} else {
	log_errx(1, "%s: invalid LUN attribute %s = %s",
	__func__, attr[i], attr[i+1]);
	}
	}
	}
	}

	static void
	cctl_end_pelement(void user_data, const char name)
	{
	struct cctl_devlist_data *devlist;
	struct cctl_port *cur_port;
	char *str;

	devlist = (struct cctl_devlist_data *)user_data;
	cur_port = devlist->cur_port;

	if ((cur_port == NULL)
	&& (strcmp(name, "ctlportlist") != 0))
	log_errx(1, "%s: cur_port == NULL! (name = %s)", __func__, name);

	if (devlist->cur_sb[devlist->level] == NULL)
	log_errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
	devlist->level, name);

	sbuf_finish(devlist->cur_sb[devlist->level]);
	str = checked_strdup(sbuf_data(devlist->cur_sb[devlist->level]));

	if (strlen(str) == 0) {
	free(str);
	str = NULL;
	}

	sbuf_delete(devlist->cur_sb[devlist->level]);
	devlist->cur_sb[devlist->level] = NULL;
	devlist->level--;

	if (strcmp(name, "frontend_type") == 0) {
	cur_port->port_frontend = str;
	str = NULL;
	} else if (strcmp(name, "port_name") == 0) {
	cur_port->port_name = str;
	str = NULL;
	} else if (strcmp(name, "physical_port") == 0) {
	cur_port->pp = strtoul(str, NULL, 0);
	} else if (strcmp(name, "virtual_port") == 0) {
	cur_port->vp = strtoul(str, NULL, 0);
	} else if (strcmp(name, "cfiscsi_target") == 0) {
	cur_port->cfiscsi_target = str;
	str = NULL;
	} else if (strcmp(name, "cfiscsi_state") == 0) {
	cur_port->cfiscsi_state = strtoul(str, NULL, 0);
	} else if (strcmp(name, "cfiscsi_portal_group_tag") == 0) {
	cur_port->cfiscsi_portal_group_tag = strtoul(str, NULL, 0);
	} else if (strcmp(name, "ctld_portal_group_name") == 0) {
	cur_port->ctld_portal_group_name = str;
	str = NULL;
	} else if (strcmp(name, "targ_port") == 0) {
	devlist->cur_port = NULL;
	} else if (strcmp(name, "ctlportlist") == 0) {
	/* Nothing. */
	} else {
	struct cctl_lun_nv *nv;

	nv = calloc(1, sizeof(*nv));
	if (nv == NULL)
	log_err(1, "%s: can't allocate %zd bytes for nv pair",
	__func__, sizeof(*nv));

	nv->name = checked_strdup(name);

	nv->value = str;
	str = NULL;
	STAILQ_INSERT_TAIL(&cur_port->attr_list, nv, links);
	}

	free(str);
	}

	static void
	cctl_char_handler(void user_data, const XML_Char str, int len)
	{
	struct cctl_devlist_data *devlist;

	devlist = (struct cctl_devlist_data *)user_data;

	sbuf_bcat(devlist->cur_sb[devlist->level], str, len);
	}

	struct conf *
	conf_new_from_kernel(void)
	{
	struct conf *conf = NULL;
	struct target *targ;
	struct portal_group *pg;
	struct pport *pp;
	struct port *cp;
	struct lun *cl;
	struct option *o;
	struct ctl_lun_list list;
	struct cctl_devlist_data devlist;
	struct cctl_lun *lun;
	struct cctl_port *port;
	XML_Parser parser;
	char str, name;
	int len, retval;

	bzero(&devlist, sizeof(devlist));
	STAILQ_INIT(&devlist.lun_list);
	STAILQ_INIT(&devlist.port_list);

	log_debugx("obtaining previously configured CTL luns from the kernel");

	str = NULL;
	len = 4096;
	retry:
	str = realloc(str, len);
	if (str == NULL)
	log_err(1, "realloc");

	bzero(&list, sizeof(list));
	list.alloc_len = len;
	list.status = CTL_LUN_LIST_NONE;
	list.lun_xml = str;

	if (ioctl(ctl_fd, CTL_LUN_LIST, &list) == -1) {
	log_warn("error issuing CTL_LUN_LIST ioctl");
	free(str);
	return (NULL);
	}

	if (list.status == CTL_LUN_LIST_ERROR) {
	log_warnx("error returned from CTL_LUN_LIST ioctl: %s",
	list.error_str);
	free(str);
	return (NULL);
	}

	if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
	len = len << 1;
	goto retry;
	}

	parser = XML_ParserCreate(NULL);
	if (parser == NULL) {
	log_warnx("unable to create XML parser");
	free(str);
	return (NULL);
	}

	XML_SetUserData(parser, &devlist);
	XML_SetElementHandler(parser, cctl_start_element, cctl_end_element);
	XML_SetCharacterDataHandler(parser, cctl_char_handler);

	retval = XML_Parse(parser, str, strlen(str), 1);
	XML_ParserFree(parser);
	free(str);
	if (retval != 1) {
	log_warnx("XML_Parse failed");
	return (NULL);
	}

	str = NULL;
	len = 4096;
	retry_port:
	str = realloc(str, len);
	if (str == NULL)
	log_err(1, "realloc");

	bzero(&list, sizeof(list));
	list.alloc_len = len;
	list.status = CTL_LUN_LIST_NONE;
	list.lun_xml = str;

	if (ioctl(ctl_fd, CTL_PORT_LIST, &list) == -1) {
	log_warn("error issuing CTL_PORT_LIST ioctl");
	free(str);
	return (NULL);
	}

	if (list.status == CTL_LUN_LIST_ERROR) {
	log_warnx("error returned from CTL_PORT_LIST ioctl: %s",
	list.error_str);
	free(str);
	return (NULL);
	}

	if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
	len = len << 1;
	goto retry_port;
	}

	parser = XML_ParserCreate(NULL);
	if (parser == NULL) {
	log_warnx("unable to create XML parser");
	free(str);
	return (NULL);
	}

	XML_SetUserData(parser, &devlist);
	XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
	XML_SetCharacterDataHandler(parser, cctl_char_handler);

	retval = XML_Parse(parser, str, strlen(str), 1);
	XML_ParserFree(parser);
	free(str);
	if (retval != 1) {
	log_warnx("XML_Parse failed");
	return (NULL);
	}

	conf = conf_new();

	name = NULL;
	STAILQ_FOREACH(port, &devlist.port_list, links) {
	if (strcmp(port->port_frontend, "ha") == 0)
	continue;
	free(name);
	if (port->pp == 0 && port->vp == 0) {
	name = checked_strdup(port->port_name);
	} else if (port->vp == 0) {
	retval = asprintf(&name, "%s/%d",
	port->port_name, port->pp);
	if (retval <= 0)
	log_err(1, "asprintf");
	} else {
	retval = asprintf(&name, "%s/%d/%d",
	port->port_name, port->pp, port->vp);
	if (retval <= 0)
	log_err(1, "asprintf");
	}

	if (port->cfiscsi_target == NULL) {
	log_debugx("CTL port %u \"%s\" wasn't managed by ctld; ",
	port->port_id, name);
	pp = pport_find(conf, name);
	if (pp == NULL) {
	#if 0
	log_debugx("found new kernel port %u \"%s\"",
	port->port_id, name);
	#endif
	pp = pport_new(conf, name, port->port_id);
	if (pp == NULL) {
	log_warnx("pport_new failed");
	continue;
	}
	}
	continue;
	}
	if (port->cfiscsi_state != 1) {
	log_debugx("CTL port %ju is not active (%d); ignoring",
	(uintmax_t)port->port_id, port->cfiscsi_state);
	continue;
	}

	targ = target_find(conf, port->cfiscsi_target);
	if (targ == NULL) {
	#if 0
	log_debugx("found new kernel target %s for CTL port %ld",
	port->cfiscsi_target, port->port_id);
	#endif
	targ = target_new(conf, port->cfiscsi_target);
	if (targ == NULL) {
	log_warnx("target_new failed");
	continue;
	}
	}

	if (port->ctld_portal_group_name == NULL)
	continue;
	pg = portal_group_find(conf, port->ctld_portal_group_name);
	if (pg == NULL) {
	#if 0
	log_debugx("found new kernel portal group %s for CTL port %ld",
	port->ctld_portal_group_name, port->port_id);
	#endif
	pg = portal_group_new(conf, port->ctld_portal_group_name);
	if (pg == NULL) {
	log_warnx("portal_group_new failed");
	continue;
	}
	}
	pg->pg_tag = port->cfiscsi_portal_group_tag;
	cp = port_new(conf, targ, pg);
	if (cp == NULL) {
	log_warnx("port_new failed");
	continue;
	}
	cp->p_ctl_port = port->port_id;
	}
	free(name);

	STAILQ_FOREACH(lun, &devlist.lun_list, links) {
	struct cctl_lun_nv *nv;

	if (lun->ctld_name == NULL) {
	log_debugx("CTL lun %ju wasn't managed by ctld; "
	"ignoring", (uintmax_t)lun->lun_id);
	continue;
	}

	cl = lun_find(conf, lun->ctld_name);
	if (cl != NULL) {
	log_warnx("found CTL lun %ju \"%s\", "
	"also backed by CTL lun %d; ignoring",
	(uintmax_t)lun->lun_id, lun->ctld_name,
	cl->l_ctl_lun);
	continue;
	}

	log_debugx("found CTL lun %ju \"%s\"",
	(uintmax_t)lun->lun_id, lun->ctld_name);

	cl = lun_new(conf, lun->ctld_name);
	if (cl == NULL) {
	log_warnx("lun_new failed");
	continue;
	}
	lun_set_backend(cl, lun->backend_type);
	lun_set_device_type(cl, lun->device_type);
	lun_set_blocksize(cl, lun->blocksize);
	lun_set_device_id(cl, lun->device_id);
	lun_set_serial(cl, lun->serial_number);
	lun_set_size(cl, lun->size_blocks * cl->l_blocksize);
	lun_set_ctl_lun(cl, lun->lun_id);

	STAILQ_FOREACH(nv, &lun->attr_list, links) {
	if (strcmp(nv->name, "file") == 0 \|\|
	strcmp(nv->name, "dev") == 0) {
	lun_set_path(cl, nv->value);
	continue;
	}
	o = option_new(&cl->l_options, nv->name, nv->value);
	if (o == NULL)
	log_warnx("unable to add CTL lun option %s "
	"for CTL lun %ju \"%s\"",
	nv->name, (uintmax_t) lun->lun_id,
	cl->l_name);
	}
	}

	return (conf);
	}

	int
	kernel_lun_add(struct lun *lun)
	{
	struct option *o;
	struct ctl_lun_req req;
	int error;

	bzero(&req, sizeof(req));

	strlcpy(req.backend, lun->l_backend, sizeof(req.backend));
	req.reqtype = CTL_LUNREQ_CREATE;

	req.reqdata.create.blocksize_bytes = lun->l_blocksize;

	if (lun->l_size != 0)
	req.reqdata.create.lun_size_bytes = lun->l_size;

	if (lun->l_ctl_lun >= 0) {
	req.reqdata.create.req_lun_id = lun->l_ctl_lun;
	req.reqdata.create.flags \|= CTL_LUN_FLAG_ID_REQ;
	}

	req.reqdata.create.flags \|= CTL_LUN_FLAG_DEV_TYPE;
	req.reqdata.create.device_type = lun->l_device_type;

	if (lun->l_serial != NULL) {
	strncpy(req.reqdata.create.serial_num, lun->l_serial,
	sizeof(req.reqdata.create.serial_num));
	req.reqdata.create.flags \|= CTL_LUN_FLAG_SERIAL_NUM;
	}

	if (lun->l_device_id != NULL) {
	strncpy(req.reqdata.create.device_id, lun->l_device_id,
	sizeof(req.reqdata.create.device_id));
	req.reqdata.create.flags \|= CTL_LUN_FLAG_DEVID;
	}

	if (lun->l_path != NULL) {
	o = option_find(&lun->l_options, "file");
	if (o != NULL) {
	option_set(o, lun->l_path);
	} else {
	o = option_new(&lun->l_options, "file", lun->l_path);
	assert(o != NULL);
	}
	}

	o = option_find(&lun->l_options, "ctld_name");
	if (o != NULL) {
	option_set(o, lun->l_name);
	} else {
	o = option_new(&lun->l_options, "ctld_name", lun->l_name);
	assert(o != NULL);
	}

	o = option_find(&lun->l_options, "scsiname");
	if (o == NULL && lun->l_scsiname != NULL) {
	o = option_new(&lun->l_options, "scsiname", lun->l_scsiname);
	assert(o != NULL);
	}

	if (!TAILQ_EMPTY(&lun->l_options)) {
	req.args_nvl = nvlist_create(0);
	if (req.args_nvl == NULL) {
	log_warn("error allocating nvlist");
	return (1);
	}

	TAILQ_FOREACH(o, &lun->l_options, o_next)
	nvlist_add_string(req.args_nvl, o->o_name, o->o_value);

	req.args = nvlist_pack(req.args_nvl, &req.args_len);
	if (req.args == NULL) {
	log_warn("error packing nvlist");
	return (1);
	}
	}

	error = ioctl(ctl_fd, CTL_LUN_REQ, &req);
	nvlist_destroy(req.args_nvl);

	if (error != 0) {
	log_warn("error issuing CTL_LUN_REQ ioctl");
	return (1);
	}

	switch (req.status) {
	case CTL_LUN_ERROR:
	log_warnx("LUN creation error: %s", req.error_str);
	return (1);
	case CTL_LUN_WARNING:
	log_warnx("LUN creation warning: %s", req.error_str);
	break;
	case CTL_LUN_OK:
	break;
	default:
	log_warnx("unknown LUN creation status: %d",
	req.status);
	return (1);
	}

	lun_set_ctl_lun(lun, req.reqdata.create.req_lun_id);
	return (0);
	}

	int
	kernel_lun_modify(struct lun *lun)
	{
	struct option *o;
	struct ctl_lun_req req;
	int error;

	bzero(&req, sizeof(req));

	strlcpy(req.backend, lun->l_backend, sizeof(req.backend));
	req.reqtype = CTL_LUNREQ_MODIFY;

	req.reqdata.modify.lun_id = lun->l_ctl_lun;
	req.reqdata.modify.lun_size_bytes = lun->l_size;

	+ if (lun->l_path != NULL) {
	+ o = option_find(&lun->l_options, "file");
	+ if (o != NULL) {
	+ option_set(o, lun->l_path);
	+ } else {
	+ o = option_new(&lun->l_options, "file", lun->l_path);
	+ assert(o != NULL);
	+ }
	+ }
	+
	+ o = option_find(&lun->l_options, "ctld_name");
	+ if (o != NULL) {
	+ option_set(o, lun->l_name);
	+ } else {
	+ o = option_new(&lun->l_options, "ctld_name", lun->l_name);
	+ assert(o != NULL);
	+ }
	+
	+ o = option_find(&lun->l_options, "scsiname");
	+ if (o == NULL && lun->l_scsiname != NULL) {
	+ o = option_new(&lun->l_options, "scsiname", lun->l_scsiname);
	+ assert(o != NULL);
	+ }
	+
	if (!TAILQ_EMPTY(&lun->l_options)) {
	req.args_nvl = nvlist_create(0);
	if (req.args_nvl == NULL) {
	log_warn("error allocating nvlist");
	return (1);
	}

	TAILQ_FOREACH(o, &lun->l_options, o_next)
	nvlist_add_string(req.args_nvl, o->o_name, o->o_value);

	req.args = nvlist_pack(req.args_nvl, &req.args_len);
	if (req.args == NULL) {
	log_warn("error packing nvlist");
	return (1);
	}
	}

	error = ioctl(ctl_fd, CTL_LUN_REQ, &req);
	nvlist_destroy(req.args_nvl);

	if (error != 0) {
	log_warn("error issuing CTL_LUN_REQ ioctl");
	return (1);
	}

	switch (req.status) {
	case CTL_LUN_ERROR:
	log_warnx("LUN modification error: %s", req.error_str);
	return (1);
	case CTL_LUN_WARNING:
	log_warnx("LUN modification warning: %s", req.error_str);
	break;
	case CTL_LUN_OK:
	break;
	default:
	log_warnx("unknown LUN modification status: %d",
	req.status);
	return (1);
	}

	return (0);
	}

	int
	kernel_lun_remove(struct lun *lun)
	{
	struct ctl_lun_req req;

	bzero(&req, sizeof(req));

	strlcpy(req.backend, lun->l_backend, sizeof(req.backend));
	req.reqtype = CTL_LUNREQ_RM;

	req.reqdata.rm.lun_id = lun->l_ctl_lun;

	if (ioctl(ctl_fd, CTL_LUN_REQ, &req) == -1) {
	log_warn("error issuing CTL_LUN_REQ ioctl");
	return (1);
	}

	switch (req.status) {
	case CTL_LUN_ERROR:
	log_warnx("LUN removal error: %s", req.error_str);
	return (1);
	case CTL_LUN_WARNING:
	log_warnx("LUN removal warning: %s", req.error_str);
	break;
	case CTL_LUN_OK:
	break;
	default:
	log_warnx("unknown LUN removal status: %d", req.status);
	return (1);
	}

	return (0);
	}

	void
	kernel_handoff(struct connection *conn)
	{
	struct ctl_iscsi req;

	bzero(&req, sizeof(req));

	req.type = CTL_ISCSI_HANDOFF;
	strlcpy(req.data.handoff.initiator_name,
	conn->conn_initiator_name, sizeof(req.data.handoff.initiator_name));
	strlcpy(req.data.handoff.initiator_addr,
	conn->conn_initiator_addr, sizeof(req.data.handoff.initiator_addr));
	if (conn->conn_initiator_alias != NULL) {
	strlcpy(req.data.handoff.initiator_alias,
	conn->conn_initiator_alias, sizeof(req.data.handoff.initiator_alias));
	}
	memcpy(req.data.handoff.initiator_isid, conn->conn_initiator_isid,
	sizeof(req.data.handoff.initiator_isid));
	strlcpy(req.data.handoff.target_name,
	conn->conn_target->t_name, sizeof(req.data.handoff.target_name));
	if (conn->conn_portal->p_portal_group->pg_offload != NULL) {
	strlcpy(req.data.handoff.offload,
	conn->conn_portal->p_portal_group->pg_offload,
	sizeof(req.data.handoff.offload));
	}
	#ifdef ICL_KERNEL_PROXY
	if (proxy_mode)
	req.data.handoff.connection_id = conn->conn_socket;
	else
	req.data.handoff.socket = conn->conn_socket;
	#else
	req.data.handoff.socket = conn->conn_socket;
	#endif
	req.data.handoff.portal_group_tag =
	conn->conn_portal->p_portal_group->pg_tag;
	if (conn->conn_header_digest == CONN_DIGEST_CRC32C)
	req.data.handoff.header_digest = CTL_ISCSI_DIGEST_CRC32C;
	if (conn->conn_data_digest == CONN_DIGEST_CRC32C)
	req.data.handoff.data_digest = CTL_ISCSI_DIGEST_CRC32C;
	req.data.handoff.cmdsn = conn->conn_cmdsn;
	req.data.handoff.statsn = conn->conn_statsn;
	req.data.handoff.max_recv_data_segment_length =
	conn->conn_max_recv_data_segment_length;
	req.data.handoff.max_send_data_segment_length =
	conn->conn_max_send_data_segment_length;
	req.data.handoff.max_burst_length = conn->conn_max_burst_length;
	req.data.handoff.first_burst_length = conn->conn_first_burst_length;
	req.data.handoff.immediate_data = conn->conn_immediate_data;

	if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1) {
	log_err(1, "error issuing CTL_ISCSI ioctl; "
	"dropping connection");
	}

	if (req.status != CTL_ISCSI_OK) {
	log_errx(1, "error returned from CTL iSCSI handoff request: "
	"%s; dropping connection", req.error_str);
	}
	}

	void
	kernel_limits(const char offload, int max_recv_dsl, int *max_send_dsl,
	int max_burst_length, int first_burst_length)
	{
	struct ctl_iscsi req;
	struct ctl_iscsi_limits_params *cilp;

	bzero(&req, sizeof(req));

	req.type = CTL_ISCSI_LIMITS;
	cilp = (struct ctl_iscsi_limits_params *)&(req.data.limits);
	if (offload != NULL) {
	strlcpy(cilp->offload, offload, sizeof(cilp->offload));
	}

	if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1) {
	log_err(1, "error issuing CTL_ISCSI ioctl; "
	"dropping connection");
	}

	if (req.status != CTL_ISCSI_OK) {
	log_errx(1, "error returned from CTL iSCSI limits request: "
	"%s; dropping connection", req.error_str);
	}

	if (cilp->max_recv_data_segment_length != 0) {
	*max_recv_dsl = cilp->max_recv_data_segment_length;
	*max_send_dsl = cilp->max_recv_data_segment_length;
	}
	if (cilp->max_send_data_segment_length != 0)
	*max_send_dsl = cilp->max_send_data_segment_length;
	if (cilp->max_burst_length != 0)
	*max_burst_length = cilp->max_burst_length;
	if (cilp->first_burst_length != 0)
	*first_burst_length = cilp->first_burst_length;
	if (max_burst_length < first_burst_length)
	first_burst_length = max_burst_length;

	if (offload != NULL) {
	log_debugx("Kernel limits for offload \"%s\" are "
	"MaxRecvDataSegment=%d, max_send_dsl=%d, "
	"MaxBurstLength=%d, FirstBurstLength=%d",
	offload, max_recv_dsl, max_send_dsl, *max_burst_length,
	*first_burst_length);
	} else {
	log_debugx("Kernel limits are "
	"MaxRecvDataSegment=%d, max_send_dsl=%d, "
	"MaxBurstLength=%d, FirstBurstLength=%d",
	max_recv_dsl, max_send_dsl, *max_burst_length,
	*first_burst_length);
	}
	}

	int
	kernel_port_add(struct port *port)
	{
	struct option *o;
	struct ctl_port_entry entry;
	struct ctl_req req;
	struct ctl_lun_map lm;
	struct target *targ = port->p_target;
	struct portal_group *pg = port->p_portal_group;
	char result_buf[NVLIST_BUFSIZE];
	int error, i;

	/* Create iSCSI port. */
	if (port->p_portal_group \|\| port->p_ioctl_port) {
	bzero(&req, sizeof(req));
	req.reqtype = CTL_REQ_CREATE;

	if (port->p_portal_group) {
	strlcpy(req.driver, "iscsi", sizeof(req.driver));
	req.args_nvl = nvlist_create(0);
	nvlist_add_string(req.args_nvl, "cfiscsi_target",
	targ->t_name);
	nvlist_add_string(req.args_nvl,
	"ctld_portal_group_name", pg->pg_name);
	nvlist_add_stringf(req.args_nvl,
	"cfiscsi_portal_group_tag", "%u", pg->pg_tag);

	if (targ->t_alias) {
	nvlist_add_string(req.args_nvl,
	"cfiscsi_target_alias", targ->t_alias);
	}

	TAILQ_FOREACH(o, &pg->pg_options, o_next)
	nvlist_add_string(req.args_nvl, o->o_name,
	o->o_value);
	}

	if (port->p_ioctl_port) {
	strlcpy(req.driver, "ioctl", sizeof(req.driver));
	req.args_nvl = nvlist_create(0);
	nvlist_add_stringf(req.args_nvl, "pp", "%d",
	port->p_ioctl_pp);
	nvlist_add_stringf(req.args_nvl, "vp", "%d",
	port->p_ioctl_vp);
	}

	req.args = nvlist_pack(req.args_nvl, &req.args_len);
	if (req.args == NULL) {
	log_warn("error packing nvlist");
	return (1);
	}

	req.result = result_buf;
	req.result_len = sizeof(result_buf);
	error = ioctl(ctl_fd, CTL_PORT_REQ, &req);
	nvlist_destroy(req.args_nvl);

	if (error != 0) {
	log_warn("error issuing CTL_PORT_REQ ioctl");
	return (1);
	}
	if (req.status == CTL_LUN_ERROR) {
	log_warnx("error returned from port creation request: %s",
	req.error_str);
	return (1);
	}
	if (req.status != CTL_LUN_OK) {
	log_warnx("unknown port creation request status %d",
	req.status);
	return (1);
	}

	req.result_nvl = nvlist_unpack(result_buf, req.result_len, 0);
	if (req.result_nvl == NULL) {
	log_warnx("error unpacking result nvlist");
	return (1);
	}

	port->p_ctl_port = nvlist_get_number(req.result_nvl, "port_id");
	nvlist_destroy(req.result_nvl);
	} else if (port->p_pport) {
	port->p_ctl_port = port->p_pport->pp_ctl_port;

	if (strncmp(targ->t_name, "naa.", 4) == 0 &&
	strlen(targ->t_name) == 20) {
	bzero(&entry, sizeof(entry));
	entry.port_type = CTL_PORT_NONE;
	entry.targ_port = port->p_ctl_port;
	entry.flags \|= CTL_PORT_WWNN_VALID;
	entry.wwnn = strtoull(targ->t_name + 4, NULL, 16);
	if (ioctl(ctl_fd, CTL_SET_PORT_WWNS, &entry) == -1)
	log_warn("CTL_SET_PORT_WWNS ioctl failed");
	}
	}

	/* Explicitly enable mapping to block any access except allowed. */
	lm.port = port->p_ctl_port;
	lm.plun = UINT32_MAX;
	lm.lun = 0;
	error = ioctl(ctl_fd, CTL_LUN_MAP, &lm);
	if (error != 0)
	log_warn("CTL_LUN_MAP ioctl failed");

	/* Map configured LUNs */
	for (i = 0; i < MAX_LUNS; i++) {
	if (targ->t_luns[i] == NULL)
	continue;
	lm.port = port->p_ctl_port;
	lm.plun = i;
	lm.lun = targ->t_luns[i]->l_ctl_lun;
	error = ioctl(ctl_fd, CTL_LUN_MAP, &lm);
	if (error != 0)
	log_warn("CTL_LUN_MAP ioctl failed");
	}

	/* Enable port */
	bzero(&entry, sizeof(entry));
	entry.targ_port = port->p_ctl_port;
	error = ioctl(ctl_fd, CTL_ENABLE_PORT, &entry);
	if (error != 0) {
	log_warn("CTL_ENABLE_PORT ioctl failed");
	return (-1);
	}

	return (0);
	}

	int
	kernel_port_update(struct port port, struct port oport)
	{
	struct ctl_lun_map lm;
	struct target *targ = port->p_target;
	struct target *otarg = oport->p_target;
	int error, i;
	uint32_t olun;

	/* Map configured LUNs and unmap others */
	for (i = 0; i < MAX_LUNS; i++) {
	lm.port = port->p_ctl_port;
	lm.plun = i;
	if (targ->t_luns[i] == NULL)
	lm.lun = UINT32_MAX;
	else
	lm.lun = targ->t_luns[i]->l_ctl_lun;
	if (otarg->t_luns[i] == NULL)
	olun = UINT32_MAX;
	else
	olun = otarg->t_luns[i]->l_ctl_lun;
	if (lm.lun == olun)
	continue;
	error = ioctl(ctl_fd, CTL_LUN_MAP, &lm);
	if (error != 0)
	log_warn("CTL_LUN_MAP ioctl failed");
	}
	return (0);
	}

	int
	kernel_port_remove(struct port *port)
	{
	struct ctl_port_entry entry;
	struct ctl_lun_map lm;
	struct ctl_req req;
	struct target *targ = port->p_target;
	struct portal_group *pg = port->p_portal_group;
	int error;

	/* Disable port */
	bzero(&entry, sizeof(entry));
	entry.targ_port = port->p_ctl_port;
	error = ioctl(ctl_fd, CTL_DISABLE_PORT, &entry);
	if (error != 0) {
	log_warn("CTL_DISABLE_PORT ioctl failed");
	return (-1);
	}

	/* Remove iSCSI or ioctl port. */
	if (port->p_portal_group \|\| port->p_ioctl_port) {
	bzero(&req, sizeof(req));
	strlcpy(req.driver, port->p_ioctl_port ? "ioctl" : "iscsi",
	sizeof(req.driver));
	req.reqtype = CTL_REQ_REMOVE;
	req.args_nvl = nvlist_create(0);
	if (req.args_nvl == NULL)
	log_err(1, "nvlist_create");

	if (port->p_ioctl_port)
	nvlist_add_stringf(req.args_nvl, "port_id", "%d",
	port->p_ctl_port);
	else {
	nvlist_add_string(req.args_nvl, "cfiscsi_target",
	targ->t_name);
	nvlist_add_stringf(req.args_nvl,
	"cfiscsi_portal_group_tag", "%u", pg->pg_tag);
	}

	req.args = nvlist_pack(req.args_nvl, &req.args_len);
	if (req.args == NULL) {
	log_warn("error packing nvlist");
	return (1);
	}

	error = ioctl(ctl_fd, CTL_PORT_REQ, &req);
	nvlist_destroy(req.args_nvl);

	if (error != 0) {
	log_warn("error issuing CTL_PORT_REQ ioctl");
	return (1);
	}
	if (req.status == CTL_LUN_ERROR) {
	log_warnx("error returned from port removal request: %s",
	req.error_str);
	return (1);
	}
	if (req.status != CTL_LUN_OK) {
	log_warnx("unknown port removal request status %d",
	req.status);
	return (1);
	}
	} else {
	/* Disable LUN mapping. */
	lm.port = port->p_ctl_port;
	lm.plun = UINT32_MAX;
	lm.lun = UINT32_MAX;
	error = ioctl(ctl_fd, CTL_LUN_MAP, &lm);
	if (error != 0)
	log_warn("CTL_LUN_MAP ioctl failed");
	}
	return (0);
	}

	#ifdef ICL_KERNEL_PROXY
	void
	kernel_listen(struct addrinfo *ai, bool iser, int portal_id)
	{
	struct ctl_iscsi req;

	bzero(&req, sizeof(req));

	req.type = CTL_ISCSI_LISTEN;
	req.data.listen.iser = iser;
	req.data.listen.domain = ai->ai_family;
	req.data.listen.socktype = ai->ai_socktype;
	req.data.listen.protocol = ai->ai_protocol;
	req.data.listen.addr = ai->ai_addr;
	req.data.listen.addrlen = ai->ai_addrlen;
	req.data.listen.portal_id = portal_id;

	if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1)
	log_err(1, "error issuing CTL_ISCSI ioctl");

	if (req.status != CTL_ISCSI_OK) {
	log_errx(1, "error returned from CTL iSCSI listen: %s",
	req.error_str);
	}
	}

	void
	kernel_accept(int connection_id, int portal_id,
	struct sockaddr client_sa, socklen_t client_salen)
	{
	struct ctl_iscsi req;
	struct sockaddr_storage ss;

	bzero(&req, sizeof(req));

	req.type = CTL_ISCSI_ACCEPT;
	req.data.accept.initiator_addr = (struct sockaddr *)&ss;

	if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1)
	log_err(1, "error issuing CTL_ISCSI ioctl");

	if (req.status != CTL_ISCSI_OK) {
	log_errx(1, "error returned from CTL iSCSI accept: %s",
	req.error_str);
	}

	*connection_id = req.data.accept.connection_id;
	*portal_id = req.data.accept.portal_id;
	*client_salen = req.data.accept.initiator_addrlen;
	memcpy(client_sa, &ss, *client_salen);
	}

	void
	kernel_send(struct pdu *pdu)
	{
	struct ctl_iscsi req;

	bzero(&req, sizeof(req));

	req.type = CTL_ISCSI_SEND;
	req.data.send.connection_id = pdu->pdu_connection->conn_socket;
	req.data.send.bhs = pdu->pdu_bhs;
	req.data.send.data_segment_len = pdu->pdu_data_len;
	req.data.send.data_segment = pdu->pdu_data;

	if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1) {
	log_err(1, "error issuing CTL_ISCSI ioctl; "
	"dropping connection");
	}

	if (req.status != CTL_ISCSI_OK) {
	log_errx(1, "error returned from CTL iSCSI send: "
	"%s; dropping connection", req.error_str);
	}
	}

	void
	kernel_receive(struct pdu *pdu)
	{
	struct connection *conn;
	struct ctl_iscsi req;

	conn = pdu->pdu_connection;
	pdu->pdu_data = malloc(conn->conn_max_recv_data_segment_length);
	if (pdu->pdu_data == NULL)
	log_err(1, "malloc");

	bzero(&req, sizeof(req));

	req.type = CTL_ISCSI_RECEIVE;
	req.data.receive.connection_id = conn->conn_socket;
	req.data.receive.bhs = pdu->pdu_bhs;
	req.data.receive.data_segment_len =
	conn->conn_max_recv_data_segment_length;
	req.data.receive.data_segment = pdu->pdu_data;

	if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1) {
	log_err(1, "error issuing CTL_ISCSI ioctl; "
	"dropping connection");
	}

	if (req.status != CTL_ISCSI_OK) {
	log_errx(1, "error returned from CTL iSCSI receive: "
	"%s; dropping connection", req.error_str);
	}

	}

	#endif /* ICL_KERNEL_PROXY */

	/*
	* XXX: I CANT INTO LATIN
	*/
	void
	kernel_capsicate(void)
	{
	cap_rights_t rights;
	const unsigned long cmds[] = { CTL_ISCSI };

	cap_rights_init(&rights, CAP_IOCTL);
	if (caph_rights_limit(ctl_fd, &rights) < 0)
	log_err(1, "cap_rights_limit");

	if (caph_ioctls_limit(ctl_fd, cmds, nitems(cmds)) < 0)
	log_err(1, "cap_ioctls_limit");

	if (caph_enter() < 0)
	log_err(1, "cap_enter");

	if (cap_sandboxed())
	log_debugx("Capsicum capability mode enabled");
	else
	log_warnx("Capsicum capability mode not supported");
	}

	diff --git a/usr.sbin/kldxref/ef_sparc64.c b/usr.sbin/kldxref/ef_sparc64.c
	deleted file mode 100644
	index 27abfc7b31ec..000000000000
	--- a/usr.sbin/kldxref/ef_sparc64.c
	+++ /dev/null
	@@ -1,71 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 2003 Jake Burkholder.
	- * All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-#include <sys/types.h>
	-#include <machine/elf.h>
	-
	-#include <err.h>
	-#include <string.h>
	-
	-#include "ef.h"
	-
	-/*
	- * Apply relocations to the values we got from the file. `relbase' is the
	- * target relocation address of the section, and `dataoff' is the target
	- * relocation address of the data in `dest'.
	- */
	-int
	-ef_reloc(struct elf_file ef, const void reldata, int reltype, Elf_Off relbase,
	- Elf_Off dataoff, size_t len, void *dest)
	-{
	- const Elf_Rela *a;
	- Elf_Size w;
	-
	- switch (reltype) {
	- case EF_RELOC_RELA:
	- a = reldata;
	- if (relbase + a->r_offset >= dataoff && relbase + a->r_offset <
	- dataoff + len) {
	- switch (ELF_R_TYPE(a->r_info)) {
	- case R_SPARC_RELATIVE:
	- w = a->r_addend + relbase;
	- memcpy((u_char *)dest + (relbase + a->r_offset -
	- dataoff), &w, sizeof(w));
	- break;
	- default:
	- warnx("unhandled relocation type %u",
	- (unsigned int)ELF_R_TYPE(a->r_info));
	- break;
	- }
	- }
	- break;
	- }
	- return (0);
	-}
	diff --git a/usr.sbin/nologin/nologin.8 b/usr.sbin/nologin/nologin.8
	index a6be6ac249fd..90b1d10f28f0 100644
	--- a/usr.sbin/nologin/nologin.8
	+++ b/usr.sbin/nologin/nologin.8
	@@ -1,74 +1,74 @@
	.\" Copyright (c) 1993
	.\" The Regents of the University of California. All rights reserved.
	.\"
	.\" Redistribution and use in source and binary forms, with or without
	.\" modification, are permitted provided that the following conditions
	.\" are met:
	.\" 1. Redistributions of source code must retain the above copyright
	.\" notice, this list of conditions and the following disclaimer.
	.\" 2. Redistributions in binary form must reproduce the above copyright
	.\" notice, this list of conditions and the following disclaimer in the
	.\" documentation and/or other materials provided with the distribution.
	.\" 3. Neither the name of the University nor the names of its contributors
	.\" may be used to endorse or promote products derived from this software
	.\" without specific prior written permission.
	.\"
	.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	.\" SUCH DAMAGE.
	.\"
	.\" @(#)nologin.8 8.1 (Berkeley) 6/19/93
	.\" $FreeBSD$
	.\"
	.Dd April 15, 2020
	.Dt NOLOGIN 8
	.Os
	.Sh NAME
	.Nm nologin
	.Nd politely refuse a login
	.Sh SYNOPSIS
	.Nm
	.Sh DESCRIPTION
	The
	.Nm
	is intended as a replacement shell field for accounts that
	have been disabled.
	.Pp
	When executed,
	.Nm
	-first writes about the login attempt to
	+first logs about the login attempt using
	.Xr syslog 3
	and then displays a message that an account is not available.
	.Pp
	To disable all logins,
	investigate
	.Xr nologin 5 .
	.Sh EXIT STATUS
	The
	.Nm
	utility always exits non-zero.
	.Sh EXAMPLES
	Here is a demonstration of executing
	.Nm :
	.Bd -literal -offset 4n
	$ nologin
	This account is currently not available.
	$ tail -n 1 /var/log/messages
	Mar 30 21:53:07 example.org nologin[65992]: Attempted login by beastie on /dev/pts/18
	.Ed
	.Sh SEE ALSO
	.Xr login 1 ,
	.Xr nologin 5
	.Sh HISTORY
	The
	.Nm
	utility appeared in
	.Bx 4.4 .

File Metadata

Mime Type: application/octet-stream
Expires: Tue, Jul 2, 11:31 AM (2 d)
Storage Engine: chunks
Storage Format: Chunks
Storage Handle: 4Z0Je2yRknrX
Default Alt Text: (6 MB)

Offset	End	Complete
0	4194304	Yes
4194304	7158228	Yes

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions